twisted.internet.threads.deferToThread

Here are the examples of the python api twisted.internet.threads.deferToThread taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

166 Examples 7

Example 1

Project: PSO2Proxy Source File: queryProtocols.py
Function: connection_made
    def connectionMade(self):
        for f in plugins.plugins.onQueryConnection:
            f(self)
        print("[ShipStatus]  PC Client connected " + str(self.transport.getPeer()) + "! Sending ship list packet...")
        d = threads.deferToThread(ships.get_ship_query, myIpAddress, self.transport.getPeer().host)
        d.addCallback(self.send_ship_list)

Example 2

Project: twango Source File: decorators.py
def call_in_thread_ignore_return(function):
    """
    Decorator: Calls the decorated function in a thread.

    Any errors will be logged using Python's logging module.
    """
    def threaded(*args, **kwargs):
        d = threads.deferToThread(function, *args, **kwargs)
        def error(failure):
            logger.error('%(error_line)s - %(error_type)s: %(error_msg)s' % {
                'error_type': str(failure.type).split("'")[1],
                'error_line': failure.getBriefTraceback().split()[-1],
                'error_msg': failure.getErrorMessage(),
            })
        d.addErrback(error)
    return threaded

Example 3

Project: palaver Source File: memcache_storage.py
    def deleteRoom(self, room, owner = None, check_persistent = False, host=''):
        def delete():
            r = self._getRoomFromCache(room, host)
            if check_persistent and r:
                p = r['persistent']            
                if p:
                    return False
        
            if r:
                self._deleteRoom(room, host)
            return True

        return threads.deferToThread(delete)

Example 4

Project: Coherence Source File: youtube_storage.py
    def retrieveSubscriptionFeedItems (self, parent, uri):
        entry = threads.deferToThread(self.yt_service.GetYouTubeSubscriptionEntry, uri)

        def gotEntry(entry):
            if entry is None:
                self.warning("Unable to retrieve subscription items %s", uri)
                return
            feed_uri = entry.feed_link[0].href
            return self.retrieveFeedItems(parent, feed_uri)

        def gotError(error):
            self.warning("ERROR: %s", error)
        entry.addCallbacks(gotEntry, gotError)
        return entry

Example 5

Project: petmail Source File: common.py
Function: cli
    def cli(self, *args, **kwargs):
        stdout, stderr = StringIO(), StringIO()
        d = threads.deferToThread(runner.run, list(args), stdout, stderr,
                                  kwargs.get("petmail"))
        def _done(rc):
            return stdout.getvalue(), stderr.getvalue(), rc
        d.addCallback(_done)
        return d

Example 6

Project: flumotion Source File: server_selection.py
Function: gethostbyname_ex
    def getHostByNameEx(self, name, timeout = (1, 3, 11, 45)):
        if timeout:
            timeoutDelay = reduce(operator.add, timeout)
        else:
            timeoutDelay = 60
        userDeferred = defer.Deferred()
#        lookupDeferred = threads.deferToThreadPool(
#            self.reactor, self.reactor.getThreadPool(),
        lookupDeferred = threads.deferToThread(
            self.socket.gethostbyname_ex, name)
        cancelCall = self.reactor.callLater(
            timeoutDelay, self._cleanup, name, lookupDeferred)
        self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
        lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
        return userDeferred

Example 7

Project: txircd Source File: ircd.py
    def save_module_data(self):
        if self.save_serialized_deferred is None or self.save_serialized_deferred.called:
            self.save_serialized_deferred = deferToThread(self.save_serialized)
            return True
        # Otherwise, there's a save currently happening.  This likely means that
        #  1. We don't need to save now; not THAT much has changed
        #  2. Saving now has the potential to cause problems.
        # We could add self.save_serialized as a callback to the Deferred, but there's
        # not a good way to check whether that's done yet without complicating things (and,
        # as mentioned, there's not a need for it).
        # The return value allows us to work around it currently saving already in the
        # cleanup step (when we absolutely must save regardless), as adding a callback
        # in IRCD.cleanup won't hurt anything.
        return False

Example 8

Project: scrapy Source File: decorators.py
Function: inthread
def inthread(func):
    """Decorator to call a function in a thread and return a deferred with the
    result
    """
    @wraps(func)
    def wrapped(*a, **kw):
        return threads.deferToThread(func, *a, **kw)
    return wrapped

Example 9

Project: gridsync Source File: sync.py
    def get_remote_metadata(self, dircap, basedir=''):
        metadata = {}
        jobs = []
        logging.debug("Getting remote metadata from %s...", dircap)
        url = '{}uri/{}/?t=json'.format(self.tahoe.node_url, dircap)
        received_data = requests.get(url).json()
        for filename, data in received_data[1]['children'].items():
            path = '/'.join([basedir, filename]).strip('/')
            metadata[path] = {
                'uri': data[1]['ro_uri'],
                'mtime': int(data[1]['metadata']['mtime'])}
            if data[0] == 'dirnode':
                jobs.append(
                    deferToThread(self.get_remote_metadata,
                                  '/'.join([dircap, filename]), path))
        results = blockingCallFromThread(reactor, gatherResults, jobs)
        for result in results:
            metadata.update(result)
        return metadata

Example 10

Project: smap Source File: republisher.py
    def republish(self, key, public, obj):
        # check perms
        if (not public and 
            key not in self.keys and 
            not settings.conf['mongo']['publish_all_private']):
            return

        col = self.mongo.smap.republish
        # pymongo mutates the argument ...
        try:
            insert = sterilize_object(obj)
        except:
            import traceback
            traceback.print_exc()
            raise
        insert['__submitted'] = int(time.time() * 1000)
        insert['__key'] = key
        return threads.deferToThread(col.save, insert)

Example 11

Project: Coherence Source File: youtube_storage.py
    def retrieveFeedItems (self, parent=None, feed_uri=''):
        feed = threads.deferToThread(self.yt_service.GetYouTubeVideoFeed, feed_uri)

        def gotFeed(feed):
            if feed is None:
                self.warning("Unable to retrieve feed %s", feed_uri)
                return
            for entry in feed.entry:
                self.appendVideoEntry(entry, parent)

        def gotError(error):
            self.warning("ERROR: %s", error)

        feed.addCallbacks(gotFeed, gotError)
        return feed

Example 12

Project: awspider Source File: base.py
    def _baseStart(self):
        LOGGER.critical("Checking S3 setup.")
        deferreds = []
        if self.scheduler_server is None:
            deferreds.append(deferToThread(self.setSchedulerServer))
        if self.aws_s3_http_cache_bucket is not None:
            self._testAWSCredentials()
            deferreds.append(
                self.s3.checkAndCreateBucket(self.aws_s3_http_cache_bucket))
        if self.aws_s3_storage_bucket is not None:
            self._testAWSCredentials()
            deferreds.append(
                self.s3.checkAndCreateBucket(self.aws_s3_storage_bucket))
        d = DeferredList(deferreds, consumeErrors=True)
        d.addCallback(self._baseStartCallback)

Example 13

Project: tahoe-lafs Source File: test_runner.py
    def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
        command = [sys.executable] + python_options + ["-m", "allmydata.scripts.runner"] + args

        if stdin is None:
            stdin_stream = None
        else:
            stdin_stream = subprocess.PIPE

        def _run():
            p = subprocess.Popen(command, stdin=stdin_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
            (out, err) = p.communicate(stdin)
            return (out, err, p.returncode)
        return threads.deferToThread(_run)

Example 14

Project: greplin-twisted-utils Source File: context_test.py
  @inline.callbacks
  def testContextIsRestoredWhenExceptionsThrown(self):
    """
    When an error occurs in a deferred, context should be restored appropriately
    This test has a follow up that imposes an ordering requirement based on method name.
    """
    self.assertFalse(context.has('foo'))
    try:
      with context.set(foo='bar'):
        self.assert_(context.has('foo'))
        yield threads.deferToThread(self.crashy)
    except AssertionError:
      pass
    self.assertFalse(context.has('foo'))

Example 15

Project: wolverine Source File: __init__.py
Function: do_request
    def doRequest(self, *args, **kw_args):
        if isinstance(args[0], Request):
            r = args[0]
        else:
            if not args[0].startswith('http://'):
                args = list(args)
                args[0] = self.getURL(args[0])
            r = Request(*args, **kw_args)
        return threads.deferToThread(urlopen, r)

Example 16

Project: deluge Source File: core.py
    def handle_custom_email_notification(self, result, eventtype):
        if not self.config['smtp_enabled']:
            return defer.succeed('SMTP notification not enabled.')
        subject, message = result
        log.debug('Spawning new thread to send email with subject: %s: %s',
                  subject, message)
        # Spawn thread because we don't want Deluge to lock up while we send the
        # email.
        return threads.deferToThread(self._notify_email, subject, message)

Example 17

Project: doubanbot Source File: scheduling.py
    def _maybe_update_prop(self, prop, mprop):
        old_val = getattr(self, prop)
        def f(x):
            new_val = getattr(self, prop)
            if old_val != new_val:
                threads.deferToThread(
                    self._deferred_write, self.short_jid, mprop, new_val)
        return f

Example 18

Project: Piped Source File: provider.py
    @defer.inlineCallbacks
    def startService(self):
        """ Start the reqest in the reactors thread pool. """
        self.running = True
        self.running_service = True

        while self.running_service:
            try:
                yield threads.deferToThread(self._run)
            except Exception as e:
                log.error()

Example 19

Project: vumi Source File: txriak_manager.py
Function: get_index_page
    def get_index_page(self, index_name, start_value, end_value=None,
                       return_terms=None, max_results=None, continuation=None):
        d = deferToThread(
            self._riak_bucket.get_index, index_name, start_value, end_value,
            return_terms=return_terms, max_results=max_results,
            continuation=continuation)
        d.addCallback(VumiTxIndexPage)
        d.addErrback(riakErrorHandler)
        return d

Example 20

Project: GlobaLeaks Source File: helpers.py
    @inlineCallbacks
    def emulate_file_upload(self, token, n):
        """
        This emulates the file upload of an incomplete submission
        """
        for i in range(0, n):
            dummyFile = self.get_dummy_file()
            dummyFile = yield threads.deferToThread(files.dump_file_fs, dummyFile)
            dummyFile['creation_date'] = datetime_null()

            f = files.serialize_memory_file(dummyFile)

            token.associate_file(dummyFile)
            dummyFile['body'].close()

            self.assertFalse({'size', 'content_type', 'name', 'creation_date'} - set(f.keys()))

Example 21

Project: palaver Source File: memcache_storage.py
    def getNicks(self, room, host =''):
        def get():
            nicks = []
            r = self._getRoomFromCache(room, host)
            if r:
                for u in self._get_room_members(room, host).values():
                    nicks.append(u['nick'])
            return nicks

        return threads.deferToThread(get)

Example 22

Project: palaver Source File: memcache_storage.py
Function: change_status
    def changeStatus(self, room, user, show = None, status = None, legacy=True, host = ''):
        return threads.deferToThread(self._changeStatus, room, user, 
                                             show=show, 
                                             status=status, 
                                             legacy=legacy, 
                                             host = host)

Example 23

Project: vumi Source File: txriak_manager.py
Function: call_and_wrap
    def _call_and_wrap(self, func):
        """
        Call a function that touches the network and wrap the result in this
        class.
        """
        d = deferToThread(func)
        d.addCallback(type(self))
        return d

Example 24

Project: ants Source File: files.py
Function: persist_file
    def persist_file(self, path, buf, info, meta=None, headers=None):
        """Upload file to S3 storage"""
        b = self._get_boto_bucket()
        key_name = '%s%s' % (self.prefix, path)
        k = b.new_key(key_name)
        if meta:
            for metakey, metavalue in meta.iteritems():
                k.set_metadata(metakey, str(metavalue))
        h = self.HEADERS.copy()
        if headers:
            h.update(headers)
        buf.seek(0)
        return threads.deferToThread(k.set_contents_from_string, buf.getvalue(),
                                     headers=h, policy=self.POLICY)

Example 25

Project: smap Source File: bacnet.py
    @defer.inlineCallbacks
    def update(self):
        for (dev, obj, path) in self._iter_points():
            try:
                val = yield threads.deferToThread(bacnet.read_prop,
                                                  dev['props'],
                                                  obj['props']['type'],
                                                  obj['props']['instance'],
                                                  bacnet.PROP_PRESENT_VALUE,
                                                  -1)
            except IOError:
                pass
            else:
                self._add(path, float(val))

Example 26

Project: Coherence Source File: youtube_storage.py
Function: upnp_init
    def upnp_init(self):
        self.current_connection_id = None

        if self.server:
            self.server.connection_manager_server.set_variable(0, 'SourceProtocolInfo',
                                                                    ['http-get:*:%s:*' % MPEG4_MIMETYPE],
                                                                    default=True)

        self.wmc_mapping = {'15': self.get_root_id()}

        self.yt_service = YouTubeService()
        self.yt_service.client_id = 'ytapi-JeanMichelSizun-youtubebackendpl-ruabstu7-0'
        self.yt_service.developer_key = 'AI39si7dv2WWffH-s3pfvmw8fTND-cPWeqF1DOcZ8rwTgTPi4fheX7jjQXpn7SG61Ido0Zm_9gYR52TcGog9Pt3iG9Sa88-1yg'
        self.yt_service.email = self.login
        self.yt_service.password = self.password
        self.yt_service.source = 'Coherence UPnP backend'
        if len(self.login) > 0:
            d = threads.deferToThread(self.yt_service.ProgrammaticLogin)

Example 27

Project: gemuo Source File: walk.py
Function: path_find
    def _path_find(self):
        if self.player.position.x == self.destination.x and \
           self.player.position.y == self.destination.y:
            if self.walk.finished():
                self._success()
            return

        from gemuo.path import path_find
        d = threads.deferToThread(path_find, self.map, self.player.position, self.destination)
        d.addCallbacks(self._path_found, self._failure)

Example 28

Project: txTelegramBot Source File: twistedclient.py
    @defer.inlineCallbacks
    def _poll_updates(self, _=None):
        while self._poll:
            yield threads.deferToThread(self._poll_updates_thread)
            if self._poll_backoff:
                d = defer.Deferred()
                reactor.callLater(self._poll_backoff, d.callback, None)
                log.msg('Backing off update poll for %s' % self._poll_backoff)
                self._poll_backoff = 0
                yield d

Example 29

Project: Coherence Source File: youtube_storage.py
    def retrievePlaylistFeedItems (self, parent, playlist_id):

        feed = threads.deferToThread(self.yt_service.GetYouTubePlaylistVideoFeed, playlist_id=playlist_id)

        def gotFeed(feed):
            if feed is None:
                self.warning("Unable to retrieve playlist items %s", feed_uri)
                return
            for entry in feed.entry:
                self.appendVideoEntry(entry, parent)

        def gotError(error):
            self.warning("ERROR: %s", error)

        feed.addCallbacks(gotFeed, gotError)
        return feed

Example 30

Project: vumi-go Source File: api.py
def spawn_celery_task_via_thread(t, *args, **kw):
    """
    Issue a task to a Celery worker using deferToThread.

    :param Task t:
        The Celery task to issue.
    :param list args:
        Postional arguments for the Celery task.
    :param dict kw:
        Keyword arguments for the Celery task.
    """
    return deferToThread(t.delay, *args, **kw)

Example 31

Project: Coherence Source File: youtube_storage.py
    def retrievePlaylistFeeds(self, parent):
        playlists_feed = threads.deferToThread(self.yt_service.GetYouTubePlaylistFeed, username=self.login)

        def gotPlaylists(playlist_video_feed):
            if playlist_video_feed is None:
                self.warning("Unable to retrieve playlists feed")
                return
            for playlist_video_entry in playlist_video_feed.entry:
                title = playlist_video_entry.title.text
                playlist_id = playlist_video_entry.id.text.split("/")[-1]  # FIXME find better way to retrieve the playlist ID

                item = LazyContainer(parent, title, playlist_id, self.refresh, self.retrievePlaylistFeedItems, playlist_id=playlist_id)
                parent.add_child(item, external_id=playlist_id)

        def gotError(error):
            self.warning("ERROR: %s", error)

        playlists_feed.addCallbacks(gotPlaylists, gotError)
        return playlists_feed

Example 32

Project: flumotion Source File: cachedprovider.py
Function: open
    def open(self):
        # Opening source file in a separate thread, as it usually involves
        # accessing a network filesystem (which would block the reactor)
        d = threads.deferToThread(open_stat, self._path)
        d.addCallbacks(self._selectDelegate, self._sourceOpenFailed)

        def _setDelegate(delegate):
            self._delegate = delegate
        d.addCallback(_setDelegate)
        d.addCallback(lambda _: self)
        return d

Example 33

Project: PSO2Proxy Source File: queryProtocols.py
Function: connection_made
    def connectionMade(self):
        port = self.transport.getHost().port
        print('[BlockQuery] %s:%i wants to load-balance on port %i!' % (self.transport.getPeer().host, self.transport.getPeer().port, port))
        d = threads.deferToThread(ships.get_first_block, port, myIpAddress)
        d.addCallback(self.send_block_scrape)
        ships.get_first_block(port, myIpAddress)

Example 34

Project: vumi Source File: txriak_manager.py
Function: next_page
    def next_page(self):
        """
        Fetch the next page of results.

        :returns:
            A new :class:`VumiTxIndexPage` object containing the next page of
            results.
        """
        if not self.has_next_page():
            return succeed(None)
        d = deferToThread(self._index_page.next_page)
        d.addCallback(type(self))
        d.addErrback(riakErrorHandler)
        return d

Example 35

Project: deluge Source File: core.py
    def on_download_complete(self, blocklist):
        """Runs any download clean up functions.

        Args:
            blocklist (str): Path of blocklist.

        Returns:
            Deferred: a Deferred which fires when clean up is done.

        """
        log.debug('Blocklist download complete: %s', blocklist)
        self.is_downloading = False
        return threads.deferToThread(self.update_info, blocklist)

Example 36

Project: flumotion Source File: test_cache_manager.py
    def testMultiThread(self):
        # FIXME: this test can deadlock....
        return

        m = cachemanager.CacheManager(self.stats, self.path,
                                      CACHE_SIZE, True, 0.4, 0.2)
        dl = []
        d = threads.deferToThread(self.fillTestCache2, m)
        dl.append(d)
        threads.deferToThread(self.fillTestCache2, m)
        dl.append(d)
        threads.deferToThread(self.fillTestCache2, m)
        dl.append(d)

        return defer.DeferredList(dl)

Example 37

Project: deluge Source File: test_component.py
Function: start
    def start(self):
        def do_sleep():
            import time
            time.sleep(1)
        d = threads.deferToThread(do_sleep)

        def on_done(result):
            self.start_count += 1
        return d.addCallback(on_done)

Example 38

Project: scrapy Source File: files.py
Function: get_boto_key
    def _get_boto_key(self, path):
        key_name = '%s%s' % (self.prefix, path)
        if self.is_botocore:
            return threads.deferToThread(
                self.s3_client.head_object,
                Bucket=self.bucket,
                Key=key_name)
        else:
            b = self._get_boto_bucket()
            return threads.deferToThread(b.get_key, key_name)

Example 39

Project: flud Source File: ClientPrimitives.py
    def _getSendStore(self, httpconn, nKu, host, port, filekey,
            datafile, metadata, params, headers):
        """
        Check the response for status. 
        """
        deferred2 = threads.deferToThread(httpconn.getresponse)
        deferred2.addCallback(self._getSendStore2, httpconn, nKu, host, port, 
                filekey, datafile, metadata, params, headers)
        deferred2.addErrback(self._errSendStore, "Couldn't get response", 
                headers, nKu, host, port, filekey, datafile, metadata,
                params, httpconn)
        return deferred2

Example 40

Project: palaver Source File: memcache_storage.py
Function: get_role
    def getRole(self, room, user, host=''):
        def role():
            role = None
            u = self._getRoster(room, user, host=host)
            if u:
                role = u['role']
                return role
            
        return threads.deferToThread(role)

Example 41

Project: Piped Source File: process_processors.py
Function: process_input
    @defer.inlineCallbacks
    def process_input(self, input, baton):
        output, stderr = yield threads.deferToThread(self._invoke_dot_in_subprocess, input, type=self.type)
        if stderr:
            e_msg = 'Error while using dot.'
            raise exceptions.PipedError(e_msg, detail=stderr)
        defer.returnValue(output)

Example 42

Project: pixelated-user-agent Source File: contacts_resource.py
    def render_GET(self, request):
        _search_engine = self.search_engine(request)
        query = request.args.get('q', [''])[-1]
        d = deferToThread(lambda: _search_engine.contacts(query))
        d.addCallback(lambda tags: respond_json_deferred(tags, request))

        def handle_error(error):
            print 'Something went wrong'
            import traceback
            traceback.print_exc()
            print error

        d.addErrback(handle_error)

        return server.NOT_DONE_YET

Example 43

Project: GlobaLeaks Source File: base.py
    @defer.inlineCallbacks
    def run(self):
        self.stats_collection_begin()

        try:
            yield threads.deferToThread(self.operation)
        except Exception as e:
            log.err("Exception while performing scheduled operation %s: %s" % \
                    (type(self).__name__, e))

            extract_exception_traceback_and_send_email(e)

        self.stats_collection_end()

Example 44

Project: palaver Source File: memcache_storage.py
Function: change_nick
    def changeNick(self, room, user, nick, host = ''):
        def nick():
            user = user.lower()
            r = self._getRoomFromCache(room, host)
            if r:
                u = self._getRoster(room, user, host)
                u['nick'] = nick
                self._setRoster(room, user, u, host)
                return nick
            else:
                raise groupchat.RoomNotFound

        return threads.deferToThread(nick)

Example 45

Project: smap Source File: data.py
    def _add_data(self, subid, ids, obj):
        """Store the data and metadata contained in a Timeseires
        """
        ids = dict(zip(map(operator.itemgetter('uuid'), obj.itervalues()), ids))
        md = SmapMetadata(self.db)
        meta_deferred = md.add(subid, ids, obj)
        data_deferred = threads.deferToThread(self._add_data_real, ids, obj)        
        d = defer.DeferredList([meta_deferred, data_deferred], 
                               fireOnOneErrback=True, consumeErrors=True)
        # propagate the original error... 
        d.addErrback(lambda x: x.value.subFailure)
        return d

Example 46

Project: pixelated-user-agent Source File: sessions.py
Function: setup_keymanager
    @defer.inlineCallbacks
    def setup_keymanager(self, provider, soledad, account_email, token, uuid):
        keymanager = yield threads.deferToThread(Keymanager,
                                                 provider,
                                                 soledad,
                                                 account_email,
                                                 token,
                                                 uuid)
        defer.returnValue(keymanager)

Example 47

Project: pixelated-user-agent Source File: tags_resource.py
Function: render_get
    def render_GET(self, request):
        _search_engine = self.search_engine(request)
        query = request.args.get('q', [''])[0]
        skip_default_tags = request.args.get('skipDefaultTags', [False])[0]

        d = deferToThread(lambda: _search_engine.tags(query=query, skip_default_tags=skip_default_tags))
        d.addCallback(lambda tags: respond_json_deferred(tags, request))
        d.addErrback(handle_error_deferred, request)

        return NOT_DONE_YET

Example 48

Project: vumi Source File: txriak_manager.py
Function: riak_map_reduce
    def riak_map_reduce(self):
        mapreduce = RiakMapReduce(self.client)
        # Hack: We replace the two methods that hit the network with
        #       deferToThread wrappers to prevent accidental sync calls in
        #       other code.
        run = mapreduce.run
        stream = mapreduce.stream
        mapreduce.run = lambda *a, **kw: deferToThread(run, *a, **kw)
        mapreduce.stream = lambda *a, **kw: deferToThread(stream, *a, **kw)
        return mapreduce

Example 49

Project: txTelegramBot Source File: twistedclient.py
Function: send_method
    def send_method(self, m):
        d = self._lock.acquire()

        def do_send(_):
            return threads.deferToThread(self._send_thread, m)

        def do_release(value):
            self._lock.release()
            return value

        d.addCallback(do_send)
        d.addBoth(do_release)

        return d

Example 50

Project: Coherence Source File: audiocd_storage.py
Function: init
    def __init__(self, server, **kwargs):
        AbstractBackendStore.__init__(self, server, **kwargs)

        self.name = 'audio CD'
        self.device_name = kwargs.get('device_name', "/dev/cdom")

        threads.deferToThread(self.extractAudioCdInfo)
See More Examples - Go to Next Page
Page 1 Selected Page 2 Page 3 Page 4