Here are the examples of the python api twisted.internet.threads.deferToThread taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
166 Examples
3
Example 1
def connectionMade(self):
for f in plugins.plugins.onQueryConnection:
f(self)
print("[ShipStatus] PC Client connected " + str(self.transport.getPeer()) + "! Sending ship list packet...")
d = threads.deferToThread(ships.get_ship_query, myIpAddress, self.transport.getPeer().host)
d.addCallback(self.send_ship_list)
3
Example 2
Project: twango Source File: decorators.py
def call_in_thread_ignore_return(function):
"""
Decorator: Calls the decorated function in a thread.
Any errors will be logged using Python's logging module.
"""
def threaded(*args, **kwargs):
d = threads.deferToThread(function, *args, **kwargs)
def error(failure):
logger.error('%(error_line)s - %(error_type)s: %(error_msg)s' % {
'error_type': str(failure.type).split("'")[1],
'error_line': failure.getBriefTraceback().split()[-1],
'error_msg': failure.getErrorMessage(),
})
d.addErrback(error)
return threaded
3
Example 3
Project: palaver Source File: memcache_storage.py
def deleteRoom(self, room, owner = None, check_persistent = False, host=''):
def delete():
r = self._getRoomFromCache(room, host)
if check_persistent and r:
p = r['persistent']
if p:
return False
if r:
self._deleteRoom(room, host)
return True
return threads.deferToThread(delete)
3
Example 4
Project: Coherence Source File: youtube_storage.py
def retrieveSubscriptionFeedItems (self, parent, uri):
entry = threads.deferToThread(self.yt_service.GetYouTubeSubscriptionEntry, uri)
def gotEntry(entry):
if entry is None:
self.warning("Unable to retrieve subscription items %s", uri)
return
feed_uri = entry.feed_link[0].href
return self.retrieveFeedItems(parent, feed_uri)
def gotError(error):
self.warning("ERROR: %s", error)
entry.addCallbacks(gotEntry, gotError)
return entry
3
Example 5
def cli(self, *args, **kwargs):
stdout, stderr = StringIO(), StringIO()
d = threads.deferToThread(runner.run, list(args), stdout, stderr,
kwargs.get("petmail"))
def _done(rc):
return stdout.getvalue(), stderr.getvalue(), rc
d.addCallback(_done)
return d
3
Example 6
def getHostByNameEx(self, name, timeout = (1, 3, 11, 45)):
if timeout:
timeoutDelay = reduce(operator.add, timeout)
else:
timeoutDelay = 60
userDeferred = defer.Deferred()
# lookupDeferred = threads.deferToThreadPool(
# self.reactor, self.reactor.getThreadPool(),
lookupDeferred = threads.deferToThread(
self.socket.gethostbyname_ex, name)
cancelCall = self.reactor.callLater(
timeoutDelay, self._cleanup, name, lookupDeferred)
self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
return userDeferred
3
Example 7
Project: txircd Source File: ircd.py
def save_module_data(self):
if self.save_serialized_deferred is None or self.save_serialized_deferred.called:
self.save_serialized_deferred = deferToThread(self.save_serialized)
return True
# Otherwise, there's a save currently happening. This likely means that
# 1. We don't need to save now; not THAT much has changed
# 2. Saving now has the potential to cause problems.
# We could add self.save_serialized as a callback to the Deferred, but there's
# not a good way to check whether that's done yet without complicating things (and,
# as mentioned, there's not a need for it).
# The return value allows us to work around it currently saving already in the
# cleanup step (when we absolutely must save regardless), as adding a callback
# in IRCD.cleanup won't hurt anything.
return False
3
Example 8
def inthread(func):
"""Decorator to call a function in a thread and return a deferred with the
result
"""
@wraps(func)
def wrapped(*a, **kw):
return threads.deferToThread(func, *a, **kw)
return wrapped
3
Example 9
Project: gridsync Source File: sync.py
def get_remote_metadata(self, dircap, basedir=''):
metadata = {}
jobs = []
logging.debug("Getting remote metadata from %s...", dircap)
url = '{}uri/{}/?t=json'.format(self.tahoe.node_url, dircap)
received_data = requests.get(url).json()
for filename, data in received_data[1]['children'].items():
path = '/'.join([basedir, filename]).strip('/')
metadata[path] = {
'uri': data[1]['ro_uri'],
'mtime': int(data[1]['metadata']['mtime'])}
if data[0] == 'dirnode':
jobs.append(
deferToThread(self.get_remote_metadata,
'/'.join([dircap, filename]), path))
results = blockingCallFromThread(reactor, gatherResults, jobs)
for result in results:
metadata.update(result)
return metadata
3
Example 10
Project: smap Source File: republisher.py
def republish(self, key, public, obj):
# check perms
if (not public and
key not in self.keys and
not settings.conf['mongo']['publish_all_private']):
return
col = self.mongo.smap.republish
# pymongo mutates the argument ...
try:
insert = sterilize_object(obj)
except:
import traceback
traceback.print_exc()
raise
insert['__submitted'] = int(time.time() * 1000)
insert['__key'] = key
return threads.deferToThread(col.save, insert)
3
Example 11
Project: Coherence Source File: youtube_storage.py
def retrieveFeedItems (self, parent=None, feed_uri=''):
feed = threads.deferToThread(self.yt_service.GetYouTubeVideoFeed, feed_uri)
def gotFeed(feed):
if feed is None:
self.warning("Unable to retrieve feed %s", feed_uri)
return
for entry in feed.entry:
self.appendVideoEntry(entry, parent)
def gotError(error):
self.warning("ERROR: %s", error)
feed.addCallbacks(gotFeed, gotError)
return feed
3
Example 12
Project: awspider Source File: base.py
def _baseStart(self):
LOGGER.critical("Checking S3 setup.")
deferreds = []
if self.scheduler_server is None:
deferreds.append(deferToThread(self.setSchedulerServer))
if self.aws_s3_http_cache_bucket is not None:
self._testAWSCredentials()
deferreds.append(
self.s3.checkAndCreateBucket(self.aws_s3_http_cache_bucket))
if self.aws_s3_storage_bucket is not None:
self._testAWSCredentials()
deferreds.append(
self.s3.checkAndCreateBucket(self.aws_s3_storage_bucket))
d = DeferredList(deferreds, consumeErrors=True)
d.addCallback(self._baseStartCallback)
3
Example 13
Project: tahoe-lafs Source File: test_runner.py
def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
command = [sys.executable] + python_options + ["-m", "allmydata.scripts.runner"] + args
if stdin is None:
stdin_stream = None
else:
stdin_stream = subprocess.PIPE
def _run():
p = subprocess.Popen(command, stdin=stdin_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
(out, err) = p.communicate(stdin)
return (out, err, p.returncode)
return threads.deferToThread(_run)
3
Example 14
Project: greplin-twisted-utils Source File: context_test.py
@inline.callbacks
def testContextIsRestoredWhenExceptionsThrown(self):
"""
When an error occurs in a deferred, context should be restored appropriately
This test has a follow up that imposes an ordering requirement based on method name.
"""
self.assertFalse(context.has('foo'))
try:
with context.set(foo='bar'):
self.assert_(context.has('foo'))
yield threads.deferToThread(self.crashy)
except AssertionError:
pass
self.assertFalse(context.has('foo'))
3
Example 15
def doRequest(self, *args, **kw_args):
if isinstance(args[0], Request):
r = args[0]
else:
if not args[0].startswith('http://'):
args = list(args)
args[0] = self.getURL(args[0])
r = Request(*args, **kw_args)
return threads.deferToThread(urlopen, r)
3
Example 16
Project: deluge Source File: core.py
def handle_custom_email_notification(self, result, eventtype):
if not self.config['smtp_enabled']:
return defer.succeed('SMTP notification not enabled.')
subject, message = result
log.debug('Spawning new thread to send email with subject: %s: %s',
subject, message)
# Spawn thread because we don't want Deluge to lock up while we send the
# email.
return threads.deferToThread(self._notify_email, subject, message)
3
Example 17
Project: doubanbot Source File: scheduling.py
def _maybe_update_prop(self, prop, mprop):
old_val = getattr(self, prop)
def f(x):
new_val = getattr(self, prop)
if old_val != new_val:
threads.deferToThread(
self._deferred_write, self.short_jid, mprop, new_val)
return f
3
Example 18
Project: Piped Source File: provider.py
@defer.inlineCallbacks
def startService(self):
""" Start the reqest in the reactors thread pool. """
self.running = True
self.running_service = True
while self.running_service:
try:
yield threads.deferToThread(self._run)
except Exception as e:
log.error()
3
Example 19
def get_index_page(self, index_name, start_value, end_value=None,
return_terms=None, max_results=None, continuation=None):
d = deferToThread(
self._riak_bucket.get_index, index_name, start_value, end_value,
return_terms=return_terms, max_results=max_results,
continuation=continuation)
d.addCallback(VumiTxIndexPage)
d.addErrback(riakErrorHandler)
return d
3
Example 20
Project: GlobaLeaks Source File: helpers.py
@inlineCallbacks
def emulate_file_upload(self, token, n):
"""
This emulates the file upload of an incomplete submission
"""
for i in range(0, n):
dummyFile = self.get_dummy_file()
dummyFile = yield threads.deferToThread(files.dump_file_fs, dummyFile)
dummyFile['creation_date'] = datetime_null()
f = files.serialize_memory_file(dummyFile)
token.associate_file(dummyFile)
dummyFile['body'].close()
self.assertFalse({'size', 'content_type', 'name', 'creation_date'} - set(f.keys()))
3
Example 21
Project: palaver Source File: memcache_storage.py
def getNicks(self, room, host =''):
def get():
nicks = []
r = self._getRoomFromCache(room, host)
if r:
for u in self._get_room_members(room, host).values():
nicks.append(u['nick'])
return nicks
return threads.deferToThread(get)
3
Example 22
def changeStatus(self, room, user, show = None, status = None, legacy=True, host = ''):
return threads.deferToThread(self._changeStatus, room, user,
show=show,
status=status,
legacy=legacy,
host = host)
3
Example 23
def _call_and_wrap(self, func):
"""
Call a function that touches the network and wrap the result in this
class.
"""
d = deferToThread(func)
d.addCallback(type(self))
return d
3
Example 24
def persist_file(self, path, buf, info, meta=None, headers=None):
"""Upload file to S3 storage"""
b = self._get_boto_bucket()
key_name = '%s%s' % (self.prefix, path)
k = b.new_key(key_name)
if meta:
for metakey, metavalue in meta.iteritems():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
h.update(headers)
buf.seek(0)
return threads.deferToThread(k.set_contents_from_string, buf.getvalue(),
headers=h, policy=self.POLICY)
3
Example 25
Project: smap Source File: bacnet.py
@defer.inlineCallbacks
def update(self):
for (dev, obj, path) in self._iter_points():
try:
val = yield threads.deferToThread(bacnet.read_prop,
dev['props'],
obj['props']['type'],
obj['props']['instance'],
bacnet.PROP_PRESENT_VALUE,
-1)
except IOError:
pass
else:
self._add(path, float(val))
3
Example 26
def upnp_init(self):
self.current_connection_id = None
if self.server:
self.server.connection_manager_server.set_variable(0, 'SourceProtocolInfo',
['http-get:*:%s:*' % MPEG4_MIMETYPE],
default=True)
self.wmc_mapping = {'15': self.get_root_id()}
self.yt_service = YouTubeService()
self.yt_service.client_id = 'ytapi-JeanMichelSizun-youtubebackendpl-ruabstu7-0'
self.yt_service.developer_key = 'AI39si7dv2WWffH-s3pfvmw8fTND-cPWeqF1DOcZ8rwTgTPi4fheX7jjQXpn7SG61Ido0Zm_9gYR52TcGog9Pt3iG9Sa88-1yg'
self.yt_service.email = self.login
self.yt_service.password = self.password
self.yt_service.source = 'Coherence UPnP backend'
if len(self.login) > 0:
d = threads.deferToThread(self.yt_service.ProgrammaticLogin)
3
Example 27
def _path_find(self):
if self.player.position.x == self.destination.x and \
self.player.position.y == self.destination.y:
if self.walk.finished():
self._success()
return
from gemuo.path import path_find
d = threads.deferToThread(path_find, self.map, self.player.position, self.destination)
d.addCallbacks(self._path_found, self._failure)
3
Example 28
Project: txTelegramBot Source File: twistedclient.py
@defer.inlineCallbacks
def _poll_updates(self, _=None):
while self._poll:
yield threads.deferToThread(self._poll_updates_thread)
if self._poll_backoff:
d = defer.Deferred()
reactor.callLater(self._poll_backoff, d.callback, None)
log.msg('Backing off update poll for %s' % self._poll_backoff)
self._poll_backoff = 0
yield d
3
Example 29
Project: Coherence Source File: youtube_storage.py
def retrievePlaylistFeedItems (self, parent, playlist_id):
feed = threads.deferToThread(self.yt_service.GetYouTubePlaylistVideoFeed, playlist_id=playlist_id)
def gotFeed(feed):
if feed is None:
self.warning("Unable to retrieve playlist items %s", feed_uri)
return
for entry in feed.entry:
self.appendVideoEntry(entry, parent)
def gotError(error):
self.warning("ERROR: %s", error)
feed.addCallbacks(gotFeed, gotError)
return feed
3
Example 30
Project: vumi-go Source File: api.py
def spawn_celery_task_via_thread(t, *args, **kw):
"""
Issue a task to a Celery worker using deferToThread.
:param Task t:
The Celery task to issue.
:param list args:
Postional arguments for the Celery task.
:param dict kw:
Keyword arguments for the Celery task.
"""
return deferToThread(t.delay, *args, **kw)
3
Example 31
Project: Coherence Source File: youtube_storage.py
def retrievePlaylistFeeds(self, parent):
playlists_feed = threads.deferToThread(self.yt_service.GetYouTubePlaylistFeed, username=self.login)
def gotPlaylists(playlist_video_feed):
if playlist_video_feed is None:
self.warning("Unable to retrieve playlists feed")
return
for playlist_video_entry in playlist_video_feed.entry:
title = playlist_video_entry.title.text
playlist_id = playlist_video_entry.id.text.split("/")[-1] # FIXME find better way to retrieve the playlist ID
item = LazyContainer(parent, title, playlist_id, self.refresh, self.retrievePlaylistFeedItems, playlist_id=playlist_id)
parent.add_child(item, external_id=playlist_id)
def gotError(error):
self.warning("ERROR: %s", error)
playlists_feed.addCallbacks(gotPlaylists, gotError)
return playlists_feed
3
Example 32
def open(self):
# Opening source file in a separate thread, as it usually involves
# accessing a network filesystem (which would block the reactor)
d = threads.deferToThread(open_stat, self._path)
d.addCallbacks(self._selectDelegate, self._sourceOpenFailed)
def _setDelegate(delegate):
self._delegate = delegate
d.addCallback(_setDelegate)
d.addCallback(lambda _: self)
return d
3
Example 33
def connectionMade(self):
port = self.transport.getHost().port
print('[BlockQuery] %s:%i wants to load-balance on port %i!' % (self.transport.getPeer().host, self.transport.getPeer().port, port))
d = threads.deferToThread(ships.get_first_block, port, myIpAddress)
d.addCallback(self.send_block_scrape)
ships.get_first_block(port, myIpAddress)
3
Example 34
def next_page(self):
"""
Fetch the next page of results.
:returns:
A new :class:`VumiTxIndexPage` object containing the next page of
results.
"""
if not self.has_next_page():
return succeed(None)
d = deferToThread(self._index_page.next_page)
d.addCallback(type(self))
d.addErrback(riakErrorHandler)
return d
3
Example 35
Project: deluge Source File: core.py
def on_download_complete(self, blocklist):
"""Runs any download clean up functions.
Args:
blocklist (str): Path of blocklist.
Returns:
Deferred: a Deferred which fires when clean up is done.
"""
log.debug('Blocklist download complete: %s', blocklist)
self.is_downloading = False
return threads.deferToThread(self.update_info, blocklist)
3
Example 36
Project: flumotion Source File: test_cache_manager.py
def testMultiThread(self):
# FIXME: this test can deadlock....
return
m = cachemanager.CacheManager(self.stats, self.path,
CACHE_SIZE, True, 0.4, 0.2)
dl = []
d = threads.deferToThread(self.fillTestCache2, m)
dl.append(d)
threads.deferToThread(self.fillTestCache2, m)
dl.append(d)
threads.deferToThread(self.fillTestCache2, m)
dl.append(d)
return defer.DeferredList(dl)
3
Example 37
def start(self):
def do_sleep():
import time
time.sleep(1)
d = threads.deferToThread(do_sleep)
def on_done(result):
self.start_count += 1
return d.addCallback(on_done)
3
Example 38
def _get_boto_key(self, path):
key_name = '%s%s' % (self.prefix, path)
if self.is_botocore:
return threads.deferToThread(
self.s3_client.head_object,
Bucket=self.bucket,
Key=key_name)
else:
b = self._get_boto_bucket()
return threads.deferToThread(b.get_key, key_name)
3
Example 39
Project: flud Source File: ClientPrimitives.py
def _getSendStore(self, httpconn, nKu, host, port, filekey,
datafile, metadata, params, headers):
"""
Check the response for status.
"""
deferred2 = threads.deferToThread(httpconn.getresponse)
deferred2.addCallback(self._getSendStore2, httpconn, nKu, host, port,
filekey, datafile, metadata, params, headers)
deferred2.addErrback(self._errSendStore, "Couldn't get response",
headers, nKu, host, port, filekey, datafile, metadata,
params, httpconn)
return deferred2
3
Example 40
def getRole(self, room, user, host=''):
def role():
role = None
u = self._getRoster(room, user, host=host)
if u:
role = u['role']
return role
return threads.deferToThread(role)
3
Example 41
@defer.inlineCallbacks
def process_input(self, input, baton):
output, stderr = yield threads.deferToThread(self._invoke_dot_in_subprocess, input, type=self.type)
if stderr:
e_msg = 'Error while using dot.'
raise exceptions.PipedError(e_msg, detail=stderr)
defer.returnValue(output)
3
Example 42
Project: pixelated-user-agent Source File: contacts_resource.py
def render_GET(self, request):
_search_engine = self.search_engine(request)
query = request.args.get('q', [''])[-1]
d = deferToThread(lambda: _search_engine.contacts(query))
d.addCallback(lambda tags: respond_json_deferred(tags, request))
def handle_error(error):
print 'Something went wrong'
import traceback
traceback.print_exc()
print error
d.addErrback(handle_error)
return server.NOT_DONE_YET
3
Example 43
Project: GlobaLeaks Source File: base.py
@defer.inlineCallbacks
def run(self):
self.stats_collection_begin()
try:
yield threads.deferToThread(self.operation)
except Exception as e:
log.err("Exception while performing scheduled operation %s: %s" % \
(type(self).__name__, e))
extract_exception_traceback_and_send_email(e)
self.stats_collection_end()
3
Example 44
def changeNick(self, room, user, nick, host = ''):
def nick():
user = user.lower()
r = self._getRoomFromCache(room, host)
if r:
u = self._getRoster(room, user, host)
u['nick'] = nick
self._setRoster(room, user, u, host)
return nick
else:
raise groupchat.RoomNotFound
return threads.deferToThread(nick)
3
Example 45
Project: smap Source File: data.py
def _add_data(self, subid, ids, obj):
"""Store the data and metadata contained in a Timeseires
"""
ids = dict(zip(map(operator.itemgetter('uuid'), obj.itervalues()), ids))
md = SmapMetadata(self.db)
meta_deferred = md.add(subid, ids, obj)
data_deferred = threads.deferToThread(self._add_data_real, ids, obj)
d = defer.DeferredList([meta_deferred, data_deferred],
fireOnOneErrback=True, consumeErrors=True)
# propagate the original error...
d.addErrback(lambda x: x.value.subFailure)
return d
3
Example 46
@defer.inlineCallbacks
def setup_keymanager(self, provider, soledad, account_email, token, uuid):
keymanager = yield threads.deferToThread(Keymanager,
provider,
soledad,
account_email,
token,
uuid)
defer.returnValue(keymanager)
3
Example 47
def render_GET(self, request):
_search_engine = self.search_engine(request)
query = request.args.get('q', [''])[0]
skip_default_tags = request.args.get('skipDefaultTags', [False])[0]
d = deferToThread(lambda: _search_engine.tags(query=query, skip_default_tags=skip_default_tags))
d.addCallback(lambda tags: respond_json_deferred(tags, request))
d.addErrback(handle_error_deferred, request)
return NOT_DONE_YET
3
Example 48
def riak_map_reduce(self):
mapreduce = RiakMapReduce(self.client)
# Hack: We replace the two methods that hit the network with
# deferToThread wrappers to prevent accidental sync calls in
# other code.
run = mapreduce.run
stream = mapreduce.stream
mapreduce.run = lambda *a, **kw: deferToThread(run, *a, **kw)
mapreduce.stream = lambda *a, **kw: deferToThread(stream, *a, **kw)
return mapreduce
3
Example 49
def send_method(self, m):
d = self._lock.acquire()
def do_send(_):
return threads.deferToThread(self._send_thread, m)
def do_release(value):
self._lock.release()
return value
d.addCallback(do_send)
d.addBoth(do_release)
return d
3
Example 50
def __init__(self, server, **kwargs):
AbstractBackendStore.__init__(self, server, **kwargs)
self.name = 'audio CD'
self.device_name = kwargs.get('device_name', "/dev/cdom")
threads.deferToThread(self.extractAudioCdInfo)