twisted.internet.task.Cooperator

Here are the examples of the python api twisted.internet.task.Cooperator taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

61 Examples 7

Example 1

Project: scrapy
Source File: defer.py
View license
def parallel(iterable, count, callable, *args, **named):
    """Execute a callable over the objects in the given iterable, in parallel,
    using no more than ``count`` concurrent calls.

    Taken from: http://jcalderone.livejournal.com/24285.html
    """
    coop = task.Cooperator()
    work = (callable(elem, *args, **named) for elem in iterable)
    return defer.DeferredList([coop.coiterate(work) for _ in range(count)])

Example 2

Project: txrdq
Source File: rdq.py
View license
    def __init__(self, func, width=0, size=None, backlog=None):
        self._func = func
        self.stopped = self.paused = False
        self._queue = DeferredPriorityQueue(size, backlog)
        self._pool = DeferredPool()
        self._coop = task.Cooperator()
        self._currentWidth = 0
        self.pendingStops = 0
        self._underway = set()
        self.width = width

Example 3

Project: treq
Source File: test_multipart.py
View license
    def setUp(self):
        """
        Create a L{Cooperator} hooked up to an easily controlled, deterministic
        scheduler to use with L{MultiPartProducer}.
        """
        self._scheduled = []
        self.cooperator = task.Cooperator(
            self._termination, self._scheduled.append)

Example 4

Project: ants
Source File: defer.py
View license
def parallel(iterable, count, callable, *args, **named):
    """Execute a callable over the objects in the given iterable, in parallel,
    using no more than ``count`` concurrent calls.

    Taken from: http://jcalderone.livejournal.com/24285.html
    """
    coop = task.Cooperator()
    work = (callable(elem, *args, **named) for elem in iterable)
    return defer.DeferredList([coop.coiterate(work) for i in xrange(count)])

Example 5

Project: twisted-gears
Source File: worker.py
View license
def worker(gearman):
    w = client.GearmanWorker(gearman)
    w.setId("exampleworker")
    w.registerFunction("test", run_test)

    coop = task.Cooperator()
    for i in range(5):
        ## Ramp up start
        reactor.callLater(0.1 * i, lambda: coop.coiterate(w.doJobs()))

Example 6

Project: curler
Source File: service.py
View license
    def start_work(self):
        worker = client.GearmanWorker(self)
        worker.registerFunction(self.job_queue, self.handle_job)

        log.msg('Firing up %d workers...' % self.num_workers)
        coop = task.Cooperator()
        for i in range(self.num_workers):
            reactor.callLater(0.1 * i, lambda: coop.coiterate(worker.doJobs()))

Example 7

Project: mythbox
Source File: test_cooperator.py
View license
    def testStoppedRejectsNewTasks(self):
        """
        Test that Cooperators refuse new tasks when they have been stopped.
        """
        def testwith(stuff):
            c = task.Cooperator()
            c.stop()
            d = c.coiterate(iter(()), stuff)
            d.addCallback(self.cbIter)
            d.addErrback(self.ebIter)
            return d.addCallback(lambda result:
                                 self.assertEquals(result, self.RESULT))
        return testwith(None).addCallback(lambda ign: testwith(defer.Deferred()))

Example 8

Project: mythbox
Source File: test_cooperator.py
View license
    def testStopRunning(self):
        """
        Test that a running iterator will not run to completion when the
        cooperator is stopped.
        """
        c = task.Cooperator()
        def myiter():
            for myiter.value in range(3):
                yield myiter.value
        myiter.value = -1
        d = c.coiterate(myiter())
        d.addCallback(self.cbIter)
        d.addErrback(self.ebIter)
        c.stop()
        def doasserts(result):
            self.assertEquals(result, self.RESULT)
            self.assertEquals(myiter.value, -1)
        d.addCallback(doasserts)
        return d

Example 9

Project: mythbox
Source File: test_cooperator.py
View license
    def testUnexpectedError(self):
        c = task.Cooperator()
        def myiter():
            if 0:
                yield None
            else:
                raise RuntimeError()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 10

Project: mythbox
Source File: test_cooperator.py
View license
    def testUnexpectedErrorActuallyLater(self):
        def myiter():
            D = defer.Deferred()
            reactor.callLater(0, D.errback, RuntimeError())
            yield D

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 11

Project: mythbox
Source File: test_cooperator.py
View license
    def testUnexpectedErrorNotActuallyLater(self):
        def myiter():
            yield defer.fail(RuntimeError())

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 12

Project: mythbox
Source File: test_cooperator.py
View license
    def testCooperation(self):
        L = []
        def myiter(things):
            for th in things:
                L.append(th)
                yield None

        groupsOfThings = ['abc', (1, 2, 3), 'def', (4, 5, 6)]

        c = task.Cooperator()
        tasks = []
        for stuff in groupsOfThings:
            tasks.append(c.coiterate(myiter(stuff)))

        return defer.DeferredList(tasks).addCallback(
            lambda ign: self.assertEquals(tuple(L), sum(zip(*groupsOfThings), ())))

Example 13

Project: mythbox
Source File: test_cooperator.py
View license
    def setUp(self):
        """
        Create a cooperator with a fake scheduler and a termination predicate
        that ensures only one unit of work will take place per tick.
        """
        self._doDeferNext = False
        self._doStopNext = False
        self._doDieNext = False
        self.work = []
        self.scheduler = FakeScheduler()
        self.cooperator = task.Cooperator(
            scheduler=self.scheduler,
            # Always stop after one iteration of work (return a function which
            # returns a function which always returns True)
            terminationPredicateFactory=lambda: lambda: True)
        self.task = self.cooperator.cooperate(self.worker())
        self.cooperator.start()

Example 14

Project: mythbox
Source File: test_cooperator.py
View license
    def testStoppedRejectsNewTasks(self):
        """
        Test that Cooperators refuse new tasks when they have been stopped.
        """
        def testwith(stuff):
            c = task.Cooperator()
            c.stop()
            d = c.coiterate(iter(()), stuff)
            d.addCallback(self.cbIter)
            d.addErrback(self.ebIter)
            return d.addCallback(lambda result:
                                 self.assertEquals(result, self.RESULT))
        return testwith(None).addCallback(lambda ign: testwith(defer.Deferred()))

Example 15

Project: mythbox
Source File: test_cooperator.py
View license
    def testStopRunning(self):
        """
        Test that a running iterator will not run to completion when the
        cooperator is stopped.
        """
        c = task.Cooperator()
        def myiter():
            for myiter.value in range(3):
                yield myiter.value
        myiter.value = -1
        d = c.coiterate(myiter())
        d.addCallback(self.cbIter)
        d.addErrback(self.ebIter)
        c.stop()
        def doasserts(result):
            self.assertEquals(result, self.RESULT)
            self.assertEquals(myiter.value, -1)
        d.addCallback(doasserts)
        return d

Example 16

Project: mythbox
Source File: test_cooperator.py
View license
    def testUnexpectedError(self):
        c = task.Cooperator()
        def myiter():
            if 0:
                yield None
            else:
                raise RuntimeError()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 17

Project: mythbox
Source File: test_cooperator.py
View license
    def testUnexpectedErrorActuallyLater(self):
        def myiter():
            D = defer.Deferred()
            reactor.callLater(0, D.errback, RuntimeError())
            yield D

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 18

Project: mythbox
Source File: test_cooperator.py
View license
    def testUnexpectedErrorNotActuallyLater(self):
        def myiter():
            yield defer.fail(RuntimeError())

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 19

Project: mythbox
Source File: test_cooperator.py
View license
    def testCooperation(self):
        L = []
        def myiter(things):
            for th in things:
                L.append(th)
                yield None

        groupsOfThings = ['abc', (1, 2, 3), 'def', (4, 5, 6)]

        c = task.Cooperator()
        tasks = []
        for stuff in groupsOfThings:
            tasks.append(c.coiterate(myiter(stuff)))

        return defer.DeferredList(tasks).addCallback(
            lambda ign: self.assertEquals(tuple(L), sum(zip(*groupsOfThings), ())))

Example 20

Project: mythbox
Source File: test_cooperator.py
View license
    def setUp(self):
        """
        Create a cooperator with a fake scheduler and a termination predicate
        that ensures only one unit of work will take place per tick.
        """
        self._doDeferNext = False
        self._doStopNext = False
        self._doDieNext = False
        self.work = []
        self.scheduler = FakeScheduler()
        self.cooperator = task.Cooperator(
            scheduler=self.scheduler,
            # Always stop after one iteration of work (return a function which
            # returns a function which always returns True)
            terminationPredicateFactory=lambda: lambda: True)
        self.task = self.cooperator.cooperate(self.worker())
        self.cooperator.start()

Example 21

Project: riko
Source File: itertools.py
View license
def get_task():
    if reactor.fake:
        task = Cooperator(scheduler=FakeReactor().callLater)
    else:
        task = real_task

    return task

Example 22

Project: riko
Source File: itertools.py
View license
def get_task():
    if reactor.fake:
        task = Cooperator(scheduler=FakeReactor().callLater)
    else:
        task = real_task

    return task

Example 23

Project: SubliminalCollaborator
Source File: test_tls.py
View license
    def buildReactor(self):
        reactor = ReactorBuilder.buildReactor(self)
        try:
            from twisted.protocols import tls
        except ImportError:
            return reactor

        # Patch twisted.protocols.tls to use this reactor, until we get
        # around to fixing #5206, or the TLS code uses an explicit reactor:
        cooperator = Cooperator(
            scheduler=lambda x: reactor.callLater(0.00001, x))
        self.patch(tls, "cooperate", cooperator.cooperate)
        return reactor

Example 24

Project: SubliminalCollaborator
Source File: test_tls.py
View license
    def buildReactor(self):
        reactor = ReactorBuilder.buildReactor(self)
        try:
            from twisted.protocols import tls
        except ImportError:
            return reactor

        # Patch twisted.protocols.tls to use this reactor, until we get
        # around to fixing #5206, or the TLS code uses an explicit reactor:
        cooperator = Cooperator(
            scheduler=lambda x: reactor.callLater(0.00001, x))
        self.patch(tls, "cooperate", cooperator.cooperate)
        return reactor

Example 25

View license
    def testStoppedRejectsNewTasks(self):
        """
        Test that Cooperators refuse new tasks when they have been stopped.
        """
        def testwith(stuff):
            c = task.Cooperator()
            c.stop()
            d = c.coiterate(iter(()), stuff)
            d.addCallback(self.cbIter)
            d.addErrback(self.ebIter)
            return d.addCallback(lambda result:
                                 self.assertEqual(result, self.RESULT))
        return testwith(None).addCallback(lambda ign: testwith(defer.Deferred()))

Example 26

View license
    def testStopRunning(self):
        """
        Test that a running iterator will not run to completion when the
        cooperator is stopped.
        """
        c = task.Cooperator()
        def myiter():
            for myiter.value in range(3):
                yield myiter.value
        myiter.value = -1
        d = c.coiterate(myiter())
        d.addCallback(self.cbIter)
        d.addErrback(self.ebIter)
        c.stop()
        def doasserts(result):
            self.assertEqual(result, self.RESULT)
            self.assertEqual(myiter.value, -1)
        d.addCallback(doasserts)
        return d

Example 27

View license
    def testUnexpectedError(self):
        c = task.Cooperator()
        def myiter():
            if 0:
                yield None
            else:
                raise RuntimeError()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 28

View license
    def testUnexpectedErrorActuallyLater(self):
        def myiter():
            D = defer.Deferred()
            reactor.callLater(0, D.errback, RuntimeError())
            yield D

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 29

View license
    def testUnexpectedErrorNotActuallyLater(self):
        def myiter():
            yield defer.fail(RuntimeError())

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 30

View license
    def testCooperation(self):
        L = []
        def myiter(things):
            for th in things:
                L.append(th)
                yield None

        groupsOfThings = ['abc', (1, 2, 3), 'def', (4, 5, 6)]

        c = task.Cooperator()
        tasks = []
        for stuff in groupsOfThings:
            tasks.append(c.coiterate(myiter(stuff)))

        return defer.DeferredList(tasks).addCallback(
            lambda ign: self.assertEqual(tuple(L), sum(zip(*groupsOfThings), ())))

Example 31

View license
    def setUp(self):
        """
        Create a cooperator with a fake scheduler and a termination predicate
        that ensures only one unit of work will take place per tick.
        """
        self._doDeferNext = False
        self._doStopNext = False
        self._doDieNext = False
        self.work = []
        self.scheduler = FakeScheduler()
        self.cooperator = task.Cooperator(
            scheduler=self.scheduler,
            # Always stop after one iteration of work (return a function which
            # returns a function which always returns True)
            terminationPredicateFactory=lambda: lambda: True)
        self.task = self.cooperator.cooperate(self.worker())
        self.cooperator.start()

Example 32

View license
    def testStoppedRejectsNewTasks(self):
        """
        Test that Cooperators refuse new tasks when they have been stopped.
        """
        def testwith(stuff):
            c = task.Cooperator()
            c.stop()
            d = c.coiterate(iter(()), stuff)
            d.addCallback(self.cbIter)
            d.addErrback(self.ebIter)
            return d.addCallback(lambda result:
                                 self.assertEqual(result, self.RESULT))
        return testwith(None).addCallback(lambda ign: testwith(defer.Deferred()))

Example 33

View license
    def testStopRunning(self):
        """
        Test that a running iterator will not run to completion when the
        cooperator is stopped.
        """
        c = task.Cooperator()
        def myiter():
            for myiter.value in range(3):
                yield myiter.value
        myiter.value = -1
        d = c.coiterate(myiter())
        d.addCallback(self.cbIter)
        d.addErrback(self.ebIter)
        c.stop()
        def doasserts(result):
            self.assertEqual(result, self.RESULT)
            self.assertEqual(myiter.value, -1)
        d.addCallback(doasserts)
        return d

Example 34

View license
    def testUnexpectedError(self):
        c = task.Cooperator()
        def myiter():
            if 0:
                yield None
            else:
                raise RuntimeError()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 35

View license
    def testUnexpectedErrorActuallyLater(self):
        def myiter():
            D = defer.Deferred()
            reactor.callLater(0, D.errback, RuntimeError())
            yield D

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 36

View license
    def testUnexpectedErrorNotActuallyLater(self):
        def myiter():
            yield defer.fail(RuntimeError())

        c = task.Cooperator()
        d = c.coiterate(myiter())
        return self.assertFailure(d, RuntimeError)

Example 37

View license
    def testCooperation(self):
        L = []
        def myiter(things):
            for th in things:
                L.append(th)
                yield None

        groupsOfThings = ['abc', (1, 2, 3), 'def', (4, 5, 6)]

        c = task.Cooperator()
        tasks = []
        for stuff in groupsOfThings:
            tasks.append(c.coiterate(myiter(stuff)))

        return defer.DeferredList(tasks).addCallback(
            lambda ign: self.assertEqual(tuple(L), sum(zip(*groupsOfThings), ())))

Example 38

View license
    def setUp(self):
        """
        Create a cooperator with a fake scheduler and a termination predicate
        that ensures only one unit of work will take place per tick.
        """
        self._doDeferNext = False
        self._doStopNext = False
        self._doDieNext = False
        self.work = []
        self.scheduler = FakeScheduler()
        self.cooperator = task.Cooperator(
            scheduler=self.scheduler,
            # Always stop after one iteration of work (return a function which
            # returns a function which always returns True)
            terminationPredicateFactory=lambda: lambda: True)
        self.task = self.cooperator.cooperate(self.worker())
        self.cooperator.start()

Example 39

Project: oxidizr
Source File: crawl.py
View license
def parallel(iterable, count, callable, *args, **named):
    coop = task.Cooperator()
    work = (callable(elem, *args, **named) for elem in iterable)
    return defer.DeferredList([coop.coiterate(work) for i in xrange(count)])

Example 40

Project: oxidizr
Source File: crawler.py
View license
def parallel(iterable, count, callable, *args, **named):
    coop = task.Cooperator()
    work = (callable(elem, *args, **named) for elem in iterable)
    return defer.DeferredList([coop.coiterate(work) for i in xrange(count)])

Example 41

Project: hookah
Source File: queue.py
View license
    def __init__(self, consumer=Consumer):
        self.q = defer.DeferredQueue()
        self.keepGoing = True
        self.consumer = consumer
        self.cooperator = task.Cooperator()

Example 42

Project: wiremaps
Source File: core.py
View license
    def startExploration(self):
        """Start to explore the range of IP.

        We try to explore several IP in parallel. The parallelism is
        defined in the configuration file.
        """

        def doWork(remaining):
            for ip, community in remaining:
                for x in list(ip):
                    if ip.net() == ip.broadcast() or (x != ip.net() and x != ip.broadcast()):
                        d = self.startExploreIP(x, community)
                        d.addErrback(self.reportError, x)
                        yield d

        # Don't explore if already exploring
        if self.exploring:
            raise exception.CollectorAlreadyRunning(
                "Exploration still running")
        self.exploring = True
        print "Start exploration..."

        # Expand list of IP to explore
        self.enumerateIP()

        # Start exploring
        dl = []
        coop = task.Cooperator()
        work = doWork(self.ips)
        for i in xrange(self.config['parallel']):
            d = coop.coiterate(work)
            dl.append(d)
        defer.DeferredList(dl).addCallback(self.stopExploration)

Example 43

Project: Tickery
Source File: ftwitter.py
View license
@defer.inlineCallbacks
def addUserByScreenname(cache, endpoint, userJob):
    # We must at least create the user.
    userJob.workToDo = WORK_TO_CREATE_A_FRIEND
    userJob.workDone = 0
    screenname = userJob.screenname
    log.msg('Adding user %r' % screenname)

    def catchUnknownScreenname(fail):
        fail.trap(error.Error)
        if int(fail.value.status) != http.NOT_FOUND:
            return fail
        return defer.fail(UnknownScreenname(screenname))

    def catchProtectedScreenname(fail):
        fail.trap(error.Error)
        if int(fail.value.status) != http.UNAUTHORIZED:
            return fail
        return defer.fail(ProtectedScreenname(screenname))

    d = cache.friendsIdCache[screenname]
    d.addErrback(catchUnknownScreenname)
    d.addErrback(catchProtectedScreenname)
    friendUids = yield d
    log.msg('Got %d friends for user %r' % (len(friendUids), screenname))

    # Make a tag for this new user to mark their friends with.
    ns = Namespace(TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME)
    d = ns.createTag(endpoint, screenname.lower(),
        "A tag used to mark %s's Twitter friends." % screenname, False)
    # TODO: check the X-FluidDB-Error-Class header in the errback to make
    # sure it really got a namespace already exists error.
    d.addErrback(_ignoreHTTPStatus, http.PRECONDITION_FAILED)
    yield d
    # Note: the call to createTag (above) will return a Tag instance when
    # txFluidDB gets fixed.
    friendTag = Tag(TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME,
                    screenname.lower())
    friendTagPath = friendTag.getPath()
    log.msg('Created Twitter friends tag %s' % friendTagPath)

    def _madeUserDone(userObject, user):
        userJob.workDone += WORK_TO_CREATE_A_FRIEND
        cache.extraTwitterTagsPool.add(
            addExtraTwitterTags(endpoint, userObject, user))
        return userObject

    def _madeUserErr(failure):
        userJob.workDone += WORK_TO_CREATE_A_FRIEND
        return failure

    def _tagFriendDone():
        userJob.workDone += WORK_TO_TAG_A_FRIEND

    def makeUser(user, thisIndex=None, totalToAdd=None):
        newName = user['screen_name']
        if thisIndex is not None:
            log.msg('Making user %r, friend %d/%d of %r.' %
                    (newName, thisIndex, totalToAdd, screenname))
        else:
            log.msg('Making user %r.' % newName)
        d = cache.oidUidScreennameCache.objectByUid(user['id'], newName)
        d.addCallbacks(_madeUserDone, _madeUserErr, callbackArgs=(user,))
        return d

    def _ignore404uid(fail, uid):
        fail.trap(error.Error)
        if int(fail.value.status) == http.NOT_FOUND:
            log.msg('Twitter uid %d is no longer found (404). Ignoring.' % uid)
            cache.userCache.removeUid(uid)
            cache.oidUidScreennameCache.removeUid(uid)
        else:
            log.msg('Failure fetching Twitter uid %d:' % uid)
            log.err(fail)

    def makeCreateUserJobs(friendsToAdd):
        nToAdd = len(friendsToAdd)
        for i, friendUid in enumerate(friendsToAdd):
            if userJob.canceled():
                log.msg('Detected cancelation of screenname %r.' % screenname)
                raise StopIteration
            d = cache.userCache.userByUid(friendUid)
            d.addCallbacks(makeUser, _ignore404uid,
                           callbackArgs=(i + 1, nToAdd),
                           errbackArgs=(friendUid,))
            yield d

    @defer.inlineCallbacks
    def addFriend(friendName, thisIndex, totalToAdd):
        log.msg('About to mark user %r as a friend %d/%d of %r.' %
                (friendName, thisIndex, totalToAdd, screenname))
        d = cache.oidUidScreennameCache.objectIdByScreenname(friendName)
        d.addErrback(log.err)
        objectId = yield d
        log.msg('Marking user %r as a friend %d/%d of %r' %
                (friendName, thisIndex, totalToAdd, screenname))
        if objectId is not None:
            o = Object(objectId)
            yield o.set(endpoint, friendTag, None)
            log.msg('Marked user %r as a friend %d/%d of %r' %
                    (friendName, thisIndex, totalToAdd, screenname))
        _tagFriendDone()

    def makeTagFriendsJobs():
        nFriendUids = len(friendUids)
        for i, friendUid in enumerate(friendUids):
            if userJob.canceled():
                log.msg('Detected cancelation of screenname %r.' % screenname)
                raise StopIteration
            d = cache.userCache.screennameByUid(friendUid)
            d.addCallbacks(addFriend, _ignore404uid,
                           callbackArgs=(i + 1, nFriendUids),
                           errbackArgs=(friendUid,))
            yield d

    # Get screename's id and add them as a Twitter user.
    user = yield cache.userCache.userByScreenname(screenname)
    userObject = yield makeUser(user)
    log.msg('User object for %r is %r' % (screenname, userObject))

    # Add the amount of work will it be to tag all friends.
    userJob.workToDo += (len(friendUids) * WORK_TO_TAG_A_FRIEND)

    # Figure out the work will it be to create whatever friends are needed.
    friendsToAdd = [fid for fid in friendUids
                    if not cache.oidUidScreennameCache.knownUid(fid)]
    nFriendsToAdd = len(friendsToAdd)
    log.msg('Must create %d new user objects as friends of %r.' %
            (nFriendsToAdd, screenname))

    if nFriendsToAdd and not userJob.canceled():
        userJob.workToDo += (nFriendsToAdd * WORK_TO_CREATE_A_FRIEND)
        start = time.time()

        # Create Fluidinfo objects for all the friends that we don't yet know
        # about.
        jobs = makeCreateUserJobs(friendsToAdd)
        deferreds = []
        coop = task.Cooperator()
        for i in xrange(MAX_SIMULTANEOUS_REQUESTS):
            d = coop.coiterate(jobs)
            d.addErrback(log.err)
            deferreds.append(d)
        yield defer.DeferredList(deferreds)

        if not userJob.canceled():
            elapsed = time.time() - start
            log.msg('Created %d new friend (of %r) objects in %.2f seconds. '
                    'Mean %.4f' % (nFriendsToAdd, screenname, elapsed,
                                   float(elapsed / nFriendsToAdd)))

    if friendUids and not userJob.canceled():
        # Tag all friends.
        start = time.time()
        jobs = makeTagFriendsJobs()
        deferreds = []
        coop = task.Cooperator()
        for i in xrange(MAX_SIMULTANEOUS_REQUESTS):
            d = coop.coiterate(jobs)
            d.addErrback(log.err)
            deferreds.append(d)
        log.msg('About to yield friend tagging DL for %r' % screenname)
        yield defer.DeferredList(deferreds)
        log.msg('Friend tagging DL finished for %r' % screenname)

        if not userJob.canceled():
            elapsed = time.time() - start
            nFriendsUids = len(friendUids)
            log.msg('Tagged %d objects as being a friend of %r in %.2f '
                    'seconds. Mean = %.4f' % (nFriendsUids, screenname,
                    elapsed, float(elapsed / nFriendsUids)))

    if userJob.canceled():
        log.msg('Canceled addUserByScreenname for %r.' % screenname)
        raise Canceled(screenname)
    else:
        # Add the updated tag to the user's object.
        log.msg('Adding updated tag to user object for %r' % screenname)
        yield userObject.set(endpoint, updatedTag, int(time.time()))
        log.msg('Successfully added screenname %r.' % (screenname,))

    userJob.workDone = userJob.workToDo

Example 44

Project: mythbox
Source File: internet.py
View license
    def __init__(self):
        self.coop = task.Cooperator(started=False)

Example 45

Project: mythbox
Source File: internet.py
View license
    def __init__(self):
        self.coop = task.Cooperator(started=False)

Example 46

Project: mythbox
Source File: test_cooperator.py
View license
    def testStopOutstanding(self):
        """
        An iterator run with L{Cooperator.coiterate} paused on a L{Deferred}
        yielded by that iterator will fire its own L{Deferred} (the one
        returned by C{coiterate}) when L{Cooperator.stop} is called.
        """
        testControlD = defer.Deferred()
        outstandingD = defer.Deferred()
        def myiter():
            reactor.callLater(0, testControlD.callback, None)
            yield outstandingD
            self.fail()
        c = task.Cooperator()
        d = c.coiterate(myiter())
        def stopAndGo(ign):
            c.stop()
            outstandingD.callback('arglebargle')

        testControlD.addCallback(stopAndGo)
        d.addCallback(self.cbIter)
        d.addErrback(self.ebIter)

        return d.addCallback(
            lambda result: self.assertEquals(result, self.RESULT))

Example 47

Project: mythbox
Source File: test_cooperator.py
View license
    def testResourceExhaustion(self):
        output = []
        def myiter():
            for i in range(100):
                output.append(i)
                if i == 9:
                    _TPF.stopped = True
                yield i

        class _TPF:
            stopped = False
            def __call__(self):
                return self.stopped

        c = task.Cooperator(terminationPredicateFactory=_TPF)
        c.coiterate(myiter()).addErrback(self.ebIter)
        c._delayedCall.cancel()
        # testing a private method because only the test case will ever care
        # about this, so we have to carefully clean up after ourselves.
        c._tick()
        c.stop()
        self.failUnless(_TPF.stopped)
        self.assertEquals(output, range(10))

Example 48

Project: mythbox
Source File: test_cooperator.py
View license
    def testCallbackReCoiterate(self):
        """
        If a callback to a deferred returned by coiterate calls coiterate on
        the same Cooperator, we should make sure to only do the minimal amount
        of scheduling work.  (This test was added to demonstrate a specific bug
        that was found while writing the scheduler.)
        """
        calls = []

        class FakeCall:
            def __init__(self, func):
                self.func = func

            def __repr__(self):
                return '<FakeCall %r>' % (self.func,)

        def sched(f):
            self.failIf(calls, repr(calls))
            calls.append(FakeCall(f))
            return calls[-1]

        c = task.Cooperator(scheduler=sched, terminationPredicateFactory=lambda: lambda: True)
        d = c.coiterate(iter(()))

        done = []
        def anotherTask(ign):
            c.coiterate(iter(())).addBoth(done.append)

        d.addCallback(anotherTask)

        work = 0
        while not done:
            work += 1
            while calls:
                calls.pop(0).func()
                work += 1
            if work > 50:
                self.fail("Cooperator took too long")

Example 49

Project: mythbox
Source File: test_cooperator.py
View license
    def testStopOutstanding(self):
        """
        An iterator run with L{Cooperator.coiterate} paused on a L{Deferred}
        yielded by that iterator will fire its own L{Deferred} (the one
        returned by C{coiterate}) when L{Cooperator.stop} is called.
        """
        testControlD = defer.Deferred()
        outstandingD = defer.Deferred()
        def myiter():
            reactor.callLater(0, testControlD.callback, None)
            yield outstandingD
            self.fail()
        c = task.Cooperator()
        d = c.coiterate(myiter())
        def stopAndGo(ign):
            c.stop()
            outstandingD.callback('arglebargle')

        testControlD.addCallback(stopAndGo)
        d.addCallback(self.cbIter)
        d.addErrback(self.ebIter)

        return d.addCallback(
            lambda result: self.assertEquals(result, self.RESULT))

Example 50

Project: mythbox
Source File: test_cooperator.py
View license
    def testResourceExhaustion(self):
        output = []
        def myiter():
            for i in range(100):
                output.append(i)
                if i == 9:
                    _TPF.stopped = True
                yield i

        class _TPF:
            stopped = False
            def __call__(self):
                return self.stopped

        c = task.Cooperator(terminationPredicateFactory=_TPF)
        c.coiterate(myiter()).addErrback(self.ebIter)
        c._delayedCall.cancel()
        # testing a private method because only the test case will ever care
        # about this, so we have to carefully clean up after ourselves.
        c._tick()
        c.stop()
        self.failUnless(_TPF.stopped)
        self.assertEquals(output, range(10))