asyncio.gather

Here are the examples of the python api asyncio.gather taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

47 Examples 7

Example 1

Project: pulsar
License: View license
Source File: __init__.py
Function: close
    def close(self):
        """Close all connections
        """
        waiters = []
        for p in self.connection_pools.values():
            waiters.append(p.close())
        self.connection_pools.clear()
        return asyncio.gather(*waiters, loop=self._loop)

Example 2

Project: asyncssh
License: View license
Source File: test_forward.py
    @asynctest
    def test_multiple_global_requests(self):
        """Test sending multiple global requests in parallel"""

        with (yield from self.connect()) as conn:
            listeners = yield from asyncio.gather(
                conn.forward_remote_port('', 0, '', 7),
                conn.forward_remote_port('', 0, '', 7))

            for listener in listeners:
                listener.close()
                yield from listener.wait_closed()

        yield from conn.wait_closed()

Example 3

Project: asyncssh
License: View license
Source File: test_process.py
    @asynctest
    def test_pause_process_reader(self):
        """Test pausing and resuming reading from another SSH process"""

        data = 4*1024*1024*'*'

        with (yield from self.connect()) as conn:
            proc1 = yield from conn.create_process(input=data)

            proc2 = yield from conn.create_process('delay', stdin=proc1.stdout,
                                                   stderr=asyncssh.DEVNULL)
            proc3 = yield from conn.create_process('delay', stdin=proc1.stderr,
                                                   stderr=asyncssh.DEVNULL)

            result2, result3 = yield from asyncio.gather(proc2.wait(),
                                                         proc3.wait())

        self.assertEqual(result2.stdout, data)
        self.assertEqual(result3.stdout, data)

Example 4

Project: aiohttp-devtools
License: View license
Source File: serve.py
Function: serve_main_app
def serve_main_app(*, main_port: int=8000, verbose: bool=False, **config):
    setup_logging(verbose)
    app = create_main_app(**config)
    loop = app.loop
    handler = app.make_handler(access_log_format='%r %s %b')
    co = asyncio.gather(loop.create_server(handler, '0.0.0.0', main_port), app.startup(), loop=loop)
    server, startup_res = loop.run_until_complete(co)

    try:
        loop.run_forever()
    except KeyboardInterrupt:  # pragma: no cover
        pass
    finally:
        server.close()
        loop.run_until_complete(server.wait_closed())
        loop.run_until_complete(app.shutdown())
        loop.run_until_complete(app.cleanup())
        loop.run_until_complete(handler.finish_connections(0.01))
    loop.close()

Example 5

Project: aioredis
License: View license
Source File: conftest.py
@pytest.yield_fixture
def _closable(loop):
    conns = []

    yield conns.append

    waiters = []
    while conns:
        conn = conns.pop(0)
        conn.close()
        waiters.append(conn.wait_closed())
    if waiters:
        loop.run_until_complete(asyncio.gather(*waiters, loop=loop))

Example 6

Project: aioredis
License: View license
Source File: pubsub_commands_test.py
@pytest.mark.run_loop
def test_subscribe_concurrency(create_redis, server, loop):
    sub = yield from create_redis(
        server.tcp_address, loop=loop)
    pub = yield from create_redis(
        server.tcp_address, loop=loop)

    res = yield from asyncio.gather(
        sub.subscribe('channel:0'),
        pub.publish('channel:0', 'Hello'),
        sub.subscribe('channel:1'),
        loop=loop)
    (ch1,), subs, (ch2,) = res

    assert ch1.name == b'channel:0'
    assert subs == 1
    assert ch2.name == b'channel:1'

Example 7

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_every_param_coroutine(self):
        # scheduled executions 1, 3, 5, 7, 9
        self.schedule = self.schedule_manager.every(self.get_coroutine,
                                                    timedelta(seconds=2),
                                                    datetime.now() + timedelta(seconds=1))
        # will be cancelled at
        self.cancel_in_seconds = 10
        try:
            self.loop.run_until_complete(
                asyncio.gather(self.schedule_canceller(), self.schedule.future))
        except asyncio.CancelledError:
            pass
        # making sure that all running jobs and the schedule are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))
        self.assertEqual(5, self.count)

Example 8

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_every_param_future(self):
        # scheduled executions 1, 3, 5, 7, 9
        self.schedule = self.schedule_manager.every(self.get_future,
                                                    timedelta(seconds=2),
                                                    datetime.now() + timedelta(seconds=1))
        # will be cancelled at
        self.cancel_in_seconds = 10
        try:
            self.loop.run_until_complete(
                asyncio.gather(self.schedule_canceller(), self.schedule.future))
        except asyncio.CancelledError:
            pass
        # making sure that all running jobs and the schedules are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))
        self.assertEqual(5, self.count)

Example 9

Project: aioxmpp
License: View license
Source File: test_testutils.py
    def test_starttls_without_callback(self):
        ssl_context = unittest.mock.MagicMock()

        self.xmlstream.transport = object()

        run_coroutine(
            asyncio.gather(
                self.xmlstream.starttls(ssl_context, None),
                self.xmlstream.run_test(
                    [
                        XMLStreamMock.STARTTLS(ssl_context, None)
                    ],
                )
            )
        )

Example 10

Project: butterfield
License: View license
Source File: core.py
Function: gather
    def gather(self):

        coros = []

        for bot in self.registry.values():
            coros.append(bot())
            coros.extend(load_plugin(x)(bot) for x in bot.daemons)

        return asyncio.gather(*coros)

Example 11

Project: vyked
License: View license
Source File: bus.py
    def _create_service_clients(self):
        futures = []
        for sc in self._service_clients:
            for host, port, node_id, service_type in self._registry_client.get_all_addresses(*sc.properties):
                if service_type == 'tcp':
                    self._node_clients[node_id] = sc
                    future = self._connect_to_client(host, node_id, port, service_type, sc)
                    futures.append(future)
        return asyncio.gather(*futures, return_exceptions=False)

Example 12

Project: motor
License: View license
Source File: test_asyncio_client.py
    @asyncio_test
    def test_client_lazy_connect(self):
        yield from self.db.test_client_lazy_connect.delete_many({})

        # Create client without connecting; connect on demand.
        cx = self.asyncio_client()
        collection = cx.motor_test.test_client_lazy_connect
        future0 = collection.insert_one({'foo': 'bar'})
        future1 = collection.insert_one({'foo': 'bar'})
        yield from asyncio.gather(future0, future1, loop=self.loop)
        resp = yield from collection.find({'foo': 'bar'}).count()
        self.assertEqual(2, resp)
        cx.close()

Example 13

Project: pulsar
License: View license
Source File: __init__.py
Function: call
    def __call__(self, actor=None):
        apps = [app(actor) for app in self.apps()]
        return asyncio.gather(*apps, loop=get_actor()._loop)

Example 14

Project: pulsar
License: View license
Source File: clients.py
Function: close
    def close(self):
        '''Close all connections

        Return a :class:`~asyncio.Future` called once all connections
        have closed
        '''
        if not self.closed:
            waiters = []
            queue = self._queue
            while queue.qsize():
                connection = queue.get_nowait()
                if connection:
                    waiters.append(connection.close())
            in_use = self._in_use_connections
            self._in_use_connections = set()
            for connection in in_use:
                if connection:
                    waiters.append(connection.close())
            self._closed = asyncio.gather(*waiters, loop=self._loop)
        return self._closed

Example 15

Project: pulsar
License: View license
Source File: futures.py
def as_gather(*args):
    """Same as :func:`~.asyncio.gather` but allows sync values
    """
    return gather(*[as_coroutine(arg) for arg in args])

Example 16

Project: pulsar-queue
License: View license
Source File: app.py
Function: teardown_class
    @classmethod
    def tearDownClass(cls):
        coros = [send('arbiter', 'kill_actor', a.name) for a in
                 (cls.tq_app, cls.rpc) if a is not None]
        return asyncio.gather(*coros)

Example 17

Project: umongo
License: View license
Source File: motor_asyncio.py
def _io_validate_data_proxy(schema, data_proxy, partial=None):
    errors = {}
    tasks = []
    tasks_field_name = []
    for name, field in schema.fields.items():
        if partial and name not in partial:
            continue
        data_name = field.attribute or name
        value = data_proxy._data[data_name]
        if value is missing:
            continue
        try:
            if field.io_validate_recursive:
                yield from field.io_validate_recursive(field, value)
            if field.io_validate:
                tasks.append(_run_validators(field.io_validate, field, value))
                tasks_field_name.append(name)
        except ValidationError as ve:
            errors[name] = ve.messages
    results = yield from asyncio.gather(*tasks, return_exceptions=True)
    for i, res in enumerate(results):
        if isinstance(res, ValidationError):
            errors[tasks_field_name[i]] = res.messages
        elif res:
            raise res
    if errors:
        raise ValidationError(errors)

Example 18

Project: vj4
License: View license
Source File: app.py
  def __init__(self):
    super(Application, self).__init__(
      handler_factory=functools.partial(web.RequestHandlerFactory, access_log=None),
      debug=options.debug)
    globals()[self.__class__.__name__] = lambda: self  # singleton

    # Initialize components.
    translation_path = path.join(path.dirname(__file__), 'locale')
    locale.load_translations(translation_path)
    self.loop.run_until_complete(asyncio.gather(tools.ensure_all_indexes(), bus.init()))
    smallcache.init()

    # Load views.
    from vj4.handler import contest
    from vj4.handler import discussion
    from vj4.handler import domain
    from vj4.handler import home
    from vj4.handler import judge
    from vj4.handler import misc
    from vj4.handler import problem
    from vj4.handler import record
    from vj4.handler import training
    from vj4.handler import user
    from vj4.handler import i18n
    if options.static:
      self.router.add_static('/', path.join(path.dirname(__file__), '.uibuild'), name='static')

Example 19

Project: aiomysql
License: View license
Source File: test_pool.py
Function: test_parallel_tasks
@pytest.mark.run_loop
def test_parallel_tasks(pool_creator, loop):
    pool = yield from pool_creator(minsize=0, maxsize=2)
    assert 2 == pool.maxsize
    assert 0 == pool.minsize
    assert 0 == pool.size
    assert 0 == pool.freesize

    fut1 = pool.acquire()
    fut2 = pool.acquire()

    conn1, conn2 = yield from asyncio.gather(fut1, fut2, loop=loop)
    assert 2 == pool.size
    assert 0 == pool.freesize
    assert {conn1, conn2} == pool._used

    pool.release(conn1)
    assert 2 == pool.size
    assert 1 == pool.freesize
    assert {conn2} == pool._used

    pool.release(conn2)
    assert 2 == pool.size
    assert 2 == pool.freesize
    assert not conn1.closed
    assert not conn2.closed

    conn3 = yield from pool.acquire()
    assert conn3 is conn1
    pool.release(conn3)

Example 20

Project: aiomysql
License: View license
Source File: test_pool.py
@pytest.mark.run_loop
def test_parallel_tasks_more(pool_creator, loop):
    pool = yield from pool_creator(minsize=0, maxsize=3)

    fut1 = pool.acquire()
    fut2 = pool.acquire()
    fut3 = pool.acquire()

    conn1, conn2, conn3 = yield from asyncio.gather(fut1, fut2, fut3,
                                                    loop=loop)
    assert 3 == pool.size
    assert 0 == pool.freesize
    assert {conn1, conn2, conn3} == pool._used

    pool.release(conn1)
    assert 3 == pool.size
    assert 1 == pool.freesize
    assert {conn2, conn3} == pool._used

    pool.release(conn2)
    assert 3 == pool.size
    assert 2 == pool.freesize
    assert {conn3} == pool._used
    assert not conn1.closed
    assert not conn2.closed

    pool.release(conn3)
    assert 3, pool.size
    assert 3, pool.freesize
    assert not pool._used
    assert not conn1.closed
    assert not conn2.closed
    assert not conn3.closed

    conn4 = yield from pool.acquire()
    assert conn4 is conn1
    pool.release(conn4)

Example 21

Project: aioredis
License: View license
Source File: connection.py
    def execute_pubsub(self, command, *channels):
        """Executes redis (p)subscribe/(p)unsubscribe commands.

        Returns asyncio.gather coroutine waiting for all channels/patterns
        to receive answers.
        """
        command = command.upper().strip()
        assert command in _PUBSUB_COMMANDS, (
            "Pub/Sub command expected", command)
        if self._reader is None or self._reader.at_eof():
            raise ConnectionClosedError("Connection closed or corrupted")
        if None in set(channels):
            raise TypeError("args must not contain None")
        if not len(channels):
            raise ValueError("No channels/patterns supplied")
        cmd = encode_command(command, *channels)
        res = []
        for ch in channels:
            fut = create_future(loop=self._loop)
            res.append(fut)
            self._waiters.append((fut, None, self._update_pubsub))
        self._writer.write(cmd)
        return asyncio.gather(*res, loop=self._loop)

Example 22

Project: aioredis
License: View license
Source File: transaction_commands_test.py
Function: test_multi_exec
@pytest.mark.run_loop
def test_multi_exec(redis, loop):
    yield from redis.delete('foo', 'bar')

    tr = redis.multi_exec()
    f1 = tr.incr('foo')
    f2 = tr.incr('bar')
    res = yield from tr.execute()
    assert res == [1, 1]
    res2 = yield from asyncio.gather(f1, f2, loop=loop)
    assert res == res2

    tr = redis.multi_exec()
    f1 = tr.incr('foo')
    f2 = tr.incr('bar')
    yield from tr.execute()
    assert (yield from f1) == 2
    assert (yield from f2) == 2

    tr = redis.multi_exec()
    f1 = tr.set('foo', 1.0)
    f2 = tr.incrbyfloat('foo', 1.2)
    res = yield from tr.execute()
    assert res == [True, 2.2]
    res2 = yield from asyncio.gather(f1, f2, loop=loop)
    assert res == res2

    tr = redis.multi_exec()
    f1 = tr.incrby('foo', 1.0)
    with pytest.raises_regex(MultiExecError, "increment must be .* int"):
        yield from tr.execute()
    with pytest.raises(TypeError):
        yield from f1

Example 23

Project: aschedule
License: View license
Source File: testing_ext.py
    def test_every_random_interval(self):
        from aschedule.ext import every_random_interval

        # scheduled executions: randrange(0, 5), randrange(5, 10), randrange(10, 15)
        times = []
        start = self.loop.time()
        cancel_in_seconds = 16
        # set the seed to avoid a scheduled execution on 16th second.
        __import__('random').seed(38042)
        async def record_times():
            times.append(round(self.loop.time() - start))
        schedule = every_random_interval(record_times, timedelta(seconds=5))

        async def schedule_canceller():
            await asyncio.sleep(cancel_in_seconds)
            aschedule.cancel(schedule)

        try:
            self.loop.run_until_complete(
                asyncio.gather(schedule_canceller(), schedule.future))
        except asyncio.CancelledError:
            pass

        self.assertEqual(3, len(times))
        for i, x in enumerate(times):
            self.assertTrue(i * 5 <= x < (i + 1) * 5)

Example 24

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_every_param_interval(self):
        # scheduled executions for schedule1, 0, 3, 6, 9
        # scheduled executions for schedule1, 0, 4, 8
        times1, times2 = [], []
        cancel_in_seconds = 10
        start = self.loop.time()

        async def record_times(times):
            times.append(round(self.loop.time() - start))

        async def cancel_schedules():
            await asyncio.sleep(cancel_in_seconds)
            self.schedule_manager.cancel(schedule1, running_jobs=True)
            self.schedule_manager.cancel(schedule2, running_jobs=True)

        schedule1 = self.schedule_manager.every(partial(record_times, times1),
                                                timedelta(seconds=3),
                                                datetime.now())
        schedule2 = self.schedule_manager.every(partial(record_times, times2),
                                                timedelta(seconds=4),
                                                datetime.now())
        try:
            self.loop.run_until_complete(
                asyncio.gather(schedule1.future, schedule2.future, cancel_schedules()))
        except asyncio.CancelledError:
            pass
        # making sure that all running jobs and the schedules are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))

        self.assertEqual([0, 3, 6, 9], times1)
        self.assertEqual([0, 4, 8], times2)

        with self.assertRaises(aschedule.AScheduleException):
            self.schedule_manager.every(self.get_coroutine,
                                        timedelta(seconds=-2),
                                        datetime.now())

Example 25

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_every_param_start_at(self):
        # scheduled executions 2, 4, 6, 8; 1, 4, 7
        times1 = []
        times2 = []
        start = self.loop.time()
        cancel_in_seconds = 9

        async def record_times(times):
            times.append(round(self.loop.time() - start))

        async def cancel_schedule():
            await asyncio.sleep(cancel_in_seconds)
            self.schedule_manager.cancel(schedule1, running_jobs=True)
            self.schedule_manager.cancel(schedule2, running_jobs=True)

        schedule1 = self.schedule_manager.every(partial(record_times, times1),
                                                interval=timedelta(seconds=2),
                                                start_at=datetime.now() + timedelta(seconds=2))
        schedule2 = self.schedule_manager.every(partial(record_times, times2),
                                                interval=timedelta(seconds=3),
                                                start_at=datetime.now() + timedelta(seconds=-2))
        try:
            self.loop.run_until_complete(
                asyncio.gather(schedule2.future, schedule1.future, cancel_schedule()))
        except asyncio.CancelledError:
            pass
        # making sure that all running jobs and the schedules are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))

        self.assertEqual([2, 4, 6, 8], times1)
        self.assertEqual([1, 4, 7], times2)

Example 26

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_every_param_count(self):
        # scheduled executions 0, 2, 4, 6, 8; without cancelling the future
        times = []
        start = self.loop.time()
        cancel_in_seconds = 16

        async def record_times():
            times.append(round(self.loop.time() - start))

        async def cancel_schedule():
            await asyncio.sleep(cancel_in_seconds)
            # should report an error that the given schedule doesn't belong to this schedule manager
            # as the schedule is completed before this arises
            with self.assertRaises(aschedule.AScheduleException):
                self.schedule_manager.cancel(schedule, running_jobs=True)

        schedule = self.schedule_manager.every(record_times, interval=timedelta(seconds=2),
                                               start_at=datetime.now(), count=5)

        self.loop.run_until_complete(
            asyncio.gather(schedule.future, cancel_schedule()))
        # making sure that all running jobs and the schedules are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))

        self.assertEqual([0, 2, 4, 6, 8], times)
        with self.assertRaises(aschedule.AScheduleException):
            self.schedule_manager.every(self.get_coroutine,
                                        timedelta(seconds=2),
                                        datetime.now(), count=0)
        with self.assertRaises(aschedule.AScheduleException):
            self.schedule_manager.every(self.get_coroutine,
                                        timedelta(seconds=2),
                                        datetime.now(), count=-1)

Example 27

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_once_at_param_coroutine(self):
        start = self.loop.time()
        times = []
        cancel_in_seconds = 10
        async def record_times():
            times.append(round(self.loop.time() - start))

        async def cancel_schedule():
            await asyncio.sleep(cancel_in_seconds)
            # should report an error that the given schedule doesn't belong to this schedule manager
            # as the schedule is completed before this arises
            with self.assertRaises(aschedule.AScheduleException):
                self.schedule_manager.cancel(schedule, running_jobs=True)

        schedule = self.schedule_manager.once_at(record_times,
                                                 datetime.now() + timedelta(seconds=5))
        self.loop.run_until_complete(
            asyncio.gather(cancel_schedule(), schedule.future))

        # making sure that all running jobs and the schedule are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))
        self.assertEqual([5], times)

Example 28

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_once_at_param_run_at(self):
        # scheduled executions 5, 7, 3
        start = self.loop.time()
        times = []
        cancel_in_seconds = 10
        async def record_times():
            times.append(round(self.loop.time() - start))

        async def cancel_schedule():
            await asyncio.sleep(cancel_in_seconds)
            # should report an error that the given schedule doesn't belong to this schedule manager
            # as the schedule is completed before this arises
            with self.assertRaises(aschedule.AScheduleException):
                self.schedule_manager.cancel(schedule1, running_jobs=True)
            with self.assertRaises(aschedule.AScheduleException):
                self.schedule_manager.cancel(schedule2, running_jobs=True)
            with self.assertRaises(aschedule.AScheduleException):
                self.schedule_manager.cancel(schedule3, running_jobs=True)

        schedule1 = self.schedule_manager.once_at(record_times,
                                                  datetime.now() + timedelta(seconds=5))
        schedule2 = self.schedule_manager.once_at(record_times,
                                                  datetime.now() + timedelta(seconds=7))
        schedule3 = self.schedule_manager.once_at(record_times,
                                                  datetime.now() + timedelta(seconds=3))
        self.loop.run_until_complete(
            asyncio.gather(cancel_schedule(), schedule1.future,
                           schedule2.future, schedule3.future))

        # making sure that all running jobs and the schedule are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))
        self.assertEqual([3, 5, 7], times)

Example 29

Project: aschedule
License: View license
Source File: test_schedule_manager.py
    def test_once_at_param_strict(self):
        # scheduled executions 0
        start = self.loop.time()
        times = []
        cancel_in_seconds = 2
        async def record_times():
            times.append(round(self.loop.time() - start))

        async def cancel_schedule():
            await asyncio.sleep(cancel_in_seconds)
            # should report an error that the given schedule doesn't belong to this schedule manager
            # as the schedule is completed before this arises
            with self.assertRaises(aschedule.AScheduleException):
                self.schedule_manager.cancel(schedule, running_jobs=True)

        with self.assertRaises(aschedule.AScheduleException):
            self.schedule_manager.once_at(record_times,
                                          datetime.now() + timedelta(seconds=-10),
                                          strict=True)

        schedule = self.schedule_manager.once_at(record_times,
                                                 datetime.now() + timedelta(seconds=-10),
                                                 strict=False)
        self.loop.run_until_complete(asyncio.gather(cancel_schedule(), schedule.future))

        # making sure that all running jobs and the schedule are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))
        self.assertEqual([0], times)

Example 30

Project: aschedule
License: View license
Source File: test_schedule_manager.py
Function: test_shut_down
    def test_shutdown(self):
        start = self.loop.time()
        times = []

        async def record_times():
            times.append(round(self.loop.time() - start))

        schedules = [
            self.schedule_manager.every(record_times,
                                        timedelta(seconds=5),
                                        datetime.now() + timedelta(seconds=x))
            for x in range(0, 5, 2)
        ]
        schedules.append(
            self.schedule_manager.once_at(record_times,
                                          datetime.now() + timedelta(seconds=3))
        )
        async def cancel_schedules():
            await asyncio.sleep(11)
            self.schedule_manager.shutdown()
        futures = [schedule.future for schedule in schedules]
        try:
            self.loop.run_until_complete(asyncio.gather(*(futures + [cancel_schedules()])))
        except asyncio.CancelledError:
            pass
        # making sure that all running jobs and the schedule are cancelled
        self.loop.run_until_complete(asyncio.sleep(10))
        self.assertEqual([0, 2, 3, 4, 5, 7, 9, 10], times)

Example 31

Project: aioxmpp
License: View license
Source File: test_service.py
    def test_query_info_deduplicate_requests(self):
        to = structs.JID.fromstr("[email protected]/res1")
        response = disco_xso.InfoQuery()

        with unittest.mock.patch.object(
                self.s,
                "send_and_decode_info_query",
                new=CoroutineMock()) as send_and_decode:
            response = {}

            send_and_decode.return_value = response

            result = run_coroutine(
                asyncio.gather(
                    self.s.query_info(to, timeout=10),
                    self.s.query_info(to, timeout=10),
                )
            )

            self.assertIs(result[0], response)
            self.assertIs(result[1], response)

        self.assertSequenceEqual(
            [
                unittest.mock.call(to, None),
            ],
            send_and_decode.mock_calls
        )

Example 32

Project: aioxmpp
License: View license
Source File: test_service.py
    def test_query_items_deduplicate_requests(self):
        to = structs.JID.fromstr("[email protected]/res1")
        response = disco_xso.ItemsQuery()

        self.cc.stream.send.return_value = response

        result = run_coroutine(
            asyncio.gather(
                self.s.query_items(to, timeout=10),
                self.s.query_items(to, timeout=10),
            )
        )

        self.assertIs(result[0], response)
        self.assertIs(result[1], response)

        self.assertSequenceEqual(
            [
                unittest.mock.call(unittest.mock.ANY),
            ],
            self.cc.stream.send.mock_calls
        )

Example 33

Project: aioxmpp
License: View license
Source File: test_testutils.py
    def test_starttls(self):
        ssl_context = unittest.mock.MagicMock()
        post_handshake_callback = unittest.mock.MagicMock()

        self.xmlstream.transport = object()

        run_coroutine(
            asyncio.gather(
                self.xmlstream.starttls(ssl_context, post_handshake_callback),
                self.xmlstream.run_test(
                    [
                        XMLStreamMock.STARTTLS(
                            ssl_context,
                            post_handshake_callback)
                    ],
                )
            )
        )

        post_handshake_callback.assert_called_once_with(
            self.xmlstream.transport)

Example 34

Project: aioxmpp
License: View license
Source File: test_testutils.py
    def test_starttls_reject_incorrect_arguments(self):
        ssl_context = unittest.mock.MagicMock()
        post_handshake_callback = unittest.mock.MagicMock()

        self.xmlstream.transport = object()

        with self.assertRaisesRegex(AssertionError,
                                    "mismatched starttls argument"):
            run_coroutine(
                asyncio.gather(
                    self.xmlstream.starttls(object(), post_handshake_callback),
                    self.xmlstream.run_test(
                        [
                            XMLStreamMock.STARTTLS(
                                ssl_context,
                                post_handshake_callback)
                        ],
                    )
                )
            )

        with self.assertRaisesRegex(AssertionError,
                                    "mismatched starttls argument"):
            run_coroutine(
                asyncio.gather(
                    self.xmlstream.starttls(ssl_context, object()),
                    self.xmlstream.run_test(
                        [
                            XMLStreamMock.STARTTLS(
                                ssl_context,
                                post_handshake_callback)
                        ],
                    )
                )
            )

Example 35

Project: aioxmpp
License: View license
Source File: test_testutils.py
    def test_starttls_propagates_exception_from_callback(self):
        ssl_context = unittest.mock.MagicMock()
        post_handshake_callback = unittest.mock.MagicMock()

        self.xmlstream.transport = object()

        exc = ValueError()
        post_handshake_callback.side_effect = exc

        caught_exception, other_result = run_coroutine(
            asyncio.gather(
                self.xmlstream.starttls(ssl_context, post_handshake_callback),
                self.xmlstream.run_test(
                    [
                        XMLStreamMock.STARTTLS(
                            ssl_context,
                            post_handshake_callback)
                    ],
                ),
                return_exceptions=True
            )
        )

        self.assertIs(caught_exception, exc)
        self.assertIs(other_result, None)

Example 36

Project: pyee
License: View license
Source File: test_async.py
def test_asyncio_emit():
    """Test that event_emitters can handle wrapping coroutines as used with
    asyncio.
    """
    loop = new_event_loop()
    ee = EventEmitter(loop=loop)

    should_call = Future(loop=loop)

    @ee.on('event')
    async def event_handler():
        should_call.set_result(True)

    async def create_timeout(loop=loop):
        await sleep(0.1, loop=loop)
        if not should_call.done():
            raise Exception('should_call timed out!')
            return should_call.cancel()

    timeout = create_timeout(loop=loop)

    @should_call.add_done_callback
    def _done(result):
        assert result

    ee.emit('event')

    loop.run_until_complete(gather(should_call, timeout, loop=loop))

    loop.close()

Example 37

Project: pyee
License: View license
Source File: test_async.py
def test_asyncio_error():
    """Test that event_emitters can handle errors when wrapping coroutines as
    used with asyncio.
    """
    loop = new_event_loop()
    ee = EventEmitter(loop=loop)

    should_call = Future(loop=loop)

    @ee.on('event')
    async def event_handler():
        raise PyeeTestException()

    @ee.on('error')
    def handle_error(exc):
        should_call.set_result(exc)

    async def create_timeout(loop=loop):
        await sleep(0.1, loop=loop)
        if not should_call.done():
            raise Exception('should_call timed out!')
            return should_call.cancel()

    timeout = create_timeout(loop=loop)

    @should_call.add_done_callback
    def _done(result):
        assert isinstance(result, PyeeTestError)

    ee.emit('event')

    loop.run_until_complete(gather(should_call, timeout, loop=loop))

    loop.close()

Example 38

Project: aiohttp
License: View license
Source File: web.py
def run_app(app, *, host='0.0.0.0', port=None,
            shutdown_timeout=60.0, ssl_context=None,
            print=print, backlog=128, access_log_format=None,
            access_log=access_logger):
    """Run an app locally"""
    if port is None:
        if not ssl_context:
            port = 8080
        else:
            port = 8443

    loop = app.loop

    make_handler_kwargs = dict()
    if access_log_format is not None:
        make_handler_kwargs['access_log_format'] = access_log_format
    handler = app.make_handler(access_log=access_log,
                               **make_handler_kwargs)
    server = loop.create_server(handler, host, port, ssl=ssl_context,
                                backlog=backlog)
    srv, startup_res = loop.run_until_complete(asyncio.gather(server,
                                                              app.startup(),
                                                              loop=loop))

    scheme = 'https' if ssl_context else 'http'
    url = URL('{}://localhost'.format(scheme))
    url = url.with_host(host).with_port(port)
    print("======== Running on {} ========\n"
          "(Press CTRL+C to quit)".format(url))

    try:
        loop.run_forever()
    except KeyboardInterrupt:  # pragma: no cover
        pass
    finally:
        srv.close()
        loop.run_until_complete(srv.wait_closed())
        loop.run_until_complete(app.shutdown())
        loop.run_until_complete(handler.shutdown(shutdown_timeout))
        loop.run_until_complete(app.cleanup())
    loop.close()

Example 39

Project: ks-email-parser
License: View license
Source File: __init__.py
def _parse_emails(settings):
    if not settings.exclusive:
        shutil.rmtree(settings.destination, ignore_errors=True)

    emails = iter(
        fs.emails(settings.source, settings.pattern, settings.exclusive))
    executor = concurrent.futures.ProcessPoolExecutor(
        max_workers=settings.workers_pool)
    tasks = []

    emails_batch = list(islice(emails, settings.workers_pool))
    while emails_batch:
        task = loop.run_in_executor(
            executor, _parse_emails_batch, emails_batch, settings)
        tasks.append(task)
        emails_batch = list(islice(emails, settings.workers_pool))
    results = yield from asyncio.gather(*tasks)
    result = reduce(
        lambda acc, result: True if acc and result else False, results)
    return result

Example 40

Project: uvloop
License: View license
Source File: test_tcp.py
    def test_create_server_1(self):
        if self.is_asyncio_loop() and sys.version_info[:3] == (3, 5, 2):
            # See https://github.com/python/asyncio/pull/366 for details.
            raise unittest.SkipTest()

        CNT = 0           # number of clients that were successful
        TOTAL_CNT = 25    # total number of clients that test will create
        TIMEOUT = 5.0     # timeout for this test

        A_DATA = b'A' * 1024 * 1024
        B_DATA = b'B' * 1024 * 1024

        async def handle_client(reader, writer):
            nonlocal CNT

            data = await reader.readexactly(len(A_DATA))
            self.assertEqual(data, A_DATA)
            writer.write(b'OK')

            data = await reader.readexactly(len(B_DATA))
            self.assertEqual(data, B_DATA)
            writer.writelines([b'S', b'P'])
            writer.write(bytearray(b'A'))
            writer.write(memoryview(b'M'))

            await writer.drain()
            writer.close()

            CNT += 1

        async def test_client(addr):
            sock = socket.socket()
            with sock:
                sock.setblocking(False)
                await self.loop.sock_connect(sock, addr)

                await self.loop.sock_sendall(sock, A_DATA)

                buf = b''
                while len(buf) != 2:
                    buf += await self.loop.sock_recv(sock, 1)
                self.assertEqual(buf, b'OK')

                await self.loop.sock_sendall(sock, B_DATA)

                buf = b''
                while len(buf) != 4:
                    buf += await self.loop.sock_recv(sock, 1)
                self.assertEqual(buf, b'SPAM')

        async def start_server():
            nonlocal CNT
            CNT = 0

            addrs = ('127.0.0.1', 'localhost')
            if not isinstance(self.loop, uvloop.Loop):
                # Hack to let tests run on Python 3.5.0
                # (asyncio doesn't support multiple hosts in 3.5.0)
                addrs = '127.0.0.1'

            srv = await asyncio.start_server(
                handle_client,
                addrs, 0,
                family=socket.AF_INET,
                loop=self.loop)

            srv_socks = srv.sockets
            self.assertTrue(srv_socks)

            addr = srv_socks[0].getsockname()

            tasks = []
            for _ in range(TOTAL_CNT):
                tasks.append(test_client(addr))

            await asyncio.wait_for(
                asyncio.gather(*tasks, loop=self.loop),
                TIMEOUT, loop=self.loop)

            self.loop.call_soon(srv.close)
            await srv.wait_closed()

            # Check that the server cleaned-up proxy-sockets
            for srv_sock in srv_socks:
                self.assertEqual(srv_sock.fileno(), -1)

        async def start_server_sock():
            nonlocal CNT
            CNT = 0

            sock = socket.socket()
            sock.bind(('127.0.0.1', 0))
            addr = sock.getsockname()

            srv = await asyncio.start_server(
                handle_client,
                None, None,
                family=socket.AF_INET,
                loop=self.loop,
                sock=sock)

            srv_socks = srv.sockets
            self.assertTrue(srv_socks)

            tasks = []
            for _ in range(TOTAL_CNT):
                tasks.append(test_client(addr))

            await asyncio.wait_for(
                asyncio.gather(*tasks, loop=self.loop),
                TIMEOUT, loop=self.loop)

            srv.close()
            await srv.wait_closed()

            # Check that the server cleaned-up proxy-sockets
            for srv_sock in srv_socks:
                self.assertEqual(srv_sock.fileno(), -1)

        self.loop.run_until_complete(start_server())
        self.assertEqual(CNT, TOTAL_CNT)

        self.loop.run_until_complete(start_server_sock())
        self.assertEqual(CNT, TOTAL_CNT)

Example 41

Project: uvloop
License: View license
Source File: test_tcp.py
    def test_create_connection_1(self):
        CNT = 0
        TOTAL_CNT = 100

        def server():
            data = yield tb.read(4)
            self.assertEqual(data, b'AAAA')
            yield tb.write(b'OK')

            data = yield tb.read(4)
            self.assertEqual(data, b'BBBB')
            yield tb.write(b'SPAM')

        async def client(addr):
            reader, writer = await asyncio.open_connection(
                *addr,
                loop=self.loop)

            writer.write(b'AAAA')
            self.assertEqual(await reader.readexactly(2), b'OK')

            re = r'(a bytes-like object)|(must be byte-ish)'
            with self.assertRaisesRegex(TypeError, re):
                writer.write('AAAA')

            writer.write(b'BBBB')
            self.assertEqual(await reader.readexactly(4), b'SPAM')

            nonlocal CNT
            CNT += 1

            writer.close()

        async def client_2(addr):
            sock = socket.socket()
            sock.connect(addr)
            reader, writer = await asyncio.open_connection(
                sock=sock,
                loop=self.loop)

            writer.write(b'AAAA')
            self.assertEqual(await reader.readexactly(2), b'OK')

            writer.write(b'BBBB')
            self.assertEqual(await reader.readexactly(4), b'SPAM')

            nonlocal CNT
            CNT += 1

            writer.close()

        def run(coro):
            nonlocal CNT
            CNT = 0

            srv = tb.tcp_server(server,
                                max_clients=TOTAL_CNT,
                                backlog=TOTAL_CNT)
            srv.start()

            tasks = []
            for _ in range(TOTAL_CNT):
                tasks.append(coro(srv.addr))

            self.loop.run_until_complete(
                asyncio.gather(*tasks, loop=self.loop))
            srv.join()
            self.assertEqual(CNT, TOTAL_CNT)

        run(client)
        run(client_2)

Example 42

Project: uvloop
License: View license
Source File: test_tcp.py
    def test_transport_shutdown(self):
        CNT = 0           # number of clients that were successful
        TOTAL_CNT = 100   # total number of clients that test will create
        TIMEOUT = 5.0     # timeout for this test

        async def handle_client(reader, writer):
            nonlocal CNT

            data = await reader.readexactly(4)
            self.assertEqual(data, b'AAAA')

            writer.write(b'OK')
            writer.write_eof()
            writer.write_eof()

            await writer.drain()
            writer.close()

            CNT += 1

        async def test_client(addr):
            reader, writer = await asyncio.open_connection(
                *addr,
                loop=self.loop)

            writer.write(b'AAAA')
            data = await reader.readexactly(2)
            self.assertEqual(data, b'OK')

            writer.close()

        async def start_server():
            nonlocal CNT
            CNT = 0

            srv = await asyncio.start_server(
                handle_client,
                '127.0.0.1', 0,
                family=socket.AF_INET,
                loop=self.loop)

            srv_socks = srv.sockets
            self.assertTrue(srv_socks)

            addr = srv_socks[0].getsockname()

            tasks = []
            for _ in range(TOTAL_CNT):
                tasks.append(test_client(addr))

            await asyncio.wait_for(
                asyncio.gather(*tasks, loop=self.loop),
                TIMEOUT, loop=self.loop)

            srv.close()
            await srv.wait_closed()

        self.loop.run_until_complete(start_server())
        self.assertEqual(CNT, TOTAL_CNT)

Example 43

Project: uvloop
License: View license
Source File: test_tcp.py
    def test_create_connection_ssl_1(self):
        CNT = 0
        TOTAL_CNT = 25

        A_DATA = b'A' * 1024 * 1024
        B_DATA = b'B' * 1024 * 1024

        sslctx = self._create_server_ssl_context(self.ONLYCERT, self.ONLYKEY)
        client_sslctx = self._create_client_ssl_context()

        def server():
            yield tb.starttls(
                sslctx,
                server_side=True)

            data = yield tb.read(len(A_DATA))
            self.assertEqual(data, A_DATA)
            yield tb.write(b'OK')

            data = yield tb.read(len(B_DATA))
            self.assertEqual(data, B_DATA)
            yield tb.write(b'SPAM')

            yield tb.close()

        async def client(addr):
            reader, writer = await asyncio.open_connection(
                *addr,
                ssl=client_sslctx,
                server_hostname='',
                loop=self.loop)

            writer.write(A_DATA)
            self.assertEqual(await reader.readexactly(2), b'OK')

            writer.write(B_DATA)
            self.assertEqual(await reader.readexactly(4), b'SPAM')

            nonlocal CNT
            CNT += 1

            writer.close()

        async def client_sock(addr):
            sock = socket.socket()
            sock.connect(addr)
            reader, writer = await asyncio.open_connection(
                sock=sock,
                ssl=client_sslctx,
                server_hostname='',
                loop=self.loop)

            writer.write(A_DATA)
            self.assertEqual(await reader.readexactly(2), b'OK')

            writer.write(B_DATA)
            self.assertEqual(await reader.readexactly(4), b'SPAM')

            nonlocal CNT
            CNT += 1

            writer.close()
            sock.close()

        def run(coro):
            nonlocal CNT
            CNT = 0

            srv = tb.tcp_server(server,
                                max_clients=TOTAL_CNT,
                                backlog=TOTAL_CNT)
            srv.start()

            tasks = []
            for _ in range(TOTAL_CNT):
                tasks.append(coro(srv.addr))

            self.loop.run_until_complete(
                asyncio.gather(*tasks, loop=self.loop))
            srv.join()
            self.assertEqual(CNT, TOTAL_CNT)

        with self._silence_eof_received_warning():
            run(client)

        with self._silence_eof_received_warning():
            run(client_sock)

Example 44

Project: uvloop
License: View license
Source File: test_unix.py
    def test_create_unix_server_1(self):
        CNT = 0           # number of clients that were successful
        TOTAL_CNT = 100   # total number of clients that test will create
        TIMEOUT = 5.0     # timeout for this test

        async def handle_client(reader, writer):
            nonlocal CNT

            data = await reader.readexactly(4)
            self.assertEqual(data, b'AAAA')
            writer.write(b'OK')

            data = await reader.readexactly(4)
            self.assertEqual(data, b'BBBB')
            writer.write(b'SPAM')

            await writer.drain()
            writer.close()

            CNT += 1

        async def test_client(addr):
            sock = socket.socket(socket.AF_UNIX)
            with sock:
                sock.setblocking(False)
                await self.loop.sock_connect(sock, addr)

                await self.loop.sock_sendall(sock, b'AAAA')

                buf = b''
                while len(buf) != 2:
                    buf += await self.loop.sock_recv(sock, 1)
                self.assertEqual(buf, b'OK')

                await self.loop.sock_sendall(sock, b'BBBB')

                buf = b''
                while len(buf) != 4:
                    buf += await self.loop.sock_recv(sock, 1)
                self.assertEqual(buf, b'SPAM')

        async def start_server():
            nonlocal CNT
            CNT = 0

            with tempfile.TemporaryDirectory() as td:
                sock_name = os.path.join(td, 'sock')
                srv = await asyncio.start_unix_server(
                    handle_client,
                    sock_name,
                    loop=self.loop)

                try:
                    srv_socks = srv.sockets
                    self.assertTrue(srv_socks)

                    tasks = []
                    for _ in range(TOTAL_CNT):
                        tasks.append(test_client(sock_name))

                    await asyncio.wait_for(
                        asyncio.gather(*tasks, loop=self.loop),
                        TIMEOUT, loop=self.loop)

                finally:
                    self.loop.call_soon(srv.close)
                    await srv.wait_closed()

                    # Check that the server cleaned-up proxy-sockets
                    for srv_sock in srv_socks:
                        self.assertEqual(srv_sock.fileno(), -1)

                # asyncio doesn't cleanup the sock file
                self.assertTrue(os.path.exists(sock_name))

        async def start_server_sock():
            nonlocal CNT
            CNT = 0

            with tempfile.TemporaryDirectory() as td:
                sock_name = os.path.join(td, 'sock')
                sock = socket.socket(socket.AF_UNIX)
                sock.bind(sock_name)

                srv = await asyncio.start_unix_server(
                    handle_client,
                    None,
                    loop=self.loop,
                    sock=sock)

                try:
                    srv_socks = srv.sockets
                    self.assertTrue(srv_socks)

                    tasks = []
                    for _ in range(TOTAL_CNT):
                        tasks.append(test_client(sock_name))

                    await asyncio.wait_for(
                        asyncio.gather(*tasks, loop=self.loop),
                        TIMEOUT, loop=self.loop)

                finally:
                    self.loop.call_soon(srv.close)
                    await srv.wait_closed()

                    # Check that the server cleaned-up proxy-sockets
                    for srv_sock in srv_socks:
                        self.assertEqual(srv_sock.fileno(), -1)

                # asyncio doesn't cleanup the sock file
                self.assertTrue(os.path.exists(sock_name))

        self.loop.run_until_complete(start_server())
        self.assertEqual(CNT, TOTAL_CNT)

        self.loop.run_until_complete(start_server_sock())
        self.assertEqual(CNT, TOTAL_CNT)

Example 45

Project: uvloop
License: View license
Source File: test_unix.py
    def test_create_unix_connection_1(self):
        CNT = 0
        TOTAL_CNT = 100

        def server():
            data = yield tb.read(4)
            self.assertEqual(data, b'AAAA')
            yield tb.write(b'OK')

            data = yield tb.read(4)
            self.assertEqual(data, b'BBBB')
            yield tb.write(b'SPAM')

        async def client(addr):
            reader, writer = await asyncio.open_unix_connection(
                addr,
                loop=self.loop)

            writer.write(b'AAAA')
            self.assertEqual(await reader.readexactly(2), b'OK')

            writer.write(b'BBBB')
            self.assertEqual(await reader.readexactly(4), b'SPAM')

            nonlocal CNT
            CNT += 1

            writer.close()

        async def client_2(addr):
            sock = socket.socket(socket.AF_UNIX)
            sock.connect(addr)
            reader, writer = await asyncio.open_unix_connection(
                sock=sock,
                loop=self.loop)

            writer.write(b'AAAA')
            self.assertEqual(await reader.readexactly(2), b'OK')

            writer.write(b'BBBB')
            self.assertEqual(await reader.readexactly(4), b'SPAM')

            nonlocal CNT
            CNT += 1

            writer.close()

        def run(coro):
            nonlocal CNT
            CNT = 0

            srv = tb.tcp_server(server,
                                family=socket.AF_UNIX,
                                max_clients=TOTAL_CNT,
                                backlog=TOTAL_CNT)
            srv.start()

            tasks = []
            for _ in range(TOTAL_CNT):
                tasks.append(coro(srv.addr))

            self.loop.run_until_complete(
                asyncio.gather(*tasks, loop=self.loop))
            srv.join()
            self.assertEqual(CNT, TOTAL_CNT)

        run(client)
        run(client_2)

Example 46

Project: uvloop
License: View license
Source File: test_unix.py
    def test_create_unix_connection_ssl_1(self):
        CNT = 0
        TOTAL_CNT = 25

        A_DATA = b'A' * 1024 * 1024
        B_DATA = b'B' * 1024 * 1024

        sslctx = self._create_server_ssl_context(self.ONLYCERT, self.ONLYKEY)
        client_sslctx = self._create_client_ssl_context()

        def server():
            yield tb.starttls(
                sslctx,
                server_side=True)

            data = yield tb.read(len(A_DATA))
            self.assertEqual(data, A_DATA)
            yield tb.write(b'OK')

            data = yield tb.read(len(B_DATA))
            self.assertEqual(data, B_DATA)
            yield tb.write(b'SPAM')

            yield tb.close()

        async def client(addr):
            reader, writer = await asyncio.open_unix_connection(
                addr,
                ssl=client_sslctx,
                server_hostname='',
                loop=self.loop)

            writer.write(A_DATA)
            self.assertEqual(await reader.readexactly(2), b'OK')

            writer.write(B_DATA)
            self.assertEqual(await reader.readexactly(4), b'SPAM')

            nonlocal CNT
            CNT += 1

            writer.close()

        def run(coro):
            nonlocal CNT
            CNT = 0

            srv = tb.tcp_server(server,
                                family=socket.AF_UNIX,
                                max_clients=TOTAL_CNT,
                                backlog=TOTAL_CNT)
            srv.start()

            tasks = []
            for _ in range(TOTAL_CNT):
                tasks.append(coro(srv.addr))

            self.loop.run_until_complete(
                asyncio.gather(*tasks, loop=self.loop))
            srv.join()
            self.assertEqual(CNT, TOTAL_CNT)

        with self._silence_eof_received_warning():
            run(client)

Example 47

Project: mlat-server
License: View license
Source File: main.py
    def run(self):
        args = self.make_arg_parser().parse_args()

        self.coordinator = coordinator.Coordinator(work_dir=args.work_dir,
                                                   pseudorange_filename=args.dump_pseudorange,
                                                   partition=args.partition,
                                                   tag=args.tag)

        subtasks = self.make_subtasks(args)

        # Start everything
        startup = asyncio.gather(*[x.start() for x in subtasks])
        self.loop.run_until_complete(startup)
        startup.result()  # provoke exceptions if something failed

        self.loop.add_signal_handler(signal.SIGINT, self.stop, "Halting on SIGINT")
        self.loop.add_signal_handler(signal.SIGTERM, self.stop, "Halting on SIGTERM")

        self.loop.run_forever()  # Well, until stop() is called anyway!

        logging.info("Server shutting down.")

        # Stop everything
        for t in reversed(subtasks):
            t.close()

        # Wait for completion
        shutdown = asyncio.gather(*[t.wait_closed() for t in subtasks], return_exceptions=True)
        self.loop.run_until_complete(shutdown)
        for e in shutdown.result():
            if isinstance(e, Exception) and not isinstance(e, asyncio.CancelledError):
                logging.error("Exception thrown during shutdown", exc_info=(type(e), e, e.__traceback__))

        self.loop.close()
        logging.info("Server shutdown done.")