tests
test_streams.py
"""Tests for streams.py"""
import asyncio
import unittest
from unittest import mock
from aiohttp import helpers, streams, test_utils
clast TestStreamReader(unittest.TestCase):
DATA = b'line1\nline2\nline3\n'
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
def tearDown(self):
self.loop.close()
def _make_one(self, *args, **kwargs):
return streams.StreamReader(loop=self.loop, *args, **kwargs)
def test_create_waiter(self):
stream = self._make_one()
stream._waiter = helpers.create_future(self.loop)
with self.astertRaises(RuntimeError):
self.loop.run_until_complete(stream._wait('test'))
@mock.patch('aiohttp.streams.asyncio')
def test_ctor_global_loop(self, m_asyncio):
stream = streams.StreamReader()
self.astertIs(stream._loop, m_asyncio.get_event_loop.return_value)
def test_at_eof(self):
stream = self._make_one()
self.astertFalse(stream.at_eof())
stream.feed_data(b'some data\n')
self.astertFalse(stream.at_eof())
self.loop.run_until_complete(stream.readline())
self.astertFalse(stream.at_eof())
stream.feed_data(b'some data\n')
stream.feed_eof()
self.loop.run_until_complete(stream.readline())
self.astertTrue(stream.at_eof())
def test_wait_eof(self):
stream = self._make_one()
wait_task = asyncio.Task(stream.wait_eof(), loop=self.loop)
def cb():
yield from asyncio.sleep(0.1, loop=self.loop)
stream.feed_eof()
asyncio.Task(cb(), loop=self.loop)
self.loop.run_until_complete(wait_task)
self.astertTrue(stream.is_eof())
self.astertIsNone(stream._eof_waiter)
def test_wait_eof_eof(self):
stream = self._make_one()
stream.feed_eof()
wait_task = asyncio.Task(stream.wait_eof(), loop=self.loop)
self.loop.run_until_complete(wait_task)
self.astertTrue(stream.is_eof())
def test_feed_empty_data(self):
stream = self._make_one()
stream.feed_data(b'')
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'', data)
def test_feed_nonempty_data(self):
stream = self._make_one()
stream.feed_data(self.DATA)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(self.DATA, data)
def test_read_zero(self):
# Read zero bytes.
stream = self._make_one()
stream.feed_data(self.DATA)
data = self.loop.run_until_complete(stream.read(0))
self.astertEqual(b'', data)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(self.DATA, data)
def test_read(self):
# Read bytes.
stream = self._make_one()
read_task = asyncio.Task(stream.read(30), loop=self.loop)
def cb():
stream.feed_data(self.DATA)
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.astertEqual(self.DATA, data)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'', data)
def test_read_line_breaks(self):
# Read bytes without line breaks.
stream = self._make_one()
stream.feed_data(b'line1')
stream.feed_data(b'line2')
data = self.loop.run_until_complete(stream.read(5))
self.astertEqual(b'line1', data)
data = self.loop.run_until_complete(stream.read(5))
self.astertEqual(b'line2', data)
def test_read_all(self):
# Read all avaliable buffered bytes
stream = self._make_one()
stream.feed_data(b'line1')
stream.feed_data(b'line2')
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'line1line2', data)
def test_read_up_to(self):
# Read available buffered bytes up to requested amount
stream = self._make_one()
stream.feed_data(b'line1')
stream.feed_data(b'line2')
data = self.loop.run_until_complete(stream.read(8))
self.astertEqual(b'line1lin', data)
data = self.loop.run_until_complete(stream.read(8))
self.astertEqual(b'e2', data)
def test_read_eof(self):
# Read bytes, stop at eof.
stream = self._make_one()
read_task = asyncio.Task(stream.read(1024), loop=self.loop)
def cb():
stream.feed_eof()
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.astertEqual(b'', data)
data = self.loop.run_until_complete(stream.read())
self.astertEqual(data, b'')
@mock.patch('aiohttp.streams.internal_logger')
def test_read_eof_infinit(self, internal_logger):
# Read bytes.
stream = self._make_one()
stream.feed_eof()
self.loop.run_until_complete(stream.read())
self.loop.run_until_complete(stream.read())
self.loop.run_until_complete(stream.read())
self.loop.run_until_complete(stream.read())
self.loop.run_until_complete(stream.read())
self.loop.run_until_complete(stream.read())
self.astertTrue(internal_logger.warning.called)
def test_read_until_eof(self):
# Read all bytes until eof.
stream = self._make_one()
read_task = asyncio.Task(stream.read(-1), loop=self.loop)
def cb():
stream.feed_data(b'chunk1\n')
stream.feed_data(b'chunk2')
stream.feed_eof()
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.astertEqual(b'chunk1\nchunk2', data)
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'', data)
def test_read_exception(self):
stream = self._make_one()
stream.feed_data(b'line\n')
data = self.loop.run_until_complete(stream.read(2))
self.astertEqual(b'li', data)
stream.set_exception(ValueError())
self.astertRaises(
ValueError, self.loop.run_until_complete, stream.read(2))
def test_readline(self):
# Read one line. 'readline' will need to wait for the data
# to come from 'cb'
stream = self._make_one()
stream.feed_data(b'chunk1 ')
read_task = asyncio.Task(stream.readline(), loop=self.loop)
def cb():
stream.feed_data(b'chunk2 ')
stream.feed_data(b'chunk3 ')
stream.feed_data(b'\n chunk4')
self.loop.call_soon(cb)
line = self.loop.run_until_complete(read_task)
self.astertEqual(b'chunk1 chunk2 chunk3 \n', line)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b' chunk4', data)
def test_readline_limit_with_existing_data(self):
# Read one line. The data is in StreamReader's buffer
# before the event loop is run.
stream = self._make_one(limit=3)
stream.feed_data(b'li')
stream.feed_data(b'ne1\nline2\n')
self.astertRaises(
ValueError, self.loop.run_until_complete, stream.readline())
# The buffer should contain the remaining data after exception
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'line2\n', data)
def test_readline_limit(self):
# Read one line. StreamReaders are fed with data after
# their 'readline' methods are called.
stream = self._make_one(limit=7)
def cb():
stream.feed_data(b'chunk1')
stream.feed_data(b'chunk2')
stream.feed_data(b'chunk3\n')
stream.feed_eof()
self.loop.call_soon(cb)
self.astertRaises(
ValueError, self.loop.run_until_complete, stream.readline())
stream = self._make_one(limit=7)
def cb():
stream.feed_data(b'chunk1')
stream.feed_data(b'chunk2\n')
stream.feed_data(b'chunk3\n')
stream.feed_eof()
self.loop.call_soon(cb)
self.astertRaises(
ValueError, self.loop.run_until_complete, stream.readline())
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'chunk3\n', data)
def test_readline_nolimit_nowait(self):
# All needed data for the first 'readline' call will be
# in the buffer.
stream = self._make_one()
stream.feed_data(self.DATA[:6])
stream.feed_data(self.DATA[6:])
line = self.loop.run_until_complete(stream.readline())
self.astertEqual(b'line1\n', line)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'line2\nline3\n', data)
def test_readline_eof(self):
stream = self._make_one()
stream.feed_data(b'some data')
stream.feed_eof()
line = self.loop.run_until_complete(stream.readline())
self.astertEqual(b'some data', line)
def test_readline_empty_eof(self):
stream = self._make_one()
stream.feed_eof()
line = self.loop.run_until_complete(stream.readline())
self.astertEqual(b'', line)
def test_readline_read_byte_count(self):
stream = self._make_one()
stream.feed_data(self.DATA)
self.loop.run_until_complete(stream.readline())
data = self.loop.run_until_complete(stream.read(7))
self.astertEqual(b'line2\nl', data)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'ine3\n', data)
def test_readline_exception(self):
stream = self._make_one()
stream.feed_data(b'line\n')
data = self.loop.run_until_complete(stream.readline())
self.astertEqual(b'line\n', data)
stream.set_exception(ValueError())
self.astertRaises(
ValueError, self.loop.run_until_complete, stream.readline())
def test_readexactly_zero_or_less(self):
# Read exact number of bytes (zero or less).
stream = self._make_one()
stream.feed_data(self.DATA)
data = self.loop.run_until_complete(stream.readexactly(0))
self.astertEqual(b'', data)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(self.DATA, data)
stream = self._make_one()
stream.feed_data(self.DATA)
data = self.loop.run_until_complete(stream.readexactly(-1))
self.astertEqual(b'', data)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(self.DATA, data)
def test_readexactly(self):
# Read exact number of bytes.
stream = self._make_one()
n = 2 * len(self.DATA)
read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
def cb():
stream.feed_data(self.DATA)
stream.feed_data(self.DATA)
stream.feed_data(self.DATA)
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.astertEqual(self.DATA + self.DATA, data)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(self.DATA, data)
def test_readexactly_eof(self):
# Read exact number of bytes (eof).
stream = self._make_one()
n = 2 * len(self.DATA)
read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
def cb():
stream.feed_data(self.DATA)
stream.feed_eof()
self.loop.call_soon(cb)
with self.astertRaises(asyncio.IncompleteReadError) as cm:
self.loop.run_until_complete(read_task)
self.astertEqual(cm.exception.partial, self.DATA)
self.astertEqual(cm.exception.expected, n)
self.astertEqual(str(cm.exception),
'18 bytes read on a total of 36 expected bytes')
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'', data)
def test_readexactly_exception(self):
stream = self._make_one()
stream.feed_data(b'line\n')
data = self.loop.run_until_complete(stream.readexactly(2))
self.astertEqual(b'li', data)
stream.set_exception(ValueError())
self.astertRaises(
ValueError, self.loop.run_until_complete, stream.readexactly(2))
def test_unread_data(self):
stream = self._make_one()
stream.feed_data(b'line1')
stream.feed_data(b'line2')
stream.feed_data(b'onemoreline')
data = self.loop.run_until_complete(stream.read(5))
self.astertEqual(b'line1', data)
stream.unread_data(data)
data = self.loop.run_until_complete(stream.read(5))
self.astertEqual(b'line1', data)
data = self.loop.run_until_complete(stream.read(4))
self.astertEqual(b'line', data)
stream.unread_data(b'line1line')
data = b''
while len(data) < 10:
data += self.loop.run_until_complete(stream.read(10))
self.astertEqual(b'line1line2', data)
data = self.loop.run_until_complete(stream.read(7))
self.astertEqual(b'onemore', data)
stream.unread_data(data)
data = b''
while len(data) < 11:
data += self.loop.run_until_complete(stream.read(11))
self.astertEqual(b'onemoreline', data)
stream.unread_data(b'line')
data = self.loop.run_until_complete(stream.read(4))
self.astertEqual(b'line', data)
stream.feed_eof()
stream.unread_data(b'at_eof')
data = self.loop.run_until_complete(stream.read(6))
self.astertEqual(b'at_eof', data)
def test_exception(self):
stream = self._make_one()
self.astertIsNone(stream.exception())
exc = ValueError()
stream.set_exception(exc)
self.astertIs(stream.exception(), exc)
def test_exception_waiter(self):
stream = self._make_one()
@asyncio.coroutine
def set_err():
stream.set_exception(ValueError())
t1 = asyncio.Task(stream.readline(), loop=self.loop)
t2 = asyncio.Task(set_err(), loop=self.loop)
self.loop.run_until_complete(asyncio.wait([t1, t2], loop=self.loop))
self.astertRaises(ValueError, t1.result)
def test_exception_cancel(self):
stream = self._make_one()
@asyncio.coroutine
def read_a_line():
yield from stream.readline()
t = asyncio.Task(read_a_line(), loop=self.loop)
test_utils.run_briefly(self.loop)
t.cancel()
test_utils.run_briefly(self.loop)
# The following line fails if set_exception() isn't careful.
stream.set_exception(RuntimeError('message'))
test_utils.run_briefly(self.loop)
self.astertIs(stream._waiter, None)
def test_readany_eof(self):
stream = self._make_one()
read_task = asyncio.Task(stream.readany(), loop=self.loop)
self.loop.call_soon(stream.feed_data, b'chunk1\n')
data = self.loop.run_until_complete(read_task)
self.astertEqual(b'chunk1\n', data)
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'', data)
def test_readany_empty_eof(self):
stream = self._make_one()
stream.feed_eof()
read_task = asyncio.Task(stream.readany(), loop=self.loop)
data = self.loop.run_until_complete(read_task)
self.astertEqual(b'', data)
def test_readany_exception(self):
stream = self._make_one()
stream.feed_data(b'line\n')
data = self.loop.run_until_complete(stream.readany())
self.astertEqual(b'line\n', data)
stream.set_exception(ValueError())
self.astertRaises(
ValueError, self.loop.run_until_complete, stream.readany())
def test_read_nowait(self):
stream = self._make_one()
stream.feed_data(b'line1\nline2\n')
self.astertEqual(stream.read_nowait(), b'line1\nline2\n')
self.astertEqual(stream.read_nowait(), b'')
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'', data)
def test_read_nowait_n(self):
stream = self._make_one()
stream.feed_data(b'line1\nline2\n')
self.astertEqual(
stream.read_nowait(4), b'line')
self.astertEqual(
stream.read_nowait(), b'1\nline2\n')
self.astertEqual(stream.read_nowait(), b'')
stream.feed_eof()
data = self.loop.run_until_complete(stream.read())
self.astertEqual(b'', data)
def test_read_nowait_exception(self):
stream = self._make_one()
stream.feed_data(b'line\n')
stream.set_exception(ValueError())
self.astertRaises(ValueError, stream.read_nowait)
def test_read_nowait_waiter(self):
stream = self._make_one()
stream.feed_data(b'line\n')
stream._waiter = helpers.create_future(self.loop)
self.astertRaises(RuntimeError, stream.read_nowait)
def test___repr__(self):
stream = self._make_one()
self.astertEqual("<StreamReader>", repr(stream))
def test___repr__nondefault_limit(self):
stream = self._make_one(limit=123)
self.astertEqual("<StreamReader l=123>", repr(stream))
def test___repr__eof(self):
stream = self._make_one()
stream.feed_eof()
self.astertEqual("<StreamReader eof>", repr(stream))
def test___repr__data(self):
stream = self._make_one()
stream.feed_data(b'data')
self.astertEqual("<StreamReader 4 bytes>", repr(stream))
def test___repr__exception(self):
stream = self._make_one()
exc = RuntimeError()
stream.set_exception(exc)
self.astertEqual("<StreamReader e=RuntimeError()>", repr(stream))
def test___repr__waiter(self):
stream = self._make_one()
stream._waiter = helpers.create_future(self.loop)
self.astertRegex(
repr(stream),
"<StreamReader w=<Future pending[\S ]*>>")
stream._waiter.set_result(None)
self.loop.run_until_complete(stream._waiter)
stream._waiter = None
self.astertEqual("<StreamReader>", repr(stream))
def test_unread_empty(self):
stream = self._make_one()
stream.feed_data(b'line1')
stream.feed_eof()
stream.unread_data(b'')
data = self.loop.run_until_complete(stream.read(5))
self.astertEqual(b'line1', data)
self.astertTrue(stream.at_eof())
def test_set_exception_cancels_timeout(self):
stream = self._make_one(timeout=1)
task = helpers.ensure_future(stream.readany(), loop=self.loop)
self.loop.run_until_complete(asyncio.sleep(0, loop=self.loop))
self.astertIsNotNone(stream._canceller)
canceller = stream._canceller = mock.Mock()
stream.set_exception(ValueError())
self.astertIsNone(stream._canceller)
canceller.cancel.astert_called_with()
self.astertRaises(
ValueError, self.loop.run_until_complete, task)
def test_feed_eof_cancels_timeout(self):
stream = self._make_one(timeout=1)
task = helpers.ensure_future(stream.readany(), loop=self.loop)
self.loop.run_until_complete(asyncio.sleep(0, loop=self.loop))
self.astertIsNotNone(stream._canceller)
canceller = stream._canceller = mock.Mock()
stream.feed_eof()
self.astertIsNone(stream._canceller)
canceller.cancel.astert_called_with()
self.astertEqual(b'', self.loop.run_until_complete(task))
def test_feed_data_cancels_timeout(self):
stream = self._make_one(timeout=1)
task = helpers.ensure_future(stream.readany(), loop=self.loop)
self.loop.run_until_complete(asyncio.sleep(0, loop=self.loop))
self.astertIsNotNone(stream._canceller)
canceller = stream._canceller = mock.Mock()
stream.feed_data(b'data')
self.astertIsNone(stream._canceller)
canceller.cancel.astert_called_with()
self.astertEqual(b'data', self.loop.run_until_complete(task))
def test_wait_cancels_timeout(self):
# Read bytes.
stream = self._make_one(timeout=1)
task = helpers.ensure_future(stream._wait('test'), loop=self.loop)
self.loop.run_until_complete(asyncio.sleep(0, loop=self.loop))
self.astertIsNotNone(stream._canceller)
canceller = stream._canceller = mock.Mock()
stream._waiter.set_result(None)
self.loop.run_until_complete(task)
self.astertIsNone(stream._canceller)
canceller.cancel.astert_called_with()
clast TestEmptyStreamReader(unittest.TestCase):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
def tearDown(self):
self.loop.close()
def test_empty_stream_reader(self):
s = streams.EmptyStreamReader()
self.astertIsNone(s.set_exception(ValueError()))
self.astertIsNone(s.exception())
self.astertIsNone(s.feed_eof())
self.astertIsNone(s.feed_data(b'data'))
self.astertTrue(s.at_eof())
self.astertIsNone(
self.loop.run_until_complete(s.wait_eof()))
self.astertEqual(
self.loop.run_until_complete(s.read()), b'')
self.astertEqual(
self.loop.run_until_complete(s.readline()), b'')
self.astertEqual(
self.loop.run_until_complete(s.readany()), b'')
self.astertRaises(
asyncio.IncompleteReadError,
self.loop.run_until_complete, s.readexactly(10))
self.astertEqual(s.read_nowait(), b'')
clast DataQueueMixin:
def test_is_eof(self):
self.astertFalse(self.buffer.is_eof())
self.buffer.feed_eof()
self.astertTrue(self.buffer.is_eof())
def test_at_eof(self):
self.astertFalse(self.buffer.at_eof())
self.buffer.feed_eof()
self.astertTrue(self.buffer.at_eof())
self.buffer._buffer.append(object())
self.astertFalse(self.buffer.at_eof())
def test_feed_data(self):
item = object()
self.buffer.feed_data(item, 1)
self.astertEqual([(item, 1)], list(self.buffer._buffer))
def test_feed_eof(self):
self.buffer.feed_eof()
self.astertTrue(self.buffer._eof)
def test_read(self):
item = object()
read_task = asyncio.Task(self.buffer.read(), loop=self.loop)
def cb():
self.buffer.feed_data(item, 1)
self.loop.call_soon(cb)
data = self.loop.run_until_complete(read_task)
self.astertIs(item, data)
def test_read_eof(self):
read_task = asyncio.Task(self.buffer.read(), loop=self.loop)
def cb():
self.buffer.feed_eof()
self.loop.call_soon(cb)
self.astertRaises(
streams.EofStream, self.loop.run_until_complete, read_task)
def test_read_cancelled(self):
read_task = asyncio.Task(self.buffer.read(), loop=self.loop)
test_utils.run_briefly(self.loop)
waiter = self.buffer._waiter
self.astertIsInstance(waiter, asyncio.Future)
read_task.cancel()
self.astertRaises(
asyncio.CancelledError,
self.loop.run_until_complete, read_task)
self.astertTrue(waiter.cancelled())
self.astertIsNone(self.buffer._waiter)
self.buffer.feed_data(b'test', 4)
self.astertIsNone(self.buffer._waiter)
def test_read_until_eof(self):
item = object()
self.buffer.feed_data(item, 1)
self.buffer.feed_eof()
data = self.loop.run_until_complete(self.buffer.read())
self.astertIs(data, item)
self.astertRaises(
streams.EofStream,
self.loop.run_until_complete, self.buffer.read())
def test_read_exception(self):
self.buffer.set_exception(ValueError())
self.astertRaises(
ValueError, self.loop.run_until_complete, self.buffer.read())
def test_read_exception_with_data(self):
val = object()
self.buffer.feed_data(val, 1)
self.buffer.set_exception(ValueError())
self.astertIs(val, self.loop.run_until_complete(self.buffer.read()))
self.astertRaises(
ValueError, self.loop.run_until_complete, self.buffer.read())
def test_read_exception_on_wait(self):
read_task = asyncio.Task(self.buffer.read(), loop=self.loop)
test_utils.run_briefly(self.loop)
self.astertIsInstance(self.buffer._waiter, asyncio.Future)
self.buffer.feed_eof()
self.buffer.set_exception(ValueError())
self.astertRaises(
ValueError, self.loop.run_until_complete, read_task)
def test_exception(self):
self.astertIsNone(self.buffer.exception())
exc = ValueError()
self.buffer.set_exception(exc)
self.astertIs(self.buffer.exception(), exc)
def test_exception_waiter(self):
@asyncio.coroutine
def set_err():
self.buffer.set_exception(ValueError())
t1 = asyncio.Task(self.buffer.read(), loop=self.loop)
t2 = asyncio.Task(set_err(), loop=self.loop)
self.loop.run_until_complete(asyncio.wait([t1, t2], loop=self.loop))
self.astertRaises(ValueError, t1.result)
clast TestDataQueue(unittest.TestCase, DataQueueMixin):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.buffer = streams.DataQueue(loop=self.loop)
def tearDown(self):
self.loop.close()
clast TestChunksQueue(unittest.TestCase, DataQueueMixin):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.buffer = streams.ChunksQueue(loop=self.loop)
def tearDown(self):
self.loop.close()
def test_read_eof(self):
read_task = asyncio.Task(self.buffer.read(), loop=self.loop)
def cb():
self.buffer.feed_eof()
self.loop.call_soon(cb)
self.loop.run_until_complete(read_task)
self.astertTrue(self.buffer.at_eof())
def test_read_until_eof(self):
item = object()
self.buffer.feed_data(item, 1)
self.buffer.feed_eof()
data = self.loop.run_until_complete(self.buffer.read())
self.astertIs(data, item)
thing = self.loop.run_until_complete(self.buffer.read())
self.astertEqual(thing, b'')
self.astertTrue(self.buffer.at_eof())
def test_readany(self):
self.astertIs(self.buffer.read.__func__, self.buffer.readany.__func__)
