pytest.mark.with_backend

Here are the examples of the python api pytest.mark.with_backend taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

38 Examples 7

Example 1

Project: s3ql Source File: t1_backends.py
Function: test_issue_114
@pytest.mark.with_backend('swift/raw')
def test_issue114(backend, monkeypatch):
    key = newname()
    value = newvalue()
    monkeypatch.setitem(backend.options, 'disable-expect100', True)
    backend[key] = value
    assert_in_index(backend, [key])

Example 2

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('*/raw', 'local/{plain,aes,zlib}')
def test_complex_meta(backend):
    key = newname()
    value = newvalue()

    metadata = { 'com\nplex: key': 42,
                 'farp_': False, 'non-farp': True,
                 'blu%rz': 23.283475,
                 'görp': b'heelo',
                 'sch.al': 'gorroobalp\nfurrö!',
                 'lo-ng': 'foobarz' * 40 }

    assert key not in backend
    backend.store(key, value, metadata)
    (value2, metadata2) = fetch_object(backend, key)

    assert value == value2
    assert metadata == metadata2
    assert lookup_object(backend, key) == metadata

Example 3

Project: s3ql Source File: t1_backends.py
Function: test_list
@pytest.mark.with_backend('*/aes')
def test_list(backend):
    keys = ([ 'prefixa' + newname() for dummy in range(6) ]
            + [ 'prefixb' + newname() for dummy in range(6) ])
    values = [ newvalue() for dummy in range(12) ]

    assert set(backend.list()) == empty_set
    for i in range(12):
        backend[keys[i]] = values[i]
    assert_in_index(backend, keys)

    assert set(backend.list('prefixa')) == set(keys[:6])
    assert set(backend.list('prefixb')) == set(keys[6:])
    assert set(backend.list('prefixc')) == empty_set

Example 4

Project: s3ql Source File: t1_backends.py
Function: test_delete
@pytest.mark.with_backend('*/aes')
def test_delete(backend):
    key = newname()
    value = newvalue()

    backend[key] = value

    # Wait for object to become visible
    assert_in_index(backend, [key])
    fetch_object(backend, key)

    # Delete it
    del backend[key]

    # Make sure that it's truly gone
    assert_not_in_index(backend, [key])
    assert_not_readable(backend, key)

Example 5

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('local/{raw,plain,aes,aes+zlib,zlib}')
def test_issue431(backend):
    key = newname()
    hdr_len = struct.calcsize(b'<I')

    def do_write(fh):
        fh.write(b'\xFF' * 50)
        fh.write(b'\xFF' * 50)
    backend.perform_write(do_write, key)

    def do_read(fh):
        fh.read(50 + 2*hdr_len)
        fh.read(50)
        assert fh.read(50) == b''
    backend.perform_read(do_read, key)

Example 6

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('local/{aes,aes+zlib}')
def test_replay(backend):
    plain_backend = backend.backend

    # Create encrypted object
    key1 = newname()
    key2 = newname()
    value = newvalue()
    backend[key1] = value

    # Retrieve compressed data
    (compr_value, meta) = fetch_object(plain_backend, key1)
    compr_value = bytearray(compr_value)

    # Copy into new object
    plain_backend.store(key2, compr_value, meta)

    assert_raises(CorruptedObjectError, fetch_object, backend, key2)

Example 7

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/raw', require_mock_server=True)
def test_delete_s3error(backend, monkeypatch):
    value = b'hello there, let us see whats going on'
    key = 'quote'
    backend[key] = value

    # Monkeypatch request handler to produce 3 errors
    handler_class = mock_server.S3CRequestHandler
    def do_DELETE(self, real_DELETE=handler_class.do_DELETE, count=[0]):
        count[0] += 1
        if count[0] > 3:
            return real_DELETE(self)
        else:
            self.send_error(503, code='OperationAborted')
    monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE)
    assert_raises(OperationAbortedError, backend.delete, key)

    enable_temp_fail(backend)
    backend.delete(key)

Example 8

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('*/*')
def test_read_write(backend):
    key = newname()
    value = newvalue()
    metadata = { 'jimmy': 'jups@42' }

    assert key not in backend
    assert_raises(NoSuchObject, backend.lookup, key)
    assert_raises(NoSuchObject, backend.fetch, key)

    def do_write(fh):
        fh.write(value)
    backend.perform_write(do_write, key, metadata)

    assert_in_index(backend, [key])
    (value2, metadata2) = fetch_object(backend, key)

    assert value == value2
    assert metadata == metadata2
    assert lookup_object(backend, key) == metadata

Example 9

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('*/raw', 'local/{plain,aes,zlib,aes+zlib}',
                          require_immediate_consistency=True)
def test_readslowly(backend):
    key = newname()
    value = newvalue()
    metadata = { 'jimmy': 'jups@42' }

    backend.store(key, value, metadata)

    s3ql.backends.common.BUFSIZE = 1
    try:
        with backend.open_read(key) as fh:
            # Force slow reading from underlying layer
            if hasattr(fh, 'fh'):
                def read_slowly(size, *, real_read=fh.fh.read):
                    return real_read(1)
                fh.fh.read = read_slowly

            buf = []
            while True:
                buf.append(fh.read(1))
                if not buf[-1]:
                    break
            value2 = b''.join(buf)
            metadata2 =  fh.metadata
    finally:
        s3ql.backends.common.BUFSIZE = BUFSIZE

    assert value == value2
    assert metadata == metadata2

Example 10

Project: s3ql Source File: t1_backends.py
Function: test_delete_multi
@pytest.mark.with_backend('*/aes')
def test_delete_multi(backend):
    keys = [ newname() for _ in range(30) ]
    value = newvalue()

    # Create objects
    for key in keys:
        backend[key] = value

    # Wait for them
    assert_in_index(backend, keys)
    for key in keys:
        fetch_object(backend, key)

    # Delete half of them
    # We don't use force=True but catch the exemption to increase the
    # chance that some existing objects are not deleted because of the
    # error.
    to_delete = keys[::2]
    to_delete.insert(7, 'not_existing')
    try:
        backend.delete_multi(to_delete)
    except NoSuchObject:
        pass

    # Without full consistency, deleting an non-existing object
    # may not give an error
    assert backend.unittest_info.retry_time or len(to_delete) > 0

    deleted = set(keys[::2]) - set(to_delete)
    assert len(deleted) > 0
    remaining = set(keys) - deleted

    assert_not_in_index(backend, deleted)
    for key in deleted:
        assert_not_readable(backend, key)

    assert_in_index(backend, remaining)
    for key in remaining:
        fetch_object(backend, key)

Example 11

Project: s3ql Source File: t1_backends.py
Function: test_clear
@pytest.mark.with_backend('*/aes')
def test_clear(backend):
    keys = [ newname() for _ in range(5) ]
    value = newvalue()

    # Create objects
    for key in keys:
        backend[key] = value

    # Wait for them
    assert_in_index(backend, keys)
    for key in keys:
        fetch_object(backend, key)

    # Delete everything
    backend.clear()

    assert_not_in_index(backend, keys)
    for key in keys:
        assert_not_readable(backend, key)

Example 12

Project: s3ql Source File: t1_backends.py
Function: test_copy
@pytest.mark.with_backend('*/raw', 'local/{plain,aes,zlib}')
def test_copy(backend):
    key1 = newname()
    key2 = newname()
    value = newvalue()
    metadata = { 'jimmy': 'jups@42' }

    backend.store(key1, value, metadata)

    # Wait for object to become visible
    assert_in_index(backend, [key1])
    fetch_object(backend, key1)

    assert_not_in_index(backend, [key2])
    assert_not_readable(backend, key2)

    backend.copy(key1, key2)

    assert_in_index(backend, [key2])
    (value2, metadata2) = fetch_object(backend, key2)

    assert value == value2
    assert metadata == metadata2

Example 13

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('*/raw')
def test_copy_special(backend):
    key1 = 'with_+_char/orig'
    key2 = 'with_+_char/dest'
    value = b'Just a couple of random bytes'

    backend.store(key1, value)

    assert_not_in_index(backend, [key2])
    assert_not_readable(backend, key2)

    # Wait for object to become visible
    assert_in_index(backend, [key1])
    fetch_object(backend, key1)

    backend.copy(key1, key2)

    assert_in_index(backend, [key2])
    value2 = backend[key2]
    assert value == value2

Example 14

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}')
def test_copy_newmeta(backend):
    key1 = newname()
    key2 = newname()
    value = newvalue()
    meta1 = { 'jimmy': 'jups@42' }
    meta2 = { 'jiy': 'jfobauske42' }

    backend.store(key1, value, meta1)

    # Wait for object to become visible
    assert_in_index(backend, [key1])
    fetch_object(backend, key1)

    assert_not_in_index(backend, [key2])
    assert_not_readable(backend, key2)

    backend.copy(key1, key2, meta2)

    assert_in_index(backend, [key2])
    (value2, meta) = fetch_object(backend, key2)

    assert value == value2
    assert meta == meta2

Example 15

Project: s3ql Source File: t1_backends.py
Function: test_rename
@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}')
def test_rename(backend):
    key1 = newname()
    key2 = newname()
    value = newvalue()
    metadata = { 'jimmy': 'jups@42' }

    backend.store(key1, value, metadata)

    # Wait for object to become visible
    assert_in_index(backend, [key1])
    fetch_object(backend, key1)

    assert_not_in_index(backend, [key2])
    assert_not_readable(backend, key2)

    backend.rename(key1, key2)

    assert_in_index(backend, [key2])
    (value2, metadata2) = fetch_object(backend, key2)

    assert value == value2
    assert metadata == metadata2

    assert_not_in_index(backend, [key1])
    assert_not_readable(backend, key1)

Example 16

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}')
def test_rename_newmeta(backend):
    key1 = newname()
    key2 = newname()
    value = newvalue()
    meta1 = { 'jimmy': 'jups@42' }
    meta2 = { 'apple': 'potatoes' }

    backend.store(key1, value, meta1)

    # Wait for object to become visible
    assert_in_index(backend, [key1])
    fetch_object(backend, key1)

    assert_not_in_index(backend, [key2])
    assert_not_readable(backend, key2)

    backend.rename(key1, key2, meta2)

    assert_in_index(backend, [key2])
    (value2, meta) = fetch_object(backend, key2)

    assert value == value2
    assert meta == meta2

Example 17

Project: s3ql Source File: t1_backends.py
Function: test_update_meta
@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}')
def test_update_meta(backend):
    key = 'simple'
    value = b'not too hard'
    meta1 = { 'jimmy': 'jups@42' }
    meta2 = { 'apple': 'potatoes' }

    backend.store(key, value, meta1)

    # Wait for object to become visible
    assert_in_index(backend, [key])
    fetch_object(backend, key)

    backend.update_meta(key, meta2)

    # Wait for updated metadata
    waited=0
    sleep_time = 1
    while True:
        (value2, meta) = fetch_object(backend, key)
        if meta != meta1:
            break
        elif waited >= backend.unittest_info.retry_time:
            pytest.fail('metadata for %s not updated after %d seconds'
                        % (key, waited))
        time.sleep(sleep_time)
        waited += sleep_time

    assert value == value2
    assert meta == meta2

Example 18

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes,zlib}',
                          require_mock_server=True)
def test_copy_error(backend, monkeypatch):
    value = b'hello there, let us see whats going on'
    key1 = 'object-key1'
    key2 = 'object-key2'
    backend[key1] = value

    # Monkeypatch request handler to produce error
    handler_class = mock_server.S3CRequestHandler
    def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]):
        count[0] += 1
        if count[0] > 3:
            return real_PUT(self)
        else:
            self.send_error(200, code='OperationAborted')
    monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
    assert_raises(OperationAbortedError, backend.copy, key1, key2)

    enable_temp_fail(backend)
    backend.copy(key1, key2)

Example 19

Project: s3ql Source File: t1_backends.py
Function: test_corruption
@pytest.mark.with_backend('local/{aes,aes+zlib,zlib,bzip2,lzma}')
def test_corruption(backend):
    plain_backend = backend.backend

    # Create compressed object
    key = newname()
    value = newvalue()
    backend[key] = value

    # Retrieve compressed data
    (compr_value, meta) = fetch_object(plain_backend, key)
    compr_value = bytearray(compr_value)

    # Overwrite with corrupted data
    # (this needs immediate consistency)
    compr_value[-3:] = b'000'
    plain_backend.store(key, compr_value, meta)

    with pytest.raises(CorruptedObjectError) as exc:
        fetch_object(backend, key)

    if backend.passphrase is None: # compression only
        assert exc.value.str == 'Invalid compressed stream'
    else:
        assert exc.value.str == 'HMAC mismatch'

Example 20

Project: s3ql Source File: t1_backends.py
Function: test_extra_data
@pytest.mark.with_backend('local/{aes,aes+zlib,zlib,bzip2,lzma}')
def test_extra_data(backend):
    plain_backend = backend.backend

    # Create compressed object
    key = newname()
    value = newvalue()
    backend[key] = value

    # Retrieve compressed data
    (compr_value, meta) = fetch_object(plain_backend, key)
    compr_value = bytearray(compr_value)

    # Overwrite with extended data
    # (this needs immediate consistency)
    compr_value += b'000'
    plain_backend.store(key, compr_value, meta)

    with pytest.raises(CorruptedObjectError) as exc:
        fetch_object(backend, key)

    if backend.passphrase is None: # compression only
        assert exc.value.str == 'Data after end of compressed stream'
    else:
        assert exc.value.str == 'Extraneous data at end of object'

Example 21

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('*/{raw,plain,aes,aes+zlib,zlib}')
def test_multi_packet(backend):
    '''Write and read packet extending over multiple chunks'''
    key = newname()

    def do_write(fh):
        for i in range(5):
            fh.write(b'\xFF' * BUFSIZE)
    backend.perform_write(do_write, key)

    def do_read(fh):
        buf = bytearray()
        while True:
            tmp = fh.read(BUFSIZE//2)
            if not tmp:
                break
            buf += tmp
        return buf
    res = backend.perform_read(do_read, key)
    assert res == b'\xFF' * (5*BUFSIZE)

Example 22

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('local/{aes,aes+zlib}')
def test_encryption(backend):
    plain_backend = backend.backend

    plain_backend['plain'] = b'foobar452'
    backend.store('encrypted', b'testdata', { 'tag': True })

    assert plain_backend['encrypted'] != b'testdata'
    assert_raises(CorruptedObjectError, backend.fetch, 'plain')
    assert_raises(CorruptedObjectError, backend.lookup, 'plain')

    backend.passphrase = None
    backend.store('not-encrypted', b'testdata2395', { 'tag': False })
    assert_raises(CorruptedObjectError, backend.fetch, 'encrypted')
    assert_raises(CorruptedObjectError, backend.lookup, 'encrypted')

    backend.passphrase = b'jobzrul'
    assert_raises(CorruptedObjectError, backend.fetch, 'encrypted')
    assert_raises(CorruptedObjectError, backend.lookup, 'encrypted')
    assert_raises(ObjectNotEncrypted, backend.fetch, 'not-encrypted')
    assert_raises(ObjectNotEncrypted, backend.lookup, 'not-encrypted')

Example 23

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/raw',
                          require_mock_server=True)
def test_list_bug(backend, monkeypatch):
    keys = ([ 'prefixa' + newname() for dummy in range(6) ]
            + [ 'prefixb' + newname() for dummy in range(6) ])
    values = [ newvalue() for dummy in range(12) ]

    assert set(backend.list()) == empty_set
    for i in range(12):
        backend[keys[i]] = values[i]
    assert_in_index(backend, keys, 0)

    # Force reconnect during list
    handler_class = mock_server.S3CRequestHandler
    def do_list(self, q, _real=handler_class.do_list):
        q.params['max_keys'] = [ '3' ]
        self.close_connection = True
        return _real(self, q)
    monkeypatch.setattr(handler_class, 'do_list', do_list)

    enable_temp_fail(backend)
    assert set(backend.list()) == set(keys)

Example 24

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/aes+zlib',
                          require_mock_server=True)
def test_corrupted_get(backend, monkeypatch):
    key = 'brafasel'
    value = b'hello there, let us see whats going on'
    backend[key] = value

    # Monkeypatch request handler to produce invalid etag
    handler_class = mock_server.S3CRequestHandler
    def send_header(self, keyword ,value, count=[0],
                    send_header_real=handler_class.send_header):
        if keyword == 'ETag':
            count[0] += 1
            if count[0] <= 3:
                value = value[::-1]
        return send_header_real(self, keyword, value)
    monkeypatch.setattr(handler_class, 'send_header', send_header)

    with assert_logs('^MD5 mismatch for', count=1, level=logging.WARNING):
        assert_raises(BadDigestError, backend.fetch, key)

    enable_temp_fail(backend)
    with assert_logs('^MD5 mismatch for', count=2, level=logging.WARNING):
        assert backend[key] == value

Example 25

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_corrupted_meta(backend, monkeypatch):
    key = 'brafasel'
    value = b'hello there, let us see whats going on'
    backend[key] = value

    # Monkeypatch request handler to mess up metadata
    handler_class = mock_server.S3CRequestHandler
    def send_header(self, keyword ,value, count=[0],
                    send_header_real=handler_class.send_header):
        if keyword == self.hdr_prefix + 'Meta-md5':
            count[0] += 1
            if count[0] <= 3:
                value = value[::-1]
        return send_header_real(self, keyword, value)
    monkeypatch.setattr(handler_class, 'send_header', send_header)

    with assert_logs('^MD5 mismatch in metadata for', count=1, level=logging.WARNING):
        assert_raises(BadDigestError, backend.fetch, key)

    enable_temp_fail(backend)
    with assert_logs('^MD5 mismatch in metadata for', count=2, level=logging.WARNING):
        assert backend[key] == value

Example 26

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_corrupted_put(backend, monkeypatch):
    key = 'brafasel'
    value = b'hello there, let us see whats going on'

    # Monkeypatch request handler to produce invalid etag
    handler_class = mock_server.S3CRequestHandler
    def send_header(self, keyword ,value, count=[0],
                    send_header_real=handler_class.send_header):
        if keyword == 'ETag':
            count[0] += 1
            if count[0] < 3:
                value = value[::-1]
        return send_header_real(self, keyword, value)
    monkeypatch.setattr(handler_class, 'send_header', send_header)

    fh = backend.open_write(key)
    fh.write(value)
    assert_raises(BadDigestError, fh.close)

    enable_temp_fail(backend)
    fh.close()

    assert backend[key] == value

Example 27

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_get_s3error(backend, monkeypatch):
    value = b'hello there, let us see whats going on'
    key = 'quote'
    backend[key] = value

    # Monkeypatch request handler to produce 3 errors
    handler_class = mock_server.S3CRequestHandler
    def do_GET(self, real_GET=handler_class.do_GET, count=[0]):
        count[0] += 1
        if count[0] > 3:
            return real_GET(self)
        else:
            self.send_error(503, code='OperationAborted')
    monkeypatch.setattr(handler_class, 'do_GET', do_GET)
    assert_raises(OperationAbortedError, backend.fetch, value)

    enable_temp_fail(backend)
    assert backend[key] == value

Example 28

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_head_s3error(backend, monkeypatch):
    value = b'hello there, let us see whats going on'
    key = 'quote'
    meta = {'bar': 42, 'foo': 42**2}
    backend.store(key, value, metadata=meta)

    # Monkeypatch request handler to produce 3 errors
    handler_class = mock_server.S3CRequestHandler
    def do_HEAD(self, real_HEAD=handler_class.do_HEAD, count=[0]):
        count[0] += 1
        if count[0] > 3:
            return real_HEAD(self)
        else:
            self.send_error(503, code='OperationAborted')
    monkeypatch.setattr(handler_class, 'do_HEAD', do_HEAD)
    with pytest.raises(HTTPError) as exc:
        backend.lookup(key)
    assert exc.value.status == 503

    enable_temp_fail(backend)
    assert backend.lookup(key) == meta

Example 29

Project: s3ql Source File: t1_backends.py
Function: test_backoff
@pytest.mark.with_backend('s3c/raw', require_mock_server=True)
def test_backoff(backend, monkeypatch):
    value = b'hello there, let us see whats going on'
    key = 'quote'
    backend[key] = value

    # Monkeypatch request handler
    handler_class = mock_server.S3CRequestHandler
    timestamps = []
    def do_DELETE(self, real_DELETE=handler_class.do_DELETE):
        timestamps.append(time.time())
        if len(timestamps) < 3:
            self.send_error(503, code='SlowDown',
                            extra_headers={'Retry-After': '1'})
        else:
            return real_DELETE(self)

    monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE)
    enable_temp_fail(backend)
    backend.delete(key)

    assert timestamps[1] - timestamps[0] > 1 - CLOCK_GRANULARITY
    assert timestamps[2] - timestamps[1] > 1 - CLOCK_GRANULARITY
    assert timestamps[2] - timestamps[0] < 10

Example 30

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/raw', require_mock_server=True)
def test_httperror(backend, monkeypatch):
    value = b'hello there, let us see whats going on'
    key = 'quote'
    backend[key] = value

    # Monkeypatch request handler to produce a HTTP Error
    handler_class = mock_server.S3CRequestHandler
    def do_DELETE(self, real_DELETE=handler_class.do_DELETE, count=[0]):
        count[0] += 1
        if count[0] >= 3:
            return real_DELETE(self)
        content = "I'm a proxy, and I messed up!".encode('utf-8')
        self.send_response(502, "Bad Gateway")
        self.send_header("Content-Type", 'text/plain; charset="utf-8"')
        self.send_header("Content-Length", str(len(content)))
        self.end_headers()
        if self.command != 'HEAD':
            self.wfile.write(content)

    monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE)
    assert_raises(HTTPError, backend.delete, key)

    enable_temp_fail(backend)
    backend.delete(key)

Example 31

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_put_s3error_early(backend, monkeypatch):
    '''Fail after expect-100'''

    data = b'hello there, let us see whats going on'
    key = 'borg'

    # Monkeypatch request handler to produce 3 errors
    handler_class = mock_server.S3CRequestHandler
    def handle_expect_100(self, real=handler_class.handle_expect_100, count=[0]):
        count[0] += 1
        if count[0] > 3:
            return real(self)
        else:
            self.send_error(503, code='OperationAborted')
            return False
    monkeypatch.setattr(handler_class, 'handle_expect_100', handle_expect_100)
    fh = backend.open_write(key)
    fh.write(data)
    assert_raises(OperationAbortedError, fh.close)

    enable_temp_fail(backend)
    fh.close()

Example 32

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_put_s3error_med(backend, monkeypatch):
    '''Fail as soon as data is received'''
    data = b'hello there, let us see whats going on'
    key = 'borg'

    # Monkeypatch request handler to produce 3 errors
    handler_class = mock_server.S3CRequestHandler
    def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]):
        count[0] += 1
        # Note: every time we return an error, the request will be retried
        # *twice*: once because of the error, and a second time because the
        # connection has been closed by the server.
        if count[0] > 2:
            return real_PUT(self)
        else:
            self.send_error(503, code='OperationAborted')

            # Since we don't read all the data, we have to close
            # the connection
            self.close_connection = True

    monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
    fh = backend.open_write(key)
    fh.write(data)
    assert_raises(OperationAbortedError, fh.close)

    enable_temp_fail(backend)
    fh.close()

Example 33

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_put_s3error_late(backend, monkeypatch):
    '''Fail after reading all data'''
    data = b'hello there, let us see whats going on'
    key = 'borg'

    # Monkeypatch request handler to produce 3 errors
    handler_class = mock_server.S3CRequestHandler
    def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]):
        count[0] += 1
        if count[0] > 3:
            return real_PUT(self)
        else:
            self.rfile.read(int(self.headers['Content-Length']))
            self.send_error(503, code='OperationAborted')

    monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
    fh = backend.open_write(key)
    fh.write(data)
    assert_raises(OperationAbortedError, fh.close)

    enable_temp_fail(backend)
    fh.close()

Example 34

Project: s3ql Source File: t1_backends.py
Function: test_issue_58
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_issue58(backend, monkeypatch):
    '''Send error while client is sending data'''

    # Monkeypatch request handler
    handler_class = mock_server.S3CRequestHandler
    def do_PUT(self, real=handler_class.do_PUT, count=[0]):
        count[0] += 1
        if count[0] > 1:
            return real(self)

        # Read half the data
        self.rfile.read(min(BUFSIZE, int(self.headers['Content-Length'])//2))

        # Then generate an error and close the connection
        self.send_error(401, code='MalformedXML')
        self.close_connection = True

    monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)

    # Write a big object. We need to write random data, or
    # compression while make the payload too small
    with pytest.raises(S3Error) as exc_info:
        with backend.open_write('borg') as fh, \
                open('/dev/urandom', 'rb') as rnd:
            for _ in range(5):
                fh.write(rnd.read(BUFSIZE))
    assert exc_info.value.code == 'MalformedXML'

    enable_temp_fail(backend)
    fh.close()

Example 35

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_issue58_b(backend, monkeypatch):
    '''Close connection while client is sending data'''

    # Monkeypatch request handler
    handler_class = mock_server.S3CRequestHandler
    def do_PUT(self, real=handler_class.do_PUT, count=[0]):
        count[0] += 1
        if count[0] > 1:
            return real(self)

        # Read half the data
        self.rfile.read(min(BUFSIZE, int(self.headers['Content-Length'])//2))

        # Then close the connection silently
        self.close_connection = True
    monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)

    # Write a big object. We need to write random data, or
    # compression while make the payload too small
    with pytest.raises(ConnectionClosed):
        with backend.open_write('borg') as fh, \
                open('/dev/urandom', 'rb') as rnd:
            for _ in range(5):
                fh.write(rnd.read(BUFSIZE))

    enable_temp_fail(backend)
    fh.close()

Example 36

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('gs/{raw,aes+zlib}',
                          require_mock_server=True)
def test_expired_token_get(backend, monkeypatch):
    '''Test handling of expired OAuth token'''

    key = 'borg'
    data = b'hello there, let us see whats going on'

    # Monkeypatch backend class to check if token is refreshed
    token_refreshed = False
    def _get_access_token(self):
        nonlocal token_refreshed
        token_refreshed = True
        self.access_token[self.password] = 'foobar'
    monkeypatch.setattr(GSBackend, '_get_access_token',
                        _get_access_token)

    # Store some data
    backend[key] = data

    # Monkeypatch request handler to produce error
    handler_class = mock_server.GSRequestHandler
    def do_GET(self, real=handler_class.do_GET, count=[0]):
        count[0] += 1
        if count[0] > 1:
            return real(self)
        else:
            self.send_error(401, code='AuthenticationRequired')
    monkeypatch.setattr(handler_class, 'do_GET', do_GET)

    token_refreshed = False
    assert backend[key] == data
    assert token_refreshed

Example 37

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('gs/{raw,aes+zlib}',
                          require_mock_server=True)
def test_expired_token_put(backend, monkeypatch):
    '''Test handling of expired OAuth token'''

    key = 'borg'
    data = b'hello there, let us see whats going on'

    # Monkeypatch backend class to check if token is refreshed
    token_refreshed = False
    def _get_access_token(self):
        nonlocal token_refreshed
        token_refreshed = True
        self.access_token[self.password] = 'foobar'
    monkeypatch.setattr(GSBackend, '_get_access_token',
                        _get_access_token)

    # Monkeypatch request handler to produce error
    handler_class = mock_server.GSRequestHandler
    def do_PUT(self, real=handler_class.do_PUT, count=[0]):
        count[0] += 1
        if count[0] > 1:
            return real(self)
        else:
            self.rfile.read(int(self.headers['Content-Length']))
            self.send_error(401, code='AuthenticationRequired')
    monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)

    token_refreshed = False
    backend[key] = data
    assert token_refreshed

Example 38

Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
                          require_mock_server=True)
def test_conn_abort(backend, monkeypatch):
    '''Close connection while sending data'''

    data = b'hello there, let us see whats going on'
    key = 'borg'
    backend[key] = data

    # Monkeypatch request handler
    handler_class = mock_server.S3CRequestHandler
    def send_data(self, data, count=[0]):
        count[0] += 1
        if count[0] >= 3:
            self.wfile.write(data)
        else:
            self.wfile.write(data[:len(data)//2])
            self.close_connection = True
    monkeypatch.setattr(handler_class, 'send_data', send_data)

    with pytest.raises(ConnectionClosed):
        with assert_logs("^Object closed prematurely, can't check MD5",
                          count=1, level=logging.WARNING):
            backend.fetch(key)

    enable_temp_fail(backend)
    assert backend[key] == data