Here are the examples of the python api mock_server.S3CRequestHandler taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
16 Examples
3
Example 1
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/raw', require_mock_server=True)
def test_delete_s3error(backend, monkeypatch):
value = b'hello there, let us see whats going on'
key = 'quote'
backend[key] = value
# Monkeypatch request handler to produce 3 errors
handler_class = mock_server.S3CRequestHandler
def do_DELETE(self, real_DELETE=handler_class.do_DELETE, count=[0]):
count[0] += 1
if count[0] > 3:
return real_DELETE(self)
else:
self.send_error(503, code='OperationAborted')
monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE)
assert_raises(OperationAbortedError, backend.delete, key)
enable_temp_fail(backend)
backend.delete(key)
0
Example 2
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes,zlib}',
require_mock_server=True)
def test_copy_error(backend, monkeypatch):
value = b'hello there, let us see whats going on'
key1 = 'object-key1'
key2 = 'object-key2'
backend[key1] = value
# Monkeypatch request handler to produce error
handler_class = mock_server.S3CRequestHandler
def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]):
count[0] += 1
if count[0] > 3:
return real_PUT(self)
else:
self.send_error(200, code='OperationAborted')
monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
assert_raises(OperationAbortedError, backend.copy, key1, key2)
enable_temp_fail(backend)
backend.copy(key1, key2)
0
Example 3
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/raw',
require_mock_server=True)
def test_list_bug(backend, monkeypatch):
keys = ([ 'prefixa' + newname() for dummy in range(6) ]
+ [ 'prefixb' + newname() for dummy in range(6) ])
values = [ newvalue() for dummy in range(12) ]
assert set(backend.list()) == empty_set
for i in range(12):
backend[keys[i]] = values[i]
assert_in_index(backend, keys, 0)
# Force reconnect during list
handler_class = mock_server.S3CRequestHandler
def do_list(self, q, _real=handler_class.do_list):
q.params['max_keys'] = [ '3' ]
self.close_connection = True
return _real(self, q)
monkeypatch.setattr(handler_class, 'do_list', do_list)
enable_temp_fail(backend)
assert set(backend.list()) == set(keys)
0
Example 4
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/aes+zlib',
require_mock_server=True)
def test_corrupted_get(backend, monkeypatch):
key = 'brafasel'
value = b'hello there, let us see whats going on'
backend[key] = value
# Monkeypatch request handler to produce invalid etag
handler_class = mock_server.S3CRequestHandler
def send_header(self, keyword ,value, count=[0],
send_header_real=handler_class.send_header):
if keyword == 'ETag':
count[0] += 1
if count[0] <= 3:
value = value[::-1]
return send_header_real(self, keyword, value)
monkeypatch.setattr(handler_class, 'send_header', send_header)
with assert_logs('^MD5 mismatch for', count=1, level=logging.WARNING):
assert_raises(BadDigestError, backend.fetch, key)
enable_temp_fail(backend)
with assert_logs('^MD5 mismatch for', count=2, level=logging.WARNING):
assert backend[key] == value
0
Example 5
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_corrupted_meta(backend, monkeypatch):
key = 'brafasel'
value = b'hello there, let us see whats going on'
backend[key] = value
# Monkeypatch request handler to mess up metadata
handler_class = mock_server.S3CRequestHandler
def send_header(self, keyword ,value, count=[0],
send_header_real=handler_class.send_header):
if keyword == self.hdr_prefix + 'Meta-md5':
count[0] += 1
if count[0] <= 3:
value = value[::-1]
return send_header_real(self, keyword, value)
monkeypatch.setattr(handler_class, 'send_header', send_header)
with assert_logs('^MD5 mismatch in metadata for', count=1, level=logging.WARNING):
assert_raises(BadDigestError, backend.fetch, key)
enable_temp_fail(backend)
with assert_logs('^MD5 mismatch in metadata for', count=2, level=logging.WARNING):
assert backend[key] == value
0
Example 6
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_corrupted_put(backend, monkeypatch):
key = 'brafasel'
value = b'hello there, let us see whats going on'
# Monkeypatch request handler to produce invalid etag
handler_class = mock_server.S3CRequestHandler
def send_header(self, keyword ,value, count=[0],
send_header_real=handler_class.send_header):
if keyword == 'ETag':
count[0] += 1
if count[0] < 3:
value = value[::-1]
return send_header_real(self, keyword, value)
monkeypatch.setattr(handler_class, 'send_header', send_header)
fh = backend.open_write(key)
fh.write(value)
assert_raises(BadDigestError, fh.close)
enable_temp_fail(backend)
fh.close()
assert backend[key] == value
0
Example 7
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_get_s3error(backend, monkeypatch):
value = b'hello there, let us see whats going on'
key = 'quote'
backend[key] = value
# Monkeypatch request handler to produce 3 errors
handler_class = mock_server.S3CRequestHandler
def do_GET(self, real_GET=handler_class.do_GET, count=[0]):
count[0] += 1
if count[0] > 3:
return real_GET(self)
else:
self.send_error(503, code='OperationAborted')
monkeypatch.setattr(handler_class, 'do_GET', do_GET)
assert_raises(OperationAbortedError, backend.fetch, value)
enable_temp_fail(backend)
assert backend[key] == value
0
Example 8
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_head_s3error(backend, monkeypatch):
value = b'hello there, let us see whats going on'
key = 'quote'
meta = {'bar': 42, 'foo': 42**2}
backend.store(key, value, metadata=meta)
# Monkeypatch request handler to produce 3 errors
handler_class = mock_server.S3CRequestHandler
def do_HEAD(self, real_HEAD=handler_class.do_HEAD, count=[0]):
count[0] += 1
if count[0] > 3:
return real_HEAD(self)
else:
self.send_error(503, code='OperationAborted')
monkeypatch.setattr(handler_class, 'do_HEAD', do_HEAD)
with pytest.raises(HTTPError) as exc:
backend.lookup(key)
assert exc.value.status == 503
enable_temp_fail(backend)
assert backend.lookup(key) == meta
0
Example 9
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/raw', require_mock_server=True)
def test_backoff(backend, monkeypatch):
value = b'hello there, let us see whats going on'
key = 'quote'
backend[key] = value
# Monkeypatch request handler
handler_class = mock_server.S3CRequestHandler
timestamps = []
def do_DELETE(self, real_DELETE=handler_class.do_DELETE):
timestamps.append(time.time())
if len(timestamps) < 3:
self.send_error(503, code='SlowDown',
extra_headers={'Retry-After': '1'})
else:
return real_DELETE(self)
monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE)
enable_temp_fail(backend)
backend.delete(key)
assert timestamps[1] - timestamps[0] > 1 - CLOCK_GRANULARITY
assert timestamps[2] - timestamps[1] > 1 - CLOCK_GRANULARITY
assert timestamps[2] - timestamps[0] < 10
0
Example 10
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/raw', require_mock_server=True)
def test_httperror(backend, monkeypatch):
value = b'hello there, let us see whats going on'
key = 'quote'
backend[key] = value
# Monkeypatch request handler to produce a HTTP Error
handler_class = mock_server.S3CRequestHandler
def do_DELETE(self, real_DELETE=handler_class.do_DELETE, count=[0]):
count[0] += 1
if count[0] >= 3:
return real_DELETE(self)
content = "I'm a proxy, and I messed up!".encode('utf-8')
self.send_response(502, "Bad Gateway")
self.send_header("Content-Type", 'text/plain; charset="utf-8"')
self.send_header("Content-Length", str(len(content)))
self.end_headers()
if self.command != 'HEAD':
self.wfile.write(content)
monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE)
assert_raises(HTTPError, backend.delete, key)
enable_temp_fail(backend)
backend.delete(key)
0
Example 11
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_put_s3error_early(backend, monkeypatch):
'''Fail after expect-100'''
data = b'hello there, let us see whats going on'
key = 'borg'
# Monkeypatch request handler to produce 3 errors
handler_class = mock_server.S3CRequestHandler
def handle_expect_100(self, real=handler_class.handle_expect_100, count=[0]):
count[0] += 1
if count[0] > 3:
return real(self)
else:
self.send_error(503, code='OperationAborted')
return False
monkeypatch.setattr(handler_class, 'handle_expect_100', handle_expect_100)
fh = backend.open_write(key)
fh.write(data)
assert_raises(OperationAbortedError, fh.close)
enable_temp_fail(backend)
fh.close()
0
Example 12
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_put_s3error_med(backend, monkeypatch):
'''Fail as soon as data is received'''
data = b'hello there, let us see whats going on'
key = 'borg'
# Monkeypatch request handler to produce 3 errors
handler_class = mock_server.S3CRequestHandler
def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]):
count[0] += 1
# Note: every time we return an error, the request will be retried
# *twice*: once because of the error, and a second time because the
# connection has been closed by the server.
if count[0] > 2:
return real_PUT(self)
else:
self.send_error(503, code='OperationAborted')
# Since we don't read all the data, we have to close
# the connection
self.close_connection = True
monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
fh = backend.open_write(key)
fh.write(data)
assert_raises(OperationAbortedError, fh.close)
enable_temp_fail(backend)
fh.close()
0
Example 13
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_put_s3error_late(backend, monkeypatch):
'''Fail after reading all data'''
data = b'hello there, let us see whats going on'
key = 'borg'
# Monkeypatch request handler to produce 3 errors
handler_class = mock_server.S3CRequestHandler
def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]):
count[0] += 1
if count[0] > 3:
return real_PUT(self)
else:
self.rfile.read(int(self.headers['Content-Length']))
self.send_error(503, code='OperationAborted')
monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
fh = backend.open_write(key)
fh.write(data)
assert_raises(OperationAbortedError, fh.close)
enable_temp_fail(backend)
fh.close()
0
Example 14
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_issue58(backend, monkeypatch):
'''Send error while client is sending data'''
# Monkeypatch request handler
handler_class = mock_server.S3CRequestHandler
def do_PUT(self, real=handler_class.do_PUT, count=[0]):
count[0] += 1
if count[0] > 1:
return real(self)
# Read half the data
self.rfile.read(min(BUFSIZE, int(self.headers['Content-Length'])//2))
# Then generate an error and close the connection
self.send_error(401, code='MalformedXML')
self.close_connection = True
monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
# Write a big object. We need to write random data, or
# compression while make the payload too small
with pytest.raises(S3Error) as exc_info:
with backend.open_write('borg') as fh, \
open('/dev/urandom', 'rb') as rnd:
for _ in range(5):
fh.write(rnd.read(BUFSIZE))
assert exc_info.value.code == 'MalformedXML'
enable_temp_fail(backend)
fh.close()
0
Example 15
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_issue58_b(backend, monkeypatch):
'''Close connection while client is sending data'''
# Monkeypatch request handler
handler_class = mock_server.S3CRequestHandler
def do_PUT(self, real=handler_class.do_PUT, count=[0]):
count[0] += 1
if count[0] > 1:
return real(self)
# Read half the data
self.rfile.read(min(BUFSIZE, int(self.headers['Content-Length'])//2))
# Then close the connection silently
self.close_connection = True
monkeypatch.setattr(handler_class, 'do_PUT', do_PUT)
# Write a big object. We need to write random data, or
# compression while make the payload too small
with pytest.raises(ConnectionClosed):
with backend.open_write('borg') as fh, \
open('/dev/urandom', 'rb') as rnd:
for _ in range(5):
fh.write(rnd.read(BUFSIZE))
enable_temp_fail(backend)
fh.close()
0
Example 16
Project: s3ql Source File: t1_backends.py
@pytest.mark.with_backend('s3c/{raw,aes+zlib}',
require_mock_server=True)
def test_conn_abort(backend, monkeypatch):
'''Close connection while sending data'''
data = b'hello there, let us see whats going on'
key = 'borg'
backend[key] = data
# Monkeypatch request handler
handler_class = mock_server.S3CRequestHandler
def send_data(self, data, count=[0]):
count[0] += 1
if count[0] >= 3:
self.wfile.write(data)
else:
self.wfile.write(data[:len(data)//2])
self.close_connection = True
monkeypatch.setattr(handler_class, 'send_data', send_data)
with pytest.raises(ConnectionClosed):
with assert_logs("^Object closed prematurely, can't check MD5",
count=1, level=logging.WARNING):
backend.fetch(key)
enable_temp_fail(backend)
assert backend[key] == data