Here are the examples of the python api twisted.web.client.Agent taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
67 Examples
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def buildAgentForWrapperTest(self, reactor):
"""
Return an Agent suitable for use in tests that wrap the Agent and want
both a fake reactor and StubHTTPProtocol.
"""
agent = client.Agent(reactor)
_oldGetEndpoint = agent._getEndpoint
agent._getEndpoint = lambda *args: (
self.StubEndpoint(_oldGetEndpoint(*args), self))
return agent
def connect(self, factory):
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def makeAgent(self):
"""
@return: a new L{twisted.web.client.Agent} instance
"""
return client.Agent(self.reactor)
def setUp(self):
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_defaultPool(self):
"""
If no pool is passed in, the L{Agent} creates a non-persistent pool.
"""
agent = client.Agent(self.reactor)
self.assertIsInstance(agent._pool, HTTPConnectionPool)
self.assertEqual(agent._pool.persistent, False)
self.assertIdentical(agent._reactor, agent._pool._reactor)
def test_persistent(self):
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_persistent(self):
"""
If C{persistent} is set to C{True} on the L{HTTPConnectionPool} (the
default), C{Request}s are created with their C{persistent} flag set to
C{True}.
"""
pool = HTTPConnectionPool(self.reactor)
agent = client.Agent(self.reactor, pool=pool)
agent._getEndpoint = lambda *args: self
agent.request(b"GET", b"http://127.0.0.1")
self.assertEqual(self.protocol.requests[0][0].persistent, True)
def test_nonPersistent(self):
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_nonPersistent(self):
"""
If C{persistent} is set to C{False} when creating the
L{HTTPConnectionPool}, C{Request}s are created with their
C{persistent} flag set to C{False}.
Elsewhere in the tests for the underlying HTTP code we ensure that
this will result in the disconnection of the HTTP protocol once the
request is done, so that the connection will not be returned to the
pool.
"""
pool = HTTPConnectionPool(self.reactor, persistent=False)
agent = client.Agent(self.reactor, pool=pool)
agent._getEndpoint = lambda *args: self
agent.request(b"GET", b"http://127.0.0.1")
self.assertEqual(self.protocol.requests[0][0].persistent, False)
def test_connectUsesConnectionPool(self):
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_connectTimeout(self):
"""
L{Agent} takes a C{connectTimeout} argument which is forwarded to the
following C{connectTCP} agent.
"""
agent = client.Agent(self.reactor, connectTimeout=5)
agent.request(b'GET', b'http://foo/')
timeout = self.reactor.tcpClients.pop()[3]
self.assertEqual(5, timeout)
def test_connectTimeoutHTTPS(self):
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_connectTimeoutHTTPS(self):
"""
L{Agent} takes a C{connectTimeout} argument which is forwarded to the
following C{connectTCP} call.
"""
agent = client.Agent(self.reactor, connectTimeout=5)
agent.request(b'GET', b'https://foo/')
timeout = self.reactor.tcpClients.pop()[3]
self.assertEqual(5, timeout)
test_connectTimeoutHTTPS.skip = skipWhenNoSSL
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_bindAddress(self):
"""
L{Agent} takes a C{bindAddress} argument which is forwarded to the
following C{connectTCP} call.
"""
agent = client.Agent(self.reactor, bindAddress='192.168.0.1')
agent.request(b'GET', b'http://foo/')
address = self.reactor.tcpClients.pop()[4]
self.assertEqual('192.168.0.1', address)
def test_bindAddressSSL(self):
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_bindAddressSSL(self):
"""
L{Agent} takes a C{bindAddress} argument which is forwarded to the
following C{connectSSL} call.
"""
agent = client.Agent(self.reactor, bindAddress='192.168.0.1')
agent.request(b'GET', b'https://foo/')
address = self.reactor.tcpClients.pop()[4]
self.assertEqual('192.168.0.1', address)
test_bindAddressSSL.skip = skipWhenNoSSL
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def attemptRequestWithMaliciousMethod(self, method):
"""
Attempt a request with the provided method.
@param method: see L{MethodInjectionTestsMixin}
"""
agent = client.Agent(self.createReactor())
uri = b"http://twisted.invalid"
agent.request(method, uri, client.Headers(), None)
class AgentURIInjectionTests(
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def attemptRequestWithMaliciousURI(self, uri):
"""
Attempt a request with the provided method.
@param uri: see L{URIInjectionTestsMixin}
"""
agent = client.Agent(self.createReactor())
method = b"GET"
agent.request(method, uri, client.Headers(), None)
class AgentHTTPSTests(TestCase, FakeReactorAndConnectMixin,
3
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def makeEndpoint(self, host=b'example.com', port=443):
"""
Create an L{Agent} with an https scheme and return its endpoint
created according to the arguments.
@param host: The host for the endpoint.
@type host: L{bytes}
@param port: The port for the endpoint.
@type port: L{int}
@return: An endpoint of an L{Agent} constructed according to args.
@rtype: L{SSL4ClientEndpoint}
"""
return client.Agent(self.createReactor())._getEndpoint(
URI.fromBytes(b'https://' + host + b":" + intToBytes(port) + b"/"))
def test_endpointType(self):
3
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_CGI(self):
cgiFilename = self.writeCGI(DUMMY_CGI)
portnum = self.startServer(cgiFilename)
url = 'http://localhost:%d/cgi' % (portnum,)
url = url.encode("ascii")
d = client.Agent(reactor).request(b"GET", url)
d.addCallback(client.readBody)
d.addCallback(self._testCGI_1)
return d
def _testCGI_1(self, res):
3
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_ReadEmptyInput(self):
cgiFilename = os.path.abspath(self.mktemp())
with open(cgiFilename, 'wt') as cgiFile:
cgiFile.write(READINPUT_CGI)
portnum = self.startServer(cgiFilename)
agent = client.Agent(reactor)
url = "http://localhost:%d/cgi" % (portnum,)
url = url.encode("ascii")
d = agent.request(b"GET", url)
d.addCallback(client.readBody)
d.addCallback(self._test_ReadEmptyInput_1)
return d
test_ReadEmptyInput.timeout = 5
3
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_ReadInput(self):
cgiFilename = os.path.abspath(self.mktemp())
with open(cgiFilename, 'wt') as cgiFile:
cgiFile.write(READINPUT_CGI)
portnum = self.startServer(cgiFilename)
agent = client.Agent(reactor)
url = "http://localhost:%d/cgi" % (portnum,)
url = url.encode("ascii")
d = agent.request(
uri=url,
method=b"POST",
bodyProducer=client.FileBodyProducer(
BytesIO(b"Here is your stdin")),
)
d.addCallback(client.readBody)
d.addCallback(self._test_ReadInput_1)
return d
test_ReadInput.timeout = 5
3
Source : test_xmlrpc.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_errorGet(self):
"""
A classic GET on the xml server should return a NOT_ALLOWED.
"""
agent = client.Agent(reactor)
d = agent.request(b"GET", networkString("http://127.0.0.1:%d/" % (self.port,)))
def checkResponse(response):
self.assertEqual(response.code, http.NOT_ALLOWED)
d.addCallback(checkResponse)
return d
def test_errorXMLContent(self):
3
Source : test_xmlrpc.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_errorXMLContent(self):
"""
Test that an invalid XML input returns an L{xmlrpc.Fault}.
"""
agent = client.Agent(reactor)
d = agent.request(
uri=networkString("http://127.0.0.1:%d/" % (self.port,)),
method=b"POST",
bodyProducer=client.FileBodyProducer(BytesIO(b"foo")))
d.addCallback(client.readBody)
def cb(result):
self.assertRaises(xmlrpc.Fault, xmlrpclib.loads, result)
d.addCallback(cb)
return d
def test_datetimeRoundtrip(self):
3
Source : api.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def _client(*args, **kwargs):
agent = kwargs.get('agent')
if agent is None:
reactor = default_reactor(kwargs.get('reactor'))
pool = default_pool(reactor,
kwargs.get('pool'),
kwargs.get('persistent'))
agent = Agent(reactor, pool=pool)
return HTTPClient(agent)
3
Source : testing.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def __init__(self, rootResource):
"""
:param rootResource: The twisted IResource at the root of the resource
tree.
"""
self._memoryReactor = MemoryReactor()
self._realAgent = Agent(reactor=self._memoryReactor)
self._rootResource = rootResource
self._pumps = set()
def request(self, method, uri, headers=None, bodyProducer=None):
3
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def buildAgentForWrapperTest(self, reactor):
"""
Return an Agent suitable for use in tests that wrap the Agent and want
both a fake reactor and StubHTTPProtocol.
"""
agent = client.Agent(reactor, self.StubPolicy())
_oldGetEndpoint = agent._getEndpoint
agent._getEndpoint = lambda *args: (
self.StubEndpoint(_oldGetEndpoint(*args), self))
return agent
def connect(self, factory):
3
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def test_connectTimeout(self):
"""
L{Agent} takes a C{connectTimeout} argument which is forwarded to the
following C{connectTCP} agent.
"""
agent = client.Agent(self.reactor, connectTimeout=5)
agent.request(b'GET', b'http://foo/')
timeout = self.reactor.tcpClients.pop()[3]
self.assertEqual(5, timeout)
def test_connectSSLTimeout(self):
3
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def test_connectSSLTimeout(self):
"""
L{Agent} takes a C{connectTimeout} argument which is forwarded to the
following C{connectSSL} call.
"""
agent = client.Agent(self.reactor, self.StubPolicy(), connectTimeout=5)
agent.request(b'GET', b'https://foo/')
timeout = self.reactor.sslClients.pop()[4]
self.assertEqual(5, timeout)
def test_bindAddress(self):
3
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def test_bindAddressSSL(self):
"""
L{Agent} takes a C{bindAddress} argument which is forwarded to the
following C{connectSSL} call.
"""
agent = client.Agent(self.reactor, self.StubPolicy(),
bindAddress='192.168.0.1')
agent.request(b'GET', b'https://foo/')
address = self.reactor.sslClients.pop()[5]
self.assertEqual('192.168.0.1', address)
def test_responseIncludesRequest(self):
3
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def makeEndpoint(self, host=b'example.com', port=443):
"""
Create an L{Agent} with an https scheme and return its endpoint
created according to the arguments.
@param host: The host for the endpoint.
@type host: L{bytes}
@param port: The port for the endpoint.
@type port: L{int}
@return: An endpoint of an L{Agent} constructed according to args.
@rtype: L{SSL4ClientEndpoint}
"""
return client.Agent(self.Reactor())._getEndpoint(
URI.fromBytes(b'https://' + host + b":" + intToBytes(port) + b"/"))
def test_endpointType(self):
3
Source : txipc.py
with Apache License 2.0
from gethue
with Apache License 2.0
from gethue
def __init__(self, host, port, remote_name=None, reactor=None):
self.url = "http://%s:%d/" % (host, port)
if remote_name is None:
# There's no easy way to get this peer's remote address
# in Twisted so I use a random UUID to identify ourselves
import uuid
self.remote_name = uuid.uuid4()
if reactor is None:
from twisted.internet import reactor
self.agent = Agent(reactor)
def read_framed_message(self, response):
3
Source : manager.py
with Apache License 2.0
from HathorNetwork
with Apache License 2.0
from HathorNetwork
def update_whitelist(self) -> Deferred:
from twisted.web.client import Agent, readBody
from twisted.web.http_headers import Headers
assert settings.WHITELIST_URL is not None
self.log.info('update whitelist')
agent = Agent(self.reactor)
d = agent.request(
b'GET',
settings.WHITELIST_URL.encode(),
Headers({'User-Agent': ['hathor-core']}),
None)
d.addCallback(readBody)
d.addErrback(self._update_whitelist_err)
d.addCallback(self._update_whitelist_cb)
def _update_whitelist_err(self, *args: Any, **kwargs: Any) -> None:
3
Source : matches_remote.py
with Apache License 2.0
from HathorNetwork
with Apache License 2.0
from HathorNetwork
def update(self) -> Deferred:
"""Update the list of items."""
from twisted.web.client import Agent, readBody
from twisted.web.http_headers import Headers
agent = Agent(self.reactor)
d = agent.request(
self.method.encode(),
self.url.encode(),
Headers(self.headers),
None
)
d.addCallback(readBody)
d.addErrback(self._update_err)
d.addCallback(self._update_cb)
def match(self, context: 'NetfilterContext') -> bool:
0
Source : twisted_test.py
with GNU General Public License v3.0
from agajdosi
with GNU General Public License v3.0
from agajdosi
def twisted_fetch(self, url, runner):
# http://twistedmatrix.com/documents/current/web/howto/client.html
chunks = []
client = Agent(self.reactor)
d = client.request(b"GET", utf8(url))
class Accumulator(Protocol):
def __init__(self, finished):
self.finished = finished
def dataReceived(self, data):
chunks.append(data)
def connectionLost(self, reason):
self.finished.callback(None)
def callback(response):
finished = Deferred()
response.deliverBody(Accumulator(finished))
return finished
d.addCallback(callback)
def shutdown(failure):
if hasattr(self, "stop_loop"):
self.stop_loop()
elif failure is not None:
# loop hasn't been initialized yet; try our best to
# get an error message out. (the runner() interaction
# should probably be refactored).
try:
failure.raiseException()
except:
logging.error("exception before starting loop", exc_info=True)
d.addBoth(shutdown)
runner()
self.assertTrue(chunks)
return b"".join(chunks)
def twisted_coroutine_fetch(self, url, runner):
0
Source : twisted_test.py
with GNU General Public License v3.0
from agajdosi
with GNU General Public License v3.0
from agajdosi
def twisted_coroutine_fetch(self, url, runner):
body = [None]
@gen.coroutine
def f():
# This is simpler than the non-coroutine version, but it cheats
# by reading the body in one blob instead of streaming it with
# a Protocol.
client = Agent(self.reactor)
response = yield client.request(b"GET", utf8(url))
with warnings.catch_warnings():
# readBody has a buggy DeprecationWarning in Twisted 15.0:
# https://twistedmatrix.com/trac/changeset/43379
warnings.simplefilter("ignore", category=DeprecationWarning)
body[0] = yield readBody(response)
self.stop_loop()
self.io_loop.add_callback(f)
runner()
return body[0]
def testTwistedServerTornadoClientReactor(self):
0
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_connectUsesConnectionPool(self):
"""
When a connection is made by the Agent, it uses its pool's
C{getConnection} method to do so, with the endpoint returned by
C{self._getEndpoint}. The key used is C{(scheme, host, port)}.
"""
endpoint = DummyEndpoint()
class MyAgent(client.Agent):
def _getEndpoint(this, uri):
self.assertEqual((uri.scheme, uri.host, uri.port),
(b"http", b"foo", 80))
return endpoint
class DummyPool(object):
connected = False
persistent = False
def getConnection(this, key, ep):
this.connected = True
self.assertEqual(ep, endpoint)
# This is the key the default Agent uses, others will have
# different keys:
self.assertEqual(key, (b"http", b"foo", 80))
return defer.succeed(StubHTTPProtocol())
pool = DummyPool()
agent = MyAgent(self.reactor, pool=pool)
self.assertIdentical(pool, agent._pool)
headers = http_headers.Headers()
headers.addRawHeader(b"host", b"foo")
bodyProducer = object()
agent.request(b'GET', b'http://foo/',
bodyProducer=bodyProducer, headers=headers)
self.assertEqual(agent._pool.connected, True)
def test_nonBytesMethod(self):
0
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_connectHTTPSCustomConnectionCreator(self):
"""
If a custom L{WebClientConnectionCreator}-like object is passed to
L{Agent.__init__} it will be used to determine the SSL parameters for
HTTPS requests. When an HTTPS request is made, the hostname and port
number of the request URL will be passed to the connection creator's
C{creatorForNetloc} method. The resulting context object will be used
to establish the SSL connection.
"""
expectedHost = b'example.org'
expectedPort = 20443
class JustEnoughConnection(object):
handshakeStarted = False
connectState = False
def do_handshake(self):
"""
The handshake started. Record that fact.
"""
self.handshakeStarted = True
def set_connect_state(self):
"""
The connection started. Record that fact.
"""
self.connectState = True
contextArgs = []
@implementer(IOpenSSLClientConnectionCreator)
class JustEnoughCreator(object):
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
def clientConnectionForTLS(self, tlsProtocol):
"""
Implement L{IOpenSSLClientConnectionCreator}.
@param tlsProtocol: The TLS protocol.
@type tlsProtocol: L{TLSMemoryBIOProtocol}
@return: C{expectedConnection}
"""
contextArgs.append((tlsProtocol, self.hostname, self.port))
return expectedConnection
expectedConnection = JustEnoughConnection()
@implementer(IPolicyForHTTPS)
class StubBrowserLikePolicyForHTTPS(object):
def creatorForNetloc(self, hostname, port):
"""
Emulate L{BrowserLikePolicyForHTTPS}.
@param hostname: The hostname to verify.
@type hostname: L{bytes}
@param port: The port number.
@type port: L{int}
@return: a stub L{IOpenSSLClientConnectionCreator}
@rtype: L{JustEnoughCreator}
"""
return JustEnoughCreator(hostname, port)
expectedCreatorCreator = StubBrowserLikePolicyForHTTPS()
reactor = self.createReactor()
agent = client.Agent(reactor, expectedCreatorCreator)
endpoint = agent._getEndpoint(URI.fromBytes(
b'https://' + expectedHost + b":" + intToBytes(expectedPort)))
endpoint.connect(Factory.forProtocol(Protocol))
tlsFactory = reactor.tcpClients[-1][2]
tlsProtocol = tlsFactory.buildProtocol(None)
tlsProtocol.makeConnection(StringTransport())
tls = contextArgs[0][0]
self.assertIsInstance(tls, TLSMemoryBIOProtocol)
self.assertEqual(contextArgs[0][1:], (expectedHost, expectedPort))
self.assertTrue(expectedConnection.handshakeStarted)
self.assertTrue(expectedConnection.connectState)
def test_deprecatedDuckPolicy(self):
0
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_deprecatedDuckPolicy(self):
"""
Passing something that duck-types I{like} a L{web client context
factory < twisted.web.client.WebClientContextFactory>} - something that
does not provide L{IPolicyForHTTPS} - to L{Agent} emits a
L{DeprecationWarning} even if you don't actually C{import
WebClientContextFactory} to do it.
"""
def warnMe():
client.Agent(deterministicResolvingReactor(MemoryReactorClock()),
"does-not-provide-IPolicyForHTTPS")
warnMe()
warnings = self.flushWarnings([warnMe])
self.assertEqual(len(warnings), 1)
[warning] = warnings
self.assertEqual(warning['category'], DeprecationWarning)
self.assertEqual(
warning['message'],
"'does-not-provide-IPolicyForHTTPS' was passed as the HTTPS "
"policy for an Agent, but it does not provide IPolicyForHTTPS. "
"Since Twisted 14.0, you must pass a provider of IPolicyForHTTPS."
)
def test_alternateTrustRoot(self):
0
Source : test_agent.py
with MIT License
from autofelix
with MIT License
from autofelix
def integrationTest(self, hostName, expectedAddress, addressType):
"""
Wrap L{AgentTestsMixin.integrationTest} with TLS.
"""
certHostName = hostName.strip(b'[]')
authority, server = certificatesForAuthorityAndServer(certHostName
.decode('ascii'))
def tlsify(serverFactory):
return TLSMemoryBIOFactory(server.options(), False, serverFactory)
def tlsagent(reactor):
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface import implementer
@implementer(IPolicyForHTTPS)
class Policy(object):
def creatorForNetloc(self, hostname, port):
return optionsForClientTLS(hostname.decode("ascii"),
trustRoot=authority)
return client.Agent(reactor, contextFactory=Policy())
(super(AgentHTTPSTests, self)
.integrationTest(hostName, expectedAddress, addressType,
serverWrapper=tlsify,
createAgent=tlsagent,
scheme=b'https'))
class WebClientContextFactoryTests(TestCase):
0
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_protectedServerAndDate(self):
"""
If the CGI script emits a I{Server} or I{Date} header, these are
ignored.
"""
cgiFilename = self.writeCGI(SPECIAL_HEADER_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum,)
url = url.encode("ascii")
agent = client.Agent(reactor)
d = agent.request(b"GET", url)
d.addCallback(discardBody)
def checkResponse(response):
self.assertNotIn('monkeys',
response.headers.getRawHeaders('server'))
self.assertNotIn('last year',
response.headers.getRawHeaders('date'))
d.addCallback(checkResponse)
return d
def test_noDuplicateContentTypeHeaders(self):
0
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_noDuplicateContentTypeHeaders(self):
"""
If the CGI script emits a I{content-type} header, make sure that the
server doesn't add an additional (duplicate) one, as per ticket 4786.
"""
cgiFilename = self.writeCGI(NO_DUPLICATE_CONTENT_TYPE_HEADER_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum,)
url = url.encode("ascii")
agent = client.Agent(reactor)
d = agent.request(b"GET", url)
d.addCallback(discardBody)
def checkResponse(response):
self.assertEqual(
response.headers.getRawHeaders('content-type'),
['text/cgi-duplicate-test'])
return response
d.addCallback(checkResponse)
return d
def test_noProxyPassthrough(self):
0
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_noProxyPassthrough(self):
"""
The CGI script is never called with the Proxy header passed through.
"""
cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum,)
url = url.encode("ascii")
agent = client.Agent(reactor)
headers = http_headers.Headers({b"Proxy": [b"foo"],
b"X-Innocent-Header": [b"bar"]})
d = agent.request(b"GET", url, headers=headers)
def checkResponse(response):
headers = json.loads(response.decode("ascii"))
self.assertEqual(
set(headers.keys()),
{"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})
d.addCallback(client.readBody)
d.addCallback(checkResponse)
return d
def test_duplicateHeaderCGI(self):
0
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_duplicateHeaderCGI(self):
"""
If a CGI script emits two instances of the same header, both are sent
in the response.
"""
cgiFilename = self.writeCGI(DUAL_HEADER_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum,)
url = url.encode("ascii")
agent = client.Agent(reactor)
d = agent.request(b"GET", url)
d.addCallback(discardBody)
def checkResponse(response):
self.assertEqual(
response.headers.getRawHeaders('header'), ['spam', 'eggs'])
d.addCallback(checkResponse)
return d
def test_malformedHeaderCGI(self):
0
Source : test_cgi.py
with MIT License
from autofelix
with MIT License
from autofelix
def test_malformedHeaderCGI(self):
"""
Check for the error message in the duplicated header
"""
cgiFilename = self.writeCGI(BROKEN_HEADER_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum,)
url = url.encode("ascii")
agent = client.Agent(reactor)
d = agent.request(b"GET", url)
d.addCallback(discardBody)
loggedMessages = []
def addMessage(eventDict):
loggedMessages.append(log.textFromEventDict(eventDict))
log.addObserver(addMessage)
self.addCleanup(log.removeObserver, addMessage)
def checkResponse(ignored):
self.assertIn("ignoring malformed CGI header: " + repr(b'XYZ'),
loggedMessages)
d.addCallback(checkResponse)
return d
def test_ReadEmptyInput(self):
0
Source : test_distrib.py
with MIT License
from autofelix
with MIT License
from autofelix
def testDistrib(self):
# site1 is the publisher
r1 = resource.Resource()
r1.putChild(b"there", static.Data(b"root", "text/plain"))
site1 = server.Site(r1)
self.f1 = PBServerFactory(distrib.ResourcePublisher(site1))
self.port1 = reactor.listenTCP(0, self.f1)
self.sub = distrib.ResourceSubscription("127.0.0.1",
self.port1.getHost().port)
r2 = resource.Resource()
r2.putChild(b"here", self.sub)
f2 = MySite(r2)
self.port2 = reactor.listenTCP(0, f2)
agent = client.Agent(reactor)
url = "http://127.0.0.1:{}/here/there".format(
self.port2.getHost().port)
url = url.encode("ascii")
d = agent.request(b"GET", url)
d.addCallback(client.readBody)
d.addCallback(self.assertEqual, b'root')
return d
def _setupDistribServer(self, child):
0
Source : test_distrib.py
with MIT License
from autofelix
with MIT License
from autofelix
def _requestTest(self, child, **kwargs):
"""
Set up a resource on a distrib site using L{ResourcePublisher} and
then retrieve it from a L{ResourceSubscription} via an HTTP client.
@param child: The resource to publish using distrib.
@param **kwargs: Extra keyword arguments to pass to L{Agent.request} when
requesting the resource.
@return: A L{Deferred} which fires with the result of the request.
"""
mainPort, mainAddr = self._setupDistribServer(child)
agent = client.Agent(reactor)
url = "http://%s:%s/child" % (mainAddr.host, mainAddr.port)
url = url.encode("ascii")
d = agent.request(b"GET", url, **kwargs)
d.addCallback(client.readBody)
return d
def _requestAgentTest(self, child, **kwargs):
0
Source : twisted_test.py
with Apache License 2.0
from awslabs
with Apache License 2.0
from awslabs
def twisted_fetch(self, url, runner):
# http://twistedmatrix.com/documents/current/web/howto/client.html
chunks = []
client = Agent(self.reactor)
d = client.request(b'GET', utf8(url))
class Accumulator(Protocol):
def __init__(self, finished):
self.finished = finished
def dataReceived(self, data):
chunks.append(data)
def connectionLost(self, reason):
self.finished.callback(None)
def callback(response):
finished = Deferred()
response.deliverBody(Accumulator(finished))
return finished
d.addCallback(callback)
def shutdown(failure):
if hasattr(self, 'stop_loop'):
self.stop_loop()
elif failure is not None:
# loop hasn't been initialized yet; try our best to
# get an error message out. (the runner() interaction
# should probably be refactored).
try:
failure.raiseException()
except:
logging.error('exception before starting loop', exc_info=True)
d.addBoth(shutdown)
runner()
self.assertTrue(chunks)
return ''.join(chunks)
def twisted_coroutine_fetch(self, url, runner):
0
Source : twisted_test.py
with Apache License 2.0
from awslabs
with Apache License 2.0
from awslabs
def twisted_coroutine_fetch(self, url, runner):
body = [None]
@gen.coroutine
def f():
# This is simpler than the non-coroutine version, but it cheats
# by reading the body in one blob instead of streaming it with
# a Protocol.
client = Agent(self.reactor)
response = yield client.request(b'GET', utf8(url))
with warnings.catch_warnings():
# readBody has a buggy DeprecationWarning in Twisted 15.0:
# https://twistedmatrix.com/trac/changeset/43379
warnings.simplefilter('ignore', category=DeprecationWarning)
body[0] = yield readBody(response)
self.stop_loop()
self.io_loop.add_callback(f)
runner()
return body[0]
def testTwistedServerTornadoClientIOLoop(self):
0
Source : snippet.py
with Apache License 2.0
from dockerizeme
with Apache License 2.0
from dockerizeme
def httpRequest(url, values={}, headers={}, method='POST'):
# Construct an Agent.
agent = Agent(reactor)
data = urllib.urlencode(values)
d = agent.request(method,
url,
Headers(headers),
StringProducer(data) if data else None)
def handle_response(response):
if response.code == 204:
d = defer.succeed('')
else:
class SimpleReceiver(protocol.Protocol):
def __init__(s, d):
s.buf = ''; s.d = d
def dataReceived(s, data):
s.buf += data
def connectionLost(s, reason):
# TODO: test if reason is twisted.web.client.ResponseDone, if not, do an errback
s.d.callback(s.buf)
d = defer.Deferred()
response.deliverBody(SimpleReceiver(d))
return d
d.addCallback(handle_response)
return d
# Sample usage:
d = httpRequest(
0
Source : test_agent.py
with The Unlicense
from dspray95
with The Unlicense
from dspray95
def test_connectUsesConnectionPool(self):
"""
When a connection is made by the Agent, it uses its pool's
C{getConnection} method to do so, with the endpoint returned by
C{self._getEndpoint}. The key used is C{(scheme, host, port)}.
"""
endpoint = DummyEndpoint()
class MyAgent(client.Agent):
def _getEndpoint(this, uri):
self.assertEqual((uri.scheme, uri.host, uri.port),
(b"http", b"foo", 80))
return endpoint
class DummyPool(object):
connected = False
persistent = False
def getConnection(this, key, ep):
this.connected = True
self.assertEqual(ep, endpoint)
# This is the key the default Agent uses, others will have
# different keys:
self.assertEqual(key, (b"http", b"foo", 80))
return defer.succeed(StubHTTPProtocol())
pool = DummyPool()
agent = MyAgent(self.reactor, pool=pool)
self.assertIdentical(pool, agent._pool)
headers = http_headers.Headers()
headers.addRawHeader(b"host", b"foo")
bodyProducer = object()
agent.request(b'GET', b'http://foo/',
bodyProducer=bodyProducer, headers=headers)
self.assertEqual(agent._pool.connected, True)
def test_unsupportedScheme(self):
0
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def test_connectHTTPSCustomContextFactory(self):
"""
If a context factory is passed to L{Agent.__init__} it will be used to
determine the SSL parameters for HTTPS requests. When an HTTPS request
is made, the hostname and port number of the request URL will be passed
to the context factory's C{getContext} method. The resulting context
object will be used to establish the SSL connection.
"""
expectedHost = b'example.org'
expectedPort = 20443
expectedContext = object()
contextArgs = []
class StubWebContextFactory(object):
def getContext(self, hostname, port):
contextArgs.append((hostname, port))
return expectedContext
agent = client.Agent(self.reactor, StubWebContextFactory())
endpoint = agent._getEndpoint(URI.fromBytes(
b'https://' + expectedHost + b":" + intToBytes(expectedPort)))
contextFactory = endpoint._sslContextFactory
context = contextFactory.getContext()
self.assertEqual(context, expectedContext)
self.assertEqual(contextArgs, [(expectedHost, expectedPort)])
def test_hostProvided(self):
0
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def test_connectHTTPSCustomConnectionCreator(self):
"""
If a custom L{WebClientConnectionCreator}-like object is passed to
L{Agent.__init__} it will be used to determine the SSL parameters for
HTTPS requests. When an HTTPS request is made, the hostname and port
number of the request URL will be passed to the connection creator's
C{creatorForNetloc} method. The resulting context object will be used
to establish the SSL connection.
"""
expectedHost = b'example.org'
expectedPort = 20443
class JustEnoughConnection(object):
handshakeStarted = False
connectState = False
def do_handshake(self):
"""
The handshake started. Record that fact.
"""
self.handshakeStarted = True
def set_connect_state(self):
"""
The connection started. Record that fact.
"""
self.connectState = True
contextArgs = []
@implementer(IOpenSSLClientConnectionCreator)
class JustEnoughCreator(object):
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
def clientConnectionForTLS(self, tlsProtocol):
"""
Implement L{IOpenSSLClientConnectionCreator}.
@param tlsProtocol: The TLS protocol.
@type tlsProtocol: L{TLSMemoryBIOProtocol}
@return: C{expectedConnection}
"""
contextArgs.append((tlsProtocol, self.hostname, self.port))
return expectedConnection
expectedConnection = JustEnoughConnection()
@implementer(IPolicyForHTTPS)
class StubBrowserLikePolicyForHTTPS(object):
def creatorForNetloc(self, hostname, port):
"""
Emulate L{BrowserLikePolicyForHTTPS}.
@param hostname: The hostname to verify.
@type hostname: L{bytes}
@param port: The port number.
@type port: L{int}
@return: a stub L{IOpenSSLClientConnectionCreator}
@rtype: L{JustEnoughCreator}
"""
return JustEnoughCreator(hostname, port)
expectedCreatorCreator = StubBrowserLikePolicyForHTTPS()
reactor = self.Reactor()
agent = client.Agent(reactor, expectedCreatorCreator)
endpoint = agent._getEndpoint(URI.fromBytes(
b'https://' + expectedHost + b":" + intToBytes(expectedPort)))
endpoint.connect(Factory.forProtocol(Protocol))
passedFactory = reactor.sslClients[-1][2]
passedContextFactory = reactor.sslClients[-1][3]
tlsFactory = TLSMemoryBIOFactory(
passedContextFactory, True, passedFactory
)
tlsProtocol = tlsFactory.buildProtocol(None)
tlsProtocol.makeConnection(StringTransport())
tls = contextArgs[0][0]
self.assertIsInstance(tls, TLSMemoryBIOProtocol)
self.assertEqual(contextArgs[0][1:], (expectedHost, expectedPort))
self.assertTrue(expectedConnection.handshakeStarted)
self.assertTrue(expectedConnection.connectState)
def test_deprecatedDuckPolicy(self):
0
Source : test_agent.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def test_deprecatedDuckPolicy(self):
"""
Passing something that duck-types I{like} a L{web client context
factory < twisted.web.client.WebClientContextFactory>} - something that
does not provide L{IPolicyForHTTPS} - to L{Agent} emits a
L{DeprecationWarning} even if you don't actually C{import
WebClientContextFactory} to do it.
"""
def warnMe():
client.Agent(MemoryReactorClock(),
"does-not-provide-IPolicyForHTTPS")
warnMe()
warnings = self.flushWarnings([warnMe])
self.assertEqual(len(warnings), 1)
[warning] = warnings
self.assertEqual(warning['category'], DeprecationWarning)
self.assertEqual(
warning['message'],
"'does-not-provide-IPolicyForHTTPS' was passed as the HTTPS "
"policy for an Agent, but it does not provide IPolicyForHTTPS. "
"Since Twisted 14.0, you must pass a provider of IPolicyForHTTPS."
)
def test_alternateTrustRoot(self):
0
Source : test_cgi.py
with MIT License
from fbla-competitive-events
with MIT License
from fbla-competitive-events
def test_noProxyPassthrough(self):
"""
The CGI script is never called with the Proxy header passed through.
"""
cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum,)
agent = client.Agent(reactor)
headers = http_headers.Headers({"Proxy": ["foo"],
"X-Innocent-Header": ["bar"]})
d = agent.request("GET", url, headers=headers)
def checkResponse(response):
headers = json.loads(response)
self.assertEqual(
set(headers.keys()),
{"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})
d.addCallback(client.readBody)
d.addCallback(checkResponse)
return d
def test_duplicateHeaderCGI(self):
0
Source : test_telegram_bot.py
with GNU General Public License v2.0
from flathub
with GNU General Public License v2.0
from flathub
def setUp(self):
table_names = [
'objects', 'object_state', 'masters',
'workers', 'configured_workers', 'connected_workers',
'builder_masters', 'builders'
]
master = fakemaster.make_master(self, wantRealReactor=True)
yield self.setUpRealDatabaseWithConnector(master, table_names=table_names,
sqlite_memory=False)
master.data = dataconnector.DataConnector()
yield master.data.setServiceParent(master)
master.config.mq = dict(type='simple')
master.mq = mqconnector.MQConnector()
yield master.mq.setServiceParent(master)
yield master.mq.setup()
yield master.mq.startService()
master.config.www = dict(
port='tcp:0:interface=127.0.0.1',
debug=True,
auth=auth.NoAuth(),
authz=authz.Authz(),
avatar_methods=[],
logfileName='http.log')
master.www = wwwservice.WWWService()
yield master.www.setServiceParent(master)
yield master.www.startService()
yield master.www.reconfigServiceWithBuildbotConfig(master.config)
session = mock.Mock()
session.uid = "0"
master.www.site.sessionFactory = mock.Mock(return_value=session)
# now that we have a port, construct the real URL and insert it into
# the config. The second reconfig isn't really required, but doesn't
# hurt.
self.url = 'http://127.0.0.1:%d/' % master.www.getPortnum()
self.url = unicode2bytes(self.url)
master.config.buildbotURL = self.url
yield master.www.reconfigServiceWithBuildbotConfig(master.config)
self.master = master
self.agent = client.Agent(reactor)
# create a telegram bot service
tb = master.config.services['TelegramBot'] = telegram.TelegramBot(
bot_token='12345:secret', useWebhook=True,
chat_ids=[-123456], notify_events=['worker']
)
tb._get_http = self.get_http
yield tb.setServiceParent(self.master)
self.bot_url = self.url + b"telegram12345:secret"
yield tb.startService()
self.sent_messages = []
def send_message(chat, message, **kwargs):
self.sent_messages.append((chat, message))
tb.bot.send_message = send_message
@defer.inlineCallbacks
0
Source : httpclientservice.py
with GNU General Public License v2.0
from flathub
with GNU General Public License v2.0
from flathub
def startService(self):
# treq only supports basicauth, so we force txrequests if the auth is
# something else
if self._auth is not None and not isinstance(self._auth, tuple):
self.PREFER_TREQ = False
if txrequests is not None and not self.PREFER_TREQ:
self._session = txrequests.Session()
self._doRequest = self._doTxRequest
elif treq is None:
raise ImportError("{classname} requires either txrequest or treq install."
" Users should call {classname}.checkAvailable() during checkConfig()"
" to properly alert the user.".format(
classname=self.__class__.__name__))
else:
self._doRequest = self._doTReq
self._pool = HTTPConnectionPool(self.master.reactor)
self._pool.maxPersistentPerHost = self.MAX_THREADS
self._agent = Agent(self.master.reactor, pool=self._pool)
return super().startService()
@defer.inlineCallbacks
See More Examples