twisted.web.client.Agent

Here are the examples of the python api twisted.web.client.Agent taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

67 Examples 7

3 Source : test_agent.py
with MIT License
from autofelix

    def buildAgentForWrapperTest(self, reactor):
        """
        Return an Agent suitable for use in tests that wrap the Agent and want
        both a fake reactor and StubHTTPProtocol.
        """
        agent = client.Agent(reactor)
        _oldGetEndpoint = agent._getEndpoint
        agent._getEndpoint = lambda *args: (
            self.StubEndpoint(_oldGetEndpoint(*args), self))
        return agent


    def connect(self, factory):

3 Source : test_agent.py
with MIT License
from autofelix

    def makeAgent(self):
        """
        @return: a new L{twisted.web.client.Agent} instance
        """
        return client.Agent(self.reactor)


    def setUp(self):

3 Source : test_agent.py
with MIT License
from autofelix

    def test_defaultPool(self):
        """
        If no pool is passed in, the L{Agent} creates a non-persistent pool.
        """
        agent = client.Agent(self.reactor)
        self.assertIsInstance(agent._pool, HTTPConnectionPool)
        self.assertEqual(agent._pool.persistent, False)
        self.assertIdentical(agent._reactor, agent._pool._reactor)


    def test_persistent(self):

3 Source : test_agent.py
with MIT License
from autofelix

    def test_persistent(self):
        """
        If C{persistent} is set to C{True} on the L{HTTPConnectionPool} (the
        default), C{Request}s are created with their C{persistent} flag set to
        C{True}.
        """
        pool = HTTPConnectionPool(self.reactor)
        agent = client.Agent(self.reactor, pool=pool)
        agent._getEndpoint = lambda *args: self
        agent.request(b"GET", b"http://127.0.0.1")
        self.assertEqual(self.protocol.requests[0][0].persistent, True)


    def test_nonPersistent(self):

3 Source : test_agent.py
with MIT License
from autofelix

    def test_nonPersistent(self):
        """
        If C{persistent} is set to C{False} when creating the
        L{HTTPConnectionPool}, C{Request}s are created with their
        C{persistent} flag set to C{False}.

        Elsewhere in the tests for the underlying HTTP code we ensure that
        this will result in the disconnection of the HTTP protocol once the
        request is done, so that the connection will not be returned to the
        pool.
        """
        pool = HTTPConnectionPool(self.reactor, persistent=False)
        agent = client.Agent(self.reactor, pool=pool)
        agent._getEndpoint = lambda *args: self
        agent.request(b"GET", b"http://127.0.0.1")
        self.assertEqual(self.protocol.requests[0][0].persistent, False)


    def test_connectUsesConnectionPool(self):

3 Source : test_agent.py
with MIT License
from autofelix

    def test_connectTimeout(self):
        """
        L{Agent} takes a C{connectTimeout} argument which is forwarded to the
        following C{connectTCP} agent.
        """
        agent = client.Agent(self.reactor, connectTimeout=5)
        agent.request(b'GET', b'http://foo/')
        timeout = self.reactor.tcpClients.pop()[3]
        self.assertEqual(5, timeout)


    def test_connectTimeoutHTTPS(self):

3 Source : test_agent.py
with MIT License
from autofelix

    def test_connectTimeoutHTTPS(self):
        """
        L{Agent} takes a C{connectTimeout} argument which is forwarded to the
        following C{connectTCP} call.
        """
        agent = client.Agent(self.reactor, connectTimeout=5)
        agent.request(b'GET', b'https://foo/')
        timeout = self.reactor.tcpClients.pop()[3]
        self.assertEqual(5, timeout)

    test_connectTimeoutHTTPS.skip = skipWhenNoSSL

3 Source : test_agent.py
with MIT License
from autofelix

    def test_bindAddress(self):
        """
        L{Agent} takes a C{bindAddress} argument which is forwarded to the
        following C{connectTCP} call.
        """
        agent = client.Agent(self.reactor, bindAddress='192.168.0.1')
        agent.request(b'GET', b'http://foo/')
        address = self.reactor.tcpClients.pop()[4]
        self.assertEqual('192.168.0.1', address)


    def test_bindAddressSSL(self):

3 Source : test_agent.py
with MIT License
from autofelix

    def test_bindAddressSSL(self):
        """
        L{Agent} takes a C{bindAddress} argument which is forwarded to the
        following C{connectSSL} call.
        """
        agent = client.Agent(self.reactor, bindAddress='192.168.0.1')
        agent.request(b'GET', b'https://foo/')
        address = self.reactor.tcpClients.pop()[4]
        self.assertEqual('192.168.0.1', address)

    test_bindAddressSSL.skip = skipWhenNoSSL

3 Source : test_agent.py
with MIT License
from autofelix

    def attemptRequestWithMaliciousMethod(self, method):
        """
        Attempt a request with the provided method.

        @param method: see L{MethodInjectionTestsMixin}
        """
        agent = client.Agent(self.createReactor())
        uri = b"http://twisted.invalid"
        agent.request(method, uri, client.Headers(), None)



class AgentURIInjectionTests(

3 Source : test_agent.py
with MIT License
from autofelix

    def attemptRequestWithMaliciousURI(self, uri):
        """
        Attempt a request with the provided method.

        @param uri: see L{URIInjectionTestsMixin}
        """
        agent = client.Agent(self.createReactor())
        method = b"GET"
        agent.request(method, uri, client.Headers(), None)



class AgentHTTPSTests(TestCase, FakeReactorAndConnectMixin,

3 Source : test_agent.py
with MIT License
from autofelix

    def makeEndpoint(self, host=b'example.com', port=443):
        """
        Create an L{Agent} with an https scheme and return its endpoint
        created according to the arguments.

        @param host: The host for the endpoint.
        @type host: L{bytes}

        @param port: The port for the endpoint.
        @type port: L{int}

        @return: An endpoint of an L{Agent} constructed according to args.
        @rtype: L{SSL4ClientEndpoint}
        """
        return client.Agent(self.createReactor())._getEndpoint(
            URI.fromBytes(b'https://' + host + b":" + intToBytes(port) + b"/"))


    def test_endpointType(self):

3 Source : test_cgi.py
with MIT License
from autofelix

    def test_CGI(self):
        cgiFilename = self.writeCGI(DUMMY_CGI)

        portnum = self.startServer(cgiFilename)
        url = 'http://localhost:%d/cgi' % (portnum,)
        url = url.encode("ascii")
        d = client.Agent(reactor).request(b"GET", url)
        d.addCallback(client.readBody)
        d.addCallback(self._testCGI_1)
        return d


    def _testCGI_1(self, res):

3 Source : test_cgi.py
with MIT License
from autofelix

    def test_ReadEmptyInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        with open(cgiFilename, 'wt') as cgiFile:
            cgiFile.write(READINPUT_CGI)

        portnum = self.startServer(cgiFilename)
        agent = client.Agent(reactor)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        d = agent.request(b"GET", url)
        d.addCallback(client.readBody)
        d.addCallback(self._test_ReadEmptyInput_1)
        return d
    test_ReadEmptyInput.timeout = 5

3 Source : test_cgi.py
with MIT License
from autofelix

    def test_ReadInput(self):
        cgiFilename = os.path.abspath(self.mktemp())
        with open(cgiFilename, 'wt') as cgiFile:
            cgiFile.write(READINPUT_CGI)

        portnum = self.startServer(cgiFilename)
        agent = client.Agent(reactor)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        d = agent.request(
            uri=url,
            method=b"POST",
            bodyProducer=client.FileBodyProducer(
                BytesIO(b"Here is your stdin")),
        )
        d.addCallback(client.readBody)
        d.addCallback(self._test_ReadInput_1)
        return d
    test_ReadInput.timeout = 5

3 Source : test_xmlrpc.py
with MIT License
from autofelix

    def test_errorGet(self):
        """
        A classic GET on the xml server should return a NOT_ALLOWED.
        """
        agent = client.Agent(reactor)
        d = agent.request(b"GET", networkString("http://127.0.0.1:%d/" % (self.port,)))
        def checkResponse(response):
            self.assertEqual(response.code, http.NOT_ALLOWED)
        d.addCallback(checkResponse)
        return d

    def test_errorXMLContent(self):

3 Source : test_xmlrpc.py
with MIT License
from autofelix

    def test_errorXMLContent(self):
        """
        Test that an invalid XML input returns an L{xmlrpc.Fault}.
        """
        agent = client.Agent(reactor)
        d = agent.request(
            uri=networkString("http://127.0.0.1:%d/" % (self.port,)),
            method=b"POST",
            bodyProducer=client.FileBodyProducer(BytesIO(b"foo")))
        d.addCallback(client.readBody)
        def cb(result):
            self.assertRaises(xmlrpc.Fault, xmlrpclib.loads, result)
        d.addCallback(cb)
        return d


    def test_datetimeRoundtrip(self):

3 Source : api.py
with MIT License
from fbla-competitive-events

def _client(*args, **kwargs):
    agent = kwargs.get('agent')
    if agent is None:
        reactor = default_reactor(kwargs.get('reactor'))
        pool = default_pool(reactor,
                            kwargs.get('pool'),
                            kwargs.get('persistent'))
        agent = Agent(reactor, pool=pool)
    return HTTPClient(agent)

3 Source : testing.py
with MIT License
from fbla-competitive-events

    def __init__(self, rootResource):
        """
        :param rootResource: The twisted IResource at the root of the resource
            tree.
        """
        self._memoryReactor = MemoryReactor()
        self._realAgent = Agent(reactor=self._memoryReactor)
        self._rootResource = rootResource
        self._pumps = set()

    def request(self, method, uri, headers=None, bodyProducer=None):

3 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def buildAgentForWrapperTest(self, reactor):
        """
        Return an Agent suitable for use in tests that wrap the Agent and want
        both a fake reactor and StubHTTPProtocol.
        """
        agent = client.Agent(reactor, self.StubPolicy())
        _oldGetEndpoint = agent._getEndpoint
        agent._getEndpoint = lambda *args: (
            self.StubEndpoint(_oldGetEndpoint(*args), self))
        return agent


    def connect(self, factory):

3 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def test_connectTimeout(self):
        """
        L{Agent} takes a C{connectTimeout} argument which is forwarded to the
        following C{connectTCP} agent.
        """
        agent = client.Agent(self.reactor, connectTimeout=5)
        agent.request(b'GET', b'http://foo/')
        timeout = self.reactor.tcpClients.pop()[3]
        self.assertEqual(5, timeout)


    def test_connectSSLTimeout(self):

3 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def test_connectSSLTimeout(self):
        """
        L{Agent} takes a C{connectTimeout} argument which is forwarded to the
        following C{connectSSL} call.
        """
        agent = client.Agent(self.reactor, self.StubPolicy(), connectTimeout=5)
        agent.request(b'GET', b'https://foo/')
        timeout = self.reactor.sslClients.pop()[4]
        self.assertEqual(5, timeout)


    def test_bindAddress(self):

3 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def test_bindAddressSSL(self):
        """
        L{Agent} takes a C{bindAddress} argument which is forwarded to the
        following C{connectSSL} call.
        """
        agent = client.Agent(self.reactor, self.StubPolicy(),
                             bindAddress='192.168.0.1')
        agent.request(b'GET', b'https://foo/')
        address = self.reactor.sslClients.pop()[5]
        self.assertEqual('192.168.0.1', address)


    def test_responseIncludesRequest(self):

3 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def makeEndpoint(self, host=b'example.com', port=443):
        """
        Create an L{Agent} with an https scheme and return its endpoint
        created according to the arguments.

        @param host: The host for the endpoint.
        @type host: L{bytes}

        @param port: The port for the endpoint.
        @type port: L{int}

        @return: An endpoint of an L{Agent} constructed according to args.
        @rtype: L{SSL4ClientEndpoint}
        """
        return client.Agent(self.Reactor())._getEndpoint(
            URI.fromBytes(b'https://' + host + b":" + intToBytes(port) + b"/"))


    def test_endpointType(self):

3 Source : txipc.py
with Apache License 2.0
from gethue

  def __init__(self, host, port, remote_name=None, reactor=None):
    self.url = "http://%s:%d/" % (host, port)

    if remote_name is None:
      # There's no easy way to get this peer's remote address
      # in Twisted so I use a random UUID to identify ourselves
      import uuid
      self.remote_name = uuid.uuid4()

    if reactor is None:
      from twisted.internet import reactor
    self.agent = Agent(reactor)

  def read_framed_message(self, response):

3 Source : manager.py
with Apache License 2.0
from HathorNetwork

    def update_whitelist(self) -> Deferred:
        from twisted.web.client import Agent, readBody
        from twisted.web.http_headers import Headers
        assert settings.WHITELIST_URL is not None
        self.log.info('update whitelist')
        agent = Agent(self.reactor)
        d = agent.request(
            b'GET',
            settings.WHITELIST_URL.encode(),
            Headers({'User-Agent': ['hathor-core']}),
            None)
        d.addCallback(readBody)
        d.addErrback(self._update_whitelist_err)
        d.addCallback(self._update_whitelist_cb)

    def _update_whitelist_err(self, *args: Any, **kwargs: Any) -> None:

3 Source : matches_remote.py
with Apache License 2.0
from HathorNetwork

    def update(self) -> Deferred:
        """Update the list of items."""
        from twisted.web.client import Agent, readBody
        from twisted.web.http_headers import Headers
        agent = Agent(self.reactor)
        d = agent.request(
            self.method.encode(),
            self.url.encode(),
            Headers(self.headers),
            None
        )
        d.addCallback(readBody)
        d.addErrback(self._update_err)
        d.addCallback(self._update_cb)

    def match(self, context: 'NetfilterContext') -> bool:

0 Source : twisted_test.py
with GNU General Public License v3.0
from agajdosi

    def twisted_fetch(self, url, runner):
        # http://twistedmatrix.com/documents/current/web/howto/client.html
        chunks = []
        client = Agent(self.reactor)
        d = client.request(b"GET", utf8(url))

        class Accumulator(Protocol):
            def __init__(self, finished):
                self.finished = finished

            def dataReceived(self, data):
                chunks.append(data)

            def connectionLost(self, reason):
                self.finished.callback(None)

        def callback(response):
            finished = Deferred()
            response.deliverBody(Accumulator(finished))
            return finished

        d.addCallback(callback)

        def shutdown(failure):
            if hasattr(self, "stop_loop"):
                self.stop_loop()
            elif failure is not None:
                # loop hasn't been initialized yet; try our best to
                # get an error message out. (the runner() interaction
                # should probably be refactored).
                try:
                    failure.raiseException()
                except:
                    logging.error("exception before starting loop", exc_info=True)

        d.addBoth(shutdown)
        runner()
        self.assertTrue(chunks)
        return b"".join(chunks)

    def twisted_coroutine_fetch(self, url, runner):

0 Source : twisted_test.py
with GNU General Public License v3.0
from agajdosi

    def twisted_coroutine_fetch(self, url, runner):
        body = [None]

        @gen.coroutine
        def f():
            # This is simpler than the non-coroutine version, but it cheats
            # by reading the body in one blob instead of streaming it with
            # a Protocol.
            client = Agent(self.reactor)
            response = yield client.request(b"GET", utf8(url))
            with warnings.catch_warnings():
                # readBody has a buggy DeprecationWarning in Twisted 15.0:
                # https://twistedmatrix.com/trac/changeset/43379
                warnings.simplefilter("ignore", category=DeprecationWarning)
                body[0] = yield readBody(response)
            self.stop_loop()

        self.io_loop.add_callback(f)
        runner()
        return body[0]

    def testTwistedServerTornadoClientReactor(self):

0 Source : test_agent.py
with MIT License
from autofelix

    def test_connectUsesConnectionPool(self):
        """
        When a connection is made by the Agent, it uses its pool's
        C{getConnection} method to do so, with the endpoint returned by
        C{self._getEndpoint}. The key used is C{(scheme, host, port)}.
        """
        endpoint = DummyEndpoint()
        class MyAgent(client.Agent):
            def _getEndpoint(this, uri):
                self.assertEqual((uri.scheme, uri.host, uri.port),
                                 (b"http", b"foo", 80))
                return endpoint

        class DummyPool(object):
            connected = False
            persistent = False
            def getConnection(this, key, ep):
                this.connected = True
                self.assertEqual(ep, endpoint)
                # This is the key the default Agent uses, others will have
                # different keys:
                self.assertEqual(key, (b"http", b"foo", 80))
                return defer.succeed(StubHTTPProtocol())

        pool = DummyPool()
        agent = MyAgent(self.reactor, pool=pool)
        self.assertIdentical(pool, agent._pool)

        headers = http_headers.Headers()
        headers.addRawHeader(b"host", b"foo")
        bodyProducer = object()
        agent.request(b'GET', b'http://foo/',
                      bodyProducer=bodyProducer, headers=headers)
        self.assertEqual(agent._pool.connected, True)


    def test_nonBytesMethod(self):

0 Source : test_agent.py
with MIT License
from autofelix

    def test_connectHTTPSCustomConnectionCreator(self):
        """
        If a custom L{WebClientConnectionCreator}-like object is passed to
        L{Agent.__init__} it will be used to determine the SSL parameters for
        HTTPS requests.  When an HTTPS request is made, the hostname and port
        number of the request URL will be passed to the connection creator's
        C{creatorForNetloc} method.  The resulting context object will be used
        to establish the SSL connection.
        """
        expectedHost = b'example.org'
        expectedPort = 20443
        class JustEnoughConnection(object):
            handshakeStarted = False
            connectState = False
            def do_handshake(self):
                """
                The handshake started.  Record that fact.
                """
                self.handshakeStarted = True
            def set_connect_state(self):
                """
                The connection started.  Record that fact.
                """
                self.connectState = True

        contextArgs = []

        @implementer(IOpenSSLClientConnectionCreator)
        class JustEnoughCreator(object):
            def __init__(self, hostname, port):
                self.hostname = hostname
                self.port = port

            def clientConnectionForTLS(self, tlsProtocol):
                """
                Implement L{IOpenSSLClientConnectionCreator}.

                @param tlsProtocol: The TLS protocol.
                @type tlsProtocol: L{TLSMemoryBIOProtocol}

                @return: C{expectedConnection}
                """
                contextArgs.append((tlsProtocol, self.hostname, self.port))
                return expectedConnection

        expectedConnection = JustEnoughConnection()
        @implementer(IPolicyForHTTPS)
        class StubBrowserLikePolicyForHTTPS(object):
            def creatorForNetloc(self, hostname, port):
                """
                Emulate L{BrowserLikePolicyForHTTPS}.

                @param hostname: The hostname to verify.
                @type hostname: L{bytes}

                @param port: The port number.
                @type port: L{int}

                @return: a stub L{IOpenSSLClientConnectionCreator}
                @rtype: L{JustEnoughCreator}
                """
                return JustEnoughCreator(hostname, port)

        expectedCreatorCreator = StubBrowserLikePolicyForHTTPS()
        reactor = self.createReactor()
        agent = client.Agent(reactor, expectedCreatorCreator)
        endpoint = agent._getEndpoint(URI.fromBytes(
            b'https://' + expectedHost + b":" + intToBytes(expectedPort)))
        endpoint.connect(Factory.forProtocol(Protocol))
        tlsFactory = reactor.tcpClients[-1][2]
        tlsProtocol = tlsFactory.buildProtocol(None)
        tlsProtocol.makeConnection(StringTransport())
        tls = contextArgs[0][0]
        self.assertIsInstance(tls, TLSMemoryBIOProtocol)
        self.assertEqual(contextArgs[0][1:], (expectedHost, expectedPort))
        self.assertTrue(expectedConnection.handshakeStarted)
        self.assertTrue(expectedConnection.connectState)


    def test_deprecatedDuckPolicy(self):

0 Source : test_agent.py
with MIT License
from autofelix

    def test_deprecatedDuckPolicy(self):
        """
        Passing something that duck-types I{like} a L{web client context
        factory   <  twisted.web.client.WebClientContextFactory>} - something that
        does not provide L{IPolicyForHTTPS} - to L{Agent} emits a
        L{DeprecationWarning} even if you don't actually C{import
        WebClientContextFactory} to do it.
        """
        def warnMe():
            client.Agent(deterministicResolvingReactor(MemoryReactorClock()),
                         "does-not-provide-IPolicyForHTTPS")
        warnMe()
        warnings = self.flushWarnings([warnMe])
        self.assertEqual(len(warnings), 1)
        [warning] = warnings
        self.assertEqual(warning['category'], DeprecationWarning)
        self.assertEqual(
            warning['message'],
            "'does-not-provide-IPolicyForHTTPS' was passed as the HTTPS "
            "policy for an Agent, but it does not provide IPolicyForHTTPS.  "
            "Since Twisted 14.0, you must pass a provider of IPolicyForHTTPS."
        )


    def test_alternateTrustRoot(self):

0 Source : test_agent.py
with MIT License
from autofelix

    def integrationTest(self, hostName, expectedAddress, addressType):
        """
        Wrap L{AgentTestsMixin.integrationTest} with TLS.
        """
        certHostName = hostName.strip(b'[]')
        authority, server = certificatesForAuthorityAndServer(certHostName
                                                              .decode('ascii'))
        def tlsify(serverFactory):
            return TLSMemoryBIOFactory(server.options(), False, serverFactory)
        def tlsagent(reactor):
            from twisted.web.iweb import IPolicyForHTTPS
            from zope.interface import implementer
            @implementer(IPolicyForHTTPS)
            class Policy(object):
                def creatorForNetloc(self, hostname, port):
                    return optionsForClientTLS(hostname.decode("ascii"),
                                               trustRoot=authority)
            return client.Agent(reactor, contextFactory=Policy())
        (super(AgentHTTPSTests, self)
         .integrationTest(hostName, expectedAddress, addressType,
                          serverWrapper=tlsify,
                          createAgent=tlsagent,
                          scheme=b'https'))



class WebClientContextFactoryTests(TestCase):

0 Source : test_cgi.py
with MIT License
from autofelix

    def test_protectedServerAndDate(self):
        """
        If the CGI script emits a I{Server} or I{Date} header, these are
        ignored.
        """
        cgiFilename = self.writeCGI(SPECIAL_HEADER_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        agent = client.Agent(reactor)
        d = agent.request(b"GET", url)
        d.addCallback(discardBody)
        def checkResponse(response):
            self.assertNotIn('monkeys',
                             response.headers.getRawHeaders('server'))
            self.assertNotIn('last year',
                             response.headers.getRawHeaders('date'))
        d.addCallback(checkResponse)
        return d


    def test_noDuplicateContentTypeHeaders(self):

0 Source : test_cgi.py
with MIT License
from autofelix

    def test_noDuplicateContentTypeHeaders(self):
        """
        If the CGI script emits a I{content-type} header, make sure that the
        server doesn't add an additional (duplicate) one, as per ticket 4786.
        """
        cgiFilename = self.writeCGI(NO_DUPLICATE_CONTENT_TYPE_HEADER_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        agent = client.Agent(reactor)
        d = agent.request(b"GET", url)
        d.addCallback(discardBody)
        def checkResponse(response):
            self.assertEqual(
                response.headers.getRawHeaders('content-type'),
                ['text/cgi-duplicate-test'])
            return response
        d.addCallback(checkResponse)
        return d


    def test_noProxyPassthrough(self):

0 Source : test_cgi.py
with MIT License
from autofelix

    def test_noProxyPassthrough(self):
        """
        The CGI script is never called with the Proxy header passed through.
        """
        cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")

        agent = client.Agent(reactor)

        headers = http_headers.Headers({b"Proxy": [b"foo"],
                                        b"X-Innocent-Header": [b"bar"]})
        d = agent.request(b"GET", url, headers=headers)

        def checkResponse(response):
            headers = json.loads(response.decode("ascii"))
            self.assertEqual(
                set(headers.keys()),
                {"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})

        d.addCallback(client.readBody)
        d.addCallback(checkResponse)
        return d


    def test_duplicateHeaderCGI(self):

0 Source : test_cgi.py
with MIT License
from autofelix

    def test_duplicateHeaderCGI(self):
        """
        If a CGI script emits two instances of the same header, both are sent
        in the response.
        """
        cgiFilename = self.writeCGI(DUAL_HEADER_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        agent = client.Agent(reactor)
        d = agent.request(b"GET", url)
        d.addCallback(discardBody)
        def checkResponse(response):
            self.assertEqual(
                response.headers.getRawHeaders('header'), ['spam', 'eggs'])
        d.addCallback(checkResponse)
        return d


    def test_malformedHeaderCGI(self):

0 Source : test_cgi.py
with MIT License
from autofelix

    def test_malformedHeaderCGI(self):
        """
        Check for the error message in the duplicated header
        """
        cgiFilename = self.writeCGI(BROKEN_HEADER_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        url = url.encode("ascii")
        agent = client.Agent(reactor)
        d = agent.request(b"GET", url)
        d.addCallback(discardBody)
        loggedMessages = []

        def addMessage(eventDict):
            loggedMessages.append(log.textFromEventDict(eventDict))

        log.addObserver(addMessage)
        self.addCleanup(log.removeObserver, addMessage)

        def checkResponse(ignored):
            self.assertIn("ignoring malformed CGI header: " + repr(b'XYZ'),
                          loggedMessages)

        d.addCallback(checkResponse)
        return d


    def test_ReadEmptyInput(self):

0 Source : test_distrib.py
with MIT License
from autofelix

    def testDistrib(self):
        # site1 is the publisher
        r1 = resource.Resource()
        r1.putChild(b"there", static.Data(b"root", "text/plain"))
        site1 = server.Site(r1)
        self.f1 = PBServerFactory(distrib.ResourcePublisher(site1))
        self.port1 = reactor.listenTCP(0, self.f1)
        self.sub = distrib.ResourceSubscription("127.0.0.1",
                                                self.port1.getHost().port)
        r2 = resource.Resource()
        r2.putChild(b"here", self.sub)
        f2 = MySite(r2)
        self.port2 = reactor.listenTCP(0, f2)
        agent = client.Agent(reactor)
        url = "http://127.0.0.1:{}/here/there".format(
            self.port2.getHost().port)
        url = url.encode("ascii")
        d = agent.request(b"GET", url)
        d.addCallback(client.readBody)
        d.addCallback(self.assertEqual, b'root')
        return d


    def _setupDistribServer(self, child):

0 Source : test_distrib.py
with MIT License
from autofelix

    def _requestTest(self, child, **kwargs):
        """
        Set up a resource on a distrib site using L{ResourcePublisher} and
        then retrieve it from a L{ResourceSubscription} via an HTTP client.

        @param child: The resource to publish using distrib.
        @param **kwargs: Extra keyword arguments to pass to L{Agent.request} when
            requesting the resource.

        @return: A L{Deferred} which fires with the result of the request.
        """
        mainPort, mainAddr = self._setupDistribServer(child)
        agent = client.Agent(reactor)
        url = "http://%s:%s/child" % (mainAddr.host, mainAddr.port)
        url = url.encode("ascii")
        d = agent.request(b"GET", url, **kwargs)
        d.addCallback(client.readBody)
        return d


    def _requestAgentTest(self, child, **kwargs):

0 Source : twisted_test.py
with Apache License 2.0
from awslabs

    def twisted_fetch(self, url, runner):
        # http://twistedmatrix.com/documents/current/web/howto/client.html
        chunks = []
        client = Agent(self.reactor)
        d = client.request(b'GET', utf8(url))

        class Accumulator(Protocol):
            def __init__(self, finished):
                self.finished = finished

            def dataReceived(self, data):
                chunks.append(data)

            def connectionLost(self, reason):
                self.finished.callback(None)

        def callback(response):
            finished = Deferred()
            response.deliverBody(Accumulator(finished))
            return finished
        d.addCallback(callback)

        def shutdown(failure):
            if hasattr(self, 'stop_loop'):
                self.stop_loop()
            elif failure is not None:
                # loop hasn't been initialized yet; try our best to
                # get an error message out. (the runner() interaction
                # should probably be refactored).
                try:
                    failure.raiseException()
                except:
                    logging.error('exception before starting loop', exc_info=True)
        d.addBoth(shutdown)
        runner()
        self.assertTrue(chunks)
        return ''.join(chunks)

    def twisted_coroutine_fetch(self, url, runner):

0 Source : twisted_test.py
with Apache License 2.0
from awslabs

    def twisted_coroutine_fetch(self, url, runner):
        body = [None]

        @gen.coroutine
        def f():
            # This is simpler than the non-coroutine version, but it cheats
            # by reading the body in one blob instead of streaming it with
            # a Protocol.
            client = Agent(self.reactor)
            response = yield client.request(b'GET', utf8(url))
            with warnings.catch_warnings():
                # readBody has a buggy DeprecationWarning in Twisted 15.0:
                # https://twistedmatrix.com/trac/changeset/43379
                warnings.simplefilter('ignore', category=DeprecationWarning)
                body[0] = yield readBody(response)
            self.stop_loop()
        self.io_loop.add_callback(f)
        runner()
        return body[0]

    def testTwistedServerTornadoClientIOLoop(self):

0 Source : snippet.py
with Apache License 2.0
from dockerizeme

def httpRequest(url, values={}, headers={}, method='POST'):
    # Construct an Agent.
    agent = Agent(reactor)
    data = urllib.urlencode(values)

    d = agent.request(method,
                      url,
                      Headers(headers),
                      StringProducer(data) if data else None)

    def handle_response(response):
        if response.code == 204:
            d = defer.succeed('')
        else:
            class SimpleReceiver(protocol.Protocol):
                def __init__(s, d):
                    s.buf = ''; s.d = d
                def dataReceived(s, data):
                    s.buf += data
                def connectionLost(s, reason):
                    # TODO: test if reason is twisted.web.client.ResponseDone, if not, do an errback
                    s.d.callback(s.buf)

            d = defer.Deferred()
            response.deliverBody(SimpleReceiver(d))
        return d

    d.addCallback(handle_response)
    return d

# Sample usage:

d = httpRequest(

0 Source : test_agent.py
with The Unlicense
from dspray95

    def test_connectUsesConnectionPool(self):
        """
        When a connection is made by the Agent, it uses its pool's
        C{getConnection} method to do so, with the endpoint returned by
        C{self._getEndpoint}. The key used is C{(scheme, host, port)}.
        """
        endpoint = DummyEndpoint()
        class MyAgent(client.Agent):
            def _getEndpoint(this, uri):
                self.assertEqual((uri.scheme, uri.host, uri.port),
                                 (b"http", b"foo", 80))
                return endpoint

        class DummyPool(object):
            connected = False
            persistent = False
            def getConnection(this, key, ep):
                this.connected = True
                self.assertEqual(ep, endpoint)
                # This is the key the default Agent uses, others will have
                # different keys:
                self.assertEqual(key, (b"http", b"foo", 80))
                return defer.succeed(StubHTTPProtocol())

        pool = DummyPool()
        agent = MyAgent(self.reactor, pool=pool)
        self.assertIdentical(pool, agent._pool)

        headers = http_headers.Headers()
        headers.addRawHeader(b"host", b"foo")
        bodyProducer = object()
        agent.request(b'GET', b'http://foo/',
                      bodyProducer=bodyProducer, headers=headers)
        self.assertEqual(agent._pool.connected, True)


    def test_unsupportedScheme(self):

0 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def test_connectHTTPSCustomContextFactory(self):
        """
        If a context factory is passed to L{Agent.__init__} it will be used to
        determine the SSL parameters for HTTPS requests.  When an HTTPS request
        is made, the hostname and port number of the request URL will be passed
        to the context factory's C{getContext} method.  The resulting context
        object will be used to establish the SSL connection.
        """
        expectedHost = b'example.org'
        expectedPort = 20443
        expectedContext = object()

        contextArgs = []
        class StubWebContextFactory(object):
            def getContext(self, hostname, port):
                contextArgs.append((hostname, port))
                return expectedContext

        agent = client.Agent(self.reactor, StubWebContextFactory())
        endpoint = agent._getEndpoint(URI.fromBytes(
            b'https://' + expectedHost + b":" + intToBytes(expectedPort)))
        contextFactory = endpoint._sslContextFactory
        context = contextFactory.getContext()
        self.assertEqual(context, expectedContext)
        self.assertEqual(contextArgs, [(expectedHost, expectedPort)])


    def test_hostProvided(self):

0 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def test_connectHTTPSCustomConnectionCreator(self):
        """
        If a custom L{WebClientConnectionCreator}-like object is passed to
        L{Agent.__init__} it will be used to determine the SSL parameters for
        HTTPS requests.  When an HTTPS request is made, the hostname and port
        number of the request URL will be passed to the connection creator's
        C{creatorForNetloc} method.  The resulting context object will be used
        to establish the SSL connection.
        """
        expectedHost = b'example.org'
        expectedPort = 20443
        class JustEnoughConnection(object):
            handshakeStarted = False
            connectState = False
            def do_handshake(self):
                """
                The handshake started.  Record that fact.
                """
                self.handshakeStarted = True
            def set_connect_state(self):
                """
                The connection started.  Record that fact.
                """
                self.connectState = True

        contextArgs = []

        @implementer(IOpenSSLClientConnectionCreator)
        class JustEnoughCreator(object):
            def __init__(self, hostname, port):
                self.hostname = hostname
                self.port = port

            def clientConnectionForTLS(self, tlsProtocol):
                """
                Implement L{IOpenSSLClientConnectionCreator}.

                @param tlsProtocol: The TLS protocol.
                @type tlsProtocol: L{TLSMemoryBIOProtocol}

                @return: C{expectedConnection}
                """
                contextArgs.append((tlsProtocol, self.hostname, self.port))
                return expectedConnection

        expectedConnection = JustEnoughConnection()
        @implementer(IPolicyForHTTPS)
        class StubBrowserLikePolicyForHTTPS(object):
            def creatorForNetloc(self, hostname, port):
                """
                Emulate L{BrowserLikePolicyForHTTPS}.

                @param hostname: The hostname to verify.
                @type hostname: L{bytes}

                @param port: The port number.
                @type port: L{int}

                @return: a stub L{IOpenSSLClientConnectionCreator}
                @rtype: L{JustEnoughCreator}
                """
                return JustEnoughCreator(hostname, port)

        expectedCreatorCreator = StubBrowserLikePolicyForHTTPS()
        reactor = self.Reactor()
        agent = client.Agent(reactor, expectedCreatorCreator)
        endpoint = agent._getEndpoint(URI.fromBytes(
            b'https://' + expectedHost + b":" + intToBytes(expectedPort)))
        endpoint.connect(Factory.forProtocol(Protocol))
        passedFactory = reactor.sslClients[-1][2]
        passedContextFactory = reactor.sslClients[-1][3]
        tlsFactory = TLSMemoryBIOFactory(
            passedContextFactory, True, passedFactory
        )
        tlsProtocol = tlsFactory.buildProtocol(None)
        tlsProtocol.makeConnection(StringTransport())
        tls = contextArgs[0][0]
        self.assertIsInstance(tls, TLSMemoryBIOProtocol)
        self.assertEqual(contextArgs[0][1:], (expectedHost, expectedPort))
        self.assertTrue(expectedConnection.handshakeStarted)
        self.assertTrue(expectedConnection.connectState)


    def test_deprecatedDuckPolicy(self):

0 Source : test_agent.py
with MIT License
from fbla-competitive-events

    def test_deprecatedDuckPolicy(self):
        """
        Passing something that duck-types I{like} a L{web client context
        factory   <  twisted.web.client.WebClientContextFactory>} - something that
        does not provide L{IPolicyForHTTPS} - to L{Agent} emits a
        L{DeprecationWarning} even if you don't actually C{import
        WebClientContextFactory} to do it.
        """
        def warnMe():
            client.Agent(MemoryReactorClock(),
                         "does-not-provide-IPolicyForHTTPS")
        warnMe()
        warnings = self.flushWarnings([warnMe])
        self.assertEqual(len(warnings), 1)
        [warning] = warnings
        self.assertEqual(warning['category'], DeprecationWarning)
        self.assertEqual(
            warning['message'],
            "'does-not-provide-IPolicyForHTTPS' was passed as the HTTPS "
            "policy for an Agent, but it does not provide IPolicyForHTTPS.  "
            "Since Twisted 14.0, you must pass a provider of IPolicyForHTTPS."
        )


    def test_alternateTrustRoot(self):

0 Source : test_cgi.py
with MIT License
from fbla-competitive-events

    def test_noProxyPassthrough(self):
        """
        The CGI script is never called with the Proxy header passed through.
        """
        cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)

        agent = client.Agent(reactor)

        headers = http_headers.Headers({"Proxy": ["foo"],
                                        "X-Innocent-Header": ["bar"]})
        d = agent.request("GET", url, headers=headers)

        def checkResponse(response):
            headers = json.loads(response)
            self.assertEqual(
                set(headers.keys()),
                {"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})

        d.addCallback(client.readBody)
        d.addCallback(checkResponse)
        return d


    def test_duplicateHeaderCGI(self):

0 Source : test_telegram_bot.py
with GNU General Public License v2.0
from flathub

    def setUp(self):
        table_names = [
            'objects', 'object_state', 'masters',
            'workers', 'configured_workers', 'connected_workers',
            'builder_masters', 'builders'
        ]

        master = fakemaster.make_master(self, wantRealReactor=True)

        yield self.setUpRealDatabaseWithConnector(master, table_names=table_names,
                                                  sqlite_memory=False)

        master.data = dataconnector.DataConnector()
        yield master.data.setServiceParent(master)

        master.config.mq = dict(type='simple')
        master.mq = mqconnector.MQConnector()
        yield master.mq.setServiceParent(master)
        yield master.mq.setup()
        yield master.mq.startService()

        master.config.www = dict(
            port='tcp:0:interface=127.0.0.1',
            debug=True,
            auth=auth.NoAuth(),
            authz=authz.Authz(),
            avatar_methods=[],
            logfileName='http.log')
        master.www = wwwservice.WWWService()
        yield master.www.setServiceParent(master)
        yield master.www.startService()
        yield master.www.reconfigServiceWithBuildbotConfig(master.config)
        session = mock.Mock()
        session.uid = "0"
        master.www.site.sessionFactory = mock.Mock(return_value=session)

        # now that we have a port, construct the real URL and insert it into
        # the config.  The second reconfig isn't really required, but doesn't
        # hurt.
        self.url = 'http://127.0.0.1:%d/' % master.www.getPortnum()
        self.url = unicode2bytes(self.url)
        master.config.buildbotURL = self.url
        yield master.www.reconfigServiceWithBuildbotConfig(master.config)

        self.master = master

        self.agent = client.Agent(reactor)

        # create a telegram bot service
        tb = master.config.services['TelegramBot'] = telegram.TelegramBot(
            bot_token='12345:secret', useWebhook=True,
            chat_ids=[-123456], notify_events=['worker']
        )
        tb._get_http = self.get_http
        yield tb.setServiceParent(self.master)
        self.bot_url = self.url + b"telegram12345:secret"

        yield tb.startService()

        self.sent_messages = []

        def send_message(chat, message, **kwargs):
            self.sent_messages.append((chat, message))
        tb.bot.send_message = send_message

    @defer.inlineCallbacks

0 Source : httpclientservice.py
with GNU General Public License v2.0
from flathub

    def startService(self):
        # treq only supports basicauth, so we force txrequests if the auth is
        # something else
        if self._auth is not None and not isinstance(self._auth, tuple):
            self.PREFER_TREQ = False
        if txrequests is not None and not self.PREFER_TREQ:
            self._session = txrequests.Session()
            self._doRequest = self._doTxRequest
        elif treq is None:
            raise ImportError("{classname} requires either txrequest or treq install."
                              " Users should call {classname}.checkAvailable() during checkConfig()"
                              " to properly alert the user.".format(
                                  classname=self.__class__.__name__))
        else:
            self._doRequest = self._doTReq
            self._pool = HTTPConnectionPool(self.master.reactor)
            self._pool.maxPersistentPerHost = self.MAX_THREADS
            self._agent = Agent(self.master.reactor, pool=self._pool)
        return super().startService()

    @defer.inlineCallbacks

See More Examples