requests.adapters.HTTPAdapter

Here are the examples of the python api requests.adapters.HTTPAdapter taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

56 Examples 7

Page 1 Selected Page 2

Example 1

Project: collectd-cloudwatch Source File: metadatareader.py
    def _get_metadata(self, request): 
        """
        This method retrieves values from metadata service.
        
        request -- The request part after the metadata service address, for example if full request is:
                   'http://169.254.169.254/latest/meta-data/placement/availability-zone/' 
                   then the request part is 'latest/meta-data/placement/availability-zone/'.
        """
        session = Session()
        session.mount("http://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
        result = session.get(self.metadata_server + request, timeout=self._REQUEST_TIMEOUT)
        if result.status_code is codes.ok:
            return str(result.text)
        else:
            self._LOGGER.error("The request: '" + str(request) + "' failed with status code: '" + str(result.status_code) + "' and message: '" + str(result.text) +"'.")
            raise MetadataRequestException("Cannot retrieve configuration from metadata service. Status code: " + str(result.status_code))

Example 2

Project: RedditImageGrab Source File: img_scrap_stuff.py
def get_reqr():
    global _common_reqr
    if _common_reqr is not None:
        return _common_reqr

    _log.info("Making a requests requester")
    from requests.adapters import HTTPAdapter

    reqr = requests.Session()
    reqr.mount('http://', HTTPAdapter(max_retries=5))
    reqr.mount('https://', HTTPAdapter(max_retries=5))
    _common_reqr = reqr
    return reqr

Example 3

Project: rekall Source File: cloud.py
Function: get_requests_session
    def get_requests_session(self):
        requests_session = self._session.GetParameter("requests_session")
        if requests_session == None:
            # To make sure we can use the requests session in the threadpool we
            # need to make sure that the connection pool can block. Otherwise it
            # will raise when it runs out of connections and the threads will be
            # terminated.
            requests_session = requests.Session()
            requests_session.mount("https://", adapters.HTTPAdapter(
                pool_connections=10, pool_maxsize=300, max_retries=10,
                pool_block=True))

            requests_session.mount("http://", adapters.HTTPAdapter(
                pool_connections=10, pool_maxsize=300, max_retries=10,
                pool_block=True))

            self._session.SetCache("requests_session", requests_session)

        return requests_session

Example 4

Project: open-synthesis Source File: tasks.py
@shared_task
def fetch_source_metadata(source_id):
    """Fetch title and description metadata for the given source."""
    source = EvidenceSource.objects.get(id=source_id)
    session = requests.session()
    session.mount('http://', HTTPAdapter(max_retries=SOURCE_METADATA_RETRY))
    session.mount('https://', HTTPAdapter(max_retries=SOURCE_METADATA_RETRY))
    html = session.get(source.source_url)
    metadata = parse_metadata(html.text)
    source.source_title = metadata.get('title', '')
    source.source_description = metadata.get('description', '')
    source.save()

Example 5

Project: exchangelib Source File: protocol.py
Function: create_session
    def create_session(self):
        session = EWSSession(self)
        session.auth = get_auth_instance(credentials=self.credentials, auth_type=self.auth_type)
        # Leave this inside the loop because headers are mutable
        headers = {'Content-Type': 'text/xml; charset=utf-8', 'Accept-Encoding': 'compress, gzip'}
        session.headers.update(headers)
        scheme = 'https' if self.has_ssl else 'http'
        # We want just one connection per session. No retries, since we wrap all requests in our own retry handler
        session.mount('%s://' % scheme, adapters.HTTPAdapter(
            pool_block=True,
            pool_connections=self.CONNECTIONS_PER_SESSION,
            pool_maxsize=self.CONNECTIONS_PER_SESSION,
            max_retries=0
        ))
        log.debug('Server %s: Created session %s', self.server, session.session_id)
        return session

Example 6

Project: socorro Source File: ftpscraper.py
Function: init
    def __init__(self, *args, **kwargs):
        super(FTPScraperCronApp, self).__init__(*args, **kwargs)
        self.session = requests.Session()
        if urlparse.urlparse(self.config.base_url).scheme == 'https':
            mount = 'https://'
        else:
            mount = 'http://'
        self.session.mount(
            mount,
            HTTPAdapter(max_retries=self.config.retries)
        )

Example 7

Project: golismero Source File: requests_ntlm.py
Function: init
    def __init__(self, username, password):
        """
            :username   - Username in 'domain\\username' format
            :password   - Password or hash in "ABCDABCDABCDABCD:ABCDABCDABCDABCD" format.
        """
        if ntlm is None:
            raise Exception("NTLM libraries unavailable")
        #parse the username
        user_parts = username.split('\\', 1)
        self.domain = user_parts[0].upper()
        self.username = user_parts[1]

        self.password = password
        self.adapter = HTTPAdapter()

Example 8

Project: lambda-webhook Source File: hook.py
Function: lambda_handler
def lambda_handler(event, context):
    print 'Webhook received'
    event['payload'] = base64.b64decode(event['payload'])
    requests_session = Session()
    retries = StaticRetry(total=40)
    requests_session.mount(event['jenkins_url'], HTTPAdapter(max_retries=retries))

    if event.get('service') == 'quay':
        relay_quay(event, requests_session)
    else:
        relay_github(event, requests_session)
    print 'Successfully relayed payload'

Example 9

Project: glance_store Source File: vmware_datastore.py
def new_session(insecure=False, ca_file=None, total_retries=None):
    session = requests.Session()
    if total_retries is not None:
        http_adapter = adapters.HTTPAdapter(
            max_retries=retry.Retry(total=total_retries))
        https_adapter = adapters.HTTPAdapter(
            max_retries=retry.Retry(total=total_retries))
        session.mount('http://', http_adapter)
        session.mount('https://', https_adapter)
    session.verify = ca_file if ca_file else not insecure
    return session

Example 10

Project: bitmask_client Source File: srpauth.py
Function: reset_session
    def _reset_session(self):
        """
        Resets the current session and sets max retries to 30.
        """
        self._session = self._fetcher.session()
        # We need to bump the default retries, otherwise logout
        # fails most of the times
        # NOTE: This is a workaround for the moment, the server
        # side seems to return correctly every time, but it fails
        # on the client end.
        if requests_has_max_retries:
            adapter = HTTPAdapter(max_retries=30)
        else:
            adapter = HTTPAdapter()
        self._session.mount('https://', adapter)

Example 11

Project: collectd-cloudwatch Source File: putclient.py
Function: run_request
    def _run_request(self, request):
        """
        Executes HTTP GET request with timeout using the endpoint defined upon client creation.
        """
        session = Session()
        session.mount("http://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
        session.mount("https://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
        result = session.get(self.endpoint + "?" + request, headers=self._get_custom_headers(), timeout=self.timeout)
        result.raise_for_status()
        return result

Example 12

Project: ceilometer Source File: http.py
    def _do_post(self, data):
        if not data:
            LOG.debug('Data set is empty!')
            return

        session = requests.Session()
        session.mount(self.target,
                      adapters.HTTPAdapter(max_retries=self.max_retries))

        content = ','.join([jsonutils.dumps(item) for item in data])
        content = '[' + content + ']'

        LOG.debug('Data to be posted by HttpPublisher: %s', content)

        res = session.post(self.target, data=content, headers=self.headers,
                           timeout=self.timeout)
        if res.status_code >= 300:
            LOG.error(_LE('Data post failed with status code %s'),
                      res.status_code)

Example 13

Project: collectd-cloudwatch Source File: setup.py
    def _get_metadata(self, request):
        """
        This method retrieves values from metadata service.

        request -- The request part after the metadata service address, for example if full request is:
                   'http://169.254.169.254/latest/meta-data/placement/availability-zone/'
                   then the request part is 'latest/meta-data/placement/availability-zone/'.
        """
        from requests import Session, codes
        from requests.adapters import HTTPAdapter
        try:
            session = Session()
            session.mount("http://", HTTPAdapter(max_retries=self._MAX_RETRIES))
            result = session.get(self.metadata_server + request, timeout=self._REQUEST_TIMEOUT)
        except Exception as e:
            raise MetadataRequestException("Cannot access metadata service. Cause: " + str(e))
        if result.status_code is not codes.ok:
            raise MetadataRequestException("Cannot retrieve configuration from metadata service. Status code: " + str(result.status_code))
        return str(result.text)

Example 14

Project: ExplainToMe Source File: textrank.py
Function: from_url
    @classmethod
    def from_url(cls, url, tokenizer,
                 useragent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0'):  # noqa
        session = Session()
        session.mount('http://', HTTPAdapter(max_retries=2))
        session.mount('https://', HTTPAdapter(max_retries=2))
        print(url)
        request = Request(method='GET',
                          url=url,
                          cookies=RequestsCookieJar())
        prepare = session.prepare_request(request)
        response = session.send(prepare, verify=True)
        return cls(response.text, tokenizer, url)

Example 15

Project: ClusterRunner Source File: network.py
    def __init__(self, min_connection_poolsize=DEFAULT_POOLSIZE):
        """
        :param min_connection_poolsize: The minimum connection pool size for this instance
        :type min_connection_poolsize: int
        """
        self._session = requests.Session()
        self._logger = get_logger(__name__)

        poolsize = max(min_connection_poolsize, DEFAULT_POOLSIZE)
        self._session.mount('http://', HTTPAdapter(pool_connections=poolsize, pool_maxsize=poolsize))

Example 16

Project: traktforalfred Source File: http.py
Function: rebuild
    def rebuild(self):
        if self.session:
            log.info('Rebuilding session and connection pools...')

        # Build the connection pool
        self.session = requests.Session()
        self.session.proxies = self.proxies

        # Mount adapters
        self.session.mount('http://', HTTPAdapter(**self.adapter_kwargs))

        if ssl is not None:
            self.session.mount('https://', HTTPSAdapter(ssl_version=ssl.PROTOCOL_TLSv1, **self.adapter_kwargs))
        else:
            log.warn('"ssl" module is not available, unable to change "ssl_version"')
            self.session.mount('https://', HTTPSAdapter(**self.adapter_kwargs))

        return self.session

Example 17

Project: ores Source File: api.py
Function: init
    def __init__(self, host, user_agent=None, batch_size=50,
                 parallel_requests=4, retries=5):
        self.host = str(host)
        self.batch_size = int(batch_size)
        self.workers = int(parallel_requests)
        self.retries = int(retries)

        self._session = requests.Session()
        self._session.mount(self.host,
                            requests.adapters.HTTPAdapter(max_retries=retries))
        self.headers = {}
        if user_agent is None:
            logger.warning("Sending requests with default User-Agent.  " +
                           "Set 'user_agent' on oresapi.Session to " +
                           "quiet this message.")
            self.headers['User-Agent'] = self.DEFAULT_USERAGENT
        else:
            self.headers['User-Agent'] = user_agent

Example 18

Project: taobaopy Source File: taobao.py
Function: session
    @property
    def session(self):
        if not self._session:
            s = requests.Session()
            s.mount('http://', HTTPAdapter(max_retries=5))
            s.mount('https://', HTTPAdapter(max_retries=5))
            self._session = s
        return self._session

Example 19

Project: MangaScrapper Source File: mangascrapper.py
    def _set_response_ins_(self, pageurl):
        """
        Sets the response for the GET request of pageurl and stores it in self.resp

        :param pageurl: url for which we store the response.
        """
        try:
            s = requests.Session()
            a = requests.adapters.HTTPAdapter(max_retries=5)
            s.mount('http://', a)
            resp = s.get(pageurl, timeout=30)
            self.__resp_obj__ = resp
            resp.close()
        except requests.exceptions.Timeout:
            logging.error("\tVery Slow Internet Connection.")
        except requests.exceptions.ConnectionError:
            logging.error("\tNetwork Unavailable. Check your connection.")
        except requests.exceptions.MissingSchema:
            logging.error("\t503 Service Unavailable. Retrying download ... ")

Example 20

Project: django-location Source File: runmeter.py
Function: init
    def __init__(self, source):
        self.source = source
        self.session = requests.Session()
        self.session.mount(
            'http://',
            HTTPAdapter(max_retries=5),
        )
        self.session.mount(
            'https://',
            HTTPAdapter(max_retries=5),
        )

Example 21

Project: django-salesforce Source File: base.py
    def make_session(self):
        """Authenticate and get the name of assigned SFDC data server"""
        with connect_lock:
            if self._sf_session is None:
                sf_session = requests.Session()
                # TODO configurable class Salesforce***Auth
                sf_session.auth = SalesforcePasswordAuth(db_alias=self.alias,
                                                         settings_dict=self.settings_dict)
                sf_instance_url = sf_session.auth.instance_url
                sf_requests_adapter = HTTPAdapter(max_retries=get_max_retries())
                sf_session.mount(sf_instance_url, sf_requests_adapter)
                # Additional header works, but the improvement is immeasurable for
                # me. (less than SF speed fluctuation)
                # sf_session.header = {'accept-encoding': 'gzip, deflate', 'connection': 'keep-alive'}
                self._sf_session = sf_session

Example 22

Project: twine Source File: repository.py
    @staticmethod
    def _make_adapter_with_retries():
        retry = util.Retry(
            connect=5,
            total=10,
            method_whitelist=['GET'],
            status_forcelist=[500, 501, 502, 503],
        )
        return adapters.HTTPAdapter(max_retries=retry)

Example 23

Project: alf Source File: adapters.py
def mount_retry_adapter(session, retries):
    adapter = requests.adapters.HTTPAdapter(max_retries=retries)

    session.mount('http://', adapter)
    session.mount('https://', adapter)

Example 24

Project: snorkel Source File: parser.py
    def __init__(self, tok_whitespace=False):
        # http://stanfordnlp.github.io/CoreNLP/corenlp-server.html
        # Spawn a StanfordCoreNLPServer process that accepts parsing requests at an HTTP port.
        # Kill it when python exits.
        # This makes sure that we load the models only once.
        # In addition, it appears that StanfordCoreNLPServer loads only required models on demand.
        # So it doesn't load e.g. coref models and the total (on-demand) initialization takes only 7 sec.
        self.port = 12345
        self.tok_whitespace = tok_whitespace
        loc = os.path.join(os.environ['SNORKELHOME'], 'parser')
        cmd = ['java -Xmx4g -cp "%s/*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer --port %d --timeout %d > /dev/null'
               % (loc, self.port, 600000)]
        self.server_pid = Popen(cmd, shell=True).pid
        atexit.register(self._kill_pserver)
        props = "\"tokenize.whitespace\": \"true\"," if self.tok_whitespace else ""
        self.endpoint = 'http://127.0.0.1:%d/?properties={%s"annotators": "tokenize,ssplit,pos,lemma,depparse,ner", "outputFormat": "json"}' % (self.port, props)

        # Following enables retries to cope with CoreNLP server boot-up latency
        # See: http://stackoverflow.com/a/35504626
        from requests.packages.urllib3.util.retry import Retry
        from requests.adapters import HTTPAdapter
        self.requests_session = requests.Session()
        retries = Retry(total=None,
                        connect=20,
                        read=0,
                        backoff_factor=0.1,
                        status_forcelist=[ 500, 502, 503, 504 ])
        self.requests_session.mount('http://', HTTPAdapter(max_retries=retries))

Example 25

Project: YouCompleteMe-x64 Source File: sessions.py
Function: init
    def __init__(self, executor=None, max_workers=2, *args, **kwargs):
        """Creates a FuturesSession

        Notes
        ~~~~~

        * ProcessPoolExecutor is not supported b/c Response objects are
          not picklable.

        * If you provide both `executor` and `max_workers`, the latter is
          ignored and provided executor is used as is.
        """
        super(FuturesSession, self).__init__(*args, **kwargs)
        if executor is None:
            executor = ThreadPoolExecutor(max_workers=max_workers)
            # set connection pool size equal to max_workers if needed
            if max_workers > DEFAULT_POOLSIZE:
                adapter_kwargs = dict(pool_connections=max_workers,
                                      pool_maxsize=max_workers)
                self.mount('https://', HTTPAdapter(**adapter_kwargs))
                self.mount('http://', HTTPAdapter(**adapter_kwargs))

        self.executor = executor

Example 26

Project: YCM_WIN_X86 Source File: test_requests.py
    def test_transport_adapter_ordering(self):
        s = requests.Session()
        order = ['https://', 'http://']
        assert order == list(s.adapters)
        s.mount('http://git', HTTPAdapter())
        s.mount('http://github', HTTPAdapter())
        s.mount('http://github.com', HTTPAdapter())
        s.mount('http://github.com/about/', HTTPAdapter())
        order = [
            'http://github.com/about/',
            'http://github.com',
            'http://github',
            'http://git',
            'https://',
            'http://',
        ]
        assert order == list(s.adapters)
        s.mount('http://gittip', HTTPAdapter())
        s.mount('http://gittip.com', HTTPAdapter())
        s.mount('http://gittip.com/about/', HTTPAdapter())
        order = [
            'http://github.com/about/',
            'http://gittip.com/about/',
            'http://github.com',
            'http://gittip.com',
            'http://github',
            'http://gittip',
            'http://git',
            'https://',
            'http://',
        ]
        assert order == list(s.adapters)
        s2 = requests.Session()
        s2.adapters = {'http://': HTTPAdapter()}
        s2.mount('https://', HTTPAdapter())
        assert 'http://' in s2.adapters
        assert 'https://' in s2.adapters

Example 27

Project: internetarchive Source File: session.py
    def _mount_http_adapter(self, protocol=None, max_retries=None,
                            status_forcelist=None, host=None):
        """Mount an HTTP adapter to the
        :class:`ArchiveSession <ArchiveSession>` object.
        """
        protocol = protocol if protocol else self.protocol
        host = host if host else 'archive.org'
        if max_retries is None:
            max_retries = self.http_adapter_kwargs.get('max_retries', 3)

        if not status_forcelist:
            status_forcelist = [500, 501, 502, 503, 504]
        if max_retries and isinstance(max_retries, (int, float)):
            max_retries = Retry(total=max_retries,
                                connect=max_retries,
                                read=max_retries,
                                redirect=False,
                                method_whitelist=Retry.DEFAULT_METHOD_WHITELIST,
                                status_forcelist=status_forcelist,
                                backoff_factor=1)
        self.http_adapter_kwargs['max_retries'] = max_retries
        max_retries_adapter = HTTPAdapter(**self.http_adapter_kwargs)
        # Don't mount on s3.us.archive.org, only archive.org!
        # IA-S3 requires a more complicated retry workflow.
        self.mount('{0}//{1}'.format(protocol, host), max_retries_adapter)

Example 28

Project: requests-foauth Source File: requests_foauth.py
Function: init
    def __init__(self, username, password):
        self.auth = (username, password)
        self.http = HTTPAdapter()

Example 29

Project: putio.py Source File: putio.py
Function: init
    def __init__(self, access_token, use_retry=False):
        self.access_token = access_token
        self.session = requests.session()

        if use_retry:
            # Retry maximum 10 times, backoff on each retry
            # Sleeps 1s, 2s, 4s, 8s, etc to a maximum of 120s between retries
            # Retries on HTTP status codes 500, 502, 503, 504
            retries = Retry(total=10,
                            backoff_factor=1,
                            status_forcelist=[500, 502, 503, 504])

            # Use the retry strategy for all HTTPS requests
            self.session.mount('https://', HTTPAdapter(max_retries=retries))

        # Keep resource classes as attributes of client.
        # Pass client to resource classes so resource object
        # can use the client.
        attributes = {'client': self}
        self.File = type('File', (_File,), attributes)
        self.Transfer = type('Transfer', (_Transfer,), attributes)
        self.Account = type('Account', (_Account,), attributes)

Example 30

Project: weboob Source File: sessions.py
Function: init
    def __init__(self, executor=None, max_workers=2, max_retries=2, *args, **kwargs):
        """Creates a FuturesSession

        Notes
        ~~~~~

        * ProcessPoolExecutor is not supported b/c Response objects are
          not picklable.

        * If you provide both `executor` and `max_workers`, the latter is
          ignored and provided executor is used as is.
        """
        super(FuturesSession, self).__init__(*args, **kwargs)
        if executor is None and ThreadPoolExecutor is not None:
            executor = ThreadPoolExecutor(max_workers=max_workers)
            # set connection pool size equal to max_workers if needed
            if max_workers > DEFAULT_POOLSIZE:
                adapter_kwargs = dict(pool_connections=max_workers,
                                      pool_maxsize=max_workers,
                                      max_retries=max_retries)
                self.mount('https://', HTTPAdapter(**adapter_kwargs))
                self.mount('http://', HTTPAdapter(**adapter_kwargs))

        self.executor = executor

Example 31

Project: pydora Source File: transport.py
Function: init
    def __init__(self):
        super(RetryingSession, self).__init__()
        self.mount('https://', HTTPAdapter(max_retries=3))
        self.mount('http://', HTTPAdapter(max_retries=3))

Example 32

Project: siskin Source File: crossref.py
    @timed
    def run(self):
        """
        The API sometimes returns a 504 or other error. We therefore cache all HTTP requests
        locally with a simple URLCache and re-attempt a URL a couple of times.
        """
        cache = URLCache(directory=os.path.join(tempfile.gettempdir(), '.urlcache'))
        adapter = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
        cache.sess.mount('http://', adapter)

        filter = "from-{self.filter}-date:{self.begin},until-{self.filter}-date:{self.end}".format(self=self)
        rows, offset = self.rows, 0

        with self.output().open('w') as output:
            while True:
                params = {"rows": rows, "offset": offset, "filter": filter}
                url = 'http://api.crossref.org/works?%s' % (urllib.urlencode(params))
                for attempt in range(1, self.attempts):
                    if not cache.is_cached(url):
                        time.sleep(self.sleep)
                    body = cache.get(url)
                    try:
                        content = json.loads(body)
                    except ValueError as err:
                        if attempt == self.attempts - 1:
                            self.logger.debug("URL was %s" % url)
                            self.logger.debug(err)
                            self.logger.debug(body[:100])
                            raise
                        if os.path.exists(cache.get_cache_file(url)):
                            self.logger.debug("trying to recover by removing cached entry")
                            os.remove(cache.get_cache_file(url))
                    else:
                        break
                items = content["message"]["items"]
                self.logger.debug("%s: %s" % (url, len(items)))
                if len(items) == 0:
                    break
                output.write(body + "\n")
                offset += rows

Example 33

Project: pytvmaze Source File: tvmaze.py
    def _endpoint_premium_get(self, url):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.get(url, auth=(self.username, self.api_key))
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code in [404, 422]:
            return None

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        results = r.json()
        return results

Example 34

Project: conda Source File: connection.py
Function: init
    def __init__(self, *args, **kwargs):
        retries = kwargs.pop('retries', RETRIES)

        super(CondaSession, self).__init__(*args, **kwargs)

        self.auth = CondaHttpAuth()  # TODO: should this just be for certain protocol adapters?

        proxies = context.proxy_servers
        if proxies:
            self.proxies = proxies

        # Configure retries
        if retries:
            http_adapter = HTTPAdapter(max_retries=retries)
            self.mount("http://", http_adapter)
            self.mount("https://", http_adapter)

        # Enable file:// urls
        self.mount("file://", LocalFSAdapter())

        # Enable ftp:// urls
        self.mount("ftp://", FTPAdapter())

        # Enable s3:// urls
        self.mount("s3://", S3Adapter())

        self.headers['User-Agent'] = user_agent

        self.verify = context.ssl_verify

        if context.client_ssl_cert_key:
            self.cert = (context.client_ssl_cert, context.client_ssl_cert_key)
        elif context.client_ssl_cert:
            self.cert = context.client_ssl_cert

Example 35

Project: siskin Source File: crossref.py
    @timed
    def run(self):
        cache = URLCache(directory=os.path.join(tempfile.gettempdir(), '.urlcache'))
        adapter = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
        cache.sess.mount('http://', adapter)

        rows, offset = self.rows, 0

        with self.output().open('w') as output:
            while True:
                params = {"rows": rows, "offset": offset}
                url = 'http://api.crossref.org/%s?%s' % (self.kind, urllib.urlencode(params))
                for attempt in range(1, 3):
                    body = cache.get(url)
                    try:
                        content = json.loads(body)
                    except ValueError as err:
                        if attempt == 2:
                            self.logger.debug("URL was %s" % url)
                            self.logger.debug(err)
                            self.logger.debug(body[:100])
                            raise
                        if os.path.exists(cache.get_cache_file(url)):
                            self.logger.debug("trying to recover by removing cached entry")
                            os.remove(cache.get_cache_file(url))
                    else:
                        break
                items = content["message"]["items"]
                self.logger.debug("%s: %s" % (url, len(items)))
                if len(items) == 0:
                    break
                output.write(body)
                output.write("\n")
                offset += rows

Example 36

Project: py-translate Source File: translator.py
def push_url(interface):
    '''
    Decorates a function returning the url of translation API.
    Creates and maintains HTTP connection state

    Returns a dict response object from the server containing the translated
    text and metadata of the request body

    :param interface: Callable Request Interface
    :type interface: Function
    '''

    @functools.wraps(interface)
    def connection(*args, **kwargs):
        """
        Extends and wraps a HTTP interface.

        :return: Response Content
        :rtype: Dictionary
        """
        session = Session()
        session.mount('http://',  HTTPAdapter(max_retries=2))
        session.mount('https://', HTTPAdapter(max_retries=2))

        request  = Request(**interface(*args, **kwargs))
        prepare  = session.prepare_request(request)
        response = session.send(prepare, verify=True)

        if response.status_code != requests.codes.ok:
            response.raise_for_status()

        cleanup = re.subn(r',(?=,)', '', response.content.decode('utf-8'))[0]

        return json.loads(cleanup.replace(r'\xA0', r' ').replace('[,', '[1,'), encoding='UTF-8')

    return connection

Example 37

Project: robobrowser Source File: responses.py
Function: on_request
    def _on_request(self, request, **kwargs):
        match = self._find_match(request)

        # TODO(dcramer): find the correct class for this
        if match is None:
            error_msg = 'Connection refused: {0}'.format(request.url)
            response = ConnectionError(error_msg)

            self._calls.add(request, response)
            raise response

        headers = {
            'Content-Type': match['content_type'],
        }
        if match['adding_headers']:
            headers.update(match['adding_headers'])

        response = HTTPResponse(
            status=match['status'],
            body=BufferIO(match['body']),
            headers=headers,
            preload_content=False,
        )

        adapter = HTTPAdapter()

        response = adapter.build_response(request, response)
        if not match['stream']:
            response.content  # NOQA

        self._calls.add(request, response)

        return response

Example 38

Project: pytvmaze Source File: tvmaze.py
    @staticmethod
    def _endpoint_standard_get(url):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.get(url)
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code in [404, 422]:
            return None

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        results = r.json()
        return results

Example 39

Project: pytvmaze Source File: tvmaze.py
    def _endpoint_premium_delete(self, url):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.delete(url, auth=(self.username, self.api_key))
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        if r.status_code == 200:
            return True

        if r.status_code == 404:
            return None

Example 40

Project: django-salesforce Source File: auth.py
    def authenticate(self):
        """
        Authenticate to the Salesforce API with the provided credentials (password).
        """
        settings_dict = self.settings_dict
        url = ''.join([settings_dict['HOST'], '/services/oauth2/token'])

        log.info("attempting authentication to %s" % settings_dict['HOST'])
        self._session.mount(settings_dict['HOST'], HTTPAdapter(max_retries=get_max_retries()))
        response = self._session.post(url, data=dict(
            grant_type      = 'password',
            client_id       = settings_dict['CONSUMER_KEY'],
            client_secret   = settings_dict['CONSUMER_SECRET'],
            username        = settings_dict['USER'],
            password        = settings_dict['PASSWORD'],
        ))
        if response.status_code == 200:
            # prefer str in Python 2 due to other API
            response_data = {str(k): str(v) for k, v in response.json().items()}
            # Verify signature (not important for this auth mechanism)
            calc_signature = (base64.b64encode(hmac.new(
                    key=settings_dict['CONSUMER_SECRET'].encode('ascii'),
                    msg=(response_data['id'] + response_data['issued_at']).encode('ascii'),
                    digestmod=hashlib.sha256).digest())).decode('ascii')
            if calc_signature == response_data['signature']:
                log.info("successfully authenticated %s" % settings_dict['USER'])
            else:
                raise IntegrityError('Invalid auth signature received')
        else:
            raise LookupError("oauth failed: %s: %s" % (settings_dict['USER'], response.text))
        return response_data

Example 41

Project: pytvmaze Source File: tvmaze.py
    def _endpoint_premium_put(self, url, payload=None):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.put(url, data=payload, auth=(self.username, self.api_key))
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        if r.status_code == 200:
            return True

        if r.status_code in [404, 422]:
            return None

Example 42

Project: droopescan Source File: base_plugin_internal.py
    def _general_init(self, opts, out=None):
        """
            Initializes a variety of variables depending on user input.
            @return: a boolean value indicating whether progressbars should be
                hidden.
        """

        self.session = Session()
        if out:
            self.out = out
        else:
            self.out = self._output(opts)

        is_cms_plugin = self._meta.label != "scan"
        if is_cms_plugin:
            self.vf = VersionsFile(self.versions_file)

        # http://stackoverflow.com/questions/23632794/in-requests-library-how-can-i-avoid-httpconnectionpool-is-full-discarding-con
        try:
            a = requests.adapters.HTTPAdapter(pool_maxsize=5000)
            self.session.mount('http://', a)
            self.session.mount('https://', a)
            self.session.cookies.set_policy(BlockAll())
        except AttributeError:
            old_req = """Running a very old version of requests! Please `pip
                install -U requests`."""
            self.out.warn(old_req)

        self.session.verify = False
        self.session.headers['User-Agent'] = self.DEFAULT_UA

        debug_requests = opts['debug_requests']
        if debug_requests:
            hide_progressbar = True
            opts['threads_identify'] = 1
            opts['threads_scan'] = 1
            opts['threads_enumerate'] = 1
            self.session = RequestsLogger(self.session)
        else:
            hide_progressbar = False

        functionality = self._functionality(opts)
        enabled_functionality = self._enabled_functionality(functionality, opts)

        return (hide_progressbar, functionality, enabled_functionality)

Example 43

Project: Pyfa Source File: eve.py
    def __init__(self, additional_headers=None, user_agent=None, cache_dir=None, cache=None):
        # Set up a Requests Session
        session = requests.Session()
        if additional_headers is None:
            additional_headers = {}
        if user_agent is None:
            user_agent = "pyfa/{0} ({1})".format(config.version, config.tag)
        session.headers.update({
            "User-Agent": user_agent,
            "Accept": "application/json",
        })
        session.headers.update(additional_headers)
        session.mount('https://public-crest.eveonline.com',
                HTTPAdapter())
        self._session = session
        if cache:
            if isinstance(cache, APICache):
                self.cache = cache  # Inherit from parents
            elif isinstance(cache, type):
                self.cache = cache()  # Instantiate a new cache
        elif cache_dir:
            self.cache_dir = cache_dir
            self.cache = FileCache(self.cache_dir)
        else:
            self.cache = DictCache()

Example 44

Project: Cactus Source File: test_cli.py
Function: test_serve
    def test_serve(self):
        cactus = self.find_cactus()

        ret, _, _ = self.run_cli(["create", self.path])
        self.assertEqual(0, ret)

        port = 12345

        p = subprocess.Popen([cactus, "serve", "-p", str(port)], cwd=self.path, stdout=subprocess.DEVNULL,
                             stderr=subprocess.DEVNULL)

        srv = "http://127.0.0.1:{0}".format(port)
        s = requests.Session()
        s.mount(srv, HTTPAdapter(max_retries=Retry(backoff_factor=0.2)))

        r = s.post("{0}/_cactus/shutdown".format(srv))
        r.raise_for_status()

        # We'd love to use p.wait(n) here, but that doesn't work on
        # some of the versions of Python we support.
        for _ in range(5):
            if p.poll() != None:
                break
            time.sleep(1)
        else:
            self.fail("Server did not exit!")

        self.assertEqual(0, p.returncode)

Example 45

Project: streamlink Source File: douyutv.py
    def _get_streams(self):
        match = _url_re.match(self.url)
        channel = match.group("channel")

        http.headers.update({'User-Agent': USER_AGENT})
        http.verify=False
        http.mount('https://', HTTPAdapter(max_retries=99))

        #Thanks to @ximellon for providing method.
        try:
            channel = int(channel)
        except ValueError:
            channel = http.get(self.url, schema=_room_id_schema)
            if channel == 0:
                channel = http.get(self.url, schema=_room_id_alt_schema)

        res = http.get(MAPI_URL.format(channel))
        room = http.json(res, schema=_room_schema)
        if not room:
            return

        if room["show_status"] != SHOW_STATUS_ONLINE:
            return

        ts = int(time.time() / 60)
        did = uuid.uuid4().hex.upper()
        sign = hashlib.md5(("{0}{1}{2}{3}".format(channel, did, LAPI_SECRET, ts)).encode("utf-8")).hexdigest()

        data = {
            "cdn": "ws",
            "rate": "0",
            "tt": ts,
            "did": did,
            "sign": sign
        }

        res = http.post(LAPI_URL.format(channel), data=data)
        room = http.json(res, schema=_lapi_schema)

        url = "{room[rtmp_url]}/{room[rtmp_live]}".format(room=room)
        stream = HTTPStream(self.session, url)
        yield "source", stream

        data = {
            "cdn": "ws",
            "rate": "2",
            "tt": ts,
            "did": did,
            "sign": sign
        }

        res = http.post(LAPI_URL.format(channel), data=data)
        room = http.json(res, schema=_lapi_schema)

        url = "{room[rtmp_url]}/{room[rtmp_live]}".format(room=room)
        stream = HTTPStream(self.session, url)
        yield "middle", stream

        data = {
            "cdn": "ws",
            "rate": "1",
            "tt": ts,
            "did": did,
            "sign": sign
        }

        res = http.post(LAPI_URL.format(channel), data=data)
        room = http.json(res, schema=_lapi_schema)

        url = "{room[rtmp_url]}/{room[rtmp_live]}".format(room=room)
        stream = HTTPStream(self.session, url)
        yield "low", stream

Example 46

Project: python-fedora Source File: openidbaseclient.py
    def __init__(self, base_url, login_url=None, useragent=None, debug=False,
                 insecure=False, openid_insecure=False, username=None,
                 cache_session=True, retries=None, timeout=None,
                 retry_backoff_factor=0):
        """Client for interacting with web services relying on fas_openid auth.

        :arg base_url: Base of every URL used to contact the server
        :kwarg login_url: The url to the login endpoint of the application.
            If none are specified, it uses the default `/login`.
        :kwarg useragent: Useragent string to use.  If not given, default to
            "Fedora OpenIdBaseClient/VERSION"
        :kwarg debug: If True, log debug information
        :kwarg insecure: If True, do not check server certificates against
            their CA's.  This means that man-in-the-middle attacks are
            possible against the `BaseClient`. You might turn this option on
            for testing against a local version of a server with a self-signed
            certificate but it should be off in production.
        :kwarg openid_insecure: If True, do not check the openid server
            certificates against their CA's.  This means that man-in-the-
            middle attacks are possible against the `BaseClient`. You might
            turn this option on for testing against a local version of a
            server with a self-signed certificate but it should be off in
            production.
        :kwarg username: Username for establishing authenticated connections
        :kwarg cache_session: If set to true, cache the user's session data on
            the filesystem between runs
        :kwarg retries: if we get an unknown or possibly transient error from
            the server, retry this many times.  Setting this to a negative
            number makes it try forever.  Defaults to zero, no retries.
            Note that this can only be set during object initialization.
        :kwarg timeout: A float describing the timeout of the connection. The
            timeout only affects the connection process itself, not the
            downloading of the response body. Defaults to 120 seconds.
        :kwarg retry_backoff_factor: Exponential backoff factor to apply in
            between retry attempts.  We will sleep for:

                `{retry_backoff_factor}*(2 ^ ({number of failed retries} - 1))`

            ...seconds inbetween attempts.  The backoff factor scales the rate
            at which we back off.   Defaults to 0 (backoff disabled).
            Note that this attribute can only be set at object initialization.
        """

        # These are also needed by OpenIdProxyClient
        self.useragent = useragent or 'Fedora BaseClient/%(version)s' % {
            'version': __version__}
        self.base_url = base_url
        self.login_url = login_url or urljoin(self.base_url, '/login')
        self.debug = debug
        self.insecure = insecure
        self.openid_insecure = openid_insecure
        self.retries = retries
        self.timeout = timeout

        # These are specific to OpenIdBaseClient
        self.username = username
        self.cache_session = cache_session
        self.cache_lock = lockfile.FileLock(b_SESSION_FILE)

        # Make sure the database for storing the session cookies exists
        if cache_session:
            self._initialize_session_cache()

        # python-requests session.  Holds onto cookies
        self._session = requests.session()

        # Also hold on to retry logic.
        # http://www.coglib.com/~icordasc/blog/2014/12/retries-in-requests.html
        server_errors = [500, 501, 502, 503, 504, 506, 507, 508, 509, 599]
        method_whitelist = Retry.DEFAULT_METHOD_WHITELIST.union(set(['POST']))
        if retries is not None:
            prefixes = ['http://', 'https://']
            for prefix in prefixes:
                self._session.mount(prefix, requests.adapters.HTTPAdapter(
                    max_retries=Retry(
                        total=retries,
                        status_forcelist=server_errors,
                        backoff_factor=retry_backoff_factor,
                        method_whitelist=method_whitelist,
                    ),
                ))

        # See if we have any cookies kicking around from a previous run
        self._load_cookies()

Example 47

Project: responses Source File: test_responses.py
def test_custom_adapter():
    @responses.activate
    def run():
        url = "http://example.com"
        responses.add(responses.GET, url, body=b'test')

        calls = [0]

        class DummyAdapter(requests.adapters.HTTPAdapter):
            def send(self, *a, **k):
                calls[0] += 1
                return super(DummyAdapter, self).send(*a, **k)

        # Test that the adapter is actually used
        session = requests.Session()
        session.mount("http://", DummyAdapter())

        resp = session.get(url, allow_redirects=False)
        assert calls[0] == 1

        # Test that the response is still correctly emulated
        session = requests.Session()
        session.mount("http://", DummyAdapter())

        resp = session.get(url)
        assert_response(resp, 'test')

    run()

Example 48

Project: azure-batch-apps-python Source File: rest_client.py
def _call(auth, *args, **kwargs):
    """Internal method to open Requests session."""

    try:
        conn_session = auth.get_session()
        conn_adptr = requests.adapters.HTTPAdapter(max_retries=RETRIES)
        conn_session.mount('https://', conn_adptr)

        LOG.info("About to make REST call with args {0}".format(args))
        LOG.debug("About to make REST call with kwargs {0}".format(kwargs))
        LOG.debug(
            "Opened requests session with max retries: {0}".format(RETRIES))

        response = conn_session.request(*args, **kwargs)
        return _check_code(response)

    except (oauth2.rfc6749.errors.InvalidGrantError,
            oauth2.rfc6749.errors.TokenExpiredError) as exp:

        LOG.info("Token expired. Attempting to refresh and continue.")
        refreshed_session  = auth.refresh_session()
        if not refreshed_session:
            raise SessionExpiredException("Please log in again. "
                                          "{0}".format(str(exp)))

        try:
            conn_adptr = requests.adapters.HTTPAdapter(max_retries=RETRIES)
            refreshed_session.mount('https://', conn_adptr)
            response = refreshed_session.request(*args, **kwargs)
            return _check_code(response)

        except Exception as exp:
            raise RestCallException(
                type(exp),
                "An {type} error occurred: {error}".format(
                    type=type(exp), error=str(exp)),
                exp)

    except (requests.RequestException,
            oauth2.rfc6749.errors.OAuth2Error) as exp:

        raise RestCallException(
            type(exp),
            "An {type} error occurred: {error}".format(type=type(exp),
                                                       error=str(exp)),
            exp)

Example 49

Project: tilequeue Source File: wof.py
def _make_requests_session_with_retries(max_retries):
    from requests.adapters import HTTPAdapter
    from requests.packages.urllib3.util import Retry

    s = requests.Session()
    a = HTTPAdapter(
        max_retries=Retry(
            total=max_retries,
            status_forcelist=[  # this is a list of statuses to consider to be
                                # an error and retry.
                429,  # Too many requests (i.e: back off)
                500,  # Generic internal server error
                502,  # Bad Gateway - i.e: upstream failure
                503,  # Unavailable, temporarily
                504,  # Gateway timeout
                522   # Origin connection timed out
            ],
            backoff_factor=1.0  # back off for 0s, 1s, 3s, 7s, etc... after
                                # each successive failure. (factor*(2^N-1))
        ))

    # use retry for both HTTP and HTTPS connections.
    s.mount('http://', a)
    s.mount('https://', a)

    return s

Example 50

Project: stream-python Source File: client.py
Function: init
    def __init__(self, api_key, api_secret, app_id, version='v1.0', timeout=6.0, base_url=None, location=None):
        '''
        Initialize the client with the given api key and secret

        :param api_key: the api key
        :param api_secret: the api secret
        :param app_id: the app id

        **Example usage**::

            import stream
            # initialize the client
            client = stream.connect('key', 'secret')
            # get a feed object
            feed = client.feed('aggregated:1')
            # write data to the feed
            activity_data = {'actor': 1, 'verb': 'tweet', 'object': 1}
            activity_id = feed.add_activity(activity_data)['id']
            activities = feed.get()

            feed.follow('flat:3')
            activities = feed.get()
            feed.unfollow('flat:3')
            feed.remove_activity(activity_id)
        '''
        self.api_key = api_key
        self.api_secret = api_secret
        self.app_id = app_id
        self.version = version
        self.timeout = timeout
        self.location = location

        if os.environ.get('LOCAL'):
            self.base_url = 'http://localhost:8000/api/'
            self.timeout = 20
        elif base_url is not None:
            self.base_url = base_url
        elif location is not None:
            self.base_url = 'https://%s-api.getstream.io/api/' % location

        self.base_analytics_url = 'https://analytics.getstream.io/analytics/'

        self.session = requests.Session()
        # TODO: turn this back on after we verify it doesnt retry on slower requests
        self.session.mount(self.base_url, HTTPAdapter(max_retries=0))
        self.auth = HTTPSignatureAuth(api_key, secret=api_secret)
See More Examples - Go to Next Page
Page 1 Selected Page 2