requests.packages.urllib3.util.retry.Retry

Here are the examples of the python api requests.packages.urllib3.util.retry.Retry taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

11 Examples 7

Example 1

Project: pulp
Source File: adapters.py
View license
    def __init__(self, pool_connections=DEFAULT_POOLSIZE,
                 pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
                 pool_block=DEFAULT_POOLBLOCK):
        if max_retries == DEFAULT_RETRIES:
            self.max_retries = Retry(0, read=False)
        else:
            self.max_retries = Retry.from_int(max_retries)
        self.config = {}
        self.proxy_manager = {}

        super(HTTPAdapter, self).__init__()

        self._pool_connections = pool_connections
        self._pool_maxsize = pool_maxsize
        self._pool_block = pool_block
        self._pool_kw_lock = RLock()

        self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)

Example 2

Project: opencraft
Source File: utils.py
View license
def get_requests_retry(total=10, connect=10, read=10, redirect=10, backoff_factor=0,
                       status_forcelist=range(500, 600)):
    """
    Returns a urllib3 `Retry` object, with the default requests retry policy
    """
    return requests.packages.urllib3.util.retry.Retry(
        total=total,
        connect=connect,
        read=read,
        redirect=redirect,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )

Example 3

Project: glance_store
Source File: vmware_datastore.py
View license
def new_session(insecure=False, ca_file=None, total_retries=None):
    session = requests.Session()
    if total_retries is not None:
        http_adapter = adapters.HTTPAdapter(
            max_retries=retry.Retry(total=total_retries))
        https_adapter = adapters.HTTPAdapter(
            max_retries=retry.Retry(total=total_retries))
        session.mount('http://', http_adapter)
        session.mount('https://', https_adapter)
    session.verify = ca_file if ca_file else not insecure
    return session

Example 4

Project: pytvmaze
Source File: tvmaze.py
View license
    @staticmethod
    def _endpoint_standard_get(url):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.get(url)
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code in [404, 422]:
            return None

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        results = r.json()
        return results

Example 5

Project: pytvmaze
Source File: tvmaze.py
View license
    def _endpoint_premium_get(self, url):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.get(url, auth=(self.username, self.api_key))
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code in [404, 422]:
            return None

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        results = r.json()
        return results

Example 6

Project: pytvmaze
Source File: tvmaze.py
View license
    def _endpoint_premium_delete(self, url):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.delete(url, auth=(self.username, self.api_key))
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        if r.status_code == 200:
            return True

        if r.status_code == 404:
            return None

Example 7

Project: pytvmaze
Source File: tvmaze.py
View license
    def _endpoint_premium_put(self, url, payload=None):
        s = requests.Session()
        retries = Retry(total=5,
                        backoff_factor=0.1,
                        status_forcelist=[429])
        s.mount('http://', HTTPAdapter(max_retries=retries))
        try:
            r = s.put(url, data=payload, auth=(self.username, self.api_key))
        except requests.exceptions.ConnectionError as e:
            raise ConnectionError(repr(e))

        s.close()

        if r.status_code == 400:
            raise BadRequest('Bad Request for url {}'.format(url))

        if r.status_code == 200:
            return True

        if r.status_code in [404, 422]:
            return None

Example 8

Project: putio.py
Source File: putio.py
View license
    def __init__(self, access_token, use_retry=False):
        self.access_token = access_token
        self.session = requests.session()

        if use_retry:
            # Retry maximum 10 times, backoff on each retry
            # Sleeps 1s, 2s, 4s, 8s, etc to a maximum of 120s between retries
            # Retries on HTTP status codes 500, 502, 503, 504
            retries = Retry(total=10,
                            backoff_factor=1,
                            status_forcelist=[500, 502, 503, 504])

            # Use the retry strategy for all HTTPS requests
            self.session.mount('https://', HTTPAdapter(max_retries=retries))

        # Keep resource classes as attributes of client.
        # Pass client to resource classes so resource object
        # can use the client.
        attributes = {'client': self}
        self.File = type('File', (_File,), attributes)
        self.Transfer = type('Transfer', (_Transfer,), attributes)
        self.Account = type('Account', (_Account,), attributes)

Example 9

Project: snorkel
Source File: parser.py
View license
    def __init__(self, tok_whitespace=False):
        # http://stanfordnlp.github.io/CoreNLP/corenlp-server.html
        # Spawn a StanfordCoreNLPServer process that accepts parsing requests at an HTTP port.
        # Kill it when python exits.
        # This makes sure that we load the models only once.
        # In addition, it appears that StanfordCoreNLPServer loads only required models on demand.
        # So it doesn't load e.g. coref models and the total (on-demand) initialization takes only 7 sec.
        self.port = 12345
        self.tok_whitespace = tok_whitespace
        loc = os.path.join(os.environ['SNORKELHOME'], 'parser')
        cmd = ['java -Xmx4g -cp "%s/*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer --port %d --timeout %d > /dev/null'
               % (loc, self.port, 600000)]
        self.server_pid = Popen(cmd, shell=True).pid
        atexit.register(self._kill_pserver)
        props = "\"tokenize.whitespace\": \"true\"," if self.tok_whitespace else ""
        self.endpoint = 'http://127.0.0.1:%d/?properties={%s"annotators": "tokenize,ssplit,pos,lemma,depparse,ner", "outputFormat": "json"}' % (self.port, props)

        # Following enables retries to cope with CoreNLP server boot-up latency
        # See: http://stackoverflow.com/a/35504626
        from requests.packages.urllib3.util.retry import Retry
        from requests.adapters import HTTPAdapter
        self.requests_session = requests.Session()
        retries = Retry(total=None,
                        connect=20,
                        read=0,
                        backoff_factor=0.1,
                        status_forcelist=[ 500, 502, 503, 504 ])
        self.requests_session.mount('http://', HTTPAdapter(max_retries=retries))

Example 10

Project: robobrowser
Source File: browser.py
View license
    def __init__(self, session=None, parser=None, user_agent=None,
                 history=True, timeout=None, allow_redirects=True, cache=False,
                 cache_patterns=None, max_age=None, max_count=None, tries=None,
                 multiplier=None):

        self.session = session or requests.Session()

        # Add default user agent string
        if user_agent is not None:
            self.session.headers['User-Agent'] = user_agent

        self.parser = parser

        self.timeout = timeout
        self.allow_redirects = allow_redirects

        # Set up caching
        if cache:
            adapter = RoboHTTPAdapter(max_age=max_age, max_count=max_count)
            cache_patterns = cache_patterns or ['http://', 'https://']
            for pattern in cache_patterns:
                self.session.mount(pattern, adapter)
        elif max_age:
            raise ValueError('Parameter `max_age` is provided, '
                             'but caching is turned off')
        elif max_count:
            raise ValueError('Parameter `max_count` is provided, '
                             'but caching is turned off')

        # Configure history
        self.history = history
        if history is True:
            self._maxlen = None
        elif not history:
            self._maxlen = 1
        else:
            self._maxlen = history
        self._states = []
        self._cursor = -1

        # Set up retries
        if tries:
            retry = Retry(tries, backoff_factor=multiplier)
            for protocol in ['http://', 'https://']:
                self.session.adapters[protocol].max_retries = retry

Example 11

Project: robobrowser
Source File: browser.py
View license
    def __init__(self, session=None, parser=None, user_agent=None,
                 history=True, timeout=None, allow_redirects=True, cache=False,
                 cache_patterns=None, max_age=None, max_count=None, tries=None,
                 multiplier=None):

        self.session = session or requests.Session()

        # Add default user agent string
        if user_agent is not None:
            self.session.headers['User-Agent'] = user_agent

        self.parser = parser

        self.timeout = timeout
        self.allow_redirects = allow_redirects

        # Set up caching
        if cache:
            adapter = RoboHTTPAdapter(max_age=max_age, max_count=max_count)
            cache_patterns = cache_patterns or ['http://', 'https://']
            for pattern in cache_patterns:
                self.session.mount(pattern, adapter)
        elif max_age:
            raise ValueError('Parameter `max_age` is provided, '
                             'but caching is turned off')
        elif max_count:
            raise ValueError('Parameter `max_count` is provided, '
                             'but caching is turned off')

        # Configure history
        self.history = history
        if history is True:
            self._maxlen = None
        elif not history:
            self._maxlen = 1
        else:
            self._maxlen = history
        self._states = []
        self._cursor = -1

        # Set up retries
        if tries:
            retry = Retry(tries, backoff_factor=multiplier)
            for protocol in ['http://', 'https://']:
                self.session.adapters[protocol].max_retries = retry