Here are the examples of the python api requests.adapters.HTTPAdapter taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
56 Examples
3
Example 1
Project: collectd-cloudwatch Source File: metadatareader.py
def _get_metadata(self, request):
"""
This method retrieves values from metadata service.
request -- The request part after the metadata service address, for example if full request is:
'http://169.254.169.254/latest/meta-data/placement/availability-zone/'
then the request part is 'latest/meta-data/placement/availability-zone/'.
"""
session = Session()
session.mount("http://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
result = session.get(self.metadata_server + request, timeout=self._REQUEST_TIMEOUT)
if result.status_code is codes.ok:
return str(result.text)
else:
self._LOGGER.error("The request: '" + str(request) + "' failed with status code: '" + str(result.status_code) + "' and message: '" + str(result.text) +"'.")
raise MetadataRequestException("Cannot retrieve configuration from metadata service. Status code: " + str(result.status_code))
3
Example 2
Project: RedditImageGrab Source File: img_scrap_stuff.py
def get_reqr():
global _common_reqr
if _common_reqr is not None:
return _common_reqr
_log.info("Making a requests requester")
from requests.adapters import HTTPAdapter
reqr = requests.Session()
reqr.mount('http://', HTTPAdapter(max_retries=5))
reqr.mount('https://', HTTPAdapter(max_retries=5))
_common_reqr = reqr
return reqr
3
Example 3
def get_requests_session(self):
requests_session = self._session.GetParameter("requests_session")
if requests_session == None:
# To make sure we can use the requests session in the threadpool we
# need to make sure that the connection pool can block. Otherwise it
# will raise when it runs out of connections and the threads will be
# terminated.
requests_session = requests.Session()
requests_session.mount("https://", adapters.HTTPAdapter(
pool_connections=10, pool_maxsize=300, max_retries=10,
pool_block=True))
requests_session.mount("http://", adapters.HTTPAdapter(
pool_connections=10, pool_maxsize=300, max_retries=10,
pool_block=True))
self._session.SetCache("requests_session", requests_session)
return requests_session
3
Example 4
Project: open-synthesis Source File: tasks.py
@shared_task
def fetch_source_metadata(source_id):
"""Fetch title and description metadata for the given source."""
source = EvidenceSource.objects.get(id=source_id)
session = requests.session()
session.mount('http://', HTTPAdapter(max_retries=SOURCE_METADATA_RETRY))
session.mount('https://', HTTPAdapter(max_retries=SOURCE_METADATA_RETRY))
html = session.get(source.source_url)
metadata = parse_metadata(html.text)
source.source_title = metadata.get('title', '')
source.source_description = metadata.get('description', '')
source.save()
3
Example 5
def create_session(self):
session = EWSSession(self)
session.auth = get_auth_instance(credentials=self.credentials, auth_type=self.auth_type)
# Leave this inside the loop because headers are mutable
headers = {'Content-Type': 'text/xml; charset=utf-8', 'Accept-Encoding': 'compress, gzip'}
session.headers.update(headers)
scheme = 'https' if self.has_ssl else 'http'
# We want just one connection per session. No retries, since we wrap all requests in our own retry handler
session.mount('%s://' % scheme, adapters.HTTPAdapter(
pool_block=True,
pool_connections=self.CONNECTIONS_PER_SESSION,
pool_maxsize=self.CONNECTIONS_PER_SESSION,
max_retries=0
))
log.debug('Server %s: Created session %s', self.server, session.session_id)
return session
3
Example 6
def __init__(self, *args, **kwargs):
super(FTPScraperCronApp, self).__init__(*args, **kwargs)
self.session = requests.Session()
if urlparse.urlparse(self.config.base_url).scheme == 'https':
mount = 'https://'
else:
mount = 'http://'
self.session.mount(
mount,
HTTPAdapter(max_retries=self.config.retries)
)
3
Example 7
def __init__(self, username, password):
"""
:username - Username in 'domain\\username' format
:password - Password or hash in "ABCDABCDABCDABCD:ABCDABCDABCDABCD" format.
"""
if ntlm is None:
raise Exception("NTLM libraries unavailable")
#parse the username
user_parts = username.split('\\', 1)
self.domain = user_parts[0].upper()
self.username = user_parts[1]
self.password = password
self.adapter = HTTPAdapter()
3
Example 8
def lambda_handler(event, context):
print 'Webhook received'
event['payload'] = base64.b64decode(event['payload'])
requests_session = Session()
retries = StaticRetry(total=40)
requests_session.mount(event['jenkins_url'], HTTPAdapter(max_retries=retries))
if event.get('service') == 'quay':
relay_quay(event, requests_session)
else:
relay_github(event, requests_session)
print 'Successfully relayed payload'
3
Example 9
Project: glance_store Source File: vmware_datastore.py
def new_session(insecure=False, ca_file=None, total_retries=None):
session = requests.Session()
if total_retries is not None:
http_adapter = adapters.HTTPAdapter(
max_retries=retry.Retry(total=total_retries))
https_adapter = adapters.HTTPAdapter(
max_retries=retry.Retry(total=total_retries))
session.mount('http://', http_adapter)
session.mount('https://', https_adapter)
session.verify = ca_file if ca_file else not insecure
return session
3
Example 10
def _reset_session(self):
"""
Resets the current session and sets max retries to 30.
"""
self._session = self._fetcher.session()
# We need to bump the default retries, otherwise logout
# fails most of the times
# NOTE: This is a workaround for the moment, the server
# side seems to return correctly every time, but it fails
# on the client end.
if requests_has_max_retries:
adapter = HTTPAdapter(max_retries=30)
else:
adapter = HTTPAdapter()
self._session.mount('https://', adapter)
3
Example 11
def _run_request(self, request):
"""
Executes HTTP GET request with timeout using the endpoint defined upon client creation.
"""
session = Session()
session.mount("http://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
session.mount("https://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
result = session.get(self.endpoint + "?" + request, headers=self._get_custom_headers(), timeout=self.timeout)
result.raise_for_status()
return result
3
Example 12
Project: ceilometer Source File: http.py
def _do_post(self, data):
if not data:
LOG.debug('Data set is empty!')
return
session = requests.Session()
session.mount(self.target,
adapters.HTTPAdapter(max_retries=self.max_retries))
content = ','.join([jsonutils.dumps(item) for item in data])
content = '[' + content + ']'
LOG.debug('Data to be posted by HttpPublisher: %s', content)
res = session.post(self.target, data=content, headers=self.headers,
timeout=self.timeout)
if res.status_code >= 300:
LOG.error(_LE('Data post failed with status code %s'),
res.status_code)
3
Example 13
Project: collectd-cloudwatch Source File: setup.py
def _get_metadata(self, request):
"""
This method retrieves values from metadata service.
request -- The request part after the metadata service address, for example if full request is:
'http://169.254.169.254/latest/meta-data/placement/availability-zone/'
then the request part is 'latest/meta-data/placement/availability-zone/'.
"""
from requests import Session, codes
from requests.adapters import HTTPAdapter
try:
session = Session()
session.mount("http://", HTTPAdapter(max_retries=self._MAX_RETRIES))
result = session.get(self.metadata_server + request, timeout=self._REQUEST_TIMEOUT)
except Exception as e:
raise MetadataRequestException("Cannot access metadata service. Cause: " + str(e))
if result.status_code is not codes.ok:
raise MetadataRequestException("Cannot retrieve configuration from metadata service. Status code: " + str(result.status_code))
return str(result.text)
3
Example 14
@classmethod
def from_url(cls, url, tokenizer,
useragent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0'): # noqa
session = Session()
session.mount('http://', HTTPAdapter(max_retries=2))
session.mount('https://', HTTPAdapter(max_retries=2))
print(url)
request = Request(method='GET',
url=url,
cookies=RequestsCookieJar())
prepare = session.prepare_request(request)
response = session.send(prepare, verify=True)
return cls(response.text, tokenizer, url)
3
Example 15
Project: ClusterRunner Source File: network.py
def __init__(self, min_connection_poolsize=DEFAULT_POOLSIZE):
"""
:param min_connection_poolsize: The minimum connection pool size for this instance
:type min_connection_poolsize: int
"""
self._session = requests.Session()
self._logger = get_logger(__name__)
poolsize = max(min_connection_poolsize, DEFAULT_POOLSIZE)
self._session.mount('http://', HTTPAdapter(pool_connections=poolsize, pool_maxsize=poolsize))
3
Example 16
def rebuild(self):
if self.session:
log.info('Rebuilding session and connection pools...')
# Build the connection pool
self.session = requests.Session()
self.session.proxies = self.proxies
# Mount adapters
self.session.mount('http://', HTTPAdapter(**self.adapter_kwargs))
if ssl is not None:
self.session.mount('https://', HTTPSAdapter(ssl_version=ssl.PROTOCOL_TLSv1, **self.adapter_kwargs))
else:
log.warn('"ssl" module is not available, unable to change "ssl_version"')
self.session.mount('https://', HTTPSAdapter(**self.adapter_kwargs))
return self.session
3
Example 17
def __init__(self, host, user_agent=None, batch_size=50,
parallel_requests=4, retries=5):
self.host = str(host)
self.batch_size = int(batch_size)
self.workers = int(parallel_requests)
self.retries = int(retries)
self._session = requests.Session()
self._session.mount(self.host,
requests.adapters.HTTPAdapter(max_retries=retries))
self.headers = {}
if user_agent is None:
logger.warning("Sending requests with default User-Agent. " +
"Set 'user_agent' on oresapi.Session to " +
"quiet this message.")
self.headers['User-Agent'] = self.DEFAULT_USERAGENT
else:
self.headers['User-Agent'] = user_agent
3
Example 18
@property
def session(self):
if not self._session:
s = requests.Session()
s.mount('http://', HTTPAdapter(max_retries=5))
s.mount('https://', HTTPAdapter(max_retries=5))
self._session = s
return self._session
3
Example 19
Project: MangaScrapper Source File: mangascrapper.py
def _set_response_ins_(self, pageurl):
"""
Sets the response for the GET request of pageurl and stores it in self.resp
:param pageurl: url for which we store the response.
"""
try:
s = requests.Session()
a = requests.adapters.HTTPAdapter(max_retries=5)
s.mount('http://', a)
resp = s.get(pageurl, timeout=30)
self.__resp_obj__ = resp
resp.close()
except requests.exceptions.Timeout:
logging.error("\tVery Slow Internet Connection.")
except requests.exceptions.ConnectionError:
logging.error("\tNetwork Unavailable. Check your connection.")
except requests.exceptions.MissingSchema:
logging.error("\t503 Service Unavailable. Retrying download ... ")
3
Example 20
def __init__(self, source):
self.source = source
self.session = requests.Session()
self.session.mount(
'http://',
HTTPAdapter(max_retries=5),
)
self.session.mount(
'https://',
HTTPAdapter(max_retries=5),
)
3
Example 21
Project: django-salesforce Source File: base.py
def make_session(self):
"""Authenticate and get the name of assigned SFDC data server"""
with connect_lock:
if self._sf_session is None:
sf_session = requests.Session()
# TODO configurable class Salesforce***Auth
sf_session.auth = SalesforcePasswordAuth(db_alias=self.alias,
settings_dict=self.settings_dict)
sf_instance_url = sf_session.auth.instance_url
sf_requests_adapter = HTTPAdapter(max_retries=get_max_retries())
sf_session.mount(sf_instance_url, sf_requests_adapter)
# Additional header works, but the improvement is immeasurable for
# me. (less than SF speed fluctuation)
# sf_session.header = {'accept-encoding': 'gzip, deflate', 'connection': 'keep-alive'}
self._sf_session = sf_session
3
Example 22
Project: twine Source File: repository.py
@staticmethod
def _make_adapter_with_retries():
retry = util.Retry(
connect=5,
total=10,
method_whitelist=['GET'],
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
0
Example 23
Project: alf Source File: adapters.py
def mount_retry_adapter(session, retries):
adapter = requests.adapters.HTTPAdapter(max_retries=retries)
session.mount('http://', adapter)
session.mount('https://', adapter)
0
Example 24
Project: snorkel Source File: parser.py
def __init__(self, tok_whitespace=False):
# http://stanfordnlp.github.io/CoreNLP/corenlp-server.html
# Spawn a StanfordCoreNLPServer process that accepts parsing requests at an HTTP port.
# Kill it when python exits.
# This makes sure that we load the models only once.
# In addition, it appears that StanfordCoreNLPServer loads only required models on demand.
# So it doesn't load e.g. coref models and the total (on-demand) initialization takes only 7 sec.
self.port = 12345
self.tok_whitespace = tok_whitespace
loc = os.path.join(os.environ['SNORKELHOME'], 'parser')
cmd = ['java -Xmx4g -cp "%s/*" edu.stanford.nlp.pipeline.StanfordCoreNLPServer --port %d --timeout %d > /dev/null'
% (loc, self.port, 600000)]
self.server_pid = Popen(cmd, shell=True).pid
atexit.register(self._kill_pserver)
props = "\"tokenize.whitespace\": \"true\"," if self.tok_whitespace else ""
self.endpoint = 'http://127.0.0.1:%d/?properties={%s"annotators": "tokenize,ssplit,pos,lemma,depparse,ner", "outputFormat": "json"}' % (self.port, props)
# Following enables retries to cope with CoreNLP server boot-up latency
# See: http://stackoverflow.com/a/35504626
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
self.requests_session = requests.Session()
retries = Retry(total=None,
connect=20,
read=0,
backoff_factor=0.1,
status_forcelist=[ 500, 502, 503, 504 ])
self.requests_session.mount('http://', HTTPAdapter(max_retries=retries))
0
Example 25
def __init__(self, executor=None, max_workers=2, *args, **kwargs):
"""Creates a FuturesSession
Notes
~~~~~
* ProcessPoolExecutor is not supported b/c Response objects are
not picklable.
* If you provide both `executor` and `max_workers`, the latter is
ignored and provided executor is used as is.
"""
super(FuturesSession, self).__init__(*args, **kwargs)
if executor is None:
executor = ThreadPoolExecutor(max_workers=max_workers)
# set connection pool size equal to max_workers if needed
if max_workers > DEFAULT_POOLSIZE:
adapter_kwargs = dict(pool_connections=max_workers,
pool_maxsize=max_workers)
self.mount('https://', HTTPAdapter(**adapter_kwargs))
self.mount('http://', HTTPAdapter(**adapter_kwargs))
self.executor = executor
0
Example 26
Project: YCM_WIN_X86 Source File: test_requests.py
def test_transport_adapter_ordering(self):
s = requests.Session()
order = ['https://', 'http://']
assert order == list(s.adapters)
s.mount('http://git', HTTPAdapter())
s.mount('http://github', HTTPAdapter())
s.mount('http://github.com', HTTPAdapter())
s.mount('http://github.com/about/', HTTPAdapter())
order = [
'http://github.com/about/',
'http://github.com',
'http://github',
'http://git',
'https://',
'http://',
]
assert order == list(s.adapters)
s.mount('http://gittip', HTTPAdapter())
s.mount('http://gittip.com', HTTPAdapter())
s.mount('http://gittip.com/about/', HTTPAdapter())
order = [
'http://github.com/about/',
'http://gittip.com/about/',
'http://github.com',
'http://gittip.com',
'http://github',
'http://gittip',
'http://git',
'https://',
'http://',
]
assert order == list(s.adapters)
s2 = requests.Session()
s2.adapters = {'http://': HTTPAdapter()}
s2.mount('https://', HTTPAdapter())
assert 'http://' in s2.adapters
assert 'https://' in s2.adapters
0
Example 27
Project: internetarchive Source File: session.py
def _mount_http_adapter(self, protocol=None, max_retries=None,
status_forcelist=None, host=None):
"""Mount an HTTP adapter to the
:class:`ArchiveSession <ArchiveSession>` object.
"""
protocol = protocol if protocol else self.protocol
host = host if host else 'archive.org'
if max_retries is None:
max_retries = self.http_adapter_kwargs.get('max_retries', 3)
if not status_forcelist:
status_forcelist = [500, 501, 502, 503, 504]
if max_retries and isinstance(max_retries, (int, float)):
max_retries = Retry(total=max_retries,
connect=max_retries,
read=max_retries,
redirect=False,
method_whitelist=Retry.DEFAULT_METHOD_WHITELIST,
status_forcelist=status_forcelist,
backoff_factor=1)
self.http_adapter_kwargs['max_retries'] = max_retries
max_retries_adapter = HTTPAdapter(**self.http_adapter_kwargs)
# Don't mount on s3.us.archive.org, only archive.org!
# IA-S3 requires a more complicated retry workflow.
self.mount('{0}//{1}'.format(protocol, host), max_retries_adapter)
0
Example 28
def __init__(self, username, password):
self.auth = (username, password)
self.http = HTTPAdapter()
0
Example 29
def __init__(self, access_token, use_retry=False):
self.access_token = access_token
self.session = requests.session()
if use_retry:
# Retry maximum 10 times, backoff on each retry
# Sleeps 1s, 2s, 4s, 8s, etc to a maximum of 120s between retries
# Retries on HTTP status codes 500, 502, 503, 504
retries = Retry(total=10,
backoff_factor=1,
status_forcelist=[500, 502, 503, 504])
# Use the retry strategy for all HTTPS requests
self.session.mount('https://', HTTPAdapter(max_retries=retries))
# Keep resource classes as attributes of client.
# Pass client to resource classes so resource object
# can use the client.
attributes = {'client': self}
self.File = type('File', (_File,), attributes)
self.Transfer = type('Transfer', (_Transfer,), attributes)
self.Account = type('Account', (_Account,), attributes)
0
Example 30
def __init__(self, executor=None, max_workers=2, max_retries=2, *args, **kwargs):
"""Creates a FuturesSession
Notes
~~~~~
* ProcessPoolExecutor is not supported b/c Response objects are
not picklable.
* If you provide both `executor` and `max_workers`, the latter is
ignored and provided executor is used as is.
"""
super(FuturesSession, self).__init__(*args, **kwargs)
if executor is None and ThreadPoolExecutor is not None:
executor = ThreadPoolExecutor(max_workers=max_workers)
# set connection pool size equal to max_workers if needed
if max_workers > DEFAULT_POOLSIZE:
adapter_kwargs = dict(pool_connections=max_workers,
pool_maxsize=max_workers,
max_retries=max_retries)
self.mount('https://', HTTPAdapter(**adapter_kwargs))
self.mount('http://', HTTPAdapter(**adapter_kwargs))
self.executor = executor
0
Example 31
def __init__(self):
super(RetryingSession, self).__init__()
self.mount('https://', HTTPAdapter(max_retries=3))
self.mount('http://', HTTPAdapter(max_retries=3))
0
Example 32
Project: siskin Source File: crossref.py
@timed
def run(self):
"""
The API sometimes returns a 504 or other error. We therefore cache all HTTP requests
locally with a simple URLCache and re-attempt a URL a couple of times.
"""
cache = URLCache(directory=os.path.join(tempfile.gettempdir(), '.urlcache'))
adapter = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
cache.sess.mount('http://', adapter)
filter = "from-{self.filter}-date:{self.begin},until-{self.filter}-date:{self.end}".format(self=self)
rows, offset = self.rows, 0
with self.output().open('w') as output:
while True:
params = {"rows": rows, "offset": offset, "filter": filter}
url = 'http://api.crossref.org/works?%s' % (urllib.urlencode(params))
for attempt in range(1, self.attempts):
if not cache.is_cached(url):
time.sleep(self.sleep)
body = cache.get(url)
try:
content = json.loads(body)
except ValueError as err:
if attempt == self.attempts - 1:
self.logger.debug("URL was %s" % url)
self.logger.debug(err)
self.logger.debug(body[:100])
raise
if os.path.exists(cache.get_cache_file(url)):
self.logger.debug("trying to recover by removing cached entry")
os.remove(cache.get_cache_file(url))
else:
break
items = content["message"]["items"]
self.logger.debug("%s: %s" % (url, len(items)))
if len(items) == 0:
break
output.write(body + "\n")
offset += rows
0
Example 33
Project: pytvmaze Source File: tvmaze.py
def _endpoint_premium_get(self, url):
s = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[429])
s.mount('http://', HTTPAdapter(max_retries=retries))
try:
r = s.get(url, auth=(self.username, self.api_key))
except requests.exceptions.ConnectionError as e:
raise ConnectionError(repr(e))
s.close()
if r.status_code in [404, 422]:
return None
if r.status_code == 400:
raise BadRequest('Bad Request for url {}'.format(url))
results = r.json()
return results
0
Example 34
def __init__(self, *args, **kwargs):
retries = kwargs.pop('retries', RETRIES)
super(CondaSession, self).__init__(*args, **kwargs)
self.auth = CondaHttpAuth() # TODO: should this just be for certain protocol adapters?
proxies = context.proxy_servers
if proxies:
self.proxies = proxies
# Configure retries
if retries:
http_adapter = HTTPAdapter(max_retries=retries)
self.mount("http://", http_adapter)
self.mount("https://", http_adapter)
# Enable file:// urls
self.mount("file://", LocalFSAdapter())
# Enable ftp:// urls
self.mount("ftp://", FTPAdapter())
# Enable s3:// urls
self.mount("s3://", S3Adapter())
self.headers['User-Agent'] = user_agent
self.verify = context.ssl_verify
if context.client_ssl_cert_key:
self.cert = (context.client_ssl_cert, context.client_ssl_cert_key)
elif context.client_ssl_cert:
self.cert = context.client_ssl_cert
0
Example 35
Project: siskin Source File: crossref.py
@timed
def run(self):
cache = URLCache(directory=os.path.join(tempfile.gettempdir(), '.urlcache'))
adapter = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
cache.sess.mount('http://', adapter)
rows, offset = self.rows, 0
with self.output().open('w') as output:
while True:
params = {"rows": rows, "offset": offset}
url = 'http://api.crossref.org/%s?%s' % (self.kind, urllib.urlencode(params))
for attempt in range(1, 3):
body = cache.get(url)
try:
content = json.loads(body)
except ValueError as err:
if attempt == 2:
self.logger.debug("URL was %s" % url)
self.logger.debug(err)
self.logger.debug(body[:100])
raise
if os.path.exists(cache.get_cache_file(url)):
self.logger.debug("trying to recover by removing cached entry")
os.remove(cache.get_cache_file(url))
else:
break
items = content["message"]["items"]
self.logger.debug("%s: %s" % (url, len(items)))
if len(items) == 0:
break
output.write(body)
output.write("\n")
offset += rows
0
Example 36
Project: py-translate Source File: translator.py
def push_url(interface):
'''
Decorates a function returning the url of translation API.
Creates and maintains HTTP connection state
Returns a dict response object from the server containing the translated
text and metadata of the request body
:param interface: Callable Request Interface
:type interface: Function
'''
@functools.wraps(interface)
def connection(*args, **kwargs):
"""
Extends and wraps a HTTP interface.
:return: Response Content
:rtype: Dictionary
"""
session = Session()
session.mount('http://', HTTPAdapter(max_retries=2))
session.mount('https://', HTTPAdapter(max_retries=2))
request = Request(**interface(*args, **kwargs))
prepare = session.prepare_request(request)
response = session.send(prepare, verify=True)
if response.status_code != requests.codes.ok:
response.raise_for_status()
cleanup = re.subn(r',(?=,)', '', response.content.decode('utf-8'))[0]
return json.loads(cleanup.replace(r'\xA0', r' ').replace('[,', '[1,'), encoding='UTF-8')
return connection
0
Example 37
def _on_request(self, request, **kwargs):
match = self._find_match(request)
# TODO(dcramer): find the correct class for this
if match is None:
error_msg = 'Connection refused: {0}'.format(request.url)
response = ConnectionError(error_msg)
self._calls.add(request, response)
raise response
headers = {
'Content-Type': match['content_type'],
}
if match['adding_headers']:
headers.update(match['adding_headers'])
response = HTTPResponse(
status=match['status'],
body=BufferIO(match['body']),
headers=headers,
preload_content=False,
)
adapter = HTTPAdapter()
response = adapter.build_response(request, response)
if not match['stream']:
response.content # NOQA
self._calls.add(request, response)
return response
0
Example 38
Project: pytvmaze Source File: tvmaze.py
@staticmethod
def _endpoint_standard_get(url):
s = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[429])
s.mount('http://', HTTPAdapter(max_retries=retries))
try:
r = s.get(url)
except requests.exceptions.ConnectionError as e:
raise ConnectionError(repr(e))
s.close()
if r.status_code in [404, 422]:
return None
if r.status_code == 400:
raise BadRequest('Bad Request for url {}'.format(url))
results = r.json()
return results
0
Example 39
Project: pytvmaze Source File: tvmaze.py
def _endpoint_premium_delete(self, url):
s = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[429])
s.mount('http://', HTTPAdapter(max_retries=retries))
try:
r = s.delete(url, auth=(self.username, self.api_key))
except requests.exceptions.ConnectionError as e:
raise ConnectionError(repr(e))
s.close()
if r.status_code == 400:
raise BadRequest('Bad Request for url {}'.format(url))
if r.status_code == 200:
return True
if r.status_code == 404:
return None
0
Example 40
Project: django-salesforce Source File: auth.py
def authenticate(self):
"""
Authenticate to the Salesforce API with the provided credentials (password).
"""
settings_dict = self.settings_dict
url = ''.join([settings_dict['HOST'], '/services/oauth2/token'])
log.info("attempting authentication to %s" % settings_dict['HOST'])
self._session.mount(settings_dict['HOST'], HTTPAdapter(max_retries=get_max_retries()))
response = self._session.post(url, data=dict(
grant_type = 'password',
client_id = settings_dict['CONSUMER_KEY'],
client_secret = settings_dict['CONSUMER_SECRET'],
username = settings_dict['USER'],
password = settings_dict['PASSWORD'],
))
if response.status_code == 200:
# prefer str in Python 2 due to other API
response_data = {str(k): str(v) for k, v in response.json().items()}
# Verify signature (not important for this auth mechanism)
calc_signature = (base64.b64encode(hmac.new(
key=settings_dict['CONSUMER_SECRET'].encode('ascii'),
msg=(response_data['id'] + response_data['issued_at']).encode('ascii'),
digestmod=hashlib.sha256).digest())).decode('ascii')
if calc_signature == response_data['signature']:
log.info("successfully authenticated %s" % settings_dict['USER'])
else:
raise IntegrityError('Invalid auth signature received')
else:
raise LookupError("oauth failed: %s: %s" % (settings_dict['USER'], response.text))
return response_data
0
Example 41
Project: pytvmaze Source File: tvmaze.py
def _endpoint_premium_put(self, url, payload=None):
s = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[429])
s.mount('http://', HTTPAdapter(max_retries=retries))
try:
r = s.put(url, data=payload, auth=(self.username, self.api_key))
except requests.exceptions.ConnectionError as e:
raise ConnectionError(repr(e))
s.close()
if r.status_code == 400:
raise BadRequest('Bad Request for url {}'.format(url))
if r.status_code == 200:
return True
if r.status_code in [404, 422]:
return None
0
Example 42
Project: droopescan Source File: base_plugin_internal.py
def _general_init(self, opts, out=None):
"""
Initializes a variety of variables depending on user input.
@return: a boolean value indicating whether progressbars should be
hidden.
"""
self.session = Session()
if out:
self.out = out
else:
self.out = self._output(opts)
is_cms_plugin = self._meta.label != "scan"
if is_cms_plugin:
self.vf = VersionsFile(self.versions_file)
# http://stackoverflow.com/questions/23632794/in-requests-library-how-can-i-avoid-httpconnectionpool-is-full-discarding-con
try:
a = requests.adapters.HTTPAdapter(pool_maxsize=5000)
self.session.mount('http://', a)
self.session.mount('https://', a)
self.session.cookies.set_policy(BlockAll())
except AttributeError:
old_req = """Running a very old version of requests! Please `pip
install -U requests`."""
self.out.warn(old_req)
self.session.verify = False
self.session.headers['User-Agent'] = self.DEFAULT_UA
debug_requests = opts['debug_requests']
if debug_requests:
hide_progressbar = True
opts['threads_identify'] = 1
opts['threads_scan'] = 1
opts['threads_enumerate'] = 1
self.session = RequestsLogger(self.session)
else:
hide_progressbar = False
functionality = self._functionality(opts)
enabled_functionality = self._enabled_functionality(functionality, opts)
return (hide_progressbar, functionality, enabled_functionality)
0
Example 43
Project: Pyfa Source File: eve.py
def __init__(self, additional_headers=None, user_agent=None, cache_dir=None, cache=None):
# Set up a Requests Session
session = requests.Session()
if additional_headers is None:
additional_headers = {}
if user_agent is None:
user_agent = "pyfa/{0} ({1})".format(config.version, config.tag)
session.headers.update({
"User-Agent": user_agent,
"Accept": "application/json",
})
session.headers.update(additional_headers)
session.mount('https://public-crest.eveonline.com',
HTTPAdapter())
self._session = session
if cache:
if isinstance(cache, APICache):
self.cache = cache # Inherit from parents
elif isinstance(cache, type):
self.cache = cache() # Instantiate a new cache
elif cache_dir:
self.cache_dir = cache_dir
self.cache = FileCache(self.cache_dir)
else:
self.cache = DictCache()
0
Example 44
def test_serve(self):
cactus = self.find_cactus()
ret, _, _ = self.run_cli(["create", self.path])
self.assertEqual(0, ret)
port = 12345
p = subprocess.Popen([cactus, "serve", "-p", str(port)], cwd=self.path, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
srv = "http://127.0.0.1:{0}".format(port)
s = requests.Session()
s.mount(srv, HTTPAdapter(max_retries=Retry(backoff_factor=0.2)))
r = s.post("{0}/_cactus/shutdown".format(srv))
r.raise_for_status()
# We'd love to use p.wait(n) here, but that doesn't work on
# some of the versions of Python we support.
for _ in range(5):
if p.poll() != None:
break
time.sleep(1)
else:
self.fail("Server did not exit!")
self.assertEqual(0, p.returncode)
0
Example 45
Project: streamlink Source File: douyutv.py
def _get_streams(self):
match = _url_re.match(self.url)
channel = match.group("channel")
http.headers.update({'User-Agent': USER_AGENT})
http.verify=False
http.mount('https://', HTTPAdapter(max_retries=99))
#Thanks to @ximellon for providing method.
try:
channel = int(channel)
except ValueError:
channel = http.get(self.url, schema=_room_id_schema)
if channel == 0:
channel = http.get(self.url, schema=_room_id_alt_schema)
res = http.get(MAPI_URL.format(channel))
room = http.json(res, schema=_room_schema)
if not room:
return
if room["show_status"] != SHOW_STATUS_ONLINE:
return
ts = int(time.time() / 60)
did = uuid.uuid4().hex.upper()
sign = hashlib.md5(("{0}{1}{2}{3}".format(channel, did, LAPI_SECRET, ts)).encode("utf-8")).hexdigest()
data = {
"cdn": "ws",
"rate": "0",
"tt": ts,
"did": did,
"sign": sign
}
res = http.post(LAPI_URL.format(channel), data=data)
room = http.json(res, schema=_lapi_schema)
url = "{room[rtmp_url]}/{room[rtmp_live]}".format(room=room)
stream = HTTPStream(self.session, url)
yield "source", stream
data = {
"cdn": "ws",
"rate": "2",
"tt": ts,
"did": did,
"sign": sign
}
res = http.post(LAPI_URL.format(channel), data=data)
room = http.json(res, schema=_lapi_schema)
url = "{room[rtmp_url]}/{room[rtmp_live]}".format(room=room)
stream = HTTPStream(self.session, url)
yield "middle", stream
data = {
"cdn": "ws",
"rate": "1",
"tt": ts,
"did": did,
"sign": sign
}
res = http.post(LAPI_URL.format(channel), data=data)
room = http.json(res, schema=_lapi_schema)
url = "{room[rtmp_url]}/{room[rtmp_live]}".format(room=room)
stream = HTTPStream(self.session, url)
yield "low", stream
0
Example 46
Project: python-fedora Source File: openidbaseclient.py
def __init__(self, base_url, login_url=None, useragent=None, debug=False,
insecure=False, openid_insecure=False, username=None,
cache_session=True, retries=None, timeout=None,
retry_backoff_factor=0):
"""Client for interacting with web services relying on fas_openid auth.
:arg base_url: Base of every URL used to contact the server
:kwarg login_url: The url to the login endpoint of the application.
If none are specified, it uses the default `/login`.
:kwarg useragent: Useragent string to use. If not given, default to
"Fedora OpenIdBaseClient/VERSION"
:kwarg debug: If True, log debug information
:kwarg insecure: If True, do not check server certificates against
their CA's. This means that man-in-the-middle attacks are
possible against the `BaseClient`. You might turn this option on
for testing against a local version of a server with a self-signed
certificate but it should be off in production.
:kwarg openid_insecure: If True, do not check the openid server
certificates against their CA's. This means that man-in-the-
middle attacks are possible against the `BaseClient`. You might
turn this option on for testing against a local version of a
server with a self-signed certificate but it should be off in
production.
:kwarg username: Username for establishing authenticated connections
:kwarg cache_session: If set to true, cache the user's session data on
the filesystem between runs
:kwarg retries: if we get an unknown or possibly transient error from
the server, retry this many times. Setting this to a negative
number makes it try forever. Defaults to zero, no retries.
Note that this can only be set during object initialization.
:kwarg timeout: A float describing the timeout of the connection. The
timeout only affects the connection process itself, not the
downloading of the response body. Defaults to 120 seconds.
:kwarg retry_backoff_factor: Exponential backoff factor to apply in
between retry attempts. We will sleep for:
`{retry_backoff_factor}*(2 ^ ({number of failed retries} - 1))`
...seconds inbetween attempts. The backoff factor scales the rate
at which we back off. Defaults to 0 (backoff disabled).
Note that this attribute can only be set at object initialization.
"""
# These are also needed by OpenIdProxyClient
self.useragent = useragent or 'Fedora BaseClient/%(version)s' % {
'version': __version__}
self.base_url = base_url
self.login_url = login_url or urljoin(self.base_url, '/login')
self.debug = debug
self.insecure = insecure
self.openid_insecure = openid_insecure
self.retries = retries
self.timeout = timeout
# These are specific to OpenIdBaseClient
self.username = username
self.cache_session = cache_session
self.cache_lock = lockfile.FileLock(b_SESSION_FILE)
# Make sure the database for storing the session cookies exists
if cache_session:
self._initialize_session_cache()
# python-requests session. Holds onto cookies
self._session = requests.session()
# Also hold on to retry logic.
# http://www.coglib.com/~icordasc/blog/2014/12/retries-in-requests.html
server_errors = [500, 501, 502, 503, 504, 506, 507, 508, 509, 599]
method_whitelist = Retry.DEFAULT_METHOD_WHITELIST.union(set(['POST']))
if retries is not None:
prefixes = ['http://', 'https://']
for prefix in prefixes:
self._session.mount(prefix, requests.adapters.HTTPAdapter(
max_retries=Retry(
total=retries,
status_forcelist=server_errors,
backoff_factor=retry_backoff_factor,
method_whitelist=method_whitelist,
),
))
# See if we have any cookies kicking around from a previous run
self._load_cookies()
0
Example 47
Project: responses Source File: test_responses.py
def test_custom_adapter():
@responses.activate
def run():
url = "http://example.com"
responses.add(responses.GET, url, body=b'test')
calls = [0]
class DummyAdapter(requests.adapters.HTTPAdapter):
def send(self, *a, **k):
calls[0] += 1
return super(DummyAdapter, self).send(*a, **k)
# Test that the adapter is actually used
session = requests.Session()
session.mount("http://", DummyAdapter())
resp = session.get(url, allow_redirects=False)
assert calls[0] == 1
# Test that the response is still correctly emulated
session = requests.Session()
session.mount("http://", DummyAdapter())
resp = session.get(url)
assert_response(resp, 'test')
run()
0
Example 48
Project: azure-batch-apps-python Source File: rest_client.py
def _call(auth, *args, **kwargs):
"""Internal method to open Requests session."""
try:
conn_session = auth.get_session()
conn_adptr = requests.adapters.HTTPAdapter(max_retries=RETRIES)
conn_session.mount('https://', conn_adptr)
LOG.info("About to make REST call with args {0}".format(args))
LOG.debug("About to make REST call with kwargs {0}".format(kwargs))
LOG.debug(
"Opened requests session with max retries: {0}".format(RETRIES))
response = conn_session.request(*args, **kwargs)
return _check_code(response)
except (oauth2.rfc6749.errors.InvalidGrantError,
oauth2.rfc6749.errors.TokenExpiredError) as exp:
LOG.info("Token expired. Attempting to refresh and continue.")
refreshed_session = auth.refresh_session()
if not refreshed_session:
raise SessionExpiredException("Please log in again. "
"{0}".format(str(exp)))
try:
conn_adptr = requests.adapters.HTTPAdapter(max_retries=RETRIES)
refreshed_session.mount('https://', conn_adptr)
response = refreshed_session.request(*args, **kwargs)
return _check_code(response)
except Exception as exp:
raise RestCallException(
type(exp),
"An {type} error occurred: {error}".format(
type=type(exp), error=str(exp)),
exp)
except (requests.RequestException,
oauth2.rfc6749.errors.OAuth2Error) as exp:
raise RestCallException(
type(exp),
"An {type} error occurred: {error}".format(type=type(exp),
error=str(exp)),
exp)
0
Example 49
Project: tilequeue Source File: wof.py
def _make_requests_session_with_retries(max_retries):
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util import Retry
s = requests.Session()
a = HTTPAdapter(
max_retries=Retry(
total=max_retries,
status_forcelist=[ # this is a list of statuses to consider to be
# an error and retry.
429, # Too many requests (i.e: back off)
500, # Generic internal server error
502, # Bad Gateway - i.e: upstream failure
503, # Unavailable, temporarily
504, # Gateway timeout
522 # Origin connection timed out
],
backoff_factor=1.0 # back off for 0s, 1s, 3s, 7s, etc... after
# each successive failure. (factor*(2^N-1))
))
# use retry for both HTTP and HTTPS connections.
s.mount('http://', a)
s.mount('https://', a)
return s
0
Example 50
def __init__(self, api_key, api_secret, app_id, version='v1.0', timeout=6.0, base_url=None, location=None):
'''
Initialize the client with the given api key and secret
:param api_key: the api key
:param api_secret: the api secret
:param app_id: the app id
**Example usage**::
import stream
# initialize the client
client = stream.connect('key', 'secret')
# get a feed object
feed = client.feed('aggregated:1')
# write data to the feed
activity_data = {'actor': 1, 'verb': 'tweet', 'object': 1}
activity_id = feed.add_activity(activity_data)['id']
activities = feed.get()
feed.follow('flat:3')
activities = feed.get()
feed.unfollow('flat:3')
feed.remove_activity(activity_id)
'''
self.api_key = api_key
self.api_secret = api_secret
self.app_id = app_id
self.version = version
self.timeout = timeout
self.location = location
if os.environ.get('LOCAL'):
self.base_url = 'http://localhost:8000/api/'
self.timeout = 20
elif base_url is not None:
self.base_url = base_url
elif location is not None:
self.base_url = 'https://%s-api.getstream.io/api/' % location
self.base_analytics_url = 'https://analytics.getstream.io/analytics/'
self.session = requests.Session()
# TODO: turn this back on after we verify it doesnt retry on slower requests
self.session.mount(self.base_url, HTTPAdapter(max_retries=0))
self.auth = HTTPSignatureAuth(api_key, secret=api_secret)