Here are the examples of the python api urllib3.Request taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
9 Examples
3
Source : utils.py
with MIT License
from coolbay
with MIT License
from coolbay
def get_blocked_videos(api=API):
api_url = '{}?action=get_blocked'.format(api)
req = urllib3.Request(api_url)
response = urllib3.urlopen(req)
return json.loads(response.read())
def interpolated_prec_rec(prec, rec):
3
Source : hipchat_api.py
with MIT License
from ergon
with MIT License
from ergon
def _fetch_with_rate_limit(url):
authorized_url = _authorize_url(url)
request = urllib3.Request(authorized_url)
try:
return urllib3.urlopen(request)
except urllib3.URLError as e:
if e.code == 429:
_mark_token_as_exceeded()
return _fetch_with_rate_limit(url)
else:
raise e
def fetch_and_parse(url, access_tokens):
3
Source : mark_as_read.py
with MIT License
from ergon
with MIT License
from ergon
def _create_request(url, params='', is_post=False):
if is_post:
request = urllib3.Request(url=url, data=params)
else:
request = urllib3.Request(url + params)
request.add_header("Authorization", "Bearer %s" % (access_token))
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
return request
def fetch_and_parse(request):
0
Source : google_imgs.py
with GNU Affero General Public License v3.0
from KeinShin
with GNU Affero General Public License v3.0
from KeinShin
def single_image(self, image_url):
main_directory = "downloads"
extensions = (".jpg", ".gif", ".png", ".bmp", ".svg", ".webp", ".ico")
url = image_url
try:
os.makedirs(main_directory)
except OSError as e:
if e.errno != 17:
raise
req = Request(
url,
headers={
"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
},
)
response = urlopen(req, None, 10)
data = response.read()
response.close()
image_name = str(url[(url.rfind("/")) + 1 :])
if "?" in image_name:
image_name = image_name[: image_name.find("?")]
# if ".jpg" in image_name or ".gif" in image_name or ".png" in image_name or ".bmp" in image_name or ".svg" in image_name or ".webp" in image_name or ".ico" in image_name:
if any(map(lambda extension: extension in image_name, extensions)):
file_name = main_directory + "/" + image_name
else:
file_name = main_directory + "/" + image_name + ".jpg"
image_name = image_name + ".jpg"
try:
output_file = open(file_name, "wb")
output_file.write(data)
output_file.close()
except IOError as e:
raise e
except OSError as e:
raise e
print(
"completed ====> " + image_name.encode("raw_unicode_escape").decode("utf-8")
)
def similar_images(self, similar_images):
0
Source : google_imgs.py
with GNU Affero General Public License v3.0
from KeinShin
with GNU Affero General Public License v3.0
from KeinShin
def download_image_thumbnail(
self,
image_url,
main_directory,
dir_name,
return_image_name,
print_urls,
socket_timeout,
print_size,
no_download,
save_source,
img_src,
ignore_urls,
):
if print_urls or no_download:
print("Image URL: " + image_url)
if no_download:
return "success", "Printed url without downloading"
try:
req = Request(
image_url,
headers={
"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
},
)
try:
# timeout time to download an image
if socket_timeout:
timeout = float(socket_timeout)
else:
timeout = 10
response = urlopen(req, None, timeout)
data = response.read()
response.close()
path = (
main_directory
+ "/"
+ dir_name
+ " - thumbnail"
+ "/"
+ return_image_name
)
try:
output_file = open(path, "wb")
output_file.write(data)
output_file.close()
if save_source:
list_path = main_directory + "/" + save_source + ".txt"
list_file = open(list_path, "a")
list_file.write(path + "\t" + img_src + "\n")
list_file.close()
except OSError as e:
download_status = "fail"
download_message = (
"OSError on an image...trying next one..." + " Error: " + str(e)
)
except IOError as e:
download_status = "fail"
download_message = (
"IOError on an image...trying next one..." + " Error: " + str(e)
)
download_status = "success"
download_message = (
"Completed Image Thumbnail ====> " + return_image_name
)
# image size parameter
if print_size:
print("Image Size: " + str(self.file_size(path)))
except UnicodeEncodeError as e:
download_status = "fail"
download_message = (
"UnicodeEncodeError on an image...trying next one..."
+ " Error: "
+ str(e)
)
except HTTPError as e: # If there is any HTTPError
download_status = "fail"
download_message = (
"HTTPError on an image...trying next one..." + " Error: " + str(e)
)
except URLError as e:
download_status = "fail"
download_message = (
"URLError on an image...trying next one..." + " Error: " + str(e)
)
except ssl.CertificateError as e:
download_status = "fail"
download_message = (
"CertificateError on an image...trying next one..."
+ " Error: "
+ str(e)
)
except IOError as e: # If there is any IOError
download_status = "fail"
download_message = (
"IOError on an image...trying next one..." + " Error: " + str(e)
)
return download_status, download_message
# Download Images
def download_image(
0
Source : google_imgs.py
with GNU Affero General Public License v3.0
from KeinShin
with GNU Affero General Public License v3.0
from KeinShin
def download_image(
self,
image_url,
image_format,
main_directory,
dir_name,
count,
print_urls,
socket_timeout,
prefix,
print_size,
no_numbering,
no_download,
save_source,
img_src,
silent_mode,
thumbnail_only,
format,
ignore_urls,
):
if not silent_mode:
if print_urls or no_download:
print("Image URL: " + image_url)
if ignore_urls:
if any(url in image_url for url in ignore_urls.split(",")):
return (
"fail",
"Image ignored due to 'ignore url' parameter",
None,
image_url,
)
if thumbnail_only:
return (
"success",
"Skipping image download...",
str(image_url[(image_url.rfind("/")) + 1 :]),
image_url,
)
if no_download:
return "success", "Printed url without downloading", None, image_url
try:
req = Request(
image_url,
headers={
"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
},
)
try:
# timeout time to download an image
if socket_timeout:
timeout = float(socket_timeout)
else:
timeout = 10
response = urlopen(req, None, timeout)
data = response.read()
response.close()
extensions = [
".jpg",
".jpeg",
".gif",
".png",
".bmp",
".svg",
".webp",
".ico",
]
# keep everything after the last '/'
image_name = str(image_url[(image_url.rfind("/")) + 1 :])
if format:
if not image_format or image_format != format:
download_status = "fail"
download_message = "Wrong image format returned. Skipping..."
return_image_name = ""
absolute_path = ""
return (
download_status,
download_message,
return_image_name,
absolute_path,
)
if (
image_format == ""
or not image_format
or "." + image_format not in extensions
):
download_status = "fail"
download_message = "Invalid or missing image format. Skipping..."
return_image_name = ""
absolute_path = ""
return (
download_status,
download_message,
return_image_name,
absolute_path,
)
if image_name.lower().find("." + image_format) < 0:
image_name = image_name + "." + image_format
else:
image_name = image_name[
: image_name.lower().find("." + image_format)
+ (len(image_format) + 1)
]
# prefix name in image
if prefix:
prefix = prefix + " "
else:
prefix = ""
if no_numbering:
path = main_directory + "/" + dir_name + "/" + prefix + image_name
else:
path = (
main_directory
+ "/"
+ dir_name
+ "/"
+ prefix
+ str(count)
+ "."
+ image_name
)
try:
output_file = open(path, "wb")
output_file.write(data)
output_file.close()
if save_source:
list_path = main_directory + "/" + save_source + ".txt"
list_file = open(list_path, "a")
list_file.write(path + "\t" + img_src + "\n")
list_file.close()
absolute_path = os.path.abspath(path)
except OSError as e:
download_status = "fail"
download_message = (
"OSError on an image...trying next one..." + " Error: " + str(e)
)
return_image_name = ""
absolute_path = ""
# return image name back to calling method to use it for thumbnail downloads
download_status = "success"
download_message = (
"Completed Image ====> " + prefix + str(count) + "." + image_name
)
return_image_name = prefix + str(count) + "." + image_name
# image size parameter
if not silent_mode:
if print_size:
print("Image Size: " + str(self.file_size(path)))
except UnicodeEncodeError as e:
download_status = "fail"
download_message = (
"UnicodeEncodeError on an image...trying next one..."
+ " Error: "
+ str(e)
)
return_image_name = ""
absolute_path = ""
except URLError as e:
download_status = "fail"
download_message = (
"URLError on an image...trying next one..." + " Error: " + str(e)
)
return_image_name = ""
absolute_path = ""
except BadStatusLine as e:
download_status = "fail"
download_message = (
"BadStatusLine on an image...trying next one..."
+ " Error: "
+ str(e)
)
return_image_name = ""
absolute_path = ""
except HTTPError as e: # If there is any HTTPError
download_status = "fail"
download_message = (
"HTTPError on an image...trying next one..." + " Error: " + str(e)
)
return_image_name = ""
absolute_path = ""
except URLError as e:
download_status = "fail"
download_message = (
"URLError on an image...trying next one..." + " Error: " + str(e)
)
return_image_name = ""
absolute_path = ""
except ssl.CertificateError as e:
download_status = "fail"
download_message = (
"CertificateError on an image...trying next one..."
+ " Error: "
+ str(e)
)
return_image_name = ""
absolute_path = ""
except IOError as e: # If there is any IOError
download_status = "fail"
download_message = (
"IOError on an image...trying next one..." + " Error: " + str(e)
)
return_image_name = ""
absolute_path = ""
except IncompleteRead as e:
download_status = "fail"
download_message = (
"IncompleteReadError on an image...trying next one..."
+ " Error: "
+ str(e)
)
return_image_name = ""
absolute_path = ""
return download_status, download_message, return_image_name, absolute_path
# Finding 'Next Image' from the given raw page
def _get_next_item(self, s):
0
Source : DyStockDataGateway.py
with MIT License
from MicroEngine
with MIT License
from MicroEngine
def _getTickDataFromTencent(code=None, date=None, retry_count=3, pause=0.001):
"""
从腾讯获取分笔数据
接口和返回的DF,保持跟tushare一致
Parameters
------
code:string
股票代码 e.g. 600848
date:string
日期 format:YYYY-MM-DD
retry_count : int, 默认 3
如遇网络等问题重复执行的次数
pause : int, 默认 0
重复请求数据过程中暂停的秒数,防止请求间隔时间太短出现的问题
return
-------
DataFrame 当日所有股票交易数据(DataFrame)
属性:成交时间、成交价格、价格变动,成交手、成交金额(元),买卖类型
"""
if code is None or len(code)!=6 or date is None:
return None
symbol = DyStockDataTicksGateway._codeToTencentSymbol(code)
yyyy, mm, dd = date.split('-')
for _ in range(retry_count):
sleep(pause)
try:
re = Request('http://stock.gtimg.cn/data/index.php?appn=detail&action=download&c={0}&d={1}'.format(symbol, yyyy+mm+dd))
lines = urlopen(re, timeout=10).read()
lines = lines.decode('GBK')
df = pd.read_table(StringIO(lines), names=['time', 'price', 'change', 'volume', 'amount', 'type'],
skiprows=[0])
except Exception as e:
print(e)
ex = e
else:
return df
raise ex
def _codeToSinaSymbol(code):
0
Source : DyStockDataGateway.py
with MIT License
from MicroEngine
with MIT License
from MicroEngine
def _getTickDataFromSina(code=None, date=None, retry_count=3, pause=0.001):
"""
获取分笔数据
Parameters
------
code:string
股票代码 e.g. 600848
date:string
日期 format:YYYY-MM-DD
retry_count : int, 默认 3
如遇网络等问题重复执行的次数
pause : int, 默认 0
重复请求数据过程中暂停的秒数,防止请求间隔时间太短出现的问题
return
-------
DataFrame 当日所有股票交易数据(DataFrame)
属性:成交时间、成交价格、价格变动,成交手、成交金额(元),买卖类型
"""
if code is None or len(code)!=6 or date is None:
return None
symbol = DyStockDataTicksGateway._codeToSinaSymbol(code)
for _ in range(retry_count):
sleep(pause)
try:
re = Request('http://market.finance.sina.com.cn/downxls.php?date={}&symbol={}'.format(date, symbol))
lines = urlopen(re, timeout=10).read()
lines = lines.decode('GBK')
if len(lines) < 20:
return None
df = pd.read_table(StringIO(lines), names=['time', 'price', 'change', 'volume', 'amount', 'type'],
skiprows=[0])
except Exception as e:
print(e)
ex = e
else:
return df
raise ex
def _getTicks(self, code, date):
0
Source : DyStockDataGateway.py
with MIT License
from MicroEngine
with MIT License
from MicroEngine
def _getDaysFrom163(self, code, startDate, endDate, retry_count=3, pause=0.001):
"""
从网易获取个股日线数据,指数和基金(ETF)除外
@code: DevilYuan Code
"""
symbol = ('0' + code[:6]) if code[-2:] == 'SH' else ('1' + code[:6])
for _ in range(retry_count):
sleep(pause)
try:
url = 'http://quotes.money.163.com/service/chddata.html?code={}&start={}&end={}&fields=TCLOSE;HIGH;LOW;TOPEN;TURNOVER;VOTURNOVER;VATURNOVER'
url = url.format(symbol, startDate.replace('-', ''), endDate.replace('-', ''))
re = Request(url)
lines = urlopen(re, timeout=10).read()
lines = lines.decode('GBK')
df = pd.read_table(StringIO(lines),
sep=',',
names=['date', 'code', 'name', 'close', 'high', 'low', 'open', 'turnover', 'volume', 'amount'],
skiprows=[0])
except Exception as e:
print(e)
ex = e
else:
df = df[['date', 'open', 'high', 'close', 'low', 'volume', 'amount', 'turnover']] # return columns
df = df.set_index('date')
df = df.sort_index(ascending=False)
return df
raise ex
def _getCodeDaysFromTuShare(self, code, startDate, endDate, fields, name=None):