Here are the examples of the python api requests.get.text taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
173 Examples
3
Example 1
def _get(self, suffix):
"""Actually make the GET request. Assume the result is JSON. Right
now, there is no error handling"""
if self.base_url.startswith('http'): # API
json_str = requests.get(self.base_url + suffix).text
else: # file system
if os.path.isdir(self.base_url + suffix):
suffix = suffix + "/index.html"
f = open(self.base_url + suffix)
json_str = f.read()
f.close()
return json.loads(json_str, object_hook=node_decode_hook)
3
Example 2
Project: bakerstreet Source File: test_bakerstreet.py
def test_resolve_service_with_path(wc_host):
"""Test that Baker Street will route traffic from the name to the proper path while also preserving URL query
parameters"""
load_watson_with_config(wc_host, "watson-test_service-path.conf")
time.sleep(5)
assert requests.get("http://localhost:8000/foo", params=dict(name="Homer")).text == "Hi, Homer!"
3
Example 3
def pre_login():
pre_login_url = 'http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su=MTUyNTUxMjY3OTY%3D&rsakt=mod&checkpin=1&client=ssologin.js%28v1.4.18%29&_=1458836718537'
pre_response = requests.get(pre_login_url).text
pre_content_regex = r'\((.*?)\)'
patten = re.search(pre_content_regex, pre_response)
nonce = None
pubkey = None
servertime = None
rsakv = None
if patten.groups():
pre_content = patten.group(1)
pre_result = json.loads(pre_content)
nonce = pre_result.get("nonce")
pubkey = pre_result.get('pubkey')
servertime = pre_result.get('servertime')
rsakv = pre_result.get("rsakv")
return nonce, pubkey, servertime, rsakv
3
Example 4
Project: spider-practice Source File: weibo.py
def get_sth(su):
# 改字典内的数据经过精简,只有在这些数据存在下才不影响获得所需的准确数据
payload = {'entry': 'weibo', 'rsakt': 'mod', 'su': su, 'checkpin': '1' }
res = requests.get('http://login.sina.com.cn/sso/prelogin.php',
params=payload).text
res = eval(res)
# print(res)
return res
3
Example 5
Project: pokemon_ai Source File: log_scraper.py
def get_replay_ids(username, page, tier='ou'):
final_links = []
url = USERNAME_URL.format(
user=username,
page=page
)
html = requests.get(url).text
soup = BeautifulSoup(html)
links = soup.find_all('a')
for link in links:
if tier in link.get("href"):
final_links.append(link.get("href").encode("utf-8")[1:])
return final_links
3
Example 6
def get_html(self, url):
try:
proxies = {
#'http': '', # ok
'http': None,
}
# proxies={} or proxies=None not work
return requests.get(url, timeout=self.time_out, proxies=proxies).text
return urllib2.urlopen(url, timeout=self.time_out).read()
except Exception as e:
print type(e), e
return ''
3
Example 7
Project: BitMeshPOC Source File: bitmesh.py
def seller_handle_domain_request():
# get the requested domain from the buyer
domain = socket.recv()
# TODO: validate the domain is a good one
html = requests.get(domain).text
print html
socket.send_string(html)
3
Example 8
def test_memory():
data = requests.get('http://localhost:5555/api/registers').text
res = api_response('registers', data=data)
url = 'http://localhost:5555/api/memory?address={}&length=64'.format(res.registers['rip'])
data = requests.get(url).text
res = api_response('memory', data=data)
assert res.is_success
assert res.memory == memory_response
3
Example 9
Project: depl Source File: test_django.py
def django_pg_test(tmpdir):
django_basic_test(tmpdir)
# django plays with the db
content = requests.get("http://localhost:8887/db_show.html").text
assert content == 'django.db.backends.postgresql_psycopg2: 1\n'
delete_pg_connection()
3
Example 10
Project: kiss.py Source File: auth.py
def get_user_info(self, request, options, access_token_result):
self.access_token = access_token_result["access_token"]
params = self.prepare_user_info_request_params(access_token_result)
user_info_response = json.loads(requests.get("%s?%s" % (options["target_uri"], url_encode(params)), auth=self.auth).text)
user_info_response = self.process_user_info_response(request, user_info_response)
user_info_response["provider"] = request.params["backend"]
user_info_response["access_token"] = params["access_token"]
return user_info_response
3
Example 11
Project: compactor Source File: test_httpd.py
def test_simple_routed_process(self):
ping = PingPongProcess()
pid = self.context.spawn(ping)
url = 'http://%s:%s/pingpong/ping' % (pid.ip, pid.port)
content = requests.get(url).text
assert content == 'pong'
assert ping.ping_event.is_set()
3
Example 12
def summarize_page(url):
import bs4
import requests
html = bs4.BeautifulSoup(requests.get(url).text)
b = find_likely_body(html)
summaries = summarize_blocks(map(lambda p: p.text, b.find_all('p')))
return Summary(url, b, html.title.text if html.title else None, summaries)
3
Example 13
Project: xbmc-addon-nrk Source File: subs.py
def get_subtitles(video_id):
html = requests.get("http://v8.psapi.nrk.no/programs/%s/subtitles/tt" % video_id).text
if not html:
return None
content = _ttml_to_srt(html)
filename = os.path.join(xbmc.translatePath("special://temp"), 'nor.srt')
with open(filename, 'w') as f:
f.write(content)
return filename
3
Example 14
Project: pypackage Source File: test_classified.py
def test_classifiers_current():
"""Ensure the shipped classifiers file is synced with pypi."""
packaged_trove_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"pypackage",
"classifiers"
)
with open(packaged_trove_file) as opent:
packaged = [t.strip() for t in opent.read().splitlines() if t]
url = "https://pypi.python.org/pypi?:action=list_classifiers"
trove_req = requests.get(url).text
current = [t.strip() for t in trove_req.splitlines() if t]
assert packaged == current, "curl {} > pypackage/classifiers".format(url)
3
Example 15
Project: eve-mlp Source File: common.py
def _webload(parent, url, eid=-1):
try:
data = requests.get(url).text
except:
data = "Couldn't get %s:\n%s" % (url, str(e))
wx.PostEvent(parent, WebLoadEvent(myEVT_WEBLOAD, eid, data))
3
Example 16
Project: glottolog3 Source File: iso.py
def changerequests(args):
html = bs(requests.get(HTML).text)
res = defaultdict(list)
for i, tr in enumerate(html.find('table', **{'class': 'stripeMe'}).find_all('tr')):
if i == 0:
cols = [text(n) for n in tr.find_all('th')]
else:
r = dict(zip(cols, [text(n) for n in tr.find_all('td')]))
res[r['CR Number']].append(r)
return res
3
Example 17
Project: cslbot Source File: wikipath.py
def gen_path(cmdargs):
epoch = datetime.now().timestamp()
params = {'a1': cmdargs.first, 'linktype': 1, 'a2': cmdargs.second, 'allowsideboxes': 1, 'submit': epoch}
html = get('http://beta.degreesofwikipedia.com/', params=params).text
path = fromstring(html).find('pre')
if path is None:
return False
output = []
for x in path.text.splitlines():
if '=>' in x:
output.append(x.split('=>')[1].strip())
return " -> ".join(output)
3
Example 18
def __init__(self):
cachedHierarchy = cachedb.gc_type_hierarchy.find_one()
if not cachedHierarchy:
rawHierarchy = requests.get("https://connect.garmin.com/proxy/activity-service-1.2/json/activity_types", headers=self._obligatory_headers).text
self._activityHierarchy = json.loads(rawHierarchy)["dictionary"]
cachedb.gc_type_hierarchy.insert({"Hierarchy": rawHierarchy})
else:
self._activityHierarchy = json.loads(cachedHierarchy["Hierarchy"])["dictionary"]
rate_lock_path = tempfile.gettempdir() + "/gc_rate.%s.lock" % HTTP_SOURCE_ADDR
# Ensure the rate lock file exists (...the easy way)
open(rate_lock_path, "a").close()
self._rate_lock = open(rate_lock_path, "r+")
3
Example 19
Project: software-factory Source File: test_gateway.py
def test_static_files_are_not_cached(self):
"""Make sure files in the 'static' dir are not cached"""
script = "topmenu.js"
url = "https://%s/static/js/%s" % (config.GATEWAY_HOST, script)
js = requests.get(url).text
# add a comment at the end of the js
cmd = "echo '// this is a useless comment' >> /var/www/static/js/%s"
ssh_run_cmd(os.path.expanduser("~/.ssh/id_rsa"),
"root",
config.GATEWAY_HOST, shlex.split(cmd % script))
newjs = requests.get(url).text
self.assertTrue(len(newjs) > len(js),
"New js is %s" % newjs)
self.assertTrue("useless comment" in newjs,
"New js has no useless comment !")
3
Example 20
Project: depl Source File: test_django.py
def django_basic_test(tmpdir):
copy_to_temp(tmpdir)
main_run(['depl', 'deploy', 'localhost'])
assert requests.get("http://localhost:8887/").text == "django rocks\n"
txt = requests.get("http://localhost:8887/static/something.txt").text
assert txt == "static files\n"
# django plays with the db
assert requests.get("http://localhost:8887/db_add.html").text == "saved\n"
3
Example 21
def get_ec2_instance_id():
"""Retrieve the instance id for the currently running EC2 instance. If
the host machine is not an EC2 instance or is for some reason unable
to make requests, return None.
:returns: (str) id of the current EC2 instance
(None) if the id could not be found"""
instance_id_url = "http://169.254.169.254/latest/meta-data/instance-id"
try:
return requests.get(instance_id_url).text
except requests.ConnectionError as err:
print err.message
3
Example 22
def get_ec2_instance_id():
"""Retrieve the instance id for the currently running EC2 instance. If
the host machine is not an EC2 instance or is for some reason unable
to make requests, return None.
:returns: (str) id of the current EC2 instance
(None) if the id could not be found"""
instance_id_url = "http://169.254.169.254/latest/meta-data/instance-id"
try:
return requests.get(instance_id_url).text
except requests.ConnectionError:
print "Could not find EC2 instance id..."
return None
3
Example 23
def time_delta(self):
"""Get the delta between this computer and RunAbove cluster."""
if self._time_delta is None:
try:
server_time = int(requests.get(self.base_url + "/time").text)
except ValueError:
raise APIError(msg='Impossible to get time from RunAbove')
self._time_delta = server_time - int(time.time())
return self._time_delta
3
Example 24
def get_logs(replay_id):
html = requests.get(REPLAY_URL.format(
replay_id=replay_id)
).text
soup = BeautifulSoup(html)
script = soup.find_all('script', {'class': 'log'})[0]
log = script.text
return log
3
Example 25
def page_rank(self):
"""
# Desc : Get PageRank of a Website, for Python
# Author : iSayme
# E-Mail : [email protected]
# Website : http://www.isayme.org
# Date : 2012-08-04
https://github.com/isayme/google_pr/blob/master/GooglePr.py
"""
gpr_hash_seed = "Mining PageRank is AGAINST GOOGLE'S TERMS OF SERVICE. Yes, I'm talking to you, scammer."
magic = 0x1020345
for i in iter(range(len(self.site))):
magic ^= ord(gpr_hash_seed[i % len(gpr_hash_seed)]) ^ ord(self.site[i])
magic = (magic >> 23 | magic << 9) & 0xFFFFFFFF
url = "http://toolbarqueries.google.com/tbr?client=navclient-auto&features=Rank&ch=%s&q=info:%s" % \
("8%08x" % (magic), self.site)
data = requests.get(url, headers=self.headers).text
data = data.split(":") if data else None
return int(data[len(data) - 1]) if data else 0
3
Example 26
def summarize_page(url):
import bs4
import requests
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
}
html = bs4.BeautifulSoup(requests.get(url, headers=headers).text)
b = find_likely_body(html)
summaries = summarize_blocks(map(lambda p: p.text, b.find_all('p')))
return Summary(url, b, html.title.text if html.title else None, summaries)
3
Example 27
def calc(eq):
query = quote(eq)
url = "https://encrypted.google.com/search?hl=en&q={0}".format(query)
soup = BeautifulSoup(requests.get(url).text, "html5lib")
answer = soup.findAll("h2", attrs={"class": "r"})
if not answer:
answer = soup.findAll("span", attrs={"class": "_m3b"})
if not answer:
return ":crying_cat_face: sorry, google doesn't have an answer for this equation :("
answer = answer[0].text.replace(u"\xa0", ",")
return answer
3
Example 28
Project: pyp2p Source File: dht_msg.py
def mutex_loop(self):
def do(args):
# Requests a mutex from the server.
call = dht_msg_endpoint + "?call=get_mutex&"
call += urlencode({"node_id": self.node_id}) + "&"
call += urlencode({"password": self.password})
# Make API call.
ret = requests.get(call, timeout=5).text
if "1" in ret or "2" in ret:
self.has_mutex = int(ret)
self.is_mutex_ready.set()
return 0
self.retry_in_thread(do, check_interval=MUTEX_TIMEOUT)
3
Example 29
Project: compactor Source File: test_httpd.py
def test_async_route(self):
web = Web('web')
pid = self.context.spawn(web)
url = 'http://%s:%s/web/ping' % (pid.ip, pid.port)
content = requests.get(url).text
assert content == 'pong'
3
Example 30
def make_request(url, ids, format, style, locale, **kwargs):
type = cn_format_headers[format]
htype = {'Accept': type}
head = dict(make_ua(), **htype)
if format == "citeproc-json":
url = "http://api.crossref.org/works/" + ids + "/" + type
return requests.get(url, headers = head, allow_redirects = True, **kwargs).text
else:
if format == "text":
type = type + "; style = " + style + "; locale = " + locale
url = url + "/" + ids
return requests.get(url, headers = head, allow_redirects = True, **kwargs).text
3
Example 31
Project: Daily_scripts Source File: CloudMusicMV.py
def mv_url(self):
html = requests.get(
'http://music.163.com/mv?id={0}'.format(self.mv_id)
).text
self.url = re.findall('murl=(.+\.mp4)', html)[0]
self.mv_name = re.findall(
'flag_title1\">(.+)</h', html)[0].replace(' ', '-')
return self.url
3
Example 32
def main(argv=None):
"""Download one lyrics sample page per referenced source.
"""
if argv is None:
argv = sys.argv
print(u'Fetching samples from:')
for s in test_lyrics.GOOGLE_SOURCES + test_lyrics.DEFAULT_SOURCES:
print(s['url'])
url = s['url'] + s['path']
fn = test_lyrics.url_to_filename(url)
if not os.path.isfile(fn):
html = requests.get(url, verify=False).text
with safe_open_w(fn) as f:
f.write(html.encode('utf-8'))
3
Example 33
def get_request(url):
for _ in range(5):
try:
return float(requests.get(url).text)
except:
pass
return None
3
Example 34
Project: arkc-client Source File: common.py
def get_ip_str():
logging.info("Getting public IP address")
try:
ip = get('https://api.ipify.org').text
logging.info("IP address to be sent is " + ip)
return ip
except Exception as err:
print(
"Error occurred in getting address. Using default 127.0.0.1 in testing environment.")
print(err)
return "127.0.0.1"
3
Example 35
Project: SmokeDetector Source File: deletionwatcher.py
@classmethod
def update_site_id_list(self):
soup = BeautifulSoup(requests.get("http://meta.stackexchange.com/topbar/site-switcher/site-list").text)
site_id_dict = {}
for site in soup.findAll("a", attrs={"data-id": True}):
site_name = site["href"][2:]
site_id = site["data-id"]
site_id_dict[site_name] = site_id
GlobalVars.site_id_dict = site_id_dict
3
Example 36
@Command('isup', ['nick'])
def cmd(send, msg, args):
"""Checks if a website is up.
Syntax: {command} <website>
"""
if not msg:
send("What are you trying to get to?")
return
nick = args['nick']
isup = get("http://isup.me/%s" % msg).text
if "looks down from here" in isup:
send("%s: %s is down" % (nick, msg))
elif "like a site on the interwho" in isup:
send("%s: %s is not a valid url" % (nick, msg))
else:
send("%s: %s is up" % (nick, msg))
3
Example 37
Project: CloudBot Source File: linux.py
@hook.command(autohelp=False)
def kernel(reply):
"""- gets a list of linux kernel versions"""
contents = requests.get("https://www.kernel.org/finger_banner").text
contents = re.sub(r'The latest(\s*)', '', contents)
contents = re.sub(r'version of the Linux kernel is:(\s*)', '- ', contents)
lines = contents.split("\n")
message = "Linux kernel versions: {}".format(", ".join(line for line in lines[:-1]))
reply(message)
3
Example 38
Project: SickGear Source File: pushbullet.py
def get_devices(self, accessToken=None):
# fill in omitted parameters
if not accessToken:
accessToken = sickbeard.PUSHBULLET_ACCESS_TOKEN
# get devices from pushbullet
try:
base64string = base64.encodestring('%s:%s' % (accessToken, ''))[:-1]
headers = {'Authorization': 'Basic %s' % base64string}
return requests.get(DEVICEAPI_ENDPOINT, headers=headers).text
except Exception as e:
return json.dumps({'error': {'message': 'Error failed to connect: %s' % e}})
3
Example 39
@cached(timeout=3600 * 24 * 3)
def requests_get(url):
""" Run :func:`requests.get` in a ``cached()`` wrapper.
The cache wrapper uses the default timeout (environment variable
``PYTHON_FTR_CACHE_TIMEOUT``, 3 days by default).
It is used in :func:`ftr_process`.
"""
LOGGER.info(u'Fetching & caching %s…', url)
return requests.get(url).text
3
Example 40
Project: pycaching Source File: util.py
def get_possible_attributes():
"""Return a dict of all possible attributes parsed from Groundspeak's website."""
# imports are here not to slow down other parts of program which normally doesn't use this method
from itertools import chain
import requests
from bs4 import BeautifulSoup
try:
page = BeautifulSoup(requests.get(_attributes_url).text, "html.parser")
except requests.exceptions.ConnectionError as e:
raise errors.Error("Cannot load attributes page.") from e
# get all <img>s containing attributes from all <dl>s with specific class
images = chain(*map(lambda i: i.find_all("img"), page.find_all("dl", "AttributesList")))
# create dict as {"machine name": "human description"}
attributes = {i.get("src").split("/")[-1].rsplit("-", 1)[0]: i.get("alt") for i in images}
return attributes
3
Example 41
def get_ip_info(self, ip):
try:
response = json.loads(requests.get(self._url % ip).text)
for k in response.keys():
response[k] = unicode(response[k])
return response
except Exception, e:
return 'Error: Cannot retrieve %s info. Exception: %s' % (ip, e.message)
3
Example 42
Project: penn-sdk-python Source File: studyspaces.py
def get_id_dict(self):
"""Extracts the ID's of the room into a dictionary. Used as a
helper for the extract_times method.
"""
group_study_codes = {}
url = BASE_URL + "/booking/vpdlc"
soup = BeautifulSoup(requests.get(url).text, 'html5lib')
options = soup.find_all('option')
for element in options:
if element['value'] != '0':
url2 = BASE_URL + str(element['value'])
soup2 = BeautifulSoup(requests.get(url2).text, 'html5lib')
id = soup2.find('input', attrs={"id": "gid"})['value']
group_study_codes[int(id)] = str(element.contents[0])
return group_study_codes
3
Example 43
Project: bakerstreet Source File: test_bakerstreet.py
def test_resolve_service_without_path(wc_host):
"""Test that Baker Street will route traffic properly"""
load_watson_with_config(wc_host, "watson-test_service-nopath.conf")
time.sleep(5)
assert requests.get("http://localhost:8000/bar").text == "Hi, everybody!"
3
Example 44
Project: twitch-chat-logger Source File: utils.py
def get_top_streams(n):
twitch_api_url = 'https://api.twitch.tv/kraken/streams/?limit=%i' % n
try:
return json.loads(requests.get(twitch_api_url).text)['streams']
except (ValueError, ConnectionError, SSLError):
time.sleep(5)
return get_top_streams(n)
3
Example 45
def test_version():
data = requests.get('http://localhost:5555/api/version').text
res = api_response('version', data=data)
assert res.is_success
assert res.api_version == 1.1
assert res.host_version == 'lldb-something'
3
Example 46
Project: penn-sdk-python Source File: studyspaces.py
def get_id_json(self):
"""Makes JSON with each element associating URL, ID, and building
name.
"""
group_study_codes = []
url = BASE_URL + "/booking/vpdlc"
soup = BeautifulSoup(requests.get(url).text, 'html5lib')
l = soup.find_all('option')
for element in l:
if element['value'] != '0':
url2 = BASE_URL + str(element['value'])
soup2 = BeautifulSoup(requests.get(url2).text, 'html5lib')
id = soup2.find('input', attrs={"id": "gid"})['value']
new_dict = {}
new_dict['id'] = int(id)
new_dict['name'] = str(element.contents[0])
new_dict['url'] = url2
group_study_codes.append(new_dict)
return group_study_codes
3
Example 47
def get_ip(debug_ip=None): # TODO: Get local network interfaces ip
logging.info("Getting public IP address")
if debug_ip:
ip = debug_ip
else:
try:
os.environ['NO_PROXY'] = 'api.ipify.org'
ip = get('https://api.ipify.org').text
except Exception as err:
logging.error(err)
logging.warning("Error getting address. Using 127.0.0.1 instead.")
ip = "127.0.0.1"
logging.info("IP address to be sent is " + ip)
return struct.unpack("!L", socket.inet_aton(ip))[0]
3
Example 48
def __init__(self):
"""
Creates a new instance of the Sirius player
At construction, we only get the global config and the channel lineup
"""
self.backend = default_backend()
self.token_cache = {}
player_page = requests.get(self.BASE_URL).text
config_url = re.search("flashvars.configURL = '(.+?)'", player_page).group(1)
self.config = ET.fromstring(requests.get(config_url).text)
lineup_url = self.config.findall("./consumerConfig/config[@name='ChannelLineUpBaseUrl']")[0].attrib['value']
lineup = json.loads(requests.get(lineup_url + '/en-us/json/lineup/200/client/ump').text)
self._parse_lineup(lineup)
3
Example 49
Project: OmgSite Source File: site_auditor.py
def safebrowsing(self):
# This site is not currently listed as suspicious.
# this site has not hosted malicious software over the past 90 days
g = requests.get('http://safebrowsing.clients.google.com/safebrowsing/diagnostic?site=%s' % self.site,
headers=self.headers).text
if 'is not currently listed':
message = 'NO - В настоящее время этот сайт не занесен в список подозрительных.'
else:
message = 'YES - В настоящее время этот сайт занесен в список подозрительных.' # may be, need test site
if 'has not hosted malicious' in g:
message = '%s NO - За последние 90 дней на этом сайте не размещалось вредоносное ПО.' % message
else:
message = '%s YES - За последние 90 дней на этом сайте размещалось вредоносное ПО.' % message
yad = requests.get('http://yandex.ru/infected?l10n=ru&url=%s' % self.site,
headers=self.headers).text.split('<title>')[1].split('</title>')[0].strip()
site_advisor = requests.get('http://www.siteadvisor.com/sites/%s' % self.site,
headers=self.headers).text.split('class="intro">')[1].split('.</p>')[0]
return {'google': message, 'yandex': yad, 's_a': site_advisor}
3
Example 50
Project: charmander Source File: google.py
def google(query):
query = quote(query)
url = "https://encrypted.google.com/search?q={0}".format(query)
soup = BeautifulSoup(requests.get(url).text, "html5lib")
answer = soup.findAll("h3", attrs={"class": "r"})
if not answer:
return ":fire: Sorry, Google doesn't have an answer for your Query :fire:"
return unquote(re.findall(r"q=(.*?)&", str(answer[0]))[0])