Here are the examples of the python api urllib.request.urlopen taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
173 Examples
5
Example 1
Project: pyowm Source File: test_weather_client.py
def test_call_API(self):
# Setup monkey patching
if 'urllib2' in context: # Python 2.x
ref_to_original_urlopen = urllib2.urlopen
urllib2.urlopen = self.mock_urlopen
else: # Python 3.x
ref_to_original_urlopen = urllib.request.urlopen
urllib.request.urlopen = self.mock_urlopen
result_output = \
self.__instance.call_API('http://tests.com/api', {'a': 1, 'b': 2})
# Tear down monkey patching
if 'urllib2' in context: # Python 2.x
urllib2.urlopen = ref_to_original_urlopen
else: # Python 3.x
urllib.request.urlopen = ref_to_original_urlopen
self.assertEqual(self.__test_output.decode('utf-8'), result_output)
4
Example 2
def check(self):
try:
with request.urlopen(self._url) as response:
body = response.read()
response = json.loads(body.decode('utf-8'))
if response["status"] in self._colors:
return True, "Cluster status is '{}'".format(response["status"])
return False, "Cluster status is '{}'".format(response["status"])
except urllib.error.HTTPError as httpe:
return False, "[{}] {}".format(httpe.code, httpe.reason)
except urllib.error.URLError as urle:
return False, urle.reason
except Exception as exc: # pylint: disable=broad-except
return False, "Unhandled error: {}".format(exc)
3
Example 3
def urlopen(*args, **kwargs):
""" Compatibility function for the urlopen function. Raises an
RuntimeError if urlopen could not be imported (which can occur in
frozen applications.
"""
try:
from urllib2 import urlopen
except ImportError:
try:
from urllib.request import urlopen # Py3k
except ImportError:
raise RuntimeError('Could not import urlopen.')
return urlopen(*args, **kwargs)
3
Example 4
def _call_api(self, param):
url = self._build_url(param)
with urllib.request.urlopen(url) as f:
data = f.read().decode('utf-8')
result = json.loads(data)
if 'results' in result['response']: # We found more than one result, returning the first.
return self._call_api(result['response']['results'][0]['zmw'])
else:
return result
3
Example 5
Project: pycapnp Source File: bundle.py
def fetch_archive(savedir, url, fname, force=False):
"""download an archive to a specific location"""
dest = pjoin(savedir, fname)
if os.path.exists(dest) and not force:
info("already have %s" % fname)
return dest
info("fetching %s into %s" % (url, savedir))
if not os.path.exists(savedir):
os.makedirs(savedir)
req = urlopen(url)
with open(dest, 'wb') as f:
f.write(req.read())
return dest
3
Example 6
Project: pgmult Source File: ap_lds.py
def download_ap():
from io import StringIO
from urllib.request import urlopen
import tarfile
print("Downloading AP data...")
response = urlopen('http://www.cs.princeton.edu/~blei/lda-c/ap.tgz')
tar = tarfile.open(fileobj=StringIO(response.read()))
return tar.extractfile('ap/ap.txt').read()
3
Example 7
def downloader(directory, iteration):
# Downloads given URL
url = 'https://virusshare.com/hashes/VirusShare_%05d.md5' % iteration
print(" Downloading {0} into {1}...".format(url, directory))
file_path = os.path.join(directory, os.path.basename(url))
contents = urlopen(url)
file_output = open(file_path,'wb')
file_output.write(contents.read())
file_output.close()
time.sleep(1)
3
Example 8
Project: palettable Source File: test_brewermap.py
def test_colorbrewer2_url_exists(self):
'''Simple check to ensure a URL is valid. Thanks to
http://stackoverflow.com/questions/4041443'''
try:
urllib.urlopen(self.bmap.colorbrewer2_url)
assert True
except:
assert False
3
Example 9
Project: docklet Source File: master_v1.py
def http_client_post(ip, port, url, entries = {}):
import urllib.request, urllib.parse, json
url = url if not url.startswith('/') else url[1:]
response = urllib.request.urlopen('http://%s:%d/%s' % (ip, port, url), urllib.parse.urlencode(entries).encode())
obj = json.loads(response.read().decode().strip())
response.close()
return obj
3
Example 10
Project: K3D-jupyter Source File: objects.py
def _to_image_src(url):
try:
response = urlopen(url)
except (IOError, ValueError):
return url
content_type = dict(response.info()).get('content-type', 'image/png')
return 'data:%s;base64,%s' % (content_type, base64.b64encode(response.read()).decode(encoding='ascii'))
3
Example 11
def read(self):
try:
log.info('weatherdata request: {}, forecast-link: {}'.format(
self.location.location_name,
self.location.forecast_link,
))
cache = Cache(self.location)
if not cache.exists() or not cache.is_fresh():
log.info('read online: {}'.format(self.location.url))
response = urllib.request.urlopen(self.location.url)
if response.status != 200:
raise
weatherdata = response.read().decode(self.encoding)
cache.dump(weatherdata)
else:
weatherdata = cache.load()
return weatherdata
except Exception as e:
raise YrException(e)
3
Example 12
Project: ungoogled-chromium Source File: _util.py
def download_if_needed(logger, file_path, url, force_download):
'''Downloads a file if necessary, unless force_download is True'''
if file_path.exists() and not file_path.is_file():
raise BuilderException("{} is an existing non-file".format(str(file_path)))
elif force_download or not file_path.is_file():
logger.info("Downloading {} ...".format(str(file_path)))
with urllib.request.urlopen(url) as response:
with file_path.open("wb") as file_obj:
shutil.copyfileobj(response, file_obj)
else:
logger.info("{} already exists. Skipping download.".format(str(file_path)))
3
Example 13
Project: CudaText Source File: workremote.py
def get_item_url(item):
try:
url = 'http://sourceforge.net/projects/synwrite-addons/files/' + item + '/download'
res = urllib.request.urlopen(url)
return res.geturl()
except:
return
3
Example 14
Project: odo Source File: url.py
@append.register(TextFile, URL(TextFile))
@append.register(JSONLines, URL(JSONLines))
@append.register(JSON, URL(JSON))
@append.register(CSV, URL(CSV))
def append_urlX_to_X(target, source, **kwargs):
with closing(urlopen(source.url, timeout=kwargs.pop('timeout', None))) as r:
chunk_size = 16 * source.chunk_size
with open(target.path, 'wb') as fp:
for chunk in iter(curry(r.read, chunk_size), b''):
fp.write(chunk)
return target
3
Example 15
def _pending_jobs(self, spider):
# Ommit scheduling new jobs if there are still pending jobs for same spider
resp = urllib.request.urlopen('http://localhost:6800/listjobs.json?project=default')
data = json.load(resp)
if 'pending' in data:
for item in data['pending']:
if item['spider'] == spider:
return True
return False
3
Example 16
def urlopen(self, url, data=None, **kwargs):
l = []
f = urllib.request.urlopen(url, data, **kwargs)
try:
# Exercise various methods
l.extend(f.readlines(200))
l.append(f.readline())
l.append(f.read(1024))
l.append(f.read())
finally:
f.close()
return b"".join(l)
3
Example 17
Project: whale-linter Source File: parser.py
def __init__(self, filename):
if self.is_url(filename) is not None:
response = urllib.request.urlopen(filename)
if self.is_content_type_plain_text(response):
self.file = response.read().decode('utf-8')
else:
print('ERROR: file format not supported\n')
elif os.path.isfile(filename):
self.file = open(filename, encoding='utf-8').read()
elif self.is_github_repo(filename):
filename = 'https://raw.githubusercontent.com/' + filename + '/master/Dockerfile'
self.file = urllib.request.urlopen(filename).read().decode('utf-8')
else:
print('ERROR: file format not supported\n')
self.TOKENS = App._config.get('all')
3
Example 18
Project: splinter Source File: run_tests.py
def wait_until_stop():
while True:
try:
results = urlopen(EXAMPLE_APP)
if results.code == 404:
break
except IOError:
break
3
Example 19
def _load_json(self):
#This is a giant hack, basically we check for http
#if it doesn't have that in the string, then in theory
#we're passing it raw json.
#this is mainly useful for testing. I'm sorry...
if 'http' in self.json_path:
data = urllib.request.urlopen(self.json_path)
str_response = data.read().decode('utf-8')
json_data = json.loads(str_response)
else:
json_data = json.loads(json.dumps(self.json_path))
return json_data
3
Example 20
Project: ros_buildfarm Source File: debian_repo.py
def load_url(url, retry=2, retry_period=1, timeout=10):
try:
fh = urlopen(url, timeout=timeout)
except HTTPError as e:
if e.code == 503 and retry:
time.sleep(retry_period)
return load_url(
url, retry=retry - 1, retry_period=retry_period,
timeout=timeout)
e.msg += ' (%s)' % url
raise
except URLError as e:
if isinstance(e.reason, socket.timeout) and retry:
time.sleep(retry_period)
return load_url(
url, retry=retry - 1, retry_period=retry_period,
timeout=timeout)
raise URLError(str(e) + ' (%s)' % url)
return fh.read()
3
Example 21
def is_url(url):
"""Determine whether a given string is a valid URL.
Parameters
----------
url : str
Returns
-------
bool
"""
try:
with urlopen(url) as r:
return r.status == 200
except:
return False
3
Example 22
Project: cppman Source File: cppreference.py
def func_test():
"""Test if there is major format changes in cplusplus.com"""
ifs = urllib.request.urlopen('http://en.cppreference.com/w/cpp/container/vector')
result = html2groff(fixupHTML(ifs.read()), 'std::vector')
assert '.SH "NAME"' in result
assert '.SH "SYNOPSIS"' in result
assert '.SH "DESCRIPTION"' in result
3
Example 23
@silencestderr
def test_broken_app(self):
def broken_app(environ, start_response):
1 // 0
server, addr = run_dev_server(broken_app)
try:
urlopen('http://%s/?foo=bar&baz=blah' % addr).read()
except HTTPError as e:
# In Python3 a 500 response causes an exception
rv = e.read()
assert b'Internal Server Error' in rv
else:
assert False, 'expected internal server error'
3
Example 24
def getLocations(self):
""" getLocations(self)
Returns a list of locations for companies that are currently advertising positions.
Parameters:
None
"""
locations = simplejson.load(urllib.request.urlopen(self.base_url + "&method=aj.jobs.getLocations"))
return self.checkResponse(locations)
3
Example 25
Project: hls-analyzer Source File: __init__.py
def _load_from_uri(uri):
resource = urlopen(uri)
base_uri = _parsed_url(_url_for(uri))
if PYTHON_MAJOR_VERSION < (3,):
content = _read_python2x(resource)
else:
content = _read_python3x(resource)
return M3U8(content, base_uri=base_uri)
3
Example 26
Project: uberwriter Source File: UberwriterInlinePreview.py
def check_url(url, item, spinner):
logger.debug("thread started, checking url")
error = False
try:
response = urllib.request.urlopen(url)
except URLError as e:
error = True
text = "Error! Reason: %s" % e.reason
if not error:
if (response.code / 100) >= 4:
logger.debug("Website not available")
text = _("Website is not available")
else:
text = _("Website is available")
logger.debug("Response: %s" % text)
spinner.destroy()
item.set_label(text)
3
Example 27
Project: python-wikiquotes Source File: utils.py
def json_from_url(url, params=None):
if params:
url += urllib.parse.quote(params)
res = urllib.request.urlopen(url)
body = res.read().decode()
return json.loads(body)
3
Example 28
Project: lizepy Source File: __init__.py
def _do_request(url):
req = Request(url, headers={ 'Accept': 'application/json' })
try:
response = urlopen(req)
except HTTPError as e:
response = e
return Response(response.url, response.headers, response.read(), response.code, response.msg)
3
Example 29
Project: orange3-text Source File: nyt.py
def api_key_valid(self):
""" Checks whether api key given at initialization is valid. """
url = self._encode_url('test')
try:
with request.urlopen(url) as connection:
if connection.getcode() == 200:
return True
except HTTPError:
return False
3
Example 30
@property
def ip(self):
"""return bot's ip as an ``ip_address`` object"""
if not self._ip:
if 'ip' in self.config:
ip = self.config['ip']
else:
ip = self.protocol.transport.get_extra_info('sockname')[0]
ip = ip_address(ip)
if ip.version == 4:
self._ip = ip
else: # pragma: no cover
response = urlopen('http://ipv4.icanhazip.com/')
ip = response.read().strip().decode()
ip = ip_address(ip)
self._ip = ip
return self._ip
3
Example 31
def parse(filename_url_or_file, guess_charset=True, parser=None):
"""Parse a filename, URL, or file-like object into an HTML docuement
tree. Note: this returns a tree, not an element. Use
``parse(...).getroot()`` to get the docuement root.
"""
if parser is None:
parser = html_parser
if not isinstance(filename_url_or_file, _strings):
fp = filename_url_or_file
elif _looks_like_url(filename_url_or_file):
fp = urlopen(filename_url_or_file)
else:
fp = open(filename_url_or_file, 'rb')
return parser.parse(fp, useChardet=guess_charset)
3
Example 32
Project: brython Source File: test_urllib2net.py
def test_file(self):
TESTFN = support.TESTFN
f = open(TESTFN, 'w')
try:
f.write('hi there\n')
f.close()
urls = [
'file:' + sanepathname2url(os.path.abspath(TESTFN)),
('file:///nonsensename/etc/passwd', None,
urllib.error.URLError),
]
self._test_urls(urls, self._extra_handlers(), retry=True)
finally:
os.remove(TESTFN)
self.assertRaises(ValueError, urllib.request.urlopen,'./relative_path/to/file')
3
Example 33
def _connected():
"""check internet connect"""
host = 'http://google.com'
try:
urlopen(host)
return True
except:
return False
3
Example 34
Project: mesos-in-action-code-samples Source File: email-weather-forecast.py
def get_forecast(zipcode):
zipcity_svc_url = 'http://forecast.weather.gov/zipcity.php?inputstring='
logging.info("Getting the weather forecast for zip code {}".format(zipcode))
try:
zipcity = urllib.request.urlopen(''.join([zipcity_svc_url, zipcode]))
forecast_url = ''.join([zipcity.geturl(), '&FcstType=text&TextType=1'])
forecast_resp = urllib.request.urlopen(forecast_url)
forecast = forecast_resp.read().decode()
logging.info('Successfully got the forecast.')
return forecast
except urllib.error.HTTPError as e:
raise WeatherForecastException(e)
else:
raise WeatherForecastException('An uncaught exception occurred.')
3
Example 35
Project: Pyro4 Source File: client.py
def pyro_call(object_name, method, callback):
request = Request("http://127.0.0.1:8080/pyro/{0}/{1}".format(object_name, method),
# headers={"x-pyro-options": "oneway", "x-pyro-gateway-key": "secretgatewaykey"}
)
with urlopen(request) as req:
charset = get_charset(req)
data = req.read().decode(charset)
if data:
callback(json.loads(data))
else:
callback(None)
3
Example 36
def run(self):
DEBUG("Searching updates...")
found = False
try:
response = request.urlopen(ui.__web_version__)
data = json.loads(response.read().decode('utf8'))
web_version, link = data['version'], data['link']
current_version = ui.__version__
if current_version < web_version:
found = True
except:
web_version, link = '', ''
INFO("No se pudo establecer la conexión")
DEBUG("Última versión disponible: {0}".format(web_version))
self.updateVersion.emit(web_version, link, found)
3
Example 37
@property
@memoize
def metadata(self):
url = self.TEMPLATE_URL.format(
path=('/actions/search/?q={name}'.format(name=self.name.replace(' ', '+'))))
lines = urllib2.urlopen(url)
first_result = str(lines.readline())
return self.Metadata(*first_result.split('|'))
3
Example 38
Project: rpaas Source File: plugin.py
def proxy_request(service_name, instance_name, path, body=None, headers=None, method='POST'):
target = get_env("TSURU_TARGET").rstrip("/")
token = get_env("TSURU_TOKEN")
url = "{}/services/{}/proxy/{}?callback={}".format(target, service_name, instance_name,
path)
request = Request(url)
request.add_header("Authorization", "bearer " + token)
request.get_method = lambda: method
if body:
try:
request.add_data(body)
except AttributeError:
request.data = body.encode('utf-8')
if headers:
for key, value in headers.items():
request.add_header(key, value)
return urlopen(request)
3
Example 39
Project: iamine Source File: core.py
def get_global_rate_limit(self):
"""Get the global rate limit per client.
:rtype: int
:returns: The global rate limit for each client.
"""
r = urllib.request.urlopen('https://archive.org/metadata/iamine-rate-limiter')
j = json.loads(r.read().decode('utf-8'))
return int(j.get('metadata', {}).get('rate_per_second', 300))
3
Example 40
def get_release():
"""Get stdiscosrv latest linux-amd64 release version from the Jenkins API.
Returns: download url
"""
jenkins_url = ("https://build.syncthing.net/job/"
"stdiscosrv/lastStableBuild/api/json")
res = urlopen(jenkins_url)
if not res:
return ""
res = res.read().decode()
res = json.loads(res)
fn = [i['fileName'] for i in res['artifacts']
if 'linux-amd64' in i['fileName']][0]
return "{}artifact/{}".format(res['url'], fn)
3
Example 41
Project: public-contracts Source File: categories_crawler.py
def get_xml():
"""
Gets the xml file from the web.
"""
from urllib.request import urlopen
request = urlopen('https://raw.githubusercontent.com/data-ac-uk/cpv/master/'
'etc/cpv_2008.xml')
tree = xml.etree.ElementTree.parse(request)
return tree.getroot()
3
Example 42
def download(filename, url):
print("Download %s into %s" % (url, filename))
response = urllib_request.urlopen(url)
with response:
content = response.read()
with open(filename, 'wb') as fp:
fp.write(content)
fp.flush()
3
Example 43
@staticmethod
def _do_retrieve(url, fname):
"""Retrieve given url to target filepath fname."""
folder = os.path.dirname(fname)
if not os.path.exists(folder):
os.makedirs(folder)
print("{}/ created.".format(folder))
if not os.path.exists(fname):
with open(fname, 'wb') as fout:
print("retrieving {}.".format(url))
resp = urlopen(url)
fout.write(resp.read())
print("{} saved.".format(fname))
else:
print("re-using artifact {}".format(fname))
return fname
3
Example 44
Project: cherrymusic Source File: cherrymodel.py
def check_for_updates(self):
try:
url = 'http://fomori.org/cherrymusic/update_check.php?version='
url += cherry.__version__
urlhandler = urllib.request.urlopen(url, timeout=5)
jsondata = codecs.decode(urlhandler.read(), 'UTF-8')
versioninfo = json.loads(jsondata)
return versioninfo
except Exception as e:
log.e(_('Error fetching version info: %s') % str(e))
return []
3
Example 45
Project: sketch-wakatime Source File: _lua_builtins.py
def get_lua_functions(version):
f = urlopen('http://www.lua.org/manual/%s/' % version)
r = re.compile(r'^<A HREF="manual.html#pdf-(.+)">\1</A>')
functions = []
for line in f:
m = r.match(line)
if m is not None:
functions.append(m.groups()[0])
return functions
3
Example 46
def open_url(url):
f = urlopen(url, timeout=30)
try:
size = f.headers.get("content-length",None)
if size is not None:
size = int(size)
except ValueError:
pass
else:
f.size = size
return f
3
Example 47
def fetch(destination):
"""fetch TLE from internet and save it to *destination*.
"""
with open(destination, "w") as dest:
for url in TLE_URLS:
response = urlopen(url)
dest.write(response.read())
3
Example 48
Project: trading-with-python Source File: cboe.py
def getPutCallRatio():
""" download current Put/Call ratio"""
urlStr = 'http://www.cboe.com/publish/ScheduledTask/MktData/datahouse/totalpc.csv'
try:
data = urllib.request.urlopen(urlStr)
except Exception as e:
s = "Failed to download:\n{0}".format(e);
print(s)
headerLine = 2
return pd.read_csv(data,header=headerLine,index_col=0,parse_dates=True)
3
Example 49
Project: trackma Source File: plex.py
def playing_file():
# returns the filename of the currently playing file
hostnport = get_config()[1]
if status() == "IDLE":
return False
session_url = "http://"+hostnport+"/status/sessions"
sdoc = xdmd.parse(urllib.request.urlopen(session_url))
attr = sdoc.getElementsByTagName("Part")[0].getAttribute("file")
name = urllib.parse.unquote(ntpath.basename(attr)[:-4])
return name
3
Example 50
def __init__(self, url):
self.url = url
if self.url.startswith('http'):
# http ressource
try:
self.fp = urllib.request.urlopen(url)
except:
_log.debug("Couldn't open: {}".format(self.url))
else:
# local file ressource
self.fp = open(url)