urllib.request.urlopen

Here are the examples of the python api urllib.request.urlopen taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

173 Examples 7

Example 1

Project: pyowm Source File: test_weather_client.py
    def test_call_API(self):
        # Setup monkey patching
        if 'urllib2' in context:  # Python 2.x
            ref_to_original_urlopen = urllib2.urlopen
            urllib2.urlopen = self.mock_urlopen
        else:  # Python 3.x
            ref_to_original_urlopen = urllib.request.urlopen
            urllib.request.urlopen = self.mock_urlopen
        result_output = \
            self.__instance.call_API('http://tests.com/api', {'a': 1, 'b': 2})
        # Tear down monkey patching
        if 'urllib2' in context:  # Python 2.x
            urllib2.urlopen = ref_to_original_urlopen
        else:   # Python 3.x
            urllib.request.urlopen = ref_to_original_urlopen
        self.assertEqual(self.__test_output.decode('utf-8'), result_output)

Example 2

Project: preflyt Source File: elasticsearch.py
Function: check
    def check(self):
        try:
            with request.urlopen(self._url) as response:
                body = response.read()
                response = json.loads(body.decode('utf-8'))
                if response["status"] in self._colors:
                    return True, "Cluster status is '{}'".format(response["status"])
                return False, "Cluster status is '{}'".format(response["status"])
        except urllib.error.HTTPError as httpe:
            return False, "[{}] {}".format(httpe.code, httpe.reason)
        except urllib.error.URLError as urle:
            return False, urle.reason
        except Exception as exc: # pylint: disable=broad-except
            return False, "Unhandled error: {}".format(exc)

Example 3

Project: imageio Source File: util.py
Function: url_open
def urlopen(*args, **kwargs):
    """ Compatibility function for the urlopen function. Raises an
    RuntimeError if urlopen could not be imported (which can occur in
    frozen applications.
    """ 
    try:
        from urllib2 import urlopen
    except ImportError:
        try:
            from urllib.request import urlopen  # Py3k
        except ImportError:
            raise RuntimeError('Could not import urlopen.')
    return urlopen(*args, **kwargs)

Example 4

Project: telex Source File: py3wu.py
Function: call_api
    def _call_api(self, param):
        url = self._build_url(param)
        with urllib.request.urlopen(url) as f:
            data = f.read().decode('utf-8')
            result = json.loads(data)
            if 'results' in result['response']: # We found more than one result, returning the first.
                return self._call_api(result['response']['results'][0]['zmw'])
            else:
                return result

Example 5

Project: pycapnp Source File: bundle.py
def fetch_archive(savedir, url, fname, force=False):
    """download an archive to a specific location"""
    dest = pjoin(savedir, fname)
    if os.path.exists(dest) and not force:
        info("already have %s" % fname)
        return dest
    info("fetching %s into %s" % (url, savedir))
    if not os.path.exists(savedir):
        os.makedirs(savedir)
    req = urlopen(url)
    with open(dest, 'wb') as f:
        f.write(req.read())
    return dest

Example 6

Project: pgmult Source File: ap_lds.py
def download_ap():
    from io import StringIO
    from urllib.request import urlopen
    import tarfile

    print("Downloading AP data...")
    response = urlopen('http://www.cs.princeton.edu/~blei/lda-c/ap.tgz')
    tar = tarfile.open(fileobj=StringIO(response.read()))
    return tar.extractfile('ap/ap.txt').read()

Example 7

Project: VirusShare-Search Source File: VirusShare-Search.py
Function: downloader
def downloader(directory, iteration):
	# Downloads given URL
	url = 'https://virusshare.com/hashes/VirusShare_%05d.md5' % iteration
	print("  Downloading {0} into {1}...".format(url, directory))
	file_path = os.path.join(directory, os.path.basename(url))
	contents = urlopen(url)
	file_output = open(file_path,'wb')
	file_output.write(contents.read())
	file_output.close()
	time.sleep(1)

Example 8

Project: palettable Source File: test_brewermap.py
    def test_colorbrewer2_url_exists(self):
        '''Simple check to ensure a URL is valid. Thanks to
        http://stackoverflow.com/questions/4041443'''
        try:
            urllib.urlopen(self.bmap.colorbrewer2_url)
            assert True
        except:
            assert False

Example 9

Project: docklet Source File: master_v1.py
def http_client_post(ip, port, url, entries = {}):
	import urllib.request, urllib.parse, json
	url = url if not url.startswith('/') else url[1:]
	response = urllib.request.urlopen('http://%s:%d/%s' % (ip, port, url), urllib.parse.urlencode(entries).encode())
	obj = json.loads(response.read().decode().strip())
	response.close()
	return obj

Example 10

Project: K3D-jupyter Source File: objects.py
def _to_image_src(url):
    try:
        response = urlopen(url)
    except (IOError, ValueError):
        return url

    content_type = dict(response.info()).get('content-type', 'image/png')

    return 'data:%s;base64,%s' % (content_type, base64.b64encode(response.read()).decode(encoding='ascii'))

Example 11

Project: python-yr Source File: utils.py
Function: read
    def read(self):
        try:
            log.info('weatherdata request: {}, forecast-link: {}'.format(
                self.location.location_name,
                self.location.forecast_link,
            ))
            cache = Cache(self.location)
            if not cache.exists() or not cache.is_fresh():
                log.info('read online: {}'.format(self.location.url))
                response = urllib.request.urlopen(self.location.url)
                if response.status != 200:
                    raise
                weatherdata = response.read().decode(self.encoding)
                cache.dump(weatherdata)
            else:
                weatherdata = cache.load()
            return weatherdata
        except Exception as e:
            raise YrException(e)

Example 12

Project: ungoogled-chromium Source File: _util.py
def download_if_needed(logger, file_path, url, force_download):
    '''Downloads a file if necessary, unless force_download is True'''

    if file_path.exists() and not file_path.is_file():
        raise BuilderException("{} is an existing non-file".format(str(file_path)))
    elif force_download or not file_path.is_file():
        logger.info("Downloading {} ...".format(str(file_path)))
        with urllib.request.urlopen(url) as response:
            with file_path.open("wb") as file_obj:
                shutil.copyfileobj(response, file_obj)
    else:
        logger.info("{} already exists. Skipping download.".format(str(file_path)))

Example 13

Project: CudaText Source File: workremote.py
def get_item_url(item):
    try:
        url = 'http://sourceforge.net/projects/synwrite-addons/files/' + item + '/download'
        res = urllib.request.urlopen(url)
        return res.geturl()
    except:
        return

Example 14

Project: odo Source File: url.py
@append.register(TextFile, URL(TextFile))
@append.register(JSONLines, URL(JSONLines))
@append.register(JSON, URL(JSON))
@append.register(CSV, URL(CSV))
def append_urlX_to_X(target, source, **kwargs):

    with closing(urlopen(source.url, timeout=kwargs.pop('timeout', None))) as r:
        chunk_size = 16 * source.chunk_size
        with open(target.path, 'wb') as fp:
            for chunk in iter(curry(r.read, chunk_size), b''):
                fp.write(chunk)
            return target

Example 15

Project: django-dynamic-scraper Source File: task_utils.py
Function: pending_jobs
    def _pending_jobs(self, spider):
        # Ommit scheduling new jobs if there are still pending jobs for same spider
        resp = urllib.request.urlopen('http://localhost:6800/listjobs.json?project=default')
        data = json.load(resp)
        if 'pending' in data:
            for item in data['pending']:
                if item['spider'] == spider:
                    return True
        return False

Example 16

Project: TrustRouter Source File: test_urllib2_localnet.py
Function: url_open
    def urlopen(self, url, data=None, **kwargs):
        l = []
        f = urllib.request.urlopen(url, data, **kwargs)
        try:
            # Exercise various methods
            l.extend(f.readlines(200))
            l.append(f.readline())
            l.append(f.read(1024))
            l.append(f.read())
        finally:
            f.close()
        return b"".join(l)

Example 17

Project: whale-linter Source File: parser.py
    def __init__(self, filename):
        if self.is_url(filename) is not None:
            response = urllib.request.urlopen(filename)
            if self.is_content_type_plain_text(response):
                self.file = response.read().decode('utf-8')
            else:
                print('ERROR: file format not supported\n')
        elif os.path.isfile(filename):
            self.file = open(filename, encoding='utf-8').read()
        elif self.is_github_repo(filename):
            filename = 'https://raw.githubusercontent.com/' + filename + '/master/Dockerfile'
            self.file = urllib.request.urlopen(filename).read().decode('utf-8')
        else:
            print('ERROR: file format not supported\n')

        self.TOKENS = App._config.get('all')

Example 18

Project: splinter Source File: run_tests.py
def wait_until_stop():
    while True:
        try:
            results = urlopen(EXAMPLE_APP)
            if results.code == 404:
                break
        except IOError:
            break

Example 19

Project: reap-get Source File: package_repository.py
Function: load_json
    def _load_json(self):
        #This is a giant hack, basically we check for http
        #if it doesn't have that in the string, then in theory
        #we're passing it raw json.
        #this is mainly useful for testing. I'm sorry...
        if 'http' in self.json_path:
            data = urllib.request.urlopen(self.json_path)
            str_response = data.read().decode('utf-8')
            json_data = json.loads(str_response)
        else:
            json_data = json.loads(json.dumps(self.json_path))
        return json_data

Example 20

Project: ros_buildfarm Source File: debian_repo.py
def load_url(url, retry=2, retry_period=1, timeout=10):
    try:
        fh = urlopen(url, timeout=timeout)
    except HTTPError as e:
        if e.code == 503 and retry:
            time.sleep(retry_period)
            return load_url(
                url, retry=retry - 1, retry_period=retry_period,
                timeout=timeout)
        e.msg += ' (%s)' % url
        raise
    except URLError as e:
        if isinstance(e.reason, socket.timeout) and retry:
            time.sleep(retry_period)
            return load_url(
                url, retry=retry - 1, retry_period=retry_period,
                timeout=timeout)
        raise URLError(str(e) + ' (%s)' % url)
    return fh.read()

Example 21

Project: pylinac Source File: io.py
Function: is_url
def is_url(url):
    """Determine whether a given string is a valid URL.

    Parameters
    ----------
    url : str

    Returns
    -------
    bool
    """
    try:
        with urlopen(url) as r:
            return r.status == 200
    except:
        return False

Example 22

Project: cppman Source File: cppreference.py
def func_test():
    """Test if there is major format changes in cplusplus.com"""
    ifs = urllib.request.urlopen('http://en.cppreference.com/w/cpp/container/vector')
    result = html2groff(fixupHTML(ifs.read()), 'std::vector')
    assert '.SH "NAME"' in result
    assert '.SH "SYNOPSIS"' in result
    assert '.SH "DESCRIPTION"' in result

Example 23

Project: gae-flask-todo Source File: serving.py
Function: test_broken_app
    @silencestderr
    def test_broken_app(self):
        def broken_app(environ, start_response):
            1 // 0
        server, addr = run_dev_server(broken_app)
        try:
            urlopen('http://%s/?foo=bar&baz=blah' % addr).read()
        except HTTPError as e:
            # In Python3 a 500 response causes an exception
            rv = e.read()
            assert b'Internal Server Error' in rv
        else:
            assert False, 'expected internal server error'

Example 24

Project: pythentic_jobs Source File: pythentic_jobs3k.py
Function: get_locations
	def getLocations(self):
		"""	getLocations(self)

			Returns a list of locations for companies that are currently advertising positions.

			Parameters:
				None
		"""
		locations = simplejson.load(urllib.request.urlopen(self.base_url + "&method=aj.jobs.getLocations"))
		return self.checkResponse(locations)

Example 25

Project: hls-analyzer Source File: __init__.py
def _load_from_uri(uri):
    resource = urlopen(uri)
    base_uri = _parsed_url(_url_for(uri))
    if PYTHON_MAJOR_VERSION < (3,):
        content = _read_python2x(resource)
    else:
        content = _read_python3x(resource)
    return M3U8(content, base_uri=base_uri)

Example 26

Project: uberwriter Source File: UberwriterInlinePreview.py
def check_url(url, item, spinner):
    logger.debug("thread started, checking url")
    error = False
    try:
        response = urllib.request.urlopen(url)
    except URLError as e:
        error = True
        text = "Error! Reason: %s" % e.reason

    if not error:
        if (response.code / 100) >= 4:
            logger.debug("Website not available")
            text = _("Website is not available")
        else:
            text = _("Website is available")
    logger.debug("Response: %s" % text)
    spinner.destroy()
    item.set_label(text)

Example 27

Project: python-wikiquotes Source File: utils.py
def json_from_url(url, params=None):
    if params:
        url += urllib.parse.quote(params)
    res = urllib.request.urlopen(url)
    body = res.read().decode()
    return json.loads(body)

Example 28

Project: lizepy Source File: __init__.py
def _do_request(url):

    req = Request(url, headers={ 'Accept': 'application/json' })

    try:
        response = urlopen(req)
    except HTTPError as e:
        response = e

    return Response(response.url, response.headers, response.read(), response.code, response.msg)

Example 29

Project: orange3-text Source File: nyt.py
    def api_key_valid(self):
        """ Checks whether api key given at initialization is valid. """
        url = self._encode_url('test')
        try:
            with request.urlopen(url) as connection:
                if connection.getcode() == 200:
                    return True
        except HTTPError:
            return False

Example 30

Project: irc3 Source File: __init__.py
Function: ip
    @property
    def ip(self):
        """return bot's ip as an ``ip_address`` object"""
        if not self._ip:
            if 'ip' in self.config:
                ip = self.config['ip']
            else:
                ip = self.protocol.transport.get_extra_info('sockname')[0]
            ip = ip_address(ip)
            if ip.version == 4:
                self._ip = ip
            else:  # pragma: no cover
                response = urlopen('http://ipv4.icanhazip.com/')
                ip = response.read().strip().decode()
                ip = ip_address(ip)
                self._ip = ip
        return self._ip

Example 31

Project: WAPT Source File: html5parser.py
Function: parse
def parse(filename_url_or_file, guess_charset=True, parser=None):
    """Parse a filename, URL, or file-like object into an HTML docuement
    tree.  Note: this returns a tree, not an element.  Use
    ``parse(...).getroot()`` to get the docuement root.
    """
    if parser is None:
        parser = html_parser
    if not isinstance(filename_url_or_file, _strings):
        fp = filename_url_or_file
    elif _looks_like_url(filename_url_or_file):
        fp = urlopen(filename_url_or_file)
    else:
        fp = open(filename_url_or_file, 'rb')
    return parser.parse(fp, useChardet=guess_charset)

Example 32

Project: brython Source File: test_urllib2net.py
    def test_file(self):
        TESTFN = support.TESTFN
        f = open(TESTFN, 'w')
        try:
            f.write('hi there\n')
            f.close()
            urls = [
                'file:' + sanepathname2url(os.path.abspath(TESTFN)),
                ('file:///nonsensename/etc/passwd', None,
                 urllib.error.URLError),
                ]
            self._test_urls(urls, self._extra_handlers(), retry=True)
        finally:
            os.remove(TESTFN)

        self.assertRaises(ValueError, urllib.request.urlopen,'./relative_path/to/file')

Example 33

Project: TermFeed Source File: feed.py
Function: connected
def _connected():
    """check internet connect"""
    host = 'http://google.com'

    try:
        urlopen(host)
        return True
    except:
        return False

Example 34

Project: mesos-in-action-code-samples Source File: email-weather-forecast.py
def get_forecast(zipcode):
    zipcity_svc_url = 'http://forecast.weather.gov/zipcity.php?inputstring='
    logging.info("Getting the weather forecast for zip code {}".format(zipcode))

    try:
        zipcity = urllib.request.urlopen(''.join([zipcity_svc_url, zipcode]))

        forecast_url = ''.join([zipcity.geturl(), '&FcstType=text&TextType=1'])
        forecast_resp = urllib.request.urlopen(forecast_url)
        forecast = forecast_resp.read().decode()

        logging.info('Successfully got the forecast.')
        return forecast

    except urllib.error.HTTPError as e:
        raise WeatherForecastException(e)

    else:
        raise WeatherForecastException('An uncaught exception occurred.')

Example 35

Project: Pyro4 Source File: client.py
def pyro_call(object_name, method, callback):
    request = Request("http://127.0.0.1:8080/pyro/{0}/{1}".format(object_name, method),
                      # headers={"x-pyro-options": "oneway", "x-pyro-gateway-key": "secretgatewaykey"}
                      )
    with urlopen(request) as req:
        charset = get_charset(req)
        data = req.read().decode(charset)
    if data:
        callback(json.loads(data))
    else:
        callback(None)

Example 36

Project: edis Source File: system_tray.py
Function: run
    def run(self):
        DEBUG("Searching updates...")
        found = False
        try:
            response = request.urlopen(ui.__web_version__)
            data = json.loads(response.read().decode('utf8'))
            web_version, link = data['version'], data['link']
            current_version = ui.__version__
            if current_version < web_version:
                found = True
        except:
            web_version, link = '', ''
            INFO("No se pudo establecer la conexión")
        DEBUG("Última versión disponible: {0}".format(web_version))
        self.updateVersion.emit(web_version, link, found)

Example 37

Project: mangopi Source File: aftv.py
Function: meta_data
        @property
        @memoize
        def metadata(self):
            url = self.TEMPLATE_URL.format(
                path=('/actions/search/?q={name}'.format(name=self.name.replace(' ', '+'))))

            lines = urllib2.urlopen(url)
            first_result = str(lines.readline())

            return self.Metadata(*first_result.split('|'))

Example 38

Project: rpaas Source File: plugin.py
def proxy_request(service_name, instance_name, path, body=None, headers=None, method='POST'):
    target = get_env("TSURU_TARGET").rstrip("/")
    token = get_env("TSURU_TOKEN")
    url = "{}/services/{}/proxy/{}?callback={}".format(target, service_name, instance_name,
                                                       path)
    request = Request(url)
    request.add_header("Authorization", "bearer " + token)
    request.get_method = lambda: method
    if body:
        try:
            request.add_data(body)
        except AttributeError:
            request.data = body.encode('utf-8')
    if headers:
        for key, value in headers.items():
            request.add_header(key, value)
    return urlopen(request)

Example 39

Project: iamine Source File: core.py
    def get_global_rate_limit(self):
        """Get the global rate limit per client.

        :rtype: int
        :returns: The global rate limit for each client.
        """
        r = urllib.request.urlopen('https://archive.org/metadata/iamine-rate-limiter')
        j = json.loads(r.read().decode('utf-8'))
        return int(j.get('metadata', {}).get('rate_per_second', 300))

Example 40

Project: dockerfiles Source File: update_release.py
Function: get_release
def get_release():
    """Get stdiscosrv latest linux-amd64 release version from the Jenkins API.

    Returns: download url

    """
    jenkins_url = ("https://build.syncthing.net/job/"
                   "stdiscosrv/lastStableBuild/api/json")
    res = urlopen(jenkins_url)
    if not res:
        return ""
    res = res.read().decode()
    res = json.loads(res)
    fn = [i['fileName'] for i in res['artifacts']
          if 'linux-amd64' in i['fileName']][0]
    return "{}artifact/{}".format(res['url'], fn)

Example 41

Project: public-contracts Source File: categories_crawler.py
def get_xml():
    """
    Gets the xml file from the web.
    """
    from urllib.request import urlopen

    request = urlopen('https://raw.githubusercontent.com/data-ac-uk/cpv/master/'
                      'etc/cpv_2008.xml')

    tree = xml.etree.ElementTree.parse(request)

    return tree.getroot()

Example 42

Project: performance Source File: venv.py
Function: download
def download(filename, url):
    print("Download %s into %s" % (url, filename))

    response = urllib_request.urlopen(url)
    with response:
        content = response.read()

    with open(filename, 'wb') as fp:
        fp.write(content)
        fp.flush()

Example 43

Project: wcwidth Source File: setup.py
Function: do_retrieve
    @staticmethod
    def _do_retrieve(url, fname):
        """Retrieve given url to target filepath fname."""
        folder = os.path.dirname(fname)
        if not os.path.exists(folder):
            os.makedirs(folder)
            print("{}/ created.".format(folder))
        if not os.path.exists(fname):
            with open(fname, 'wb') as fout:
                print("retrieving {}.".format(url))
                resp = urlopen(url)
                fout.write(resp.read())
            print("{} saved.".format(fname))
        else:
            print("re-using artifact {}".format(fname))
        return fname

Example 44

Project: cherrymusic Source File: cherrymodel.py
    def check_for_updates(self):
        try:
            url = 'http://fomori.org/cherrymusic/update_check.php?version='
            url += cherry.__version__
            urlhandler = urllib.request.urlopen(url, timeout=5)
            jsondata = codecs.decode(urlhandler.read(), 'UTF-8')
            versioninfo = json.loads(jsondata)
            return versioninfo
        except Exception as e:
            log.e(_('Error fetching version info: %s') % str(e))
            return []

Example 45

Project: sketch-wakatime Source File: _lua_builtins.py
    def get_lua_functions(version):
        f = urlopen('http://www.lua.org/manual/%s/' % version)
        r = re.compile(r'^<A HREF="manual.html#pdf-(.+)">\1</A>')
        functions = []
        for line in f:
            m = r.match(line)
            if m is not None:
                functions.append(m.groups()[0])
        return functions

Example 46

Project: CorpusTools Source File: versioning.py
Function: open_url
def open_url(url):
    f = urlopen(url, timeout=30)
    try:
        size = f.headers.get("content-length",None)
        if size is not None:
            size = int(size)
    except ValueError:
        pass
    else:
        f.size = size
    return f

Example 47

Project: pyorbital Source File: tlefile.py
Function: fetch
def fetch(destination):
    """fetch TLE from internet and save it to *destination*.
   """
    with open(destination, "w") as dest:
        for url in TLE_URLS:
            response = urlopen(url)
            dest.write(response.read())

Example 48

Project: trading-with-python Source File: cboe.py
def getPutCallRatio():
    """ download current Put/Call ratio"""
    urlStr = 'http://www.cboe.com/publish/ScheduledTask/MktData/datahouse/totalpc.csv'

    try:
        data = urllib.request.urlopen(urlStr)
    except Exception as e:
        s = "Failed to download:\n{0}".format(e);
        print(s)
       
    headerLine = 2
    
    return pd.read_csv(data,header=headerLine,index_col=0,parse_dates=True)

Example 49

Project: trackma Source File: plex.py
def playing_file():
    # returns the filename of the currently playing file
    hostnport = get_config()[1]

    if status() == "IDLE":
        return False

    session_url = "http://"+hostnport+"/status/sessions"
    sdoc = xdmd.parse(urllib.request.urlopen(session_url))

    attr = sdoc.getElementsByTagName("Part")[0].getAttribute("file")
    name = urllib.parse.unquote(ntpath.basename(attr)[:-4])

    return name

Example 50

Project: vulnix Source File: main.py
Function: init
    def __init__(self, url):
        self.url = url
        if self.url.startswith('http'):
            # http ressource
            try:
                self.fp = urllib.request.urlopen(url)
            except:
                _log.debug("Couldn't open: {}".format(self.url))
        else:
            # local file ressource
            self.fp = open(url)
See More Examples - Go to Next Page
Page 1 Selected Page 2 Page 3 Page 4