django.utils.six.moves.urllib.request.build_opener

Here are the examples of the python api django.utils.six.moves.urllib.request.build_opener taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

1 Examples 7

Example 1

Project: django-linkcheck
Source File: models.py
View license
    def _check_external(self, tested_url, external_recheck_interval):
        logger.info('checking external link: %s' % tested_url)
        external_recheck_datetime = now() - timedelta(minutes=external_recheck_interval)

        if self.last_checked and (self.last_checked > external_recheck_datetime):
            return self.status

        opener = build_opener(RedirectHandler)
        # Remove URL fragment identifiers
        url = tested_url.rsplit('#')[0]
        # Check that non-ascii chars are properly encoded
        try:
            url.encode('ascii')
        except UnicodeEncodeError:
            url = iri_to_uri(url)

        try:
            if tested_url.count('#'):
                # We have to get the content so we can check the anchors
                response = opener.open(
                    url,
                    timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT
                )
            else:
                # Might as well just do a HEAD request
                req = HeadRequest(url, headers={'User-Agent' : "http://%s Linkchecker" % settings.SITE_DOMAIN})
                try:
                    response = opener.open(
                        req,
                        timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT
                    )
                except (ValueError, HTTPError) as error:
                    # ...except sometimes it triggers a bug in urllib2
                    if hasattr(error, 'code') and error.code == METHOD_NOT_ALLOWED:
                        req = GetRequest(url, headers={'User-Agent' : "http://%s Linkchecker" % settings.SITE_DOMAIN})
                    else:
                        req = url
                    response = opener.open(
                        req,
                        timeout=LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT
                    )

            self.message = ' '.join([str(response.code), response.msg])
            self.status = True

            if tested_url.count('#'):

                anchor = tested_url.split('#')[1]
                from linkcheck import parse_anchors
                try:
                    names = parse_anchors(response.read())
                    if anchor in names:
                        self.message = 'Working external hash anchor'
                        self.status = True
                    else:
                        self.message = 'Broken external hash anchor'
                        self.status = False

                except:
                    # The external web page is mal-formatted #or maybe other parse errors like encoding
                    # I reckon a broken anchor on an otherwise good URL should count as a pass
                    self.message = "Page OK but anchor can't be checked"
                    self.status = True

        except http_client.BadStatusLine:
                self.message = "Bad Status Line"

        except HTTPError as e:
            if hasattr(e, 'code') and hasattr(e, 'msg'):
                self.message = ' '.join([str(e.code), e.msg])
            else:
                self.message = "Unknown Error"

        except URLError as e:
            if hasattr(e, 'reason'):
                self.message = 'Unreachable: '+str(e.reason)
            elif hasattr(e, 'code') and e.code!=301:
                self.message = 'Error: '+str(e.code)
            else:
                self.message = 'Redirect. Check manually: '+str(e.code)
        except Exception as e:
            self.message = 'Other Error: %s' % e
        else:
            if response.getcode() == 301 and response.geturl() != url:
                self.redirect_to = response.geturl()
            elif self.redirect_to:
                self.redirect_to = ''

        self.last_checked = now()
        self.save()