requests.exceptions.TooManyRedirects

Here are the examples of the python api requests.exceptions.TooManyRedirects taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

6 Examples 7

Example 1

Project: whatlastgenre
Source File: dataprovider.py
View license
    def _query(self, params):
        """Query What.CD API."""
        # lazy login
        if not self.session.cookies.get('session', None):
            self.log.debug('no session cookie, login')
            self.login()
        try:
            result = self._request_json('https://what.cd/ajax.php', params)
        except requests.exceptions.TooManyRedirects:
            self.log.debug('session cookie expired, relogin')
            self.login()
            return self._query(params)
        try:
            response = result['response']
        except KeyError:
            raise DataProviderError('request failure')
        return response

Example 2

Project: whatlastgenre
Source File: dataprovider.py
View license
    def _request(self, url, params, method='GET'):
        """Send a request.

        Honor rate limits and record some timings for stats.

        :param url: url string
        :param params: dict of call parameters
        :param method: request method
        """
        self._wait_rate_limit()
        time_ = time.time()
        try:
            if method == 'POST':
                res = self.session.post(url, data=params)
            else:
                res = self.session.get(url, params=params)
        except requests.exceptions.TooManyRedirects as err:
            raise err
        except requests.exceptions.RequestException as err:
            self.log.debug(err)
            raise DataProviderError("request: %s" % err.message)
        if not getattr(res, 'from_cache', False):
            self.stats['reqs_web'] += 1
            self.stats['time_resp'] += time.time() - time_
            self.last_request = time_
        else:
            self.stats['reqs_lowcache'] += 1
        if res.status_code not in [200, 404]:
            raise DataProviderError(
                'status code %d: %s' % (res.status_code, res.reason))
        return res

Example 3

Project: whatlastgenre
Source File: dataprovider.py
View license
    def login(self):
        """Login to What.CD."""

        def login():
            """Send a login request with username and password."""
            self.session.cookies.clear()
            self._request('https://what.cd/login.php',
                          {'username': username, 'password': password}, 'POST')
            assert self.session.cookies.get('session', None)

        username, password = self.get_credentials()
        try:
            if requests_cache:
                with self.session.cache_disabled():
                    login()
            else:
                login()
        except (requests.exceptions.TooManyRedirects, AssertionError):
            self.log.critical('WhatCD login failed')
            exit()
        # save session cookie to config
        cookie = base64.b64encode(self.session.cookies['session'])
        if not self.conf.has_section('whatcd'):
            self.conf.add_section('whatcd')
        self.conf.set('whatcd', 'session', cookie)
        self.conf.save()

Example 4

Project: kickass-get
Source File: kickass_parse.py
View license
def page_torrents_traverser(options):
    """ get total_counts number of torrents (in Torrent objects) on kat.cr
        maximum: 10000
    """
    # check whether arguments are valid
    command_line.check_args(options)

    root_url = network.choose_mirror(data.mirrors)


    if root_url == None:
        sys.exit("No working mirror. Try another time.")

    all_torrents = []

    # print 'Connecting ...'
    # status_code = network.check_connection(root_url)
    # if status_code != 0 or status_code != 404:
    #     print 'Connection success!'
    # else:
    #     raise requests.exceptions.ConnectionError('Connection failed, code {}. Try a proxy.'.format(str(status_code)))

    # enable custom searching
    if options.keyword != None:
        index_url = root_url + '/usearch' + '/{}'.format(' '.join(options.keyword))
        if options.category != 'all':
            index_url += ' category:{}'.format(options.category)
    else:
        index_url = root_url + categories[options.category]

    try:
        # base index url
        # change this to enable scrapping of torrents in searched results
        # eg: index_url = 'https://kat.cr/usearch/revenant/'
        # index_url = root_url + categories[options.category]

        start = timeit.default_timer()

        page_links = get_page_torrent_links(index_url, root_url)
        # per page torrents
        per_page_torrents = len(page_links)

        # total counts
        total_counts = options.counts
        assert (total_counts <= 10000), "Maximum Counts Exceeded! Maximum 10000 links."
        page_torrents = []

        if total_counts < per_page_torrents:
            page_torrents = get_page_torrents(page_links,options.workers,total_counts)
        else:
            # torrents on the first page
            page_torrents = get_page_torrents(page_links,options.workers,per_page_torrents)

        all_torrents += page_torrents

        # floor the number of pages
        pages = int(total_counts / len(page_torrents))

        for i in range(1, pages):
            page_links = get_page_torrent_links(index_url + '/' +
                                                str(i+1),root_url)
            page_torrents= get_page_torrents(page_links, options.workers, per_page_torrents)
            all_torrents += page_torrents

        # add the remaining torrents
        if (total_counts-len(all_torrents)) != 0:
            page_links = get_page_torrent_links(index_url + '/' +
                                                str(pages+1),root_url)
            page_torrents = get_page_torrents(page_links, options.workers, total_counts-len(all_torrents))
            all_torrents += page_torrents

        stop = timeit.default_timer()

        print 'Time taken: ', stop-start

    except UnicodeEncodeError:
        print 'UnicodeEncodeError. Prepare to dump current data.'

    except requests.exceptions.TooManyRedirects:
        print 'Too many redirects. Prepare to dump current data.'

    # except requests.exceptions.ConnectionError:
    #     print 'ConnectionError. Prepare to dump current data.'

    # transmission block
    # based on pirate-get:
    # https://github.com/vikstrous/pirate-get
    #
    if options.transmission:
        transmission_command = ['transmission-remote']
        if options.port:
            transmission_command.append(options.port)

        ret = subprocess.call(transmission_command + ['-l'],
                              stdout=subprocess.DEVNULL,
                              stderr=subprocess.DEVNULL)
        if ret != 0:
            print('Transmission is not running.')
            sys.exit(1)

        for torrent in all_torrents:
            url = torrent.magnet
            subprocess.call(transmission_command + ['--add', url])


    if options.csvfile or options.magnet2file or options.torrents:
        while True:
            if options.destpath:
                dest_path = options.destpath
            else:
                dest_path = raw_input("Enter the path to store file(s) to (q to quit): ").strip()
            if os.path.exists(dest_path):

                if options.csvfile:
                    write_torrents_to_file(all_torrents, dest_path) # csv data output to file

                if options.magnet2file:
                    write_torrents_to_file(all_torrents, dest_path, True) # write magnet links to file

                if options.torrents:
                    pool = Pool(processes=options.workers)

                    # for torrent in all_torrents:
                    #     torrent.save_to_file(dest_path)
                    #
                    # multiprocessing save torrents to files

                    pool.map(lambda x: x.save_to_file(dest_path), all_torrents)

                    pool.close()
                    pool.join()
                    break;

            elif dest_path == 'q':
                break;
            else:
                print termcolor.colored('Path does not exist. \n', 'red')

    else: # print torrents to screen
        print termcolor.colored('Torrents displayed: {}'.format(len(all_torrents)).center(50, '='), 'red', attrs=['bold'])

        # for i in range(len(all_torrents)):
        #     print termcolor.colored('Torrent {}: \n'.format(i), 'red', attrs=['blink'])
        #     print str(all_torrents[i]) + '\n'
        printer.print_torrents(all_torrents)


        index = command_line.readInt("Choose a torrent using its index (q to quit): ", 'q', 0, len(all_torrents))

        # choose magnet or torrent
        while index != None:
            usr_input = raw_input("Choose magnet and/or torrent file (magnet, torrent, both; q to quit): ").strip()
            if (usr_input == 'q'):
                break
            if (usr_input == 'magnet') or (usr_input == 'both'):
                print 'Magnet link: \n'
                print all_torrents[index].getMagnet() + '\n \n'
                print termcolor.colored('Enjoy the ride!', 'magenta', attrs=['blink'])
                #Open the magnet link
                os.system("open %s" % all_torrents[index].getMagnet())

                if (usr_input == 'both'):
                    usr_input = 'torrent'
                if (usr_input == 'magnet'):
                    break

            if (usr_input == 'torrent'):
                while True:
                    dest_path = raw_input("Enter the path to store file to (q to quit): ").strip()

                    if os.path.exists(dest_path):
                        all_torrents[index].save_to_file(dest_path)
                        break

                    elif dest_path == 'q':
                        break
                    else:
                        print termcolor.colored('Path does not exist. \n', 'red')


    return all_torrents

Example 5

Project: kickass-get
Source File: kickass_parse.py
View license
def page_torrents_traverser(options):
    """ get total_counts number of torrents (in Torrent objects) on kat.cr
        maximum: 10000
    """
    # check whether arguments are valid
    command_line.check_args(options)

    root_url = network.choose_mirror(data.mirrors)


    if root_url == None:
        sys.exit("No working mirror. Try another time.")

    all_torrents = []

    # print 'Connecting ...'
    # status_code = network.check_connection(root_url)
    # if status_code != 0 or status_code != 404:
    #     print 'Connection success!'
    # else:
    #     raise requests.exceptions.ConnectionError('Connection failed, code {}. Try a proxy.'.format(str(status_code)))

    # enable custom searching
    if options.keyword != None:
        index_url = root_url + '/usearch' + '/{}'.format(' '.join(options.keyword))
        if options.category != 'all':
            index_url += ' category:{}'.format(options.category)
    else:
        index_url = root_url + categories[options.category]

    try:
        # base index url
        # change this to enable scrapping of torrents in searched results
        # eg: index_url = 'https://kat.cr/usearch/revenant/'
        # index_url = root_url + categories[options.category]

        start = timeit.default_timer()

        page_links = get_page_torrent_links(index_url, root_url)
        # per page torrents
        per_page_torrents = len(page_links)

        # total counts
        total_counts = options.counts
        assert (total_counts <= 10000), "Maximum Counts Exceeded! Maximum 10000 links."
        page_torrents = []

        if total_counts < per_page_torrents:
            page_torrents = get_page_torrents(page_links,options.workers,total_counts)
        else:
            # torrents on the first page
            page_torrents = get_page_torrents(page_links,options.workers,per_page_torrents)

        all_torrents += page_torrents

        # floor the number of pages
        pages = int(total_counts / len(page_torrents))

        for i in range(1, pages):
            page_links = get_page_torrent_links(index_url + '/' +
                                                str(i+1),root_url)
            page_torrents= get_page_torrents(page_links, options.workers, per_page_torrents)
            all_torrents += page_torrents

        # add the remaining torrents
        if (total_counts-len(all_torrents)) != 0:
            page_links = get_page_torrent_links(index_url + '/' +
                                                str(pages+1),root_url)
            page_torrents = get_page_torrents(page_links, options.workers, total_counts-len(all_torrents))
            all_torrents += page_torrents

        stop = timeit.default_timer()

        print 'Time taken: ', stop-start

    except UnicodeEncodeError:
        print 'UnicodeEncodeError. Prepare to dump current data.'

    except requests.exceptions.TooManyRedirects:
        print 'Too many redirects. Prepare to dump current data.'

    # except requests.exceptions.ConnectionError:
    #     print 'ConnectionError. Prepare to dump current data.'

    # transmission block
    # based on pirate-get:
    # https://github.com/vikstrous/pirate-get
    #
    if options.transmission:
        transmission_command = ['transmission-remote']
        if options.port:
            transmission_command.append(options.port)

        ret = subprocess.call(transmission_command + ['-l'],
                              stdout=subprocess.DEVNULL,
                              stderr=subprocess.DEVNULL)
        if ret != 0:
            print('Transmission is not running.')
            sys.exit(1)

        for torrent in all_torrents:
            url = torrent.magnet
            subprocess.call(transmission_command + ['--add', url])


    if options.csvfile or options.magnet2file or options.torrents:
        while True:
            if options.destpath:
                dest_path = options.destpath
            else:
                dest_path = raw_input("Enter the path to store file(s) to (q to quit): ").strip()
            if os.path.exists(dest_path):

                if options.csvfile:
                    write_torrents_to_file(all_torrents, dest_path) # csv data output to file

                if options.magnet2file:
                    write_torrents_to_file(all_torrents, dest_path, True) # write magnet links to file

                if options.torrents:
                    pool = Pool(processes=options.workers)

                    # for torrent in all_torrents:
                    #     torrent.save_to_file(dest_path)
                    #
                    # multiprocessing save torrents to files

                    pool.map(lambda x: x.save_to_file(dest_path), all_torrents)

                    pool.close()
                    pool.join()
                    break;

            elif dest_path == 'q':
                break;
            else:
                print termcolor.colored('Path does not exist. \n', 'red')

    else: # print torrents to screen
        print termcolor.colored('Torrents displayed: {}'.format(len(all_torrents)).center(50, '='), 'red', attrs=['bold'])

        # for i in range(len(all_torrents)):
        #     print termcolor.colored('Torrent {}: \n'.format(i), 'red', attrs=['blink'])
        #     print str(all_torrents[i]) + '\n'
        printer.print_torrents(all_torrents)


        index = command_line.readInt("Choose a torrent using its index (q to quit): ", 'q', 0, len(all_torrents))

        # choose magnet or torrent
        while index != None:
            usr_input = raw_input("Choose magnet and/or torrent file (magnet, torrent, both; q to quit): ").strip()
            if (usr_input == 'q'):
                break
            if (usr_input == 'magnet') or (usr_input == 'both'):
                print 'Magnet link: \n'
                print all_torrents[index].getMagnet() + '\n \n'
                print termcolor.colored('Enjoy the ride!', 'magenta', attrs=['blink'])
                #Open the magnet link
                os.system("open %s" % all_torrents[index].getMagnet())

                if (usr_input == 'both'):
                    usr_input = 'torrent'
                if (usr_input == 'magnet'):
                    break

            if (usr_input == 'torrent'):
                while True:
                    dest_path = raw_input("Enter the path to store file to (q to quit): ").strip()

                    if os.path.exists(dest_path):
                        all_torrents[index].save_to_file(dest_path)
                        break

                    elif dest_path == 'q':
                        break
                    else:
                        print termcolor.colored('Path does not exist. \n', 'red')


    return all_torrents

Example 6

Project: unisubs
Source File: tests.py
View license
    def test_network_errors(self):
        self.check_network_error(ConnectionError(), 'Connection error')
        self.check_network_error(Timeout(), 'Request timeout')
        self.check_network_error(TooManyRedirects(), 'Too many redirects')