scrapy.utils.conf.build_component_list

Here are the examples of the python api scrapy.utils.conf.build_component_list taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

11 Examples 7

Example 1

Project: scrapy
Source File: test_utils_conf.py
View license
    def test_backwards_compatible_build_dict(self):
        base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
        custom = {'two': None, 'three': 8, 'four': 4}
        self.assertEqual(build_component_list(base, custom,
                                              convert=lambda x: x),
                         ['one', 'four', 'five', 'three'])

Example 2

Project: scrapy
Source File: test_utils_conf.py
View license
    def test_duplicate_components_in_basesettings(self):
        # Higher priority takes precedence
        duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
        duplicate_bs.set('ONE', 4, priority=10)
        self.assertEqual(build_component_list(duplicate_bs,
                                              convert=lambda x: x.lower()),
                         ['two', 'one'])
        duplicate_bs.set('one', duplicate_bs['one'], priority=20)
        self.assertEqual(build_component_list(duplicate_bs,
                                              convert=lambda x: x.lower()),
                         ['one', 'two'])
        # Same priority raises ValueError
        duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
        self.assertRaises(ValueError, build_component_list, duplicate_bs,
                          convert=lambda x: x.lower())

Example 3

Project: scrapy
Source File: check.py
View license
    def run(self, args, opts):
        # load contracts
        contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
        conman = ContractsManager(load_object(c) for c in contracts)
        runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
        result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)

        # contract requests
        contract_reqs = defaultdict(list)

        spider_loader = self.crawler_process.spider_loader

        for spidername in args or spider_loader.list():
            spidercls = spider_loader.load(spidername)
            spidercls.start_requests = lambda s: conman.from_spider(s, result)

            tested_methods = conman.tested_methods_from_spidercls(spidercls)
            if opts.list:
                for method in tested_methods:
                    contract_reqs[spidercls.name].append(method)
            elif tested_methods:
                self.crawler_process.crawl(spidercls)

        # start checks
        if opts.list:
            for spider, methods in sorted(contract_reqs.items()):
                if not methods and not opts.verbose:
                    continue
                print(spider)
                for method in sorted(methods):
                    print('  * %s' % method)
        else:
            start = time.time()
            self.crawler_process.start()
            stop = time.time()

            result.printErrors()
            result.printSummary(start, stop)
            self.exitcode = int(not result.wasSuccessful())

Example 4

Project: scrapy
Source File: middleware.py
View license
    @classmethod
    def _get_mwlist_from_settings(cls, settings):
        return build_component_list(
            settings.getwithbase('DOWNLOADER_MIDDLEWARES'))

Example 5

Project: scrapy
Source File: spidermw.py
View license
    @classmethod
    def _get_mwlist_from_settings(cls, settings):
        return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))

Example 6

Project: scrapy
Source File: extension.py
View license
    @classmethod
    def _get_mwlist_from_settings(cls, settings):
        return build_component_list(settings.getwithbase('EXTENSIONS'))

Example 7

Project: scrapy
Source File: __init__.py
View license
    @classmethod
    def _get_mwlist_from_settings(cls, settings):
        return build_component_list(settings.getwithbase('ITEM_PIPELINES'))

Example 8

Project: scrapy
Source File: test_utils_conf.py
View license
    def test_build_dict(self):
        d = {'one': 1, 'two': None, 'three': 8, 'four': 4}
        self.assertEqual(build_component_list(d, convert=lambda x: x),
                         ['one', 'four', 'three'])

Example 9

Project: scrapy
Source File: test_utils_conf.py
View license
    def test_return_list(self):
        custom = ['a', 'b', 'c']
        self.assertEqual(build_component_list(None, custom,
                                              convert=lambda x: x),
                         custom)

Example 10

Project: scrapy
Source File: test_utils_conf.py
View license
    def test_map_dict(self):
        custom = {'one': 1, 'two': 2, 'three': 3}
        self.assertEqual(build_component_list({}, custom,
                                              convert=lambda x: x.upper()),
                         ['ONE', 'TWO', 'THREE'])

Example 11

Project: scrapy
Source File: test_utils_conf.py
View license
    def test_map_list(self):
        custom = ['a', 'b', 'c']
        self.assertEqual(build_component_list(None, custom,
                                              lambda x: x.upper()),
                         ['A', 'B', 'C'])