Here are the examples of the python api frontera.contrib.scrapy.settings_adapter.ScrapySettingsAdapter taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
3 Examples
3
Example 1
def __init__(self, crawler, manager=None):
self.crawler = crawler
self.stats_manager = StatsManager(crawler.stats)
self._pending_requests = deque()
self.redirect_enabled = crawler.settings.get('REDIRECT_ENABLED')
settings = ScrapySettingsAdapter(crawler.settings)
self.frontier = ScrapyFrontierManager(settings, manager)
self._delay_on_empty = self.frontier.manager.settings.get('DELAY_ON_EMPTY')
self._delay_next_call = 0.0
self.logger = getLogger('frontera.contrib.scrapy.schedulers.FronteraScheduler')
0
Example 2
Project: frontera Source File: test_scrapy_settings_adapter.py
def test_fallsback_to_crawler_settings():
settings = ScrapySettingsAdapter({'DELAY_ON_EMPTY': 10})
assert settings.get('DELAY_ON_EMPTY') == 10
0
Example 3
Project: frontera Source File: test_scrapy_settings_adapter.py
def test_frontera_settings_have_precedence_over_crawler_settings():
crawler_settings = {'MAX_REQUESTS': 10,
'FRONTERA_SETTINGS': 'tests.scrapy_spider.frontera.settings'}
settings = ScrapySettingsAdapter(crawler_settings)
assert settings.get('MAX_REQUESTS') == 5