scrapy.signals.engine_stopped

Here are the examples of the python api scrapy.signals.engine_stopped taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

4 Examples 7

Example 1

Project: scrapy
Source File: __init__.py
View license
    def __init__(self, crawler):
        self._crawler = crawler
        self._schemes = {}  # stores acceptable schemes on instancing
        self._handlers = {}  # stores instanced handlers for schemes
        self._notconfigured = {}  # remembers failed handlers
        handlers = without_none_values(
            crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
        for scheme, clspath in six.iteritems(handlers):
            self._schemes[scheme] = clspath

        crawler.signals.connect(self._close, signals.engine_stopped)

Example 2

Project: scrapyrt
Source File: core.py
View license
def monkey_patch_and_connect_log_observer(crawler, log_observer):
    """Ugly hack to close log file.

    Monkey patch log_observer.stop method to close file each time
    log observer is closed.
    I prefer this to be fixed in Scrapy itself, but as
    Scrapy is going to switch to standart python logging soon
    https://github.com/scrapy/scrapy/pull/1060
    this change wouldn't be accepted in preference of merging
    new logging sooner.

    """
    def stop_and_close_log_file(self):
        self.__stop()
        self.write.__self__.close()

    log_observer.__stop = log_observer.stop
    log_observer.stop = types.MethodType(
        stop_and_close_log_file, log_observer)
    crawler.signals.connect(log_observer.stop, signals.engine_stopped)

Example 3

Project: scrapy
Source File: engine.py
View license
    @defer.inlineCallbacks
    def _finish_stopping_engine(self):
        yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
        self._closewait.callback(None)

Example 4

Project: scrapy
Source File: crawler.py
View license
    def __init__(self, spidercls, settings=None):
        if isinstance(settings, dict) or settings is None:
            settings = Settings(settings)

        self.spidercls = spidercls
        self.settings = settings.copy()
        self.spidercls.update_settings(self.settings)

        self.signals = SignalManager(self)
        self.stats = load_object(self.settings['STATS_CLASS'])(self)

        handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
        logging.root.addHandler(handler)
        # lambda is assigned to Crawler attribute because this way it is not
        # garbage collected after leaving __init__ scope
        self.__remove_handler = lambda: logging.root.removeHandler(handler)
        self.signals.connect(self.__remove_handler, signals.engine_stopped)

        lf_cls = load_object(self.settings['LOG_FORMATTER'])
        self.logformatter = lf_cls.from_crawler(self)
        self.extensions = ExtensionManager.from_crawler(self)

        self.settings.freeze()
        self.crawling = False
        self.spider = None
        self.engine = None