Here are the examples of the python api scrapy.signals.engine_stopped taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

4 Examples 7

Example 1

Project: scrapy
Source File:
View license
    def __init__(self, crawler):
        self._crawler = crawler
        self._schemes = {}  # stores acceptable schemes on instancing
        self._handlers = {}  # stores instanced handlers for schemes
        self._notconfigured = {}  # remembers failed handlers
        handlers = without_none_values(
        for scheme, clspath in six.iteritems(handlers):
            self._schemes[scheme] = clspath

        crawler.signals.connect(self._close, signals.engine_stopped)

Example 2

Project: scrapyrt
Source File:
View license
def monkey_patch_and_connect_log_observer(crawler, log_observer):
    """Ugly hack to close log file.

    Monkey patch log_observer.stop method to close file each time
    log observer is closed.
    I prefer this to be fixed in Scrapy itself, but as
    Scrapy is going to switch to standart python logging soon
    this change wouldn't be accepted in preference of merging
    new logging sooner.

    def stop_and_close_log_file(self):

    log_observer.__stop = log_observer.stop
    log_observer.stop = types.MethodType(
        stop_and_close_log_file, log_observer)
    crawler.signals.connect(log_observer.stop, signals.engine_stopped)

Example 3

Project: scrapy
Source File:
View license
    def _finish_stopping_engine(self):
        yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)

Example 4

Project: scrapy
Source File:
View license
    def __init__(self, spidercls, settings=None):
        if isinstance(settings, dict) or settings is None:
            settings = Settings(settings)

        self.spidercls = spidercls
        self.settings = settings.copy()

        self.signals = SignalManager(self)
        self.stats = load_object(self.settings['STATS_CLASS'])(self)

        handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
        # lambda is assigned to Crawler attribute because this way it is not
        # garbage collected after leaving __init__ scope
        self.__remove_handler = lambda: logging.root.removeHandler(handler)
        self.signals.connect(self.__remove_handler, signals.engine_stopped)

        lf_cls = load_object(self.settings['LOG_FORMATTER'])
        self.logformatter = lf_cls.from_crawler(self)
        self.extensions = ExtensionManager.from_crawler(self)

        self.crawling = False
        self.spider = None
        self.engine = None