scrapy.signals.spider_closed

Here are the examples of the python api scrapy.signals.spider_closed taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

16 Examples 7

3 Source : memdebug.py
with MIT License
from autofelix

    def from_crawler(cls, crawler):
        if not crawler.settings.getbool('MEMDEBUG_ENABLED'):
            raise NotConfigured
        o = cls(crawler.stats)
        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
        return o

    def spider_closed(self, spider, reason):

3 Source : spiderstate.py
with MIT License
from autofelix

    def from_crawler(cls, crawler):
        jobdir = job_dir(crawler.settings)
        if not jobdir:
            raise NotConfigured

        obj = cls(jobdir)
        crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)
        crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
        return obj

    def spider_closed(self, spider):

3 Source : statsmailer.py
with MIT License
from autofelix

    def from_crawler(cls, crawler):
        recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
        if not recipients:
            raise NotConfigured
        mail = MailSender.from_settings(crawler.settings)
        o = cls(crawler.stats, recipients, mail)
        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
        return o
        
    def spider_closed(self, spider):

3 Source : __init__.py
with MIT License
from autofelix

    def _set_crawler(self, crawler):
        self.crawler = crawler
        self.settings = crawler.settings
        crawler.signals.connect(self.close, signals.spider_closed)

    def start_requests(self):

3 Source : item_spider.py
with MIT License
from BreakBB

    def from_crawler(cls, crawler, *args, **kwargs) -> Spider:
        spider = super(ItemSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def parse(self, response) -> Dict[str, Union[int, str]]:

3 Source : npc_spider.py
with MIT License
from BreakBB

    def from_crawler(cls, crawler, *args, **kwargs) -> Spider:
        spider = super(NPCSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def parse(self, response) -> Dict[str, Union[int, str]]:

3 Source : object_spider.py
with MIT License
from BreakBB

    def from_crawler(cls, crawler, *args, **kwargs) -> Spider:
        spider = super(ObjectSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def parse(self, response) -> Dict[str, Union[int, str]]:

3 Source : quest_spider.py
with MIT License
from BreakBB

    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(QuestSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def parse(self, response):

3 Source : quest_xp_spider.py
with MIT License
from BreakBB

    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(QuestXpSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def parse(self, response):

3 Source : status.py
with MIT License
from City-Bureau

    def from_crawler(cls, crawler: Crawler):
        """Generate an extension from a crawler

        :param crawler: Current scrapy crawler
        """
        ext = cls(crawler)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
        crawler.signals.connect(ext.spider_error, signal=signals.spider_error)
        return ext

    def spider_closed(self):

3 Source : middlewares.py
with MIT License
from clemfromspace

    async def _from_crawler(cls, crawler):
        """Start the browser"""

        middleware = cls()
        middleware.browser = await launch({'logLevel': crawler.settings.get('LOG_LEVEL')})
        crawler.signals.connect(middleware.spider_closed, signals.spider_closed)

        return middleware

    @classmethod

3 Source : middlewares.py
with MIT License
from Open-Speech-EkStep

    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def spider_closed(self, spider):

3 Source : middleware.py
with BSD 3-Clause "New" or "Revised" License
from scrapinghub

    def from_crawler(cls: Type[InjectionMiddlewareTV], crawler: Crawler) -> InjectionMiddlewareTV:
        o = cls(crawler)
        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
        return o

    def spider_closed(self, spider: Spider) -> None:

3 Source : middlewares.py
with MIT License
from vyvojer

    def from_crawler(cls, crawler: Crawler):
        drivers_number = crawler.settings.getint("SELENIUM_POOL_SIZE", 1)
        middleware = cls(drivers_number)
        crawler.signals.connect(middleware.spider_closed, signals.spider_closed)
        return middleware

    def process_request(self, request: SeleniumRequest, spider):

0 Source : engine.py
with MIT License
from autofelix

    def close_spider(self, spider, reason='cancelled'):
        """Close (cancel) spider and clear all its outstanding requests"""

        slot = self.slot
        if slot.closing:
            return slot.closing
        logger.info("Closing spider (%(reason)s)",
                    {'reason': reason},
                    extra={'spider': spider})

        dfd = slot.close()

        def log_failure(msg):
            def errback(failure):
                logger.error(
                    msg,
                    exc_info=failure_to_exc_info(failure),
                    extra={'spider': spider}
                )
            return errback

        dfd.addBoth(lambda _: self.downloader.close())
        dfd.addErrback(log_failure('Downloader close failure'))

        dfd.addBoth(lambda _: self.scraper.close_spider(spider))
        dfd.addErrback(log_failure('Scraper close failure'))

        dfd.addBoth(lambda _: slot.scheduler.close(reason))
        dfd.addErrback(log_failure('Scheduler close failure'))

        dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
            signal=signals.spider_closed, spider=spider, reason=reason))
        dfd.addErrback(log_failure('Error while sending spider_close signal'))

        dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
        dfd.addErrback(log_failure('Stats close failure'))

        dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)",
                                          {'reason': reason},
                                          extra={'spider': spider}))

        dfd.addBoth(lambda _: setattr(self, 'slot', None))
        dfd.addErrback(log_failure('Error while unassigning slot'))

        dfd.addBoth(lambda _: setattr(self, 'spider', None))
        dfd.addErrback(log_failure('Error while unassigning spider'))

        dfd.addBoth(lambda _: self._spider_closed_callback(spider))

        return dfd

    def _close_all_spiders(self):

0 Source : middlewares.py
with GNU General Public License v2.0
from pip-uninstaller-python

    def from_crawler(cls, crawler):
        s = cls()  # 实例化
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s
    def process_request(self, request, spider):