Here are the examples of the python api scrapy.signals.spider_closed taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
16 Examples
3
Source : memdebug.py
with MIT License
from autofelix
with MIT License
from autofelix
def from_crawler(cls, crawler):
if not crawler.settings.getbool('MEMDEBUG_ENABLED'):
raise NotConfigured
o = cls(crawler.stats)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider, reason):
3
Source : spiderstate.py
with MIT License
from autofelix
with MIT License
from autofelix
def from_crawler(cls, crawler):
jobdir = job_dir(crawler.settings)
if not jobdir:
raise NotConfigured
obj = cls(jobdir)
crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)
crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
return obj
def spider_closed(self, spider):
3
Source : statsmailer.py
with MIT License
from autofelix
with MIT License
from autofelix
def from_crawler(cls, crawler):
recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
if not recipients:
raise NotConfigured
mail = MailSender.from_settings(crawler.settings)
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider):
3
Source : __init__.py
with MIT License
from autofelix
with MIT License
from autofelix
def _set_crawler(self, crawler):
self.crawler = crawler
self.settings = crawler.settings
crawler.signals.connect(self.close, signals.spider_closed)
def start_requests(self):
3
Source : item_spider.py
with MIT License
from BreakBB
with MIT License
from BreakBB
def from_crawler(cls, crawler, *args, **kwargs) -> Spider:
spider = super(ItemSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def parse(self, response) -> Dict[str, Union[int, str]]:
3
Source : npc_spider.py
with MIT License
from BreakBB
with MIT License
from BreakBB
def from_crawler(cls, crawler, *args, **kwargs) -> Spider:
spider = super(NPCSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def parse(self, response) -> Dict[str, Union[int, str]]:
3
Source : object_spider.py
with MIT License
from BreakBB
with MIT License
from BreakBB
def from_crawler(cls, crawler, *args, **kwargs) -> Spider:
spider = super(ObjectSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def parse(self, response) -> Dict[str, Union[int, str]]:
3
Source : quest_spider.py
with MIT License
from BreakBB
with MIT License
from BreakBB
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(QuestSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def parse(self, response):
3
Source : quest_xp_spider.py
with MIT License
from BreakBB
with MIT License
from BreakBB
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(QuestXpSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def parse(self, response):
3
Source : status.py
with MIT License
from City-Bureau
with MIT License
from City-Bureau
def from_crawler(cls, crawler: Crawler):
"""Generate an extension from a crawler
:param crawler: Current scrapy crawler
"""
ext = cls(crawler)
crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
crawler.signals.connect(ext.spider_error, signal=signals.spider_error)
return ext
def spider_closed(self):
3
Source : middlewares.py
with MIT License
from clemfromspace
with MIT License
from clemfromspace
async def _from_crawler(cls, crawler):
"""Start the browser"""
middleware = cls()
middleware.browser = await launch({'logLevel': crawler.settings.get('LOG_LEVEL')})
crawler.signals.connect(middleware.spider_closed, signals.spider_closed)
return middleware
@classmethod
3
Source : middlewares.py
with MIT License
from Open-Speech-EkStep
with MIT License
from Open-Speech-EkStep
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
return s
def spider_closed(self, spider):
3
Source : middleware.py
with BSD 3-Clause "New" or "Revised" License
from scrapinghub
with BSD 3-Clause "New" or "Revised" License
from scrapinghub
def from_crawler(cls: Type[InjectionMiddlewareTV], crawler: Crawler) -> InjectionMiddlewareTV:
o = cls(crawler)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider: Spider) -> None:
3
Source : middlewares.py
with MIT License
from vyvojer
with MIT License
from vyvojer
def from_crawler(cls, crawler: Crawler):
drivers_number = crawler.settings.getint("SELENIUM_POOL_SIZE", 1)
middleware = cls(drivers_number)
crawler.signals.connect(middleware.spider_closed, signals.spider_closed)
return middleware
def process_request(self, request: SeleniumRequest, spider):
0
Source : engine.py
with MIT License
from autofelix
with MIT License
from autofelix
def close_spider(self, spider, reason='cancelled'):
"""Close (cancel) spider and clear all its outstanding requests"""
slot = self.slot
if slot.closing:
return slot.closing
logger.info("Closing spider (%(reason)s)",
{'reason': reason},
extra={'spider': spider})
dfd = slot.close()
def log_failure(msg):
def errback(failure):
logger.error(
msg,
exc_info=failure_to_exc_info(failure),
extra={'spider': spider}
)
return errback
dfd.addBoth(lambda _: self.downloader.close())
dfd.addErrback(log_failure('Downloader close failure'))
dfd.addBoth(lambda _: self.scraper.close_spider(spider))
dfd.addErrback(log_failure('Scraper close failure'))
dfd.addBoth(lambda _: slot.scheduler.close(reason))
dfd.addErrback(log_failure('Scheduler close failure'))
dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
signal=signals.spider_closed, spider=spider, reason=reason))
dfd.addErrback(log_failure('Error while sending spider_close signal'))
dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
dfd.addErrback(log_failure('Stats close failure'))
dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)",
{'reason': reason},
extra={'spider': spider}))
dfd.addBoth(lambda _: setattr(self, 'slot', None))
dfd.addErrback(log_failure('Error while unassigning slot'))
dfd.addBoth(lambda _: setattr(self, 'spider', None))
dfd.addErrback(log_failure('Error while unassigning spider'))
dfd.addBoth(lambda _: self._spider_closed_callback(spider))
return dfd
def _close_all_spiders(self):
0
Source : middlewares.py
with GNU General Public License v2.0
from pip-uninstaller-python
with GNU General Public License v2.0
from pip-uninstaller-python
def from_crawler(cls, crawler):
s = cls() # 实例化
crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
return s
def process_request(self, request, spider):