Here are the examples of the python api scrapy.signalmanager.dispatcher.connect taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
3 Examples
0
Source : test_downloader_mw_process_request_exception_nacked.py
with MIT License
from groupbwt
with MIT License
from groupbwt
def test_crawler_successfully(self, rabbit_setup: PikaBlockingConnection, crawler: CrawlerProcess):
successfully_handled = False
def nack_callback(rmq_message: BaseRmqMessage):
logging.info('NACK_CALLBACK')
nonlocal successfully_handled
successfully_handled = True
crawler.stop()
def ack_callback(rmq_message: BaseRmqMessage):
logging.info('ACK_CALLBACK')
crawler.stop()
dispatcher.connect(ack_callback, CustomSignals.message_ack)
dispatcher.connect(nack_callback, CustomSignals.message_nack)
crawler.crawl(MySpider)
crawler.start()
assert successfully_handled
queue = rabbit_setup.rabbit_channel.queue_declare(queue=QUEUE_NAME, durable=True)
assert queue.method.message_count == 1
0
Source : test_filtered_request_nacked.py
with MIT License
from groupbwt
with MIT License
from groupbwt
def test_crawler_successfully(self, rabbit_setup: PikaBlockingConnection, crawler: CrawlerProcess):
count_ack = 0
count_nack = 0
def nack_callback(rmq_message: BaseRmqMessage):
logging.info('NACK_CALLBACK')
nonlocal count_nack
count_nack += 1
crawler.stop()
def ack_callback(rmq_message: BaseRmqMessage):
logging.info('ACK_CALLBACK')
nonlocal count_ack
count_ack += 1
if count_ack == 2:
crawler.stop()
dispatcher.connect(ack_callback, CustomSignals.message_ack)
dispatcher.connect(nack_callback, CustomSignals.message_nack)
crawler.crawl(MySpider)
crawler.start()
assert count_ack == 2
assert count_nack == 0
queue = rabbit_setup.rabbit_channel.queue_declare(queue=QUEUE_NAME, durable=True)
assert queue.method.message_count == 0
0
Source : test_spider_errback_successfully_acked.py
with MIT License
from groupbwt
with MIT License
from groupbwt
def test_crawler_successfully(self, rabbit_setup: PikaBlockingConnection, crawler: CrawlerProcess):
successfully_handled = False
def nack_callback(rmq_message: BaseRmqMessage):
logging.info('NACK_CALLBACK')
crawler.stop()
def ack_callback(rmq_message: BaseRmqMessage):
logging.info('ACK_CALLBACK')
nonlocal successfully_handled
successfully_handled = True
crawler.stop()
dispatcher.connect(ack_callback, CustomSignals.message_ack)
dispatcher.connect(nack_callback, CustomSignals.message_nack)
crawler.crawl(MySpider)
crawler.start()
assert successfully_handled
queue = rabbit_setup.rabbit_channel.queue_declare(queue=QUEUE_NAME, durable=True)
assert queue.method.message_count == 0