scraper.scraper_test.EventSpider

Here are the examples of the python api scraper.scraper_test.EventSpider taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

5 Examples 7

Example 1

Project: django-dynamic-scraper Source File: scraper_run_test.py
    def test_testmode(self):
        kwargs = {
            'id': 1,
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()
        
        self.assertEqual(len(Event.objects.all()), 0)

Example 2

Project: django-dynamic-scraper Source File: scraper_run_test.py
    def test_task_run_type(self):
        self.event_website.url = os.path.join(self.SERVER_URL, 'not_existing_site/event_main.html')
        self.event_website.save()
        
        kwargs = {
            'id': 1,
            'do_action': 'yes',
            'run_type': 'TASK',
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()
        
        self.assertEqual(spider.scheduler_runtime.num_zero_actions, 1)
        
        spider.log("Test message", logging.ERROR)
        self.assertGreater(Log.objects.count(), 0)

Example 3

Project: django-dynamic-scraper Source File: scraper_run_test.py
    def test_no_task_run_type(self):
        self.event_website.url = os.path.join(self.SERVER_URL, 'not_existing_site/event_main.html')
        self.event_website.save()
        
        kwargs = {
            'id': 1,
            'do_action': 'yes',
            'run_type': 'SHELL',
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()
        
        self.assertEqual(spider.scheduler_runtime.num_zero_actions, 0)
        
        spider.log("Test message", logging.ERROR)
        self.assertEqual(Log.objects.count(), 0)

Example 4

Project: django-dynamic-scraper Source File: scraper_run_test.py
    def test_runtime_config_max_items_read(self):        
        kwargs = {
            'id': 1,
            'do_action': 'yes',
            'run_type': 'SHELL',
            'max_items_read': '3',
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()

        self.assertEqual(len(Event.objects.all()), 3)

Example 5

Project: django-dynamic-scraper Source File: scraper_run_test.py
    def test_runtime_config_max_items_save(self):        
        kwargs = {
            'id': 1,
            'do_action': 'yes',
            'run_type': 'SHELL',
            'max_items_save': '3',
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()

        self.assertEqual(len(Event.objects.all()), 3)