scraper.models.Event.objects.all

Here are the examples of the python api scraper.models.Event.objects.all taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

58 Examples 7

Example 1

View license
    def test_x_path_type_keep(self):
        self.event.url = 'http://localhost:8010/static/site_for_checker/event1.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 2

View license
    def test_x_path_type_keep_double(self):
        self.setUpWithSecondChecker()
        self.event.url = 'http://localhost:8010/static/site_for_checker/event1.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 3

View license
    def test_x_path_type_blank_result_field_keep(self):
        self.scraper.checker_x_path_result = ''
        self.event.url = 'http://localhost:8010/static/site_for_checker/event1.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 4

View license
    def test_x_path_type_404_delete(self):
        self.event.url = 'http://localhost:8010/static/site_for_checker/event_which_is_not_there.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 5

View license
    def test_x_path_type_404_delete_with_zero_actions(self):
        self.event.url = 'http://localhost:8010/static/site_for_checker/event_which_is_not_there.html'
        self.event.save()
        
        self.event.checker_runtime.num_zero_actions = 3
        self.event.checker_runtime.save()
        
        kwargs = {
            'id': 1,
            'do_action': 'yes',
            'run_type': 'TASK',
        }
        checker = EventChecker(**kwargs)
        self.process.crawl(checker, **kwargs)
        self.process.start()
        
        self.assertEqual(len(Event.objects.all()), 1)

Example 6

View license
    def test_x_path_type_x_path_delete(self):
        self.event.url = 'http://localhost:8010/static/site_for_checker/event2.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 7

View license
    def test_x_path_type_x_path_first_delete_double(self):
        self.setUpWithSecondChecker()
        self.event.url = 'http://localhost:8010/static/site_for_checker/event2.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 8

View license
    def test_x_path_type_x_path_second_delete_double(self):
        self.setUpWithSecondChecker()
        self.checker.checker_x_path = '//div[@class="oh_my_wrong_xpath_for_delete"]/div/text()'
        self.checker.save()
        self.event.url = 'http://localhost:8010/static/site_for_checker/event2.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 9

View license
    def test_x_path_type_blank_result_field_x_path_delete(self):
        self.scraper.checker_x_path_result = ''
        self.event.url = 'http://localhost:8010/static/site_for_checker/event2.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 10

View license
    def _run_img_test_with_dirs(self, img_dirs):
        img_paths = self._create_imgs_in_dirs(img_dirs)

        self.se_desc.mandatory = True
        self.se_desc.save()
        self.soa_desc.attr_type = 'I'
        self.soa_desc.save()
        
        self.event.url = 'http://localhost:8010/static/site_for_checker/event_which_is_not_there.html'
        self.event.save()

        for path in img_paths:
            self.assertTrue(os.path.exists(path))
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)
        for path in img_paths:
            self.assertFalse(os.path.exists(path))

Example 11

View license
    def test_404_type_404_delete(self):
        self.checker.checker_type = '4'
        self.checker.save()
        self.event.url = 'http://localhost:8010/static/site_for_checker/event_which_is_not_there.html'
        self.event.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 12

View license
    def test_404_type_x_path_delete(self):
        self.checker.checker_type = '4'
        self.checker.save()
        
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 13

View license
    def test_config_empty_append_str(self):
        self.setUpPaginationRangeFunctTypeScraper()
        self.event_website.url = os.path.join(self.SERVER_URL, 'site_generic/event_main.html')
        self.event_website.save()
        self.scraper.pagination_append_str = ''
        self.scraper.save()

        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 4)

Example 14

View license
    def test_config_append_str_without_page(self):
        self.setUpPaginationRangeFunctTypeScraper()
        self.scraper.pagination_append_str = '.html'
        self.scraper.save()

        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 4)

Example 15

View license
    def test_range_funct_type_one_page(self):
        self.setUpPaginationRangeFunctTypeScraper()
        self.scraper.pagination_page_replace = '1,2'
        self.scraper.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 4)

Example 16

View license
    def test_missing_img_when_img_field_not_mandatory(self):
        self.se_desc.mandatory = False
        self.se_desc.save()
        self.soa_desc.attr_type = 'I'
        self.soa_desc.save()
        
        self.event_website.url = os.path.join(self.SERVER_URL, 'site_with_imgs/event_main2.html')
        self.event_website.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 1)

Example 17

View license
    def test_json_array(self):
        self.setUpScraperJSONDefaultScraper()
        
        self.event_website.url = os.path.join(self.SERVER_URL, 'site_with_json_content_type/event_main_array.json')
        self.event_website.save()
        
        self.se_base.x_path = '$'
        self.se_base.save()
        
        self.run_event_spider(1)
        #logging.info(unicode(Event.objects.all()))
        self.assertEqual(len(Event.objects.all()), 3)

Example 18

View license
    def test_checker_x_path_type_x_path_delete(self):
        self.setUpScraperJSONDefaultScraper()
        self.extraSetUpHTMLChecker()
        self.assertEqual(len(Event.objects.all()), 1)
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 19

View license
    def test_checker_x_path_type_x_path_no_delete(self):
        self.setUpScraperJSONDefaultScraper()
        self.extraSetUpHTMLChecker()
        self.checker.checker_x_path = '//div[@class="this_is_the_wrong_xpath"]/div/text()'
        self.checker.save()
        self.assertEqual(len(Event.objects.all()), 1)
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 20

View license
    def test_json_checker_x_path_type_x_path_delete(self):
        self.setUpScraperJSONDefaultScraper()
        self.extraSetUpJSONChecker()
        self.assertEqual(len(Event.objects.all()), 1)
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 21

View license
    def test_json_checker_x_path_type_x_path_no_delete(self):
        self.setUpScraperJSONDefaultScraper()
        self.extraSetUpJSONChecker()
        self.checker.checker_x_path = 'this_is_the_wrong_xpath'
        self.checker.save()
        self.assertEqual(len(Event.objects.all()), 1)
        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 22

View license
    def test_only_main_page_scrapyjs_main_page(self):
        self.setUpScraperJSDockerScraper()
        self.event_website.url = os.path.join('http://10.0.2.2:8010/static/', 'site_with_js/event_main.html')
        self.event_website.save()
        self.rpt_dp1.render_javascript = False
        self.rpt_dp1.save()

        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 2)
        self.assertEqual(len(Event.objects.filter(description='Event 1 description')), 1)
        self.assertEqual(len(Event.objects.filter(description='Event 1 JS description')), 0)

Example 23

View license
    def test_default_no_scrapyjs_checker_delete(self):
        self.setUpScraperJSDefaultChecker()
        self.checker.checker_x_path_result = 'Event not found'
        self.checker.save()

        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 24

View license
    def test_default_no_scrapyjs_checker_no_delete(self):
        self.setUpScraperJSDefaultChecker()
        self.checker.checker_x_path_result = 'Event JS not found'
        self.checker.save()

        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 25

View license
    def test_activated_scrapyjs_checker_delete(self):
        self.setUpScraperJSDockerChecker()
        self.checker.checker_x_path_result = 'Event JS not found'
        self.checker.save()

        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 0)

Example 26

View license
    def test_activated_scrapyjs_checker_no_delete(self):
        self.setUpScraperJSDockerChecker()
        self.checker.checker_x_path_result = 'Event not found'
        self.checker.save()

        self.run_event_checker(1)
        self.assertEqual(len(Event.objects.all()), 1)

Example 27

View license
    def test_replace_processor_wrong_x_path(self):
        self.setUpProcessorTest()
        self.se_title.x_path = '/div[@class="class_which_is_not_there"]/text()'
        self.se_title.processors = 'replace'
        self.se_title.proc_ctxt = "'replace': 'This text is a replacement'"
        self.se_title.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 0)

Example 28

View license
    def test_replace_processor_correct_x_path(self):
        self.setUpProcessorTest()
        self.se_title.processors = 'replace'
        self.se_title.proc_ctxt = "'replace': 'This text is a replacement'"
        self.se_title.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)

Example 29

View license
    def test_replace_processor_unicode_replace(self):
        self.setUpProcessorTest()
        self.se_title.processors = 'replace'
        self.se_title.proc_ctxt = "'replace': 'Replacement with beautiful unicode ❤ ☀ ★ ☂ ☻ ♞ ☯ ☭ ☢'"
        self.se_title.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)

Example 30

View license
    def test_static_processor_wrong_x_path(self):
        self.setUpProcessorTest()
        self.se_title.x_path = '/div[@class="class_which_is_not_there"]/text()'
        self.se_title.processors = 'static'
        self.se_title.proc_ctxt = "'static': 'This text should always be there'"
        self.se_title.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)

Example 31

View license
    def test_static_processor_correct_x_path(self):
        self.setUpProcessorTest()
        self.se_title.processors = 'static'
        self.se_title.proc_ctxt = "'static': 'This text should always be there'"
        self.se_title.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)

Example 32

View license
    def test_reg_exp(self):
        self.se_desc.reg_exp = '(\d{6})'
        self.se_desc.save()
        self.event_website.url = os.path.join(self.SERVER_URL, 'site_with_reg_exp/event_main.html')
        self.event_website.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)
        self.assertEqual(Event.objects.get(title='Event 1').description, '563423')

Example 33

View license
    def test_processor_with_detail_page_url_placeholder(self):
        self.setUpProcessorTestWithDetailPageUrlPlaceholder()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 1)
        self.assertEqual(
            Event.objects.get(title='site_with_processor').url,
            'http://localhost:8010/static/site_with_processor/event1.html')

Example 34

View license
    def test_with_request_type_post(self):
        self.setUpWithRequestTypeMethodScraper('post')
        self.rpt_mp.method = "POST"
        self.rpt_mp.save()

        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 4)

Example 35

View license
    def test_with_custom_header(self):
        self.setUpWithHeaderBodyDataScraper('header')
        self.rpt_mp.headers = '{ "Referer": "http://comingfromhere.io" }'
        self.rpt_mp.save()

        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 4)

Example 36

View license
    def test_with_custom_body(self):
        self.setUpWithHeaderBodyDataScraper('body')
        self.rpt_mp.method = "POST"
        self.rpt_mp.body = 'This is the HTTP request body content.'
        self.rpt_mp.save()

        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 4)

Example 37

View license
    def test_with_form_data_simple(self):
        self.setUpWithFormDataScraper('simple')
        self.rpt_mp.form_data = '{ "simple": "SIMPLE_VALUE" }'
        self.rpt_mp.save()
        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 4)

Example 38

View license
    def test_standard_update_field(self):
        self.soa_title.attr_type = 'T'
        self.soa_title.save()
        
        self.run_event_spider(1)
        self.assertEqual(len(Event.objects.all()), 4)

Example 39

View license
    def test_save_to_db(self):
        self.soa_desc.save_to_db = False
        self.soa_desc.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 4)
        self.assertEqual(Event.objects.filter(description='Event 1 description').count(), 0)

Example 40

View license
    def test_save_to_db_non_model_attribute(self):
        self.soa_desc.name='non_model_attribute'
        self.soa_desc.save_to_db = False
        self.soa_desc.mandatory = True
        self.soa_desc.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 4)

Example 41

View license
    def test_testmode(self):
        kwargs = {
            'id': 1,
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()
        
        self.assertEqual(len(Event.objects.all()), 0)

Example 42

View license
    def test_xml_content_type(self):
        self.se_base.x_path = '//item'
        self.se_base.save()
        self.se_title.x_path = 'title/text()'
        self.se_title.save()
        self.se_url.x_path = 'link/text()'
        self.se_url.save()
        self.se_desc.x_path = 'description/text()'
        self.se_desc.from_detail_page = False
        self.se_desc.save()
        
        self.scraper.content_type = 'X'
        self.scraper.save()
        
        self.event_website.url = os.path.join(self.SERVER_URL, 'site_with_xml_content_type/event_main.xml')
        self.event_website.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 3)

Example 43

View license
    def test_runtime_config_max_items_read(self):        
        kwargs = {
            'id': 1,
            'do_action': 'yes',
            'run_type': 'SHELL',
            'max_items_read': '3',
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()

        self.assertEqual(len(Event.objects.all()), 3)

Example 44

View license
    def test_runtime_config_max_items_save(self):        
        kwargs = {
            'id': 1,
            'do_action': 'yes',
            'run_type': 'SHELL',
            'max_items_save': '3',
        }
        spider = EventSpider(**kwargs)
        self.process.crawl(spider, **kwargs)
        self.process.start()

        self.assertEqual(len(Event.objects.all()), 3)

Example 45

View license
    def test_max_items_read(self):
        self.scraper.max_items_read = 3
        self.scraper.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 3)

Example 46

View license
    def test_max_items_save(self):
        self.scraper.max_items_read = 3
        self.scraper.max_items_save = 2
        self.scraper.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)

Example 47

View license
    def test_missing_mandatory(self):
        self.se_desc.mandatory = True
        self.se_desc.save()
        self.event_website.url = os.path.join(self.SERVER_URL, 'site_missing_mandatory/event_main.html')
        self.event_website.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)

Example 48

View license
    def test_p_on_start(self):
        self.setUpPaginationRangeFunctTypeScraper()   
        self.run_event_spider(1)

        self.assertEqual(len(Event.objects.all()), 7)

Example 49

View license
    def test_free_list_type_scraper_run(self):
        self.setUpPaginationFreeListTypeScraper()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 7)

Example 50

View license
    def process_image_test(self, expected_dirs, non_expected_dirs):
        imgs = [
            '1d7c0c2ea752d7aa951e88f2bc90a3f17058c473.jpg',
            '3cfa4d48e423c5eb3d4f6e9b5e5d373036ac5192.jpg',
        ]
        
        expected_paths = []
        for expected_dir in expected_dirs:
            for img in imgs:
                expected_paths.append(os.path.join(self.PROJECT_ROOT, expected_dir, img))
        non_expected_paths = []
        for non_expected_dir in non_expected_dirs:
            for img in imgs:
                non_expected_paths.append(os.path.join(self.PROJECT_ROOT, non_expected_dir, img))
        self.se_desc.mandatory = True
        self.se_desc.save()
        self.soa_desc.attr_type = 'I'
        self.soa_desc.save()
        
        self.event_website.url = os.path.join(self.SERVER_URL, 'site_with_imgs/event_main.html')
        self.event_website.save()
        self.run_event_spider(1)
        
        self.assertEqual(len(Event.objects.all()), 2)
        self.assertEqual(Event.objects.get(title='Event 1').description, imgs[0])
        for path in expected_paths:
            self.assertTrue(os.access(path, os.F_OK), "Expected image path {p} not found!".format(p=path))
        for path in non_expected_paths:
            self.assertFalse(os.access(path, os.F_OK), "Not expected image path {p} found!".format(p=path))