Here are the examples of the python api scraper.models.Event.objects.filter taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
13 Examples
3
Example 1
Project: django-dynamic-scraper Source File: scraper_json_run_test.py
Function: test_static_processor_empty_x_path
Function: test_static_processor_empty_x_path
def test_static_processor_empty_x_path(self):
self.setUpScraperJSONDefaultScraper()
self.se_title.x_path = ''
self.se_title.processors = 'static'
self.se_title.proc_ctxt = "'static': 'This text should always be there'"
self.se_title.save()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(title='This text should always be there')), 3)
3
Example 2
def test_detail_page(self):
self.setUpScraperJSONDefaultScraper()
self.se_desc.x_path = '//div/div[@class="description"]/text()'
self.se_desc.request_page_type = 'DP1'
self.se_desc.save()
self.run_event_spider(1)
#logging.info(unicode(Event.objects.all()))
self.assertEqual(len(Event.objects.filter(description='Event Detail Page 1 Description')), 1)
3
Example 3
Project: django-dynamic-scraper Source File: scraper_json_run_test.py
def test_detail_page_json(self):
self.setUpScraperJSONDefaultScraper()
self.rpt_dp1.content_type = 'J'
self.rpt_dp1.save()
self.se_url.x_path = 'json_url'
self.se_url.save()
self.se_desc.x_path = 'event_details.description'
self.se_desc.request_page_type = 'DP1'
self.se_desc.save()
self.run_event_spider(1)
#logging.info(unicode(Event.objects.all()))
self.assertEqual(len(Event.objects.filter(description='Event Detail Page 1 Description')), 1)
3
Example 4
Project: django-dynamic-scraper Source File: scraper_processor_run_test.py
def test_substr_replace_processor(self):
self.setUpProcessorTest()
self.se_title.processors = 'substr_replace'
self.se_title.proc_ctxt = "'substr_replace': 'Event 1:New Event 1'"
self.se_title.save()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(title='New Event 1')), 1)
3
Example 5
Project: django-dynamic-scraper Source File: scraper_processor_run_test.py
def test_substr_replace_processor_with_colon(self):
self.setUpProcessorTest()
self.event_website.url = os.path.join(self.SERVER_URL, 'site_with_processor/event_main_substr_replace_with_colon.html')
self.event_website.save()
self.se_title.processors = 'substr_replace'
self.se_title.proc_ctxt = "'substr_replace': 'Event 1\: This:New Event 1\: That'"
self.se_title.save()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(title='New Event 1: That')), 1)
3
Example 6
Project: django-dynamic-scraper Source File: scraper_processor_run_test.py
Function: test_static_processor_empty_x_path
Function: test_static_processor_empty_x_path
def test_static_processor_empty_x_path(self):
self.setUpProcessorTest()
self.se_title.x_path = ''
self.se_title.processors = 'static'
self.se_title.proc_ctxt = "'static': 'This text should always be there'"
self.se_title.save()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(title='This text should always be there')), 2)
3
Example 7
Project: django-dynamic-scraper Source File: scraper_processor_run_test.py
def test_static_processor_unicode_text(self):
self.setUpProcessorTest()
self.se_title.processors = 'static'
self.se_title.proc_ctxt = "'static': 'This text should always be there ❤ ☀ ★ ☂ ☻ ♞ ☯ ☭ ☢'"
self.se_title.save()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(title='This text should always be there ❤ ☀ ★ ☂ ☻ ♞ ☯ ☭ ☢')), 2)
3
Example 8
Project: django-dynamic-scraper Source File: scraper_run_test.py
def test_unicode_standard_field_main_page(self):
self.event_website.url = os.path.join(self.SERVER_URL, 'site_unicode/event_main.html')
self.event_website.save()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(title='Event 1 ❤ ☀ ★ ☂ ☻ ♞ ☯ ☭ ☢')), 1)
self.assertEqual(len(Event.objects.filter(title='Event 2 雨雪天开车尤其危险')), 1)
3
Example 9
Project: django-dynamic-scraper Source File: scraper_run_test.py
def test_unicode_standard_field_detail_page(self):
self.event_website.url = os.path.join(self.SERVER_URL, 'site_unicode/event_main.html')
self.event_website.save()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(description='Event 1 description ♖ ☦ ✝ ❖ ➎ ♠ ♣ ♥')), 1)
self.assertEqual(len(Event.objects.filter(description='Event 2 description Камеры наблюдения заглянули')), 1)
0
Example 10
Project: django-dynamic-scraper Source File: scraper_json_run_test.py
def test_non_repetition(self):
self.setUpScraperJSONDefaultScraper()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(title='Event 1')), 1)
0
Example 11
Project: django-dynamic-scraper Source File: scraper_json_run_test.py
def test_multiple_detail_pages(self):
self.setUpScraperJSONDefaultScraper()
self.se_desc.x_path = '//div/div[@class="description2"]/text()'
self.se_desc.request_page_type = 'DP1'
self.se_desc.save()
self.soa_url.id_field = False
self.soa_url.save_to_db = False
self.soa_url.save()
self.soa_url2.save_to_db = False
self.soa_url2.save()
self.rpt_dp2 = RequestPageType(page_type='DP2', scraper=self.scraper, scraped_obj_attr=self.soa_url2, content_type='J')
self.rpt_dp2.save()
self.se_url2 = ScraperElem(scraped_obj_attr=self.soa_url2, scraper=self.scraper,
x_path='json_url', request_page_type='MP')
self.se_url2.save()
self.se_desc2 = ScraperElem(scraped_obj_attr=self.soa_desc2, scraper=self.scraper,
x_path='event_details.description2', request_page_type='DP2', mandatory=False)
self.se_desc2.save()
self.run_event_spider(1)
#logging.info(unicode(Event.objects.all()))
events = Event.objects.filter(
title='Event 1',
#url='http://localhost:8010/static/site_with_json_content_type/event1.html',
#url2='http://localhost:8010/static/site_with_json_content_type/event1.json',
#description='Event Detail Page 1 Description HTML',
description2='Event Detail Page 1 Description JSON',
)
self.assertEqual(len(events), 1)
0
Example 12
Project: django-dynamic-scraper Source File: scraper_js_run_test.py
def test_default_no_scrapyjs_detail_page(self):
self.setUpScraperJSDefaultScraper()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(description='Event 1 description')), 1)
0
Example 13
Project: django-dynamic-scraper Source File: scraper_js_run_test.py
def test_activated_scrapyjs_detail_page(self):
self.setUpScraperJSDockerScraper()
self.run_event_spider(1)
self.assertEqual(len(Event.objects.filter(description='Event 1 JS description')), 1)