dynamic_scraper.models.RequestPageType

Here are the examples of the python api dynamic_scraper.models.RequestPageType taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

1 Examples 7

Example 1

View license
    def test_multiple_detail_pages(self):
        self.setUpScraperJSONDefaultScraper()
        self.se_desc.x_path = '//div/div[@class="description2"]/text()'
        self.se_desc.request_page_type = 'DP1'
        self.se_desc.save()

        self.soa_url.id_field = False
        self.soa_url.save_to_db = False
        self.soa_url.save()

        self.soa_url2.save_to_db = False
        self.soa_url2.save()

        self.rpt_dp2 = RequestPageType(page_type='DP2', scraper=self.scraper, scraped_obj_attr=self.soa_url2, content_type='J')
        self.rpt_dp2.save()
        
        self.se_url2 = ScraperElem(scraped_obj_attr=self.soa_url2, scraper=self.scraper, 
            x_path='json_url', request_page_type='MP')
        self.se_url2.save()
        
        self.se_desc2 = ScraperElem(scraped_obj_attr=self.soa_desc2, scraper=self.scraper, 
            x_path='event_details.description2', request_page_type='DP2', mandatory=False)
        self.se_desc2.save()
        

        self.run_event_spider(1)
        #logging.info(unicode(Event.objects.all()))
        events = Event.objects.filter(
            title='Event 1',
            #url='http://localhost:8010/static/site_with_json_content_type/event1.html',
            #url2='http://localhost:8010/static/site_with_json_content_type/event1.json',
            #description='Event Detail Page 1 Description HTML',
            description2='Event Detail Page 1 Description JSON',
        )
        self.assertEqual(len(events), 1)