dynamic_scraper.models.Scraper.objects.all

Here are the examples of the python api dynamic_scraper.models.Scraper.objects.all taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

1 Examples 7

Example 1

View license
    def handle(self, *args, **options):
        '''
        if options.get('only_active'):
            scraper_list = Scraper.objects.filter(
                checker_x_path__isnull=False, 
                checker_ref_url__isnull=False,
                status='A'
            )
        else:
            scraper_list = Scraper.objects.filter(
                checker_x_path__isnull=False, 
                checker_ref_url__isnull=False
            )
        '''
        mail_to_admins = False
        msg = ''
        for scraper in Scraper.objects.all():
            if not (options.get('only_active') and scraper.status != 'A') and scraper.checker_set.count() > 0:
                scraper_str  = str(scraper) + " "
                scraper_str += "(ID:" + str(scraper.pk) + ", Status: " + scraper.get_status_display() + ")"
                print("Run checker test for scraper {}".format(scraper_str))
                
                cmd  = 'scrapy crawl checker_test '
                if options.get('report_only_errors'):
                    cmd += '-L ERROR '
                else:
                    cmd += '-L WARNING '
                cmd += '-a id=' + str(scraper.pk)
                
                p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
                stderr = p.communicate()[1]
                
                if stderr != '':
                    print(stderr)
                    msg += 'Checker test for scraper {s} failed:\n'.format(s=scraper_str)
                    msg += stderr + '\n\n'
                    mail_to_admins = True
                else:
                    print("Checker configuration working.")
        
        if options.get('send_admin_mail') and mail_to_admins:
            print("Send mail to admins...")
            if 'django.contrib.sites' in settings.INSTALLED_APPS:
                from django.contrib.sites.models import Site
                subject = Site.objects.get_current().name
            else:
                subject = 'DDS Scraper Site'
            subject += " - Errors while running checker configuration tests"
            
            mail_admins(subject, msg)