dynamic_scraper.models.Scraper.objects.filter

Here are the examples of the python api dynamic_scraper.models.Scraper.objects.filter taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

2 Examples 7

Example 1

Project: django-dynamic-scraper Source File: check_last_checker_deletes.py
    def handle(self, *args, **options):
        mail_to_admins = False
        msg = ''
        
        if options.get('with_next_alert'):
            scrapers = Scraper.objects.filter(next_last_checker_delete_alert__lte=datetime.datetime.now())
            print("{num} scraper(s) with future next alert timestamp found in DB...\n".format(num=len(scrapers)))
        else:
            scrapers = Scraper.objects.all()
            print("{num} scraper(s) found in DB...\n".format(num=len(scrapers)))
        
        for s in scrapers:
            if not (options.get('only_active') and s.status != 'A'):
                td = s.get_last_checker_delete_alert_period_timedelta()
                if td:
                    period = s.last_checker_delete_alert_period
                    s_str = "SCRAPER: {scraper}\nID:{id}, Status:{status}, Alert Period:{period}".format(
                        scraper=str(s), id=s.pk, status=s.get_status_display(), period=period)
                    print(s_str)
                    
                    if options.get('with_next_alert'):
                        s.next_last_checker_delete_alert = datetime.datetime.now() + td
                        s.save()
                    
                    if not s.last_checker_delete or \
                        (s.last_checker_delete < (datetime.datetime.now() - td)):
                        if s.last_checker_delete:
                            error_str = "Last checker delete older than alert period ({date_str})!".format(
                                date_str=s.last_checker_delete.strftime('%Y-%m-%d %H:%m'),)
                        else:
                            error_str = "Last checker delete not available!"
                        print(error_str)
                        msg += s_str + '\n' + error_str + '\n\n'
                        mail_to_admins = True
                    else:
                        print("OK")
                    print()
                else:
                    print("Ommitting scraper {scraper}, no (valid) time period set.\n".format(scraper=str(s)))
            else:
                print("Ommitting scraper {scraper}, not active.\n".format(scraper=str(s)))
        
        if options.get('send_admin_mail') and mail_to_admins:
            print("Send mail to admins...")
            if 'django.contrib.sites' in settings.INSTALLED_APPS:
                from django.contrib.sites.models import Site
                subject = Site.objects.get_current().name
            else:
                subject = 'DDS Scraper Site'
            subject += " - Last checker delete check for scraper(s) failed"
            
            mail_admins(subject, msg)

Example 2

Project: django-dynamic-scraper Source File: check_last_scraper_saves.py
    def handle(self, *args, **options):
        mail_to_admins = False
        msg = ''
        
        if options.get('with_next_alert'):
            scrapers = Scraper.objects.filter(next_last_scraper_save_alert__lte=datetime.datetime.now())
            print("{num} scraper(s) with future next alert timestamp found in DB...\n".format(num=len(scrapers)))
        else:
            scrapers = Scraper.objects.all()
            print("{num} scraper(s) found in DB...\n".format(num=len(scrapers)))
        
        for s in scrapers:
            if not (options.get('only_active') and s.status != 'A'):
                td = s.get_last_scraper_save_alert_period_timedelta()
                if td:
                    period = s.last_scraper_save_alert_period
                    s_str = "SCRAPER: {scraper}\nID:{id}, Status:{status}, Alert Period:{period}".format(
                        scraper=str(s), id=s.pk, status=s.get_status_display(), period=period)
                    print(s_str)
                    
                    if options.get('with_next_alert'):
                        s.next_last_scraper_save_alert = datetime.datetime.now() + td
                        s.save()
                    
                    if not s.last_scraper_save or \
                        (s.last_scraper_save < (datetime.datetime.now() - td)):
                        date_str = 'None'
                        if s.last_scraper_save:
                            error_str = "Last scraper save older than alert period ({date_str})!".format(
                                date_str=s.last_scraper_save.strftime('%Y-%m-%d %H:%m'),)
                        else:
                            error_str = "Last scraper save not available!"
                        print(error_str)
                        msg += s_str + '\n' + error_str + '\n\n'
                        mail_to_admins = True
                    else:
                        print("OK")
                    print()
                else:
                    print("Ommitting scraper {scraper}, no (valid) time period set.\n".format(scraper=str(s)))
            else:
                print("Ommitting scraper {scraper}, not active.\n".format(scraper=str(s)))
        
        if options.get('send_admin_mail') and mail_to_admins:
            print("Send mail to admins...")
            if 'django.contrib.sites' in settings.INSTALLED_APPS:
                from django.contrib.sites.models import Site
                subject = Site.objects.get_current().name
            else:
                subject = 'DDS Scraper Site'
            subject += " - Last scraper save check for scraper(s) failed"
            
            mail_admins(subject, msg)