dynamic_scraper.models.Log

Here are the examples of the python api dynamic_scraper.models.Log taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

1 Examples 7

Example 1

View license
    def log(self, message, level=logging.DEBUG):
        if self.conf['RUN_TYPE'] == 'TASK' and self.conf['DO_ACTION']:
            
            if self.conf['LOG_ENABLED'] and level >= getattr(logging, self.conf['LOG_LEVEL']):
                l = Log()
                l.message = message
                l.ref_object = self.ref_object.__class__.__name__ + " (" + str(self.ref_object.pk) + ")"
                l.type = 'None'
                l.level = int(level)
                l.spider_name = self.name
                l.scraper = self.scraper
                
                # Look for corresponding log markers
                lms = LogMarker.objects.filter(
                    Q(ref_object=l.ref_object) | Q(ref_object=''),
                    Q(spider_name=l.spider_name) | Q(spider_name=''),
                    Q(scraper=l.scraper) | Q(scraper__isnull=True),
                )
                for lm in lms:
                    if lm.message_contains in l.message:
                        if lm.custom_type:
                            l.type = lm.custom_type
                        else:
                            l.type = lm.get_mark_with_type_display()
                l.save()
                
                #Delete old logs
                if Log.objects.count() > self.conf['LOG_LIMIT']:
                    items = Log.objects.all()[self.conf['LOG_LIMIT']:]
                    for item in items:
                        item.delete()
        
        logging.log(level, message)