Here are the examples of the python api scrapy.exceptions.NotConfigured taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
11 Examples
3
Example 1
def parse(self, response):
if not hasattr(self, 'parse_node'):
raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
response = self.adapt_response(response)
if self.iterator == 'iternodes':
nodes = self._iternodes(response)
elif self.iterator == 'xml':
selector = Selector(response, type='xml')
self._register_namespaces(selector)
nodes = selector.xpath('//%s' % self.itertag)
elif self.iterator == 'html':
selector = Selector(response, type='html')
self._register_namespaces(selector)
nodes = selector.xpath('//%s' % self.itertag)
else:
raise NotSupported('Unsupported node iterator')
return self.parse_nodes(response, nodes)
3
Example 2
Project: scrapy Source File: boto.py
def is_botocore():
try:
import botocore
return True
except ImportError:
if six.PY2:
try:
import boto
return False
except ImportError:
raise NotConfigured('missing botocore or boto library')
else:
raise NotConfigured('missing botocore library')
3
Example 3
Project: scrapy Source File: project.py
def project_data_dir(project='default'):
"""Return the current project data dir, creating it if it doesn't exist"""
if not inside_project():
raise NotConfigured("Not inside a project")
cfg = get_config()
if cfg.has_option(DATADIR_CFG_SECTION, project):
d = cfg.get(DATADIR_CFG_SECTION, project)
else:
scrapy_cfg = closest_scrapy_cfg()
if not scrapy_cfg:
raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
d = abspath(join(dirname(scrapy_cfg), '.scrapy'))
if not exists(d):
os.makedirs(d)
return d
3
Example 4
@classmethod
def from_crawler(cls, crawler):
splash_base_url = crawler.settings.get('SPLASH_URL',
cls.default_splash_url)
log_400 = crawler.settings.getbool('SPLASH_LOG_400', True)
slot_policy = crawler.settings.get('SPLASH_SLOT_POLICY',
cls.default_policy)
if slot_policy not in SlotPolicy._known:
raise NotConfigured("Incorrect slot policy: %r" % slot_policy)
return cls(crawler, splash_base_url, slot_policy, log_400)
3
Example 5
@classmethod
def from_crawler(cls, crawler):
try:
return cls(crawler)
except Exception as e:
raise NotConfigured('WEBDRIVER_BROWSER is misconfigured: %r (%r)'
% (crawler.settings.get('WEBDRIVER_BROWSER'), e))
3
Example 6
@classmethod
def from_crawler(cls, crawler):
dsn = crawler.settings.get("SENTRY_DSN", None)
if dsn is None:
raise NotConfigured('No SENTRY_DSN configured')
o = cls(dsn=dsn)
return o
3
Example 7
@classmethod
def from_crawler(cls, crawler, client=None, dsn=None):
dsn = os.environ.get(
"SENTRY_DSN", crawler.settings.get("SENTRY_DSN", None))
if dsn is None:
raise NotConfigured('No SENTRY_DSN configured')
o = cls(dsn=dsn)
crawler.signals.connect(o.spider_error, signal=signals.spider_error)
return o
3
Example 8
@classmethod
def from_crawler(cls, crawler):
dsn = os.environ.get(
"SENTRY_DSN", crawler.settings.get("SENTRY_DSN", None))
if dsn is None:
raise NotConfigured('No SENTRY_DSN configured')
return cls(dsn)
0
Example 9
def __init__(self):
# Open database connection
self.db = mysql.connect(host=ROJAK_DB_HOST, port=ROJAK_DB_PORT,
user=ROJAK_DB_USER, passwd=ROJAK_DB_PASS, db=ROJAK_DB_NAME)
self.cursor = self.db.cursor()
self.media = {}
try:
# Get media information from the database
self.logger.info('Fetching media information')
self.cursor.execute(sql_get_media, [self.name])
row = self.cursor.fetchone()
self.media['id'] = row[0]
self.media['last_scraped_at'] = row[1]
except mysql.Error as err:
self.logger.error('Unable to fetch media data: %s', err)
raise NotConfigured('Unable to fetch media data: %s' % err)
if ROJAK_SLACK_TOKEN != '':
self.is_slack = True
self.slack = Slacker(ROJAK_SLACK_TOKEN)
else:
self.is_slack = False
self.logger.info('Post error to #rojak-pantau-errors is disabled')
0
Example 10
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
# unless 'anon' was set explicitly (True/False).
anon = kw.get('anon')
if anon is None and not aws_access_key_id and not aws_secret_access_key:
kw['anon'] = True
self.anon = kw.get('anon')
self._signer = None
if is_botocore():
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError('Unexpected keyword arguments: %s' % kw)
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
else:
_S3Connection = _get_boto_connection()
try:
self.conn = _S3Connection(
aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
self._download_http = httpdownloadhandler(settings).download_request
0
Example 11
def parse(self, response):
if not hasattr(self, 'parse_row'):
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
response = self.adapt_response(response)
return self.parse_rows(response)