Here are the examples of the python api scrapy.commands.ScrapyCommand.process_options taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
5 Examples
3
Source : parse.py
with MIT License
from autofelix
with MIT License
from autofelix
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
self.process_spider_arguments(opts)
self.process_request_meta(opts)
self.process_request_cb_kwargs(opts)
def process_spider_arguments(self, opts):
3
Source : parse.py
with The Unlicense
from dspray95
with The Unlicense
from dspray95
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
self.process_spider_arguments(opts)
self.process_request_meta(opts)
def process_spider_arguments(self, opts):
0
Source : crawl.py
with MIT License
from autofelix
with MIT License
from autofelix
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
if opts.output == '-':
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(
self.settings.getwithbase('FEED_EXPORTERS'))
valid_output_formats = feed_exporters.keys()
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
if opts.output_format not in valid_output_formats:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
tuple(valid_output_formats)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
0
Source : runspider.py
with MIT License
from autofelix
with MIT License
from autofelix
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
if opts.output == '-':
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
valid_output_formats = feed_exporters.keys()
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
if opts.output_format not in valid_output_formats:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
tuple(valid_output_formats)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
0
Source : crawl.py
with MIT License
from tonywu7
with MIT License
from tonywu7
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
opts.spargs = {}
if len(args) == 2:
self.settings['RSS'] = args.pop()
self._takeover_logging()
if opts.output:
self.settings['OUTPUT'] = opts.output[0]
self.settings.pop('FEEDS')
self.settings['CMDLINE_ARGS'] = {'args': args, 'opts': vars(opts)}
if opts.verbose:
self.settings['VERBOSE'] = True
self.settings.set('LOG_VIOLATIONS', True, priority='cmdline')
self.settings.set('STATS_DUMP', True, priority='cmdline')