scrapy.commands.ScrapyCommand.process_options

Here are the examples of the python api scrapy.commands.ScrapyCommand.process_options taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

5 Examples 7

3 Source : parse.py
with MIT License
from autofelix

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)

        self.process_spider_arguments(opts)
        self.process_request_meta(opts)
        self.process_request_cb_kwargs(opts)

    def process_spider_arguments(self, opts):

3 Source : parse.py
with The Unlicense
from dspray95

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)

        self.process_spider_arguments(opts)
        self.process_request_meta(opts)

    def process_spider_arguments(self, opts):

0 Source : crawl.py
with MIT License
from autofelix

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        try:
            opts.spargs = arglist_to_dict(opts.spargs)
        except ValueError:
            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
        if opts.output:
            if opts.output == '-':
                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
            else:
                self.settings.set('FEED_URI', opts.output, priority='cmdline')
            feed_exporters = without_none_values(
                self.settings.getwithbase('FEED_EXPORTERS'))
            valid_output_formats = feed_exporters.keys()
            if not opts.output_format:
                opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
            if opts.output_format not in valid_output_formats:
                raise UsageError("Unrecognized output format '%s', set one"
                                 " using the '-t' switch or as a file extension"
                                 " from the supported list %s" % (opts.output_format,
                                                                  tuple(valid_output_formats)))
            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')

    def run(self, args, opts):

0 Source : runspider.py
with MIT License
from autofelix

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        try:
            opts.spargs = arglist_to_dict(opts.spargs)
        except ValueError:
            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
        if opts.output:
            if opts.output == '-':
                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
            else:
                self.settings.set('FEED_URI', opts.output, priority='cmdline')
            feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
            valid_output_formats = feed_exporters.keys()
            if not opts.output_format:
                opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
            if opts.output_format not in valid_output_formats:
                raise UsageError("Unrecognized output format '%s', set one"
                                 " using the '-t' switch or as a file extension"
                                 " from the supported list %s" % (opts.output_format,
                                                                  tuple(valid_output_formats)))
            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')

    def run(self, args, opts):

0 Source : crawl.py
with MIT License
from tonywu7

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)

        opts.spargs = {}

        if len(args) == 2:
            self.settings['RSS'] = args.pop()

        self._takeover_logging()

        if opts.output:
            self.settings['OUTPUT'] = opts.output[0]
        self.settings.pop('FEEDS')

        self.settings['CMDLINE_ARGS'] = {'args': args, 'opts': vars(opts)}

        if opts.verbose:
            self.settings['VERBOSE'] = True
            self.settings.set('LOG_VIOLATIONS', True, priority='cmdline')
            self.settings.set('STATS_DUMP', True, priority='cmdline')