scrapy.commands.ScrapyCommand.add_options

Here are the examples of the python api scrapy.commands.ScrapyCommand.add_options taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

18 Examples 7

3 Source : check.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-l", "--list", dest="list", action="store_true",
                          help="only list contracts, without checking them")
        parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
                          help="print contract tests for all spiders")

    def run(self, args, opts):

3 Source : crawl.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                          help="set spider argument (may be repeated)")
        parser.add_option("-o", "--output", metavar="FILE",
                          help="dump scraped items into FILE (use - for stdout)")
        parser.add_option("-t", "--output-format", metavar="FORMAT",
                          help="format to use for dumping items with -o")

    def process_options(self, args, opts):

3 Source : fetch.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--spider", dest="spider",
            help="use this spider")
        parser.add_option("--headers", dest="headers", action="store_true", \
            help="print response HTTP headers instead of body")
        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
            default=False, help="do not handle HTTP 3xx status codes and print response as-is")

    def _print_headers(self, headers, prefix):

3 Source : genspider.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-l", "--list", dest="list", action="store_true",
            help="List available templates")
        parser.add_option("-e", "--edit", dest="edit", action="store_true",
            help="Edit spider after creating it")
        parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
            help="Dump template to standard output")
        parser.add_option("-t", "--template", dest="template", default="basic",
            help="Uses a custom template.")
        parser.add_option("--force", dest="force", action="store_true",
            help="If the spider already exists, overwrite it with the template")

    def run(self, args, opts):

3 Source : settings.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--get", dest="get", metavar="SETTING",
            help="print raw setting value")
        parser.add_option("--getbool", dest="getbool", metavar="SETTING",
            help="print setting value, interpreted as a boolean")
        parser.add_option("--getint", dest="getint", metavar="SETTING",
            help="print setting value, interpreted as an integer")
        parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
            help="print setting value, interpreted as a float")
        parser.add_option("--getlist", dest="getlist", metavar="SETTING",
            help="print setting value, interpreted as a list")

    def run(self, args, opts):

3 Source : shell.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-c", dest="code",
            help="evaluate the code in the shell, print the result and exit")
        parser.add_option("--spider", dest="spider",
            help="use this spider")
        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
            default=False, help="do not handle HTTP 3xx status codes and print response as-is")

    def update_vars(self, vars):

3 Source : version.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
            help="also display twisted/python/platform info (useful for bug reports)")

    def run(self, args, opts):

3 Source : validate.py
with MIT License
from City-Bureau

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option(
            "--all",
            dest="all",
            action="store_true",
            help="Run validation on all scrapers",
        )

    def run(self, args, opts):

3 Source : migrate.py
with MIT License
from DakotaNelson

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--mongo_uri", dest="mongo_uri", metavar="URI",
                help="connection string for MongoDB containing Strava data",
                default="mongodb://localhost:27017/strava")
        parser.add_option("--postgres_uri", dest="postgres_uri", metavar="URI",
                help="connection string for PostgreSQL to put Strava data into",
                default="postgresql:///strava")

    def run(self, args, opts):

3 Source : postgres_setup.py
with MIT License
from DakotaNelson

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--postgres_uri", dest="postgres_uri", metavar="URI",
                help="connection string for PostgreSQL to put Strava data into",
                default="postgresql:///strava")

    def run(self, args, opts):

3 Source : strava_crawl.py
with MIT License
from DakotaNelson

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--postgres_uri", dest="postgres_uri", metavar="URI",
                help="connection string for PostgreSQL to put data into",
                default="postgresql:///strava")
        parser.add_option("-a", dest="spargs", action="append", default={},
                metavar="NAME=VALUE",
                help="set spider argument (may be repeated)")
        parser.add_option("-m", "--max", action="store_true", dest="max",
                help="start crawling from the largest ID found in the DB")
        parser.add_option("-s", "--start", dest="start_val",
                default=None, metavar="ID", type="int",
                help="start crawling from the ID equal to this value")

    def run(self, args, opts):

3 Source : crawlall.py
with BSD 3-Clause "New" or "Revised" License
from open-contracting

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_argument('--dry-run', action='store_true', help='Runs the spiders without writing any files')
        parser.add_argument('--sample', type=int, help='The number of files to write')

    def run(self, args, opts):

3 Source : pluck.py
with BSD 3-Clause "New" or "Revised" License
from open-contracting

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_argument('-p', '--package-pointer', help='The JSON Pointer to the value in the package')
        parser.add_argument('-r', '--release-pointer', help='The JSON Pointer to the value in the release')
        parser.add_argument('-t', '--truncate', type=int, help='Truncate the value to this number of characters')
        parser.add_argument('--max-bytes', type=int,
                            help='Stop downloading an OCDS file after reading this many bytes')

    def run(self, args, opts):

0 Source : parse.py
with MIT License
from autofelix

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--spider", dest="spider", default=None,
            help="use this spider without looking for one")
        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
            help="set spider argument (may be repeated)")
        parser.add_option("--pipelines", action="store_true",
            help="process items through pipelines")
        parser.add_option("--nolinks", dest="nolinks", action="store_true",
            help="don't show links to follow (extracted requests)")
        parser.add_option("--noitems", dest="noitems", action="store_true",
            help="don't show scraped items")
        parser.add_option("--nocolour", dest="nocolour", action="store_true",
            help="avoid using pygments to colorize the output")
        parser.add_option("-r", "--rules", dest="rules", action="store_true",
            help="use CrawlSpider rules to discover the callback")
        parser.add_option("-c", "--callback", dest="callback",
            help="use this callback for parsing, instead looking for a callback")
        parser.add_option("-m", "--meta", dest="meta",
            help="inject extra meta into the Request, it must be a valid raw json string")
        parser.add_option("--cbkwargs", dest="cbkwargs",
            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
        parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
            help="maximum depth for parsing requests [default: %default]")
        parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
            help="print each depth level one by one")

    @property

0 Source : parse.py
with The Unlicense
from dspray95

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("--spider", dest="spider", default=None,
            help="use this spider without looking for one")
        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
            help="set spider argument (may be repeated)")
        parser.add_option("--pipelines", action="store_true",
            help="process items through pipelines")
        parser.add_option("--nolinks", dest="nolinks", action="store_true",
            help="don't show links to follow (extracted requests)")
        parser.add_option("--noitems", dest="noitems", action="store_true",
            help="don't show scraped items")
        parser.add_option("--nocolour", dest="nocolour", action="store_true",
            help="avoid using pygments to colorize the output")
        parser.add_option("-r", "--rules", dest="rules", action="store_true",
            help="use CrawlSpider rules to discover the callback")
        parser.add_option("-c", "--callback", dest="callback",
            help="use this callback for parsing, instead looking for a callback")
        parser.add_option("-m", "--meta", dest="meta",
            help="inject extra meta into the Request, it must be a valid raw json string")
        parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
            help="maximum depth for parsing requests [default: %default]")
        parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
            help="print each depth level one by one")


    @property

0 Source : consumer.py
with MIT License
from groupbwt

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option(
            "-q",
            "--queue",
            type="str",
            dest="queue_name",
            help="Queue name to consume messages",
            action="callback",
            callback=self.queue_option_callback,
        )
        parser.add_option(
            "-m",
            "--mode",
            type="choice",
            choices=self.action_modes,
            default="action",
            dest="mode",
            help="Command run mode: action for one time execution and exit or worker",
        )
        parser.add_option(
            "-p",
            "--prefetch_count",
            type="int",
            default=None,
            dest="prefetch_count",
            help="RabbitMQ consumer prefetch count setting",
        )

    def queue_option_callback(self, _option, opt, value, parser):

0 Source : producer.py
with MIT License
from groupbwt

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option(
            "-t",
            "--task_queue",
            type="str",
            dest="task_queue_name",
            help="Queue name to produce tasks",
            action="callback",
            callback=self.task_queue_option_callback,
        )
        parser.add_option(
            "-r",
            "--reply_to_queue",
            type="str",
            dest="reply_to_queue_name",
            help="Queue name to return replies",
            action="callback",
            callback=self.reply_to_queue_option_callback,
        )
        parser.add_option(
            "-m",
            "--mode",
            type="choice",
            choices=self.action_modes,
            default="action",
            dest="mode",
            help="Command run mode: action for one time execution and exit or worker",
        )
        parser.add_option(
            "-c",
            "--chunk_size",
            type="int",
            default=Producer._DEFAULT_CHUNK_SIZE,
            dest="chunk_size",
            help="number of tasks to produce at one iteration",
        )

    def task_queue_option_callback(self, _option, opt, value, parser):

0 Source : resume.py
with MIT License
from tonywu7

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)

    def process_options(self, args, opts):