Here are the examples of the python api scrapy.commands.ScrapyCommand.add_options taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
18 Examples
3
Source : check.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
def run(self, args, opts):
3
Source : crawl.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
3
Source : fetch.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true", \
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
3
Source : genspider.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
3
Source : settings.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
def run(self, args, opts):
3
Source : shell.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):
3
Source : version.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts):
3
Source : validate.py
with MIT License
from City-Bureau
with MIT License
from City-Bureau
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option(
"--all",
dest="all",
action="store_true",
help="Run validation on all scrapers",
)
def run(self, args, opts):
3
Source : migrate.py
with MIT License
from DakotaNelson
with MIT License
from DakotaNelson
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--mongo_uri", dest="mongo_uri", metavar="URI",
help="connection string for MongoDB containing Strava data",
default="mongodb://localhost:27017/strava")
parser.add_option("--postgres_uri", dest="postgres_uri", metavar="URI",
help="connection string for PostgreSQL to put Strava data into",
default="postgresql:///strava")
def run(self, args, opts):
3
Source : postgres_setup.py
with MIT License
from DakotaNelson
with MIT License
from DakotaNelson
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--postgres_uri", dest="postgres_uri", metavar="URI",
help="connection string for PostgreSQL to put Strava data into",
default="postgresql:///strava")
def run(self, args, opts):
3
Source : strava_crawl.py
with MIT License
from DakotaNelson
with MIT License
from DakotaNelson
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--postgres_uri", dest="postgres_uri", metavar="URI",
help="connection string for PostgreSQL to put data into",
default="postgresql:///strava")
parser.add_option("-a", dest="spargs", action="append", default={},
metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-m", "--max", action="store_true", dest="max",
help="start crawling from the largest ID found in the DB")
parser.add_option("-s", "--start", dest="start_val",
default=None, metavar="ID", type="int",
help="start crawling from the ID equal to this value")
def run(self, args, opts):
3
Source : crawlall.py
with BSD 3-Clause "New" or "Revised" License
from open-contracting
with BSD 3-Clause "New" or "Revised" License
from open-contracting
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument('--dry-run', action='store_true', help='Runs the spiders without writing any files')
parser.add_argument('--sample', type=int, help='The number of files to write')
def run(self, args, opts):
3
Source : pluck.py
with BSD 3-Clause "New" or "Revised" License
from open-contracting
with BSD 3-Clause "New" or "Revised" License
from open-contracting
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument('-p', '--package-pointer', help='The JSON Pointer to the value in the package')
parser.add_argument('-r', '--release-pointer', help='The JSON Pointer to the value in the release')
parser.add_argument('-t', '--truncate', type=int, help='Truncate the value to this number of characters')
parser.add_argument('--max-bytes', type=int,
help='Stop downloading an OCDS file after reading this many bytes')
def run(self, args, opts):
0
Source : parse.py
with MIT License
from autofelix
with MIT License
from autofelix
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
@property
0
Source : parse.py
with The Unlicense
from dspray95
with The Unlicense
from dspray95
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
@property
0
Source : consumer.py
with MIT License
from groupbwt
with MIT License
from groupbwt
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option(
"-q",
"--queue",
type="str",
dest="queue_name",
help="Queue name to consume messages",
action="callback",
callback=self.queue_option_callback,
)
parser.add_option(
"-m",
"--mode",
type="choice",
choices=self.action_modes,
default="action",
dest="mode",
help="Command run mode: action for one time execution and exit or worker",
)
parser.add_option(
"-p",
"--prefetch_count",
type="int",
default=None,
dest="prefetch_count",
help="RabbitMQ consumer prefetch count setting",
)
def queue_option_callback(self, _option, opt, value, parser):
0
Source : producer.py
with MIT License
from groupbwt
with MIT License
from groupbwt
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option(
"-t",
"--task_queue",
type="str",
dest="task_queue_name",
help="Queue name to produce tasks",
action="callback",
callback=self.task_queue_option_callback,
)
parser.add_option(
"-r",
"--reply_to_queue",
type="str",
dest="reply_to_queue_name",
help="Queue name to return replies",
action="callback",
callback=self.reply_to_queue_option_callback,
)
parser.add_option(
"-m",
"--mode",
type="choice",
choices=self.action_modes,
default="action",
dest="mode",
help="Command run mode: action for one time execution and exit or worker",
)
parser.add_option(
"-c",
"--chunk_size",
type="int",
default=Producer._DEFAULT_CHUNK_SIZE,
dest="chunk_size",
help="number of tasks to produce at one iteration",
)
def task_queue_option_callback(self, _option, opt, value, parser):
0
Source : resume.py
with MIT License
from tonywu7
with MIT License
from tonywu7
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def process_options(self, args, opts):