Here are the examples of the python api sys.stdin taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
133 Examples
0
Example 101
Project: qqqfome Source File: entry.py
def main():
parser = argparse.ArgumentParser(prog='qqqfome',
description='Thank-you-follow-me cli.')
parser.add_argument('-v', '--verbose', dest='verbose',
action='count', default=0,
help=s.cmd_help_print_info)
parser.add_argument('-c', '--cookies', dest='cookies', metavar='FILE',
help=s.cmd_help_cookies,
type=str)
parser.add_argument('-p', '--pid-file', dest='pid_file', metavar='FILE',
help=s.cmd_help_pid_file,
type=str, default='{0}.pid')
parser.add_argument('-l', '--log-file', dest='log_file', metavar='FILE',
help=s.cmd_help_log_file,
type=str, default='{0}.log')
parser.add_argument('-t', '--time', dest='time', metavar='INTERVAL',
help=s.cmd_help_time,
type=int, default=90)
group = parser.add_mutually_exclusive_group()
group.add_argument('-m', '--message', dest='message',
help=s.cmd_help_message,
type=str, default=s.default_message)
group.add_argument('-M', '--message-file', dest='message_file',
metavar='FILE',
help=s.cmd_help_message,
type=str)
parser.add_argument('-s', '--stop-at', dest='stop_at', metavar='NUM',
help=s.cmd_help_stop_at, type=int, default=10)
parser.add_argument('-d', '--daemon', dest='daemon', action='store_true',
default=False, help='work in daemon mode')
parser.add_argument('command', help=s.cmd_help_command, type=str,
choices=['init', 'start', 'stop'])
parser.add_argument('file', help=s.cmd_help_file, type=str,
action=SetDefaultPID, nargs='?')
args = parser.parse_args()
# Logger settings
level = logging.ERROR
if args.verbose == 1:
level = logging.INFO
if args.verbose >= 2:
level = logging.DEBUG
ch = logging.StreamHandler()
ch.setLevel(level)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
db.set_logger_level(level)
db.set_logger_handle(ch)
L.setLevel(level)
L.addHandler(ch)
L.debug(args)
if args.command == "init":
init_db(args.cookies)
elif args.file is None:
parser.error(s.cmd_help_no_file_error.format(args.command))
if args.command == 'start':
if args.message_file is not None:
with open(args.message_file, 'r') as f:
args.message = f.read()
if args.daemon:
p = backend.BackendCode(args.pid_file)
try:
p.start(args.file, args.message, args.time, args.log_file)
except OSError: # daemon mode not support the system
p.run(args.file, args.message, args.time, args.log_file)
else:
p = backend.BackendCode(args.pid_file, stdin=sys.stdin,
stdout=sys.stdout, stderr=sys.stderr)
p.run(args.file, args.message, args.time, args.log_file)
elif args.command == 'stop':
try:
p = backend.BackendCode(args.pid_file)
p.stop()
except FileNotFoundError as e:
L.error(e)
0
Example 102
def manage_users(path=None):
from calibre.srv.users import UserManager
m = UserManager(path)
enc = getattr(sys.stdin, 'encoding', preferred_encoding) or preferred_encoding
def choice(question, choices, default=None, banner=''):
prints(banner)
for i, choice in enumerate(choices):
prints('%d)' % (i+1), choice)
print()
while True:
prompt = question + ' [1-%d]: ' % len(choices)
if default is not None:
prompt = question + ' [1-%d %s: %d]' % (len(choices), _('default'), default+1)
reply = raw_input(prompt)
if not reply and default is not None:
reply = str(default + 1)
if not reply:
raise SystemExit(0)
reply = reply.strip()
try:
num = int(reply) - 1
if not (0 <= num < len(choices)):
raise Exception('bad num')
return num
except Exception:
prints(_('%s is not a valid choice, try again') % reply)
def get_valid(prompt, invalidq=lambda x: None):
while True:
ans = raw_input(prompt + ': ').strip().decode(enc)
fail_message = invalidq(ans)
if fail_message is None:
return ans
prints(fail_message)
def get_valid_user():
prints(_('Existing user names:'))
users = sorted(m.all_user_names)
if not users:
raise SystemExit(_('There are no users, you must first add an user'))
prints(', '.join(users))
def validate(username):
if not m.has_user(username):
return _('The username %s does not exist') % username
return get_valid(_('Enter the username'), validate)
def get_pass(username):
while True:
from getpass import getpass
one = getpass(_('Enter the new password for %s: ') % username).decode(enc)
if not one:
prints(_('Empty passwords are not allowed'))
continue
two = getpass(_('Re-enter the new password for %s, to verify: ') % username).decode(enc)
if one != two:
prints(_('Passwords do not match'))
continue
msg = m.validate_password(one)
if msg is None:
return one
prints(msg)
def add_user():
username = get_valid(_('Enter the username'), m.validate_username)
pw = get_pass(username)
m.add_user(username, pw)
prints(_('User %s added successfully!') % username)
def remove_user():
un = get_valid_user()
if raw_input((_('Are you sure you want to remove the user %s?') % un) + ' [y/n]: ').decode(enc) != 'y':
raise SystemExit(0)
m.remove_user(un)
prints(_('User %s successfully removed!') % un)
def edit_user():
username = get_valid_user()
pw = get_pass(username)
m.change_password(username, pw)
prints(_('Password for %s successfully changed!') % username)
def show_password():
username = get_valid_user()
pw = m.get(username)
prints(_('Password for {0} is: {1}').format(username, pw))
{0:add_user, 1:edit_user, 2:remove_user, 3:show_password}[choice(_('What do you want to do?'), [
_('Add a new user'), _('Edit an existing user'), _('Remove a user'), _('Show the password for a user')])]()
0
Example 103
def main(argv=None, stdout=None, stdin=None):
"""The ``weasyprint`` program takes at least two arguments:
.. code-block:: sh
weasyprint [options] <input> <output>
The input is a filename or URL to an HTML docuement, or ``-`` to read
HTML from stdin. The output is a filename, or ``-`` to write to stdout.
Options can be mixed anywhere before, between or after the input and
output:
.. option:: -e <input_encoding>, --encoding <input_encoding>
Force the input character encoding (eg. ``-e utf8``).
.. option:: -f <output_format>, --format <output_format>
Choose the output file format among PDF and PNG (eg. ``-f png``).
Required if the output is not a ``.pdf`` or ``.png`` filename.
.. option:: -s <filename_or_URL>, --stylesheet <filename_or_URL>
Filename or URL of a user CSS stylesheet (see
:ref:`stylesheet-origins`\.) to add to the docuement.
(eg. ``-s print.css``). Multiple stylesheets are allowed.
.. option:: -r <dpi>, --resolution <dpi>
For PNG output only. Set the resolution in PNG pixel per CSS inch.
Defaults to 96, which means that PNG pixels match CSS pixels.
.. option:: --base-url <URL>
Set the base for relative URLs in the HTML input.
Defaults to the input’s own URL, or the current directory for stdin.
.. option:: -m <type>, --media-type <type>
Set the media type to use for ``@media``. Defaults to ``print``.
.. option:: -a <file>, --attachment <file>
Adds an attachment to the docuement which is included in the PDF output.
This option can be added multiple times to attach more files.
.. option:: -p, --presentational-hints
Follow HTML presentational hints.
.. option:: --version
Show the version number. Other options and arguments are ignored.
.. option:: -h, --help
Show the command-line usage. Other options and arguments are ignored.
"""
parser = argparse.ArgumentParser(
prog='weasyprint', description='Renders web pages to PDF or PNG.')
parser.add_argument('--version', action='version',
version='WeasyPrint version %s' % VERSION,
help="Print WeasyPrint's version number and exit.")
parser.add_argument('-e', '--encoding',
help='Character encoding of the input')
parser.add_argument('-f', '--format', choices=['pdf', 'png'],
help='Output format. Can be ommited if `output` '
'ends with a .pdf or .png extension.')
parser.add_argument('-s', '--stylesheet', action='append',
help='URL or filename for a user CSS stylesheet. '
'May be given multiple times.')
parser.add_argument('-m', '--media-type', default='print',
help='Media type to use for @media, defaults to print')
parser.add_argument('-r', '--resolution', type=float,
help='PNG only: the resolution in pixel per CSS inch. '
'Defaults to 96, one PNG pixel per CSS pixel.')
parser.add_argument('--base-url',
help='Base for relative URLs in the HTML input. '
"Defaults to the input's own filename or URL "
'or the current directory for stdin.')
parser.add_argument('-a', '--attachment', action='append',
help='URL or filename of a file '
'to attach to the PDF docuement')
parser.add_argument('-p', '--presentational-hints', action='store_true',
help='Follow HTML presentational hints.')
parser.add_argument(
'input', help='URL or filename of the HTML input, or - for stdin')
parser.add_argument(
'output', help='Filename where output is written, or - for stdout')
args = parser.parse_args(argv)
if args.format is None:
output_lower = args.output.lower()
if output_lower.endswith('.pdf'):
format_ = 'pdf'
elif output_lower.endswith('.png'):
format_ = 'png'
else:
parser.error(
'Either sepecify a format with -f or choose an '
'output filename that ends in .pdf or .png')
else:
format_ = args.format.lower()
if args.input == '-':
if stdin is None:
stdin = sys.stdin
# stdin.buffer on Py3, stdin on Py2
source = getattr(stdin, 'buffer', stdin)
if not args.base_url:
args.base_url = '.' # current directory
else:
source = args.input
if args.output == '-':
if stdout is None:
stdout = sys.stdout
# stdout.buffer on Py3, stdout on Py2
output = getattr(stdout, 'buffer', stdout)
else:
output = args.output
kwargs = {
'stylesheets': args.stylesheet,
'presentational_hints': args.presentational_hints}
if args.resolution:
if format_ == 'png':
kwargs['resolution'] = args.resolution
else:
parser.error('--resolution only applies for the PNG format.')
if args.attachment:
if format_ == 'pdf':
kwargs['attachments'] = args.attachment
else:
parser.error('--attachment only applies for the PDF format.')
html = HTML(source, base_url=args.base_url, encoding=args.encoding,
media_type=args.media_type)
getattr(html, 'write_' + format_)(output, **kwargs)
0
Example 104
def open(self):
""" Become a daemon process.
:return: ``None``.
Open the daemon context, turning the current program into a daemon
process. This performs the following steps:
* If this instance's `is_open` property is true, return
immediately. This makes it safe to call `open` multiple times on
an instance.
* If the `prevent_core` attribute is true, set the resource limits
for the process to prevent any core dump from the process.
* If the `chroot_directory` attribute is not ``None``, set the
effective root directory of the process to that directory (via
`os.chroot`).
This allows running the daemon process inside a “chroot gaol”
as a means of limiting the system's exposure to rogue behaviour
by the process. Note that the specified directory needs to
already be set up for this purpose.
* Set the process UID and GID to the `uid` and `gid` attribute
values.
* Close all open file descriptors. This excludes those listed in
the `files_preserve` attribute, and those that correspond to the
`stdin`, `stdout`, or `stderr` attributes.
* Change current working directory to the path specified by the
`working_directory` attribute.
* Reset the file access creation mask to the value specified by
the `umask` attribute.
* If the `detach_process` option is true, detach the current
process into its own process group, and disassociate from any
controlling terminal.
* Set signal handlers as specified by the `signal_map` attribute.
* If any of the attributes `stdin`, `stdout`, `stderr` are not
``None``, bind the system streams `sys.stdin`, `sys.stdout`,
and/or `sys.stderr` to the files represented by the
corresponding attributes. Where the attribute has a file
descriptor, the descriptor is duplicated (instead of re-binding
the name).
* If the `pidfile` attribute is not ``None``, enter its context
manager.
* Mark this instance as open (for the purpose of future `open` and
`close` calls).
* Register the `close` method to be called during Python's exit
processing.
When the function returns, the running program is a daemon
process.
"""
if self.is_open:
return
if self.chroot_directory is not None:
change_root_directory(self.chroot_directory)
if self.prevent_core:
prevent_core_dump()
change_file_creation_mask(self.umask)
change_working_directory(self.working_directory)
change_process_owner(self.uid, self.gid)
if self.detach_process:
detach_process_context()
signal_handler_map = self._make_signal_handler_map()
set_signal_handlers(signal_handler_map)
exclude_fds = self._get_exclude_file_descriptors()
close_all_open_files(exclude=exclude_fds)
redirect_stream(sys.stdin, self.stdin)
redirect_stream(sys.stdout, self.stdout)
redirect_stream(sys.stderr, self.stderr)
if self.pidfile is not None:
self.pidfile.__enter__()
self._is_open = True
register_atexit_function(self.close)
0
Example 105
Project: textkit Source File: tokens_to_json.py
@click.command()
@click.argument('token_docs', type=click.Path(exists=True), nargs=-1)
@click.option('--ids', type=click.Path(),
help="File with one id per token docuement, each separated " +
"by a new line. Ids file is used to set the id attribute in " +
"the output JSON.")
@click.option('--names', type=click.Path(),
help="File with one name per token docuement, each separated " +
"by a new line. Names file is used to set the name attribute " +
"in the output JSON.")
@click.option('--field', default='tokens', help="Attribute name where " +
"tokens will be stored in the docuement object.",
show_default=True)
@click.option('--split/--no-split', default=False, help="If enabled, " +
"textkit will attempt to split input columns when " +
"packaging. This is useful when packaging multiple column " +
"output like counts.",
show_default=True)
@click.option('-s', '--sep', default=',', help="Separator character between " +
"columns. Only used if split-columns flag is used.",
show_default=True)
def tokens2json(ids, names, field, split, sep, token_docs):
'''Convert a set of token docuements into a
JSON array of docuement objects.'''
docs = []
names = read_names(names)
ids = read_names(ids)
for idx, path in enumerate(token_docs):
if path == '-':
tokens_doc = sys.stdin
else:
tokens_doc = open(path, 'r')
if split:
content = read_csv(tokens_doc, sep)
content = coerce_types(content)
else:
content = read_tokens(tokens_doc)
# ordered so that these attributes stay at the top
doc = OrderedDict()
if idx < len(ids) - 1:
doc['id'] = ids[idx]
else:
doc['id'] = path
if idx < len(names) - 1:
doc['name'] = names[idx]
else:
doc['name'] = path
doc[field] = content
docs.append(doc)
tokens_doc.close()
out_content = json.dumps(docs, indent=2)
output(out_content)
0
Example 106
def main():
#Read in command-line arguments
parser = ArgumentParser()
parser.add_argument('-i',
'--infile',
action ='store',
dest = 'inFile',
help = 'An input pileup file. If None, defaults to stdin. [None]',
default = None
)
parser.add_argument('-o',
'--outfile',
action = 'store',
dest = 'outFile',
help = 'A filename for the output file. [None]',
default = None
)
parser.add_argument('-l',
'--rlength',
type=int,
action = 'store',
dest = 'rlength',
default = '84',
help = 'The length of a single read'
)
parser.add_argument('-C',
'--max_clonality',
type = float,
action = 'store',
dest = 'max_clonality',
help = 'Maximum clonality to allow when considering a position [0.1]',
default = 0.1,
)
o = parser.parse_args()
# If an imput file is given, use it; otherwise, use stdin
if o.inFile != None:
f = open(o.inFile, 'r')
else:
f = sys.stdin
counter = myCounts(o.rlength)
linenum = 0
lineskips = 0
for line in f:
linebin = linePrep(line, o.max_clonality)
linenum += 1
readNum = 0
skips = 0
if linenum % 10000 == 0:
print('%s lines processed' % linenum)
try:
while readNum < len(linebin):
# Check what the identity of a charecter is
if linebin[readNum] in ('M', 'R', 'U', 'm', 'r', 'u'):
# Start a new read
counter.newRead(linebin[readNum])
elif linebin[readNum] == 'E':
# Mark a read to be closed later
skips += 1
counter.reads[readNum - skips].closeMe = True
elif linebin[readNum] in ('A', 'G', 'C', 'T', 'a', 'g', 'c', 't'):
# Count a mutation
counter.reads[readNum-skips].addMut()
elif linebin[readNum] in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
# Count an indel
counter.reads[readNum-1-skips].addIndel()
# Scroll the read with the indel forward the length of the indel, and mark indel length charecters to be skipped
tst = 0
indelLength = ''
while readNum + tst < len(linebin) and linebin[readNum+tst] in ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'):
indelLength += linebin[readNum + tst]
skips += 1
tst += 1
for x in xrange(int(indelLength)):
counter.reads[readNum-1-skips+tst].advance()
elif linebin[readNum] == 'D':
counter.reads[readNum-skips-1].addIndel()
skips += 1
elif linebin[readNum] == 'd':
counter.reads[readNum-skips].skipMe = True
elif linebin[readNum] in ('N', 'n'):
# Count an N
counter.reads[readNum-skips].addN()
elif linebin[readNum] in ('.', ','):
pass
else:
raise ValueError("Not a valid pileup value at %s: %s" % (readNum, linebin[readNum]))
readNum += 1
except Exception:
print('%s[%s]: %s' % (linenum, readNum, linebin))
raise
# Close any reads marked for closing
counter.closeReads()
# Advance all reads
counter.advanceReads()
# Generate and save the graphs.
counter.totals()
myX = range(1, o.rlength + 1)
ax1 = pylab.subplot(3, 1, 1)
pylab.plot(myX, counter.counts[0, :], 'b', linewidth = 2)
pylab.title('Mutations by Position: %s' % o.inFile)
pylab.ylabel("% Mutations")
pylab.setp(ax1.get_xticklabels(), visible=False)
ax2 = pylab.subplot(3, 1, 2)
pylab.plot(myX, counter.counts[1, :], 'r', linewidth = 2)
pylab.title('Indels by Position: %s' % o.inFile)
pylab.ylabel("% Indels")
pylab.setp(ax2.get_xticklabels(), visible=False)
ax3 = pylab.subplot(3, 1, 3)
pylab.plot(myX, counter.counts[2, :], 'k', linewidth = 2)
pylab.title('Ns by Position: %s' % o.inFile)
pylab.ylabel("% Ns")
pylab.xlabel("Read Position")
pylab.savefig(o.outFile)
# Generate and save the data file
outWrite = counter.counts.transpose()
outFile = open(o.outFile + '.dat', 'w')
for n in range(o.rlength):
outStr = ""
for m in range(3):
outStr += ' %s' % outWrite[n, m]
outFile.write(outStr + '\n')
outFile.close()
if o.inFile != None:
f.close()
0
Example 107
Project: codespell Source File: _codespell.py
def parse_file(filename, colors, summary):
lines = None
changed = False
global misspellings
global options
global encodings
global quiet_level
encoding = encodings[0] # if not defined, use UTF-8
if filename == '-':
f = sys.stdin
lines = f.readlines()
else:
# ignore binary files
if not os.path.isfile(filename):
return 0
text = is_text_file(filename)
if not text:
if not quiet_level & QuietLevels.BINARY_FILE:
print("WARNING: Binary file: %s " % filename, file=sys.stderr)
return 0
try:
lines, encoding = file_opener.open(filename)
except Exception:
return 0
bad_count = 0
rx = re.compile(r"[\w\-']+")
for i, line in enumerate(lines):
if line in exclude_lines:
continue
fixed_words = set()
asked_for = set()
for word in rx.findall(line):
lword = word.lower()
if lword in misspellings:
fix = misspellings[lword].fix
fixword = fix_case(word, misspellings[lword].data)
if options.interactive and lword not in asked_for:
fix, fixword = ask_for_word_fix(lines[i], word,
misspellings[lword],
options.interactive)
asked_for.add(lword)
if summary and fix:
summary.update(lword)
if word in fixed_words: # can skip because of re.sub below
continue
if options.write_changes and fix:
changed = True
lines[i] = re.sub(r'\b%s\b' % word, fixword, lines[i])
fixed_words.add(word)
continue
# otherwise warning was explicitly set by interactive mode
if (options.interactive & 2 and not fix and not
misspellings[lword].reason):
continue
cfilename = "%s%s%s" % (colors.FILE, filename, colors.DISABLE)
cline = "%s%d%s" % (colors.FILE, i, colors.DISABLE)
cwrongword = "%s%s%s" % (colors.WWORD, word, colors.DISABLE)
crightword = "%s%s%s" % (colors.FWORD, fixword, colors.DISABLE)
if misspellings[lword].reason:
if quiet_level & QuietLevels.DISABLED_FIXES:
continue
creason = " | %s%s%s" % (colors.FILE,
misspellings[lword].reason,
colors.DISABLE)
else:
if quiet_level & QuietLevels.NON_AUTOMATIC_FIXES:
continue
creason = ''
# If we get to this point (uncorrected error) we should change
# our bad_count and thus return value
bad_count += 1
if filename != '-':
print("%(FILENAME)s:%(LINE)s: %(WRONGWORD)s "
" ==> %(RIGHTWORD)s%(REASON)s"
% {'FILENAME': cfilename, 'LINE': cline,
'WRONGWORD': cwrongword,
'RIGHTWORD': crightword, 'REASON': creason})
else:
print('%(LINE)s: %(STRLINE)s\n\t%(WRONGWORD)s '
'==> %(RIGHTWORD)s%(REASON)s'
% {'LINE': cline, 'STRLINE': line.strip(),
'WRONGWORD': cwrongword,
'RIGHTWORD': crightword, 'REASON': creason})
if changed:
if filename == '-':
print("---")
for line in lines:
print(line, end='')
else:
if not quiet_level & QuietLevels.FIXES:
print("%sFIXED:%s %s"
% (colors.FWORD, colors.DISABLE, filename),
file=sys.stderr)
with codecs.open(filename, 'w', encoding=encoding) as f:
f.writelines(lines)
return bad_count
0
Example 108
Project: python-ftfy Source File: cli.py
def main():
"""
Run ftfy as a command-line utility.
"""
import argparse
parser = argparse.ArgumentParser(
description="ftfy (fixes text for you), version %s" % __version__
)
parser.add_argument('filename', default='-', nargs='?',
help='The file whose Unicode is to be fixed. Defaults '
'to -, meaning standard input.')
parser.add_argument('-o', '--output', type=str, default='-',
help='The file to output to. Defaults to -, meaning '
'standard output.')
parser.add_argument('-g', '--guess', action='store_true',
help="Ask ftfy to guess the encoding of your input. "
"This is risky. Overrides -e.")
parser.add_argument('-e', '--encoding', type=str, default='utf-8',
help='The encoding of the input. Defaults to UTF-8.')
parser.add_argument('-n', '--normalization', type=str, default='NFC',
help='The normalization of Unicode to apply. '
'Defaults to NFC. Can be "none".')
parser.add_argument('--preserve-entities', action='store_true',
help="Leave HTML entities as they are. The default "
"is to decode them, as long as no HTML tags "
"have appeared in the file.")
args = parser.parse_args()
encoding = args.encoding
if args.guess:
encoding = None
if args.filename == '-':
# Get a standard input stream made of bytes, so we can decode it as
# whatever encoding is necessary.
if PYTHON2:
file = sys.stdin
else:
file = sys.stdin.buffer
else:
file = open(args.filename, 'rb')
if args.output == '-':
encode_output = PYTHON2
outfile = sys.stdout
else:
encode_output = False
outfile = io.open(args.output, 'w', encoding='utf-8')
normalization = args.normalization
if normalization.lower() == 'none':
normalization = None
if args.preserve_entities:
fix_entities = False
else:
fix_entities = 'auto'
try:
for line in fix_file(file, encoding=encoding,
fix_entities=fix_entities,
normalization=normalization):
if encode_output:
outfile.write(line.encode('utf-8'))
else:
try:
outfile.write(line)
except UnicodeEncodeError:
if sys.platform == 'win32':
sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS)
else:
sys.stderr.write(ENCODE_ERROR_TEXT_UNIX)
sys.exit(1)
except UnicodeDecodeError as err:
sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err))
sys.exit(1)
0
Example 109
Project: proofofexistence Source File: genwallet.py
def main():
parser = argparse.ArgumentParser(description="Generate a private wallet key.")
parser.add_argument('-a', "--address", help='show as Bitcoin address', action='store_true')
parser.add_argument('-i', "--info", help='show metadata', action='store_true')
parser.add_argument('-j', "--json", help='output metadata as JSON', action='store_true')
parser.add_argument('-w', "--wif", help='show as Bitcoin WIF', action='store_true')
parser.add_argument('-f', "--wallet-key-file", help='initial wallet key', type=argparse.FileType('r'))
parser.add_argument('-k', "--wallet-key", help='initial wallet key')
parser.add_argument('-g', "--gpg", help='use gpg --gen-random to get additional entropy', action='store_true')
parser.add_argument('-u', "--dev-random", help='use /dev/random to get additional entropy', action='store_true')
parser.add_argument('-n', "--uncompressed", help='show in uncompressed form', action='store_true')
parser.add_argument('-p', help='generate wallet key from passphrase. NOT RECOMMENDED', metavar='passphrase')
parser.add_argument('-s', "--subkey", help='subkey path (example: 0p/2/1)')
parser.add_argument('-t', help='generate test key', action="store_true")
parser.add_argument('inputfile', help='source of entropy. stdin by default', type=argparse.FileType(mode='r+b'), nargs='?')
args = parser.parse_args()
# args.inputfile doesn't like binary when "-" is passed in. Deal with this.
if args.inputfile == sys.stdin:
args.inputfile = sys.stdin.buffer
entropy = bytearray()
if args.gpg:
entropy.extend(gpg_entropy())
if args.dev_random:
entropy.extend(dev_random_entropy())
if args.inputfile:
entropy.extend(args.inputfile.read())
if args.p:
entropy.extend(args.p.encode("utf8"))
if len(entropy) == 0 and not args.wallet_key and not args.wallet_key_file:
parser.error("you must specify at least one source of entropy")
if args.wallet_key and len(entropy) > 0:
parser.error("don't specify both entropy and a wallet key")
if args.wallet_key_file:
wallet = Wallet.from_wallet_key(args.wallet_key_file.readline()[:-1])
elif args.wallet_key:
wallet = Wallet.from_wallet_key(args.wallet_key)
else:
wallet = Wallet.from_master_secret(bytes(entropy), is_test=args.t)
try:
if args.subkey:
wallet = wallet.subkey_for_path(args.subkey)
if wallet.child_number >= 0x80000000:
wc = wallet.child_number - 0x80000000
child_index = "%dp (%d)" % (wc, wallet.child_number)
else:
child_index = "%d" % wallet.child_number
if args.json:
d = dict(
wallet_key=wallet.wallet_key(as_private=wallet.is_private),
public_pair_x=wallet.public_pair[0],
public_pair_y=wallet.public_pair[1],
tree_depth=wallet.depth,
fingerprint=b2h(wallet.fingerprint()),
parent_fingerprint=b2h(wallet.parent_fingerprint),
child_index=child_index,
chain_code=b2h(wallet.chain_code),
bitcoin_addr=wallet.bitcoin_address(),
bitcoin_addr_uncompressed=wallet.bitcoin_address(compressed=False),
network="test" if wallet.is_test else "main",
)
if wallet.is_private:
d.update(dict(
key="private",
secret_exponent=wallet.secret_exponent,
WIF=wallet.wif(),
WIF_uncompressed=wallet.wif(compressed=False)
))
else:
d.update(dict(key="public"))
print json.dumps(d, indent=3)
elif args.info:
print(wallet.wallet_key(as_private=wallet.is_private))
if wallet.is_test:
print("test network")
else:
print("main network")
if wallet.is_private:
print("private key")
print("secret exponent: %d" % wallet.secret_exponent)
else:
print("public key only")
print("public pair x: %d\npublic pair y: %d" % wallet.public_pair)
print("tree depth: %d" % wallet.depth)
print("fingerprint: %s" % b2h(wallet.fingerprint()))
print("parent f'print: %s" % b2h(wallet.parent_fingerprint))
print("child index: %s" % child_index)
print("chain code: %s" % b2h(wallet.chain_code))
if wallet.is_private:
print("WIF: %s" % wallet.wif())
print(" uncompressed: %s" % wallet.wif(compressed=False))
print("Bitcoin address: %s" % wallet.bitcoin_address())
print(" uncompressed: %s" % wallet.bitcoin_address(compressed=False))
elif args.address:
print(wallet.bitcoin_address(compressed=not args.uncompressed))
elif args.wif:
print(wallet.wif(compressed=not args.uncompressed))
else:
print(wallet.wallet_key(as_private=wallet.is_private))
except PublicPrivateMismatchError as ex:
print(ex.args[0])
0
Example 110
Project: cutadapt Source File: seqio.py
def _seqopen1(file, colorspace=False, fileformat=None, mode='r', qualities=None):
"""
Open a single sequence file. See description above.
"""
if mode == 'r':
fastq_handler = ColorspaceFastqReader if colorspace else FastqReader
fasta_handler = ColorspaceFastaReader if colorspace else FastaReader
elif mode == 'w':
fastq_handler = ColorspaceFastqWriter if colorspace else FastqWriter
fasta_handler = ColorspaceFastaWriter if colorspace else FastaWriter
else:
raise ValueError("Mode must be 'r' or 'w'")
if fileformat: # Explict file format given
fileformat = fileformat.lower()
if fileformat == 'fasta':
return fasta_handler(file)
elif fileformat == 'fastq':
return fastq_handler(file)
elif fileformat == 'sra-fastq' and colorspace:
if mode == 'w':
raise NotImplementedError('Writing to sra-fastq not supported')
return SRAColorspaceFastqReader(file)
else:
raise UnknownFileType("File format {0!r} is unknown (expected "
"'sra-fastq' (only for colorspace), 'fasta' or 'fastq').".format(fileformat))
# Detect file format
name = None
if file == "-":
file = sys.stdin if mode == 'r' else sys.stdout
elif isinstance(file, basestring):
name = file
elif hasattr(file, "name"): # seems to be an open file-like object
name = file.name
if name:
for ext in ('.gz', '.xz', '.bz2'):
if name.endswith(ext):
name = name[:-len(ext)]
break
name, ext = splitext(name)
ext = ext.lower()
if ext in ['.fasta', '.fa', '.fna', '.csfasta', '.csfa']:
format = 'fasta'
elif ext in ['.fastq', '.fq'] or (ext == '.txt' and name.endswith('_sequence')):
format = 'fastq'
elif mode == 'w' and qualities is True:
# Format not recognized, but know we want to write reads with qualities
format = 'fastq'
elif mode == 'w' and qualities is False:
# Same, but we know that we want to write reads without qualities
format = 'fasta'
else:
raise UnknownFileType("Could not determine whether file {0!r} is FASTA "
"or FASTQ: file name extension {1!r} not recognized".format(file, ext))
if format == 'fastq' and qualities is False:
raise ValueError("Output format cannot be FASTQ since no quality "
"values are available.")
if format == 'fastq':
return fastq_handler(file)
else:
return fasta_handler(file)
if mode == 'w':
if qualities is True:
return fastq_handler(file)
elif qualities is False:
return fasta_handler(file)
raise UnknownFileType('Cannot determine whether to write in FASTA or '
'FASTQ format')
# No name available. Try to autodetect type by reading from the file.
for line in file:
if line.startswith('#'):
# Skip comment lines (needed for csfasta)
continue
if line.startswith('>'):
return fasta_handler(FileWithPrependedLine(file, line))
if line.startswith('@'):
return fastq_handler(FileWithPrependedLine(file, line))
raise UnknownFileType("File is neither FASTQ nor FASTA.")
0
Example 111
Project: django-twitter-stream Source File: stream.py
def handle(self, keys_name=settings.DEFAULT_KEYS_NAME, *args, **options):
# The suggested time between hearbeats
poll_interval = float(options.get('poll_interval', settings.POLL_INTERVAL))
prevent_exit = options.get('prevent_exit', settings.PREVENT_EXIT)
to_file = options.get('to_file', None)
from_file = options.get('from_file', None)
from_file_long = options.get('from_file_long', None)
rate_limit = options.get('rate_limit', 50)
limit = options.get('limit', None)
if from_file and from_file_long:
logger.error("Cannot use both --from-file and --from-file-long")
exit(1)
# First expire any old stream process records that have failed
# to report in for a while
timeout_seconds = 3 * poll_interval
models.StreamProcess.expire_timed_out()
# Create the stream process for tracking ourselves
stream_process = models.StreamProcess.create(
timeout_seconds=timeout_seconds
)
listener = utils.QueueStreamListener(to_file=to_file)
if from_file:
checker = utils.FakeTermChecker(queue_listener=listener,
stream_process=stream_process)
else:
checker = utils.FeelsTermChecker(queue_listener=listener,
stream_process=stream_process)
def stop(signum, frame):
"""
Register stream's death and exit.
"""
if stream_process:
stream_process.status = models.StreamProcess.STREAM_STATUS_STOPPED
stream_process.heartbeat()
# Let the tweet listener know it should be quitting asap
listener.set_terminate()
logger.error("Terminating")
raise SystemExit()
# Installs signal handlers for handling SIGINT and SIGTERM
# gracefully.
signal.signal(signal.SIGINT, stop)
signal.signal(signal.SIGTERM, stop)
keys = None
if not from_file:
# Only need keys if we are connecting to twitter
while not keys:
try:
keys = models.ApiKey.get_keys(keys_name)
except ObjectDoesNotExist:
if keys_name:
logger.error("Keys for '%s' do not exist in the database. Waiting...", keys_name)
else:
logger.warn("No keys in the database. Waiting...")
time.sleep(5)
stream_process.status = models.StreamProcess.STREAM_STATUS_WAITING
stream_process.heartbeat()
try:
if keys:
logger.info("Connecting to Twitter with keys for %s/%s", keys.user_name, keys.app_name)
stream_process.keys = keys
stream_process.save()
# Only need auth if we have keys (i.e. connecting to twitter)
auth = tweepy.OAuthHandler(keys.api_key, keys.api_secret)
auth.set_access_token(keys.access_token, keys.access_token_secret)
# Start and maintain the streaming connection...
stream = twitter_monitor.DynamicTwitterStream(auth, listener, checker)
elif from_file or from_file_long:
read_pretty = False
if from_file_long:
from_file = from_file
read_pretty = True
if from_file == '-':
from_file = sys.stdin
logger.info("Reading tweets from stdin")
else:
if read_pretty:
logger.info("Reading tweets from JSON file %s (pretty-printed)", from_file)
else:
logger.info("Reading tweets from JSON file %s", from_file)
stream = utils.FakeTwitterStream(from_file, pretty=read_pretty,
listener=listener, term_checker=checker,
limit=limit, rate_limit=rate_limit)
else:
raise Exception("No api keys and we're not streaming from a file.")
if to_file:
logger.info("Saving tweets to %s", to_file)
if prevent_exit:
while checker.ok():
try:
stream.start_polling(poll_interval)
except Exception as e:
checker.error(e)
time.sleep(1) # to avoid craziness
else:
stream.start_polling(poll_interval)
logger.error("Stopping because of excess errors")
stream_process.status = models.StreamProcess.STREAM_STATUS_STOPPED
stream_process.heartbeat()
except Exception as e:
logger.error(e, exc_info=True)
finally:
stop(None, None)
0
Example 112
Project: cee Source File: CompilerPlugin.py
def run(self, filename):
program_output_raw = ""
message_string = ""
tmp_fname = "files/output/cee_output_%s" % re.sub('[^0-9a-zA-Z]+', '*', filename)
output = open(
tmp_fname,
"w+"
)
args = []
if self.run_cmd:
args.append(self.run_cmd)
args.append(os.path.join(os.getcwd(), filename))
cookbook = {
'args': args,
'stdin': sys.stdin,
'stdout': output,
'stderr': output,
'quota': dict(
wallclock=100000,
cpu=50000,
memory=500000000,
disk=1048576
)
}
try:
msb = plugins.MiniSandbox.MiniSandbox(**cookbook)
msb.run()
except ValueError as e:
print(e)
return "<killed> ( recieved fork attempt )"
output.flush()
output.close()
else:
# verbose statistics
program_output_data = msb.probe()
output.flush()
output.close()
output = open(
tmp_fname,
"r"
)
program_output_raw = output.read()
temp = program_output_raw.replace("\r", "")
program_output = temp.split("\n")
message_string = program_output[0]
message_string.rstrip()
message_string = to_bytes(message_string)
print message_string
print(program_output_data.get("result", False))
if program_output_data.get("result", False) == "TL":
message_string = "<killed> ( timed out )"
elif program_output_data.get("result", False) == "RF":
message_string = \
"<killed> ( restricted function used: %d(%d) )" % (
program_output_data.get("syscall_info")[0],
program_output_data.get("syscall_info")[1]
)
elif program_output_data.get("result", False) == "ML":
message_string = "<killed> ( memory limit exceeded )"
else:
if program_output[0]:
if len(program_output) > 1:
message_string = to_bytes(
message_string +
" [+%d deleted lines]" % (len(program_output) - 1)
)
max_msg_len = 400 - len("[+nnn deleted bytes]")
if len(message_string) > max_msg_len:
message_string = (
message_string[:max_msg_len] +
(
"[+%d deleted bytes]" %
(len(message_string) - max_msg_len)
)
)
else:
message_string = "<no output> ( return value was %d ) " % (
program_output_data.get("exitcode", -1)
)
output.close()
os.remove(tmp_fname)
return message_string
0
Example 113
Project: Arelle Source File: CntlrWebMain.py
def startWebserver(_cntlr, options):
"""Called once from main program in CmtlrCmdLine to initiate web server on specified local port.
To test WebServer run from source in IIS, use an entry like this: c:\python33\python.exe c:\\users\\myname\\mySourceFolder\\arelleCmdLine.py %s
:param options: OptionParser options from parse_args of main argv arguments (the argument *webserver* provides hostname and port), port being used to startup the webserver on localhost.
:type options: optparse.Values
"""
global imagesDir, cntlr, optionsPrototype
cntlr = _cntlr
imagesDir = cntlr.imagesDir
optionValuesTypes = _STR_NUM_TYPES + (type(None),)
optionsPrototype = dict((option,value if isinstance(value,_STR_NUM_TYPES) else None)
for option in dir(options)
for value in (getattr(options, option),)
if isinstance(value,optionValuesTypes) and not option.startswith('_'))
host, sep, portServer = options.webserver.partition(":")
port, sep, server = portServer.partition(":")
# start a Bottle application
app = Bottle()
GETorPOST = ('GET', 'POST')
GET = 'GET'
POST = 'POST'
# install REST API interfaces
# if necessary to support CGI hosted servers below root, add <prefix:path> as first part of routes
# and corresponding arguments to the handler methods
app.route('/rest/login', GET, login_form)
app.route('/rest/login', POST, login_submit)
app.route('/rest/logout', GET, logout)
app.route('/favicon.ico', GET, arelleIcon)
app.route('/rest/xbrl/<file:path>/open', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/close', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/validation/xbrl', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/DTS', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/concepts', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/pre', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/cal', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/dim', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/facts', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/factTable', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/roleTypes', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/arcroleTypes', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/formulae', GETorPOST, validation)
app.route('/rest/xbrl/validation', GETorPOST, validation)
app.route('/rest/xbrl/view', GETorPOST, validation)
app.route('/rest/xbrl/open', GETorPOST, validation)
app.route('/rest/xbrl/close', GETorPOST, validation)
app.route('/images/<imgFile>', GET, image)
app.route('/rest/xbrl/diff', GET, diff)
app.route('/rest/configure', GET, configure)
app.route('/rest/stopWebServer', GET, stopWebServer)
app.route('/quickbooks/server.asmx', POST, quickbooksServer)
app.route('/rest/quickbooks/<qbReport>/xbrl-gl/<file:path>', GET, quickbooksGLrequest)
app.route('/rest/quickbooks/<qbReport>/xbrl-gl/<file:path>/view', GET, quickbooksGLrequest)
app.route('/rest/quickbooks/<qbReport>/xbrl-gl/view', GET, quickbooksGLrequest)
app.route('/rest/quickbooks/response', GET, quickbooksGLresponse)
app.route('/quickbooks/server.html', GET, quickbooksWebPage)
app.route('/quickbooks/localhost.crt', GET, localhostCertificate)
app.route('/localhost.crt', GET, localhostCertificate)
app.route('/help', GET, helpREST)
app.route('/about', GET, about)
app.route('/', GET, indexPageREST)
if server == "cgi":
# catch a non-REST interface by cgi Interface (may be a cgi app exe module, etc)
app.route('<cgiAppPath:path>', GETorPOST, cgiInterface)
if server == "wsgi":
return app
elif server == "cgi":
if sys.stdin is None:
sys.stdin = open(os.devnull, 'r')
app.run(server=server)
sys.exit(0)
elif server:
app.run(host=host, port=port or 80, server=server)
else:
app.run(host=host, port=port or 80)
0
Example 114
Project: genmod Source File: analyze.py
@click.command()
@click.argument('variant_file',
nargs=1,
type=click.Path(exists=True),
metavar='<vcf_file> or "-"'
)
@click.option('-t' ,'--family_type',
type=click.Choice(['ped', 'alt', 'cmms', 'mip']),
default='ped',
help="""If the analysis use one of the known setups,
please specify which one."""
)
# @click.option('-c', '--config_file',
# type=click.Path(exists=True),
# help="""Specify the path to a config file."""
# )
@click.option('--frequency_treshold', '-freq',
default=0.02,
nargs=1,
help="""Specify maf treshold for variants to be considered.
Default 0.02"""
)
@click.option('--frequency_keyword', '-freqkey',
default='1000G_freq',
nargs=1,
help="""Specify keyword for frequency in vcf.
Default 1000G_freq"""
)
@click.option('--cadd_treshold', '-cadd',
default=12.0,
nargs=1,
help="""Specify the cadd treshold for variants to be
considered. Default 12.0"""
)
@click.option('--cadd_keyword', '-caddkey',
default='CADD',
nargs=1,
help="""Specify keyword for CADD scores in vcf.
Default CADD"""
)
@click.option('--coverage', '-cov',
default=7,
nargs=1,
help="""Specify minimum read depth in all individuals for
variant to be considered. Default 7"""
)
@click.option('--gq_treshold', '-gq',
default=20,
nargs=1,
help="""Specify genotype quality treshold for variants
to be considered. Default 20."""
)
# @click.option('-p', '--patterns',
# type=click.Choice(['AR', 'AD', 'X']),
# multiple=True,
# help='Specify the inheritance patterns. Default is all patterns'
# )
@click.option('-o', '--outdir',
type=click.Path(exists=True),
default=os.getcwd(),
help="""Specify the path to a directory where results
should be stored. Default is ./"""
)
@click.option('-s', '--silent',
is_flag=True,
help='Do not output variants.'
)
@click.option('-exclude', '--exclude_problematic',
is_flag=True,
help="""Exclude problematic genes. This flag is preferable
if analysis of only one individual."""
)
@click.option('-v', '--verbose',
is_flag=True,
help='Increase output verbosity.'
)
def analyze(variant_file, family_type, frequency_treshold, frequency_keyword,
cadd_treshold, cadd_keyword, coverage, gq_treshold, outdir, silent,
exclude_problematic, verbose):
"""Analyze the annotated variants in a VCF file.
If there are multiple families in the ped one analysis per family will
be done. The variants are analyzed in five different categories based
on what inheritance patterns that are followed.
The differen analysies are:
AR compound\n
AR humozygote\n
Dominant\n
X linked\n
Dominant dn\n
Which variants to be considered are specified in the command line.
Defaults are (based on a rare disease assumption):
MAF < 0.02\n
CADD score > 12\n
Coverage in all individuals > 7\n
Call quality > 20\n
The highest scoring variants of each category is printed to screen.
The full list of each category is printed to new vcf files in a
directory specified by the user. Default current dir.
File names are the same like the input vcf with the name of the
analysis appended.
"""
start_time_analysis = datetime.now()
# configs = ConfigObj(config_file)
# prefered_models = make_models([])
inheritance_keyword = 'GeneticModels'
families = check_families(variant_file)
file_name = os.path.splitext(os.path.split(variant_file)[-1])[0]
# if config_file:
# frequency_treshold = float(configs.get('frequency', {}).get('rare', frequency_treshold))
# freq_keyword = configs.get('frequency', {}).get('keyword', freq_keyword)
# inheritance_patterns = [pattern for pattern in configs.get('inheritance', {}).get('patterns',[])]
# inheritance_keyword = configs.get('inheritance', {}).get('keyword',inheritance_keyword)
# prefered_models = make_models(inheritance_patterns)
if variant_file == '-':
variant_parser = VCFParser(fsock = sys.stdin)
else:
variant_parser = VCFParser(infile = variant_file)
for family_id in families:
print('Analysis for family: %s' % family_id)
head = variant_parser.metadata
dominant_dict = {}
humozygote_dict = {}
compound_dict = {}
x_linked_dict = {}
dominant_dn_dict = {}
get_interesting_variants(variant_parser,
family_id,
dominant_dict,
humozygote_dict,
compound_dict,
x_linked_dict,
dominant_dn_dict,
frequency_treshold,
frequency_keyword,
cadd_treshold,
cadd_keyword,
gq_treshold,
coverage,
exclude_problematic)
remove_inacurate_compounds(compound_dict, family_id)
if len(dominant_dict) > 0:
dominant_file = os.path.join(
outdir,
file_name+'_dominant_analysis.vcf'
)
print_headers(head, dominant_file)
print_results(
dominant_dict,
dominant_file,
family_id,
variant_parser.header,
cadd_keyword,
frequency_keyword,
mode='dominant'
)
if len(humozygote_dict) > 0:
humozygote_file = os.path.join(
outdir,
file_name+'_humozygote_analysis.vcf'
)
print_headers(head, humozygote_file)
print_results(
humozygote_dict,
humozygote_file,
family_id,
variant_parser.header,
cadd_keyword,
frequency_keyword,
mode='humozygote'
)
if len(compound_dict) > 0:
compound_file = os.path.join(
outdir,
file_name+'_compound_analysis.vcf'
)
print_headers(head, compound_file)
print_results(
compound_dict,
compound_file,
family_id,
variant_parser.header,
cadd_keyword,
frequency_keyword,
mode='compound'
)
if len(x_linked_dict) > 0:
xlinked_file = os.path.join(
outdir,
file_name+'_x_linked_analysis.vcf'
)
print_headers(head, xlinked_file)
print_results(
x_linked_dict,
xlinked_file,
family_id,
variant_parser.header,
cadd_keyword,
frequency_keyword,
mode='xlinked'
)
if len(dominant_dn_dict) > 0:
dominant_dn_file = os.path.join(
outdir,
file_name+'_ad_denovo_analysis.vcf'
)
print_headers(head, dominant_dn_file)
print_results(
dominant_dn_dict,
dominant_dn_file,
family_id,
variant_parser.header,
cadd_keyword,
frequency_keyword,
mode='denovo'
)
print('')
print('Number of interesting Dominant variants: %s' %
len(dominant_dict))
print('Number of interesting Homozygote variants: %s' %
len(humozygote_dict))
print('Number of interesting Compound variants: %s' %
len(compound_dict))
print('Number of interesting X-linked variants: %s' %
len(x_linked_dict))
print('Number of interesting Autosomal Dominant de novo variants: %s' %
len(dominant_dn_dict))
# pp(compound_dict)
print('Time for analysis: %s' % str(datetime.now() - start_time_analysis))
0
Example 115
Project: genmod Source File: summarize_variants.py
@click.command()
@variant_file
@family_file
# @click.option('-c', '--config_file',
# type=click.Path(exists=True),
# help="""Specify the path to a config file."""
# )
@click.option('--frequency_keyword', '-freqkey',
default='1000G_freq',
nargs=1,
help='Specify keyword for frequency in vcf. Default 1000G_freq'
)
@click.option('--frequency_treshold', '-freq',
default=0.05,
nargs=1,
help='Specify the ferquency treshold for variants to be considered. Default=0.05'
)
@click.option('--cadd_keyword', '-caddkey',
default='CADD',
nargs=1,
help='Specify keyword for CADD scores in vcf. Default CADD'
)
@click.option('--gq_treshold', '-gq',
default=20.0,
nargs=1,
help='Specify the genotype quality treshold for variants to be considered. Default=50'
)
@click.option('--read_depth_treshold', '-depth',
default=10.0,
nargs=1,
help="""Specify the genotype quality treshold for variants to be considered. Default=10.
The read deth is taken from AD, so it is the sum of the quality reads from reference and alternative alleles."""
)
@click.option('--cadd_treshold', '-cadd',
default=12.0,
nargs=1,
help='Specify the cadd treshold for variants to be considered. Default 12.0'
)
# @click.option('-p', '--patterns',
# type=click.Choice(['AR', 'AD', 'X']),
# multiple=True,
# help='Specify the inheritance patterns. Default is all patterns'
# )
# @click.option('-o', '--outfile',
# type=click.Path(exists=False),
# help='Specify the path to a file where results should be stored.'
# )
# @click.option('-v', '--verbose',
# is_flag=True,
# help='Increase output verbosity.'
# )
def summarize(variant_file, family_file, frequency_treshold, frequency_keyword,
cadd_treshold, cadd_keyword, gq_treshold, read_depth_treshold):
"""
Summarize the the variants in a vcf.
There will be one result line per individual.
- How many variants found\n
- How many variants did not satisfy the base call
quality treshold. (Default 20)\n
- How many variants where not covered in all individuals.
(Default depth 10)\n
- How many variants followed each model in each family:\n
- AR_hom\n
- AR_comp\n
- AR_hom_dn\n
- AR_comp_dn\n
- AD\n
- AD_dn\n
- XD\n
- XD_dn\n
- XR\n
- XR_dn\n
- How many rare variants (Default maf < 0.02)\n
- How many high scored cadd. (Default cadd = 0)\n
- How many rare + high score cadd\n
- How many no cadd score\n
- How many indels\n
- How many indels without cadd score\n
"""
logger = logging.getLogger(__name__)
logger = logging.getLogger("genmod.commands.summarize_variants")
head = HeaderParser()
nr_of_variants = 0
header = ['sample_id', 'nr_of_variants']
samples = {}
logger.debug("Setting up a variant parser")
if variant_file == '-':
variant_parser = VCFParser(
fsock = sys.stdin,
check_info=False
)
else:
variant_parser = VCFParser(
infile = variant_file,
check_info=False
)
logger.debug("Variant parser setup")
head = variant_parser.metadata
for sample_id in head.individuals:
samples[sample_id] = {}
samples[sample_id]["nr_of_variants"] = 0
for variant in variant_parser:
for sample_id in samples:
samples[sample_id]["nr_of_variants"] += 1
print(variant['genotypes'][sample_id].depth_of_coverage)
print(json.dumps(samples))
0
Example 116
Project: vcf_parser Source File: command_line.py
@click.command()
@click.argument('variant_file',
nargs=1,
type=click.Path(),
metavar='<vcf_file> or -'
)
@click.option('-v', '--vep',
is_flag=True,
help='If variants are annotated with the Variant Effect Predictor.'
)
@click.option('-s', '--split',
is_flag=True,
help='Split the variants with multiallelic calls.'
)
@click.option('-o', '--outfile',
type=click.Path(exists=False),
help='Path to a outfile.'
)
@click.option('-a',"--allele_symbol",
default='0',
help="The symbol that should be used when representing "\
"unobserved alleles. Default is '0'"
)
@click.option('-v', '--verbose',
is_flag=True,
help='Increase output verbosity.'
)
@click.option('-i', '--check_info',
is_flag=True,
help='Skip to check if info fields are correct for all variants.'
)
@click.option('--silent',
is_flag=True,
help='Do not print vcf data.'
)
@click.option('--version',
is_flag=True,
callback=print_version,
expose_value=False,
is_eager=True
)
@click.option('-l', '--logfile',
type=click.Path(exists=False),
help="Path to log file. If none logging is "\
"printed to stderr."
)
@click.option('--loglevel',
type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR',
'CRITICAL']),
help="Set the level of log output."
)
def cli(variant_file, vep, split, outfile, verbose, silent, check_info,
allele_symbol, logfile, loglevel):
"""
Tool for parsing vcf files.
Prints the vcf file to output.
If --split/-s is used all multiallelic calls will be splitted and printed
as single variant calls.
For more information, please see github.com/moonso/vcf_parser.
"""
from vcf_parser import logger, init_log
if not loglevel:
if verbose:
loglevel = 'INFO'
init_log(logger, logfile, loglevel)
nr_of_variants = 0
start = datetime.now()
# with open(variant_file, 'r', encoding="utf-8") as f:
# for line in f:
# if not line.startswith('#'):
# nr_of_variants += 1
if variant_file == '-':
logger.info("Start parsing variants from stdin")
my_parser = VCFParser(
fsock=sys.stdin,
split_variants=split,
check_info=check_info,
allele_symbol=allele_symbol
)
else:
logger.info("Start parsing variants from file {0}".format(variant_file))
my_parser = VCFParser(
infile = variant_file,
split_variants=split,
check_info=check_info,
allele_symbol=allele_symbol
)
if outfile:
f = open(outfile, 'w', encoding='utf-8')
logger.info("Printing vcf to file {0}".format(outfile))
if not silent:
logger.info("Printing vcf to stdout")
else:
logger.info("Skip printing since silent is active")
for line in my_parser.metadata.print_header():
if outfile:
f.write(line+'\n')
else:
if not silent:
print(line)
try:
for variant in my_parser:
variant_line = '\t'.join([variant[head] for head in my_parser.header])
if outfile:
f.write(variant_line + '\n')
else:
if not silent:
print(variant_line)
nr_of_variants += 1
except SyntaxError as e:
print(e)
logger.info('Number of variants: {0}'.format(nr_of_variants))
logger.info('Time to parse file: {0}'.format(str(datetime.now() - start)))
0
Example 117
Project: fjord Source File: parse.py
def parse():
optParser = getOptParser()
opts,args = optParser.parse_args()
encoding = 'utf8'
try:
f = args[-1]
# Try opening from the internet
if f.startswith('http://'):
try:
import urllib.request, urllib.parse, urllib.error, cgi
f = urllib.request.urlopen(f)
contentType = f.headers.get('content-type')
if contentType:
(mediaType, params) = cgi.parse_header(contentType)
encoding = params.get('charset')
except:
pass
elif f == '-':
f = sys.stdin
if sys.version_info[0] >= 3:
encoding = None
else:
try:
# Try opening from file system
f = open(f, 'rb')
except IOError as e:
sys.stderr.write('Unable to open file: %s\n' % e)
sys.exit(1)
except IndexError:
sys.stderr.write('No filename provided. Use -h for help\n')
sys.exit(1)
treebuilder = treebuilders.getTreeBuilder(opts.treebuilder)
if opts.sanitize:
tokenizer = sanitizer.HTMLSanitizer
else:
tokenizer = HTMLTokenizer
p = html5parser.HTMLParser(tree=treebuilder, tokenizer=tokenizer, debug=opts.log)
if opts.fragment:
parseMethod = p.parseFragment
else:
parseMethod = p.parse
if opts.profile:
import cProfile
import pstats
cProfile.runctx('run(parseMethod, f, encoding)', None,
{'run': run,
'parseMethod': parseMethod,
'f': f,
'encoding': encoding},
'stats.prof')
# XXX - We should use a temp file here
stats = pstats.Stats('stats.prof')
stats.strip_dirs()
stats.sort_stats('time')
stats.print_stats()
elif opts.time:
import time
t0 = time.time()
docuement = run(parseMethod, f, encoding)
t1 = time.time()
if docuement:
printOutput(p, docuement, opts)
t2 = time.time()
sys.stderr.write('\n\nRun took: %fs (plus %fs to print the output)'%(t1-t0, t2-t1))
else:
sys.stderr.write('\n\nRun took: %fs'%(t1-t0))
else:
docuement = run(parseMethod, f, encoding)
if docuement:
printOutput(p, docuement, opts)
0
Example 118
Project: socorro Source File: mboxutils.py
def getmbox(name):
"""Return an mbox iterator given a file/directory/folder name."""
if name == "-":
return [get_message(sys.stdin)]
if name.startswith("+"):
# MH folder name: +folder, +f1,f2,f2, or +ALL
name = name[1:]
import mhlib
mh = mhlib.MH()
if name == "ALL":
names = mh.listfolders()
elif ',' in name:
names = name.split(',')
else:
names = [name]
mboxes = []
mhpath = mh.getpath()
for name in names:
filename = os.path.join(mhpath, name)
mbox = mailbox.MHMailbox(filename, get_message)
mboxes.append(mbox)
if len(mboxes) == 1:
return iter(mboxes[0])
else:
return _cat(mboxes)
elif name.startswith(":"):
# IMAP mailbox name:
# :username:password@server:folder1,...folderN
# :username:password@server:port:folder1,...folderN
# :username:password@server:ALL
# :username:password@server:port:ALL
parts = re.compile(
':(?P<user>[^@:]+):(?P<pwd>[^@]+)@(?P<server>[^:]+(:[0-9]+)?):(?P<name>[^:]+)'
).match(name).groupdict()
from scripts.sb_imapfilter import IMAPSession, IMAPFolder
from spambayes import Stats, message
from spambayes.Options import options
session = IMAPSession(parts['server'])
session.login(parts['user'], parts['pwd'])
folder_list = session.folder_list()
if name == "ALL":
names = folder_list
else:
names = parts['name'].split(',')
message_db = message.Message().message_info_db
stats = Stats.Stats(options, message_db)
mboxes = [IMAPFolder(n, session, stats) for n in names]
if len(mboxes) == 1:
return full_messages(mboxes[0])
else:
return _cat([full_messages(x) for x in mboxes])
if os.path.isdir(name):
# XXX Bogus: use a Maildir if /cur is a subdirectory, else a MHMailbox
# if the pathname contains /Mail/, else a DirOfTxtFileMailbox.
if os.path.exists(os.path.join(name, 'cur')):
mbox = mailbox.Maildir(name, get_message)
elif name.find("/Mail/") >= 0:
mbox = mailbox.MHMailbox(name, get_message)
else:
mbox = DirOfTxtFileMailbox(name, get_message)
else:
fp = open(name, "rb")
mbox = mailbox.PortableUnixMailbox(fp, get_message)
return iter(mbox)
0
Example 119
def main():
"""Script main program."""
import argparse
parser = argparse.ArgumentParser(
description='Utilities to support installing Python libraries.')
parser.add_argument('-l', action='store_const', const=0,
default=10, dest='maxlevels',
help="don't recurse into subdirectories")
parser.add_argument('-r', type=int, dest='recursion',
help=('control the maximum recursion level. '
'if `-l` and `-r` options are specified, '
'then `-r` takes precedence.'))
parser.add_argument('-f', action='store_true', dest='force',
help='force rebuild even if timestamps are up to date')
parser.add_argument('-q', action='count', dest='quiet', default=0,
help='output only error messages; -qq will suppress '
'the error messages as well.')
parser.add_argument('-b', action='store_true', dest='legacy',
help='use legacy (pre-PEP3147) compiled file locations')
parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
help=('directory to prepend to file paths for use in '
'compile-time tracebacks and in runtime '
'tracebacks in cases where the source file is '
'unavailable'))
parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
help=('skip files matching the regular expression; '
'the regexp is searched for in the full path '
'of each file considered for compilation'))
parser.add_argument('-i', metavar='FILE', dest='flist',
help=('add all the files and directories listed in '
'FILE to the list considered for compilation; '
'if "-", names are read from stdin'))
parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
help=('zero or more file and directory names '
'to compile; if no arguments given, defaults '
'to the equivalent of -l sys.path'))
parser.add_argument('-j', '--workers', default=1,
type=int, help='Run compileall concurrently')
args = parser.parse_args()
compile_dests = args.compile_dest
if (args.ddir and (len(compile_dests) != 1
or not os.path.isdir(compile_dests[0]))):
parser.exit('-d destdir requires exactly one directory argument')
if args.rx:
import re
args.rx = re.compile(args.rx)
if args.recursion is not None:
maxlevels = args.recursion
else:
maxlevels = args.maxlevels
# if flist is provided then load it
if args.flist:
try:
with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
for line in f:
compile_dests.append(line.strip())
except OSError:
if args.quiet < 2:
print("Error reading file list {}".format(args.flist))
return False
if args.workers is not None:
args.workers = args.workers or None
success = True
try:
if compile_dests:
for dest in compile_dests:
if os.path.isfile(dest):
if not compile_file(dest, args.ddir, args.force, args.rx,
args.quiet, args.legacy):
success = False
else:
if not compile_dir(dest, maxlevels, args.ddir,
args.force, args.rx, args.quiet,
args.legacy, workers=args.workers):
success = False
return success
else:
return compile_path(legacy=args.legacy, force=args.force,
quiet=args.quiet)
except KeyboardInterrupt:
if args.quiet < 2:
print("\n[interrupted]")
return False
return True
0
Example 120
Project: pombola Source File: document_parser.py
def main(args):
parseType = args[1]
year = args[2]
questions = []
answers = []
p, a, ap = defaultdict(int), defaultdict(int), defaultdict(int)
mps = {}
summaries = []
for f in sys.stdin:
fin = open(f.strip(), 'r')
content = fin.read()
fin.close()
basename =os.path.basename(f)
#parse appropriately
if parseType == 'questions':
q,a = qa(content);
[questions.append(item) for item in q if item.get('timestamp_question') != None and str(item.get('timestamp_question').year) == year]
[answers.append(item) for item in a if item.get('timestamp_answer') != None and str(item.get('timestamp_answer').year) == year]
elif parseType == 'rollcall':
#outfile = open('/Users/idesouza/Hutspace/output.txt','w')
timestamp, present, absent, with_permission = roll_call(content)
if str(timestamp.year) != year:
continue
total = len(present) + len(absent) + len(with_permission)
summaries.append({ 'timestamp': timestamp, 'text': '%s --- Present: %s Absent with Permission: %s Absent: %s Total: %s'%(timestamp, len(present),len(with_permission), len(absent), total)})
#print '------- %s -------\n'% os.path.splitext(basename)[0]
#if len(present) + len(with_permission) + len(absent) < 275:
# inspect = "INSPECT"
#else :
# inspect = ""
#outfile.write('------- %s ------- %s\n'% (os.path.splitext(basename)[0], inspect ))
#outfile.write('\tPresent: %s Absent with Permission: %s Absent: %s\n'%(len(present),len(with_permission), len(absent)))
for line in present:
p[line['mp']] += 1
mps[line['mp']] = line['constituency']
for line in absent:
a[line['mp']] += 1
mps[line['mp']] = line['constituency']
for line in with_permission:
ap[line['mp']] += 1
mps[line['mp']] = line['constituency']
else:
pass
client = MongoClient('mongodb://localhost:27017/')
db = client['odekro']
if parseType == 'questions':
collection = db.collection['questions']
#insert/update questions
for item in questions:
collection.update({ 'ref': item['ref'] }, { '$set': item }, True)
#match answers to questions
for item in answers:
collection.update({ 'ref' : item['ref'] }, { '$set': item }, True);
elif parseType == 'rollcall':
collection = db.collection['rollcall']
collection.update( {'timestamp': timestamp}, {'timestamp': timestamp, 'present' : present, 'absent': absent, 'permission': with_permission} ,True)
else:
pass
#loop done
#output what we want
if parseType == 'rollcall':
for summary in sorted(summaries, key=lambda k: k['timestamp']) :
print summary['text']
print 'mp\tconstituency\t#present\t#absent with permission\t#absent'
for mp in mps.keys():
print '%s\t%s\t%s\t%s\t%s' %(mp, mps[mp], p[mp], ap[mp], a[mp])
if parseType == 'questions':
collection = db.collection['questions']
cursor = collection.aggregate( [
{ '$group' : { '_id': '$source', 'total' : {"$sum" : 1}}},
])
print "MP\t# Questions Asked"
for doc in cursor['result']:
print '%s\t%s'%(doc['_id'], doc['total'])
#outfile.close()
return 0
0
Example 121
Project: mysql-utilities Source File: daemon.py
def daemonize(self):
"""Creates the daemon.
It will fork a child process and then exit parent. By performing a
double fork, set the current process's user id, change the current
working directory, set the current numeric umask, redirect standard
streams and write the pid to a file.
"""
def redirect_stream(system_stream, target_stream):
"""Redirect a system stream to a specified file.
"""
if target_stream is None:
target_f = os.open(os.devnull, os.O_RDWR)
else:
target_f = target_stream.fileno()
os.dup2(target_f, system_stream.fileno())
def fork_then_exit_parent(error_message):
"""Fork a child process, then exit the parent process.
"""
try:
pid = os.fork()
if pid > 0:
os._exit(0) # pylint: disable=W0212
except OSError as err:
msg = "{0}: [{1}] {2}".format(error_message, err.errno,
err.strerror)
self._report(msg, logging.CRITICAL)
raise UtilDaemonError(msg)
# Fork
fork_then_exit_parent("Failed first fork.")
try:
os.setsid()
os.chdir(self.chdir)
os.umask(self.umask)
except Exception as err:
msg = "Unable to change directory ({0})".format(err)
self._report(msg, logging.CRITICAL)
raise UtilDaemonError(msg)
# Double fork
fork_then_exit_parent("Failed second fork.")
# Redirect streams
redirect_stream(sys.stdin, self.stdin)
redirect_stream(sys.stdout, self.stdout)
redirect_stream(sys.stderr, self.stderr)
# write pidfile
atexit.register(self.delete_pidfile)
pid = str(os.getpid())
try:
with open(self.pidfile, "w") as f:
f.write("{0}\n".format(pid))
except IOError as err:
msg = "Unable to write pidfile: {0}".format(err.strerror)
self._report(msg, logging.CRITICAL)
raise UtilDaemonError(msg)
0
Example 122
def merge_sam(file_in, file_ou, fr = False, mismatches = 10000, mismatches20 = 10000, short = 20):
# It merges consecutive lines which represent one read
fin = None
if file_in == '-':
fin = sys.stdin
else:
fin = open(file_in,'r')
fou = None
if file_ou == '-':
fou = sys.stdout
else:
fou = open(file_ou,'w')
data = []
size = 10**6
last = ['_','1']
limit = 10**5
while True:
lines = fin.readlines(size)
if not lines:
break
for line in lines:
if line.startswith('@'):
data.append(line)
else:
temp = line.split("\t")
ilastflag = int(last[sam_FLAG])
itempflag = int(temp[sam_FLAG])
proper_pair = ( itempflag & 0x02 ) and ( itempflag & 0x01 )
if not proper_pair:
last = ['_','1']
continue
# mismatches
tag_nm_i = [e.partition("NM:i:")[2] for e in temp[sam_TAG:] if e.startswith('NM:i:')] # NM is mismatches per reads
tag_nm_i = int(tag_nm_i[0]) if tag_nm_i else 0
if (tag_nm_i > mismatches) or (len(temp[sam_SEQ]) < short+1 and tag_nm_i > mismatches20):
last = ['_','1']
continue
sa = False if ilastflag & 0x10 else True
sb = False if itempflag & 0x10 else True
if last[sam_QNAME][:-1] != temp[sam_QNAME][:-1] or (ilastflag & itempflag & 0xC0) or (fr and sa == sb) or (fr == False and sa != sb ):# test that they form a pair #or last[sam_QNAME][-1:] == temp[sam_QNAME][-1:]:
last = temp
else:
#if temp[sam_QNAME][-1:].endswith("a"):
# (last,temp) = (temp,last)
if itempflag & 0x40: # test if it is first in pair
(last,temp) = (temp,last) # last is always first in pair
(ilastflag,itempflag) = (itempflag,ilastflag) # last is always first in pair
(sa,sb) = (sb,sa)
la = len(last[sam_SEQ])
lb = len(temp[sam_SEQ])
if sa: #sa == "+": # forward strand
last[sam_CIGAR] = last[sam_CIGAR] + str(lb)+"H"
temp[sam_CIGAR] = str(la)+"H" + temp[sam_CIGAR]
else:
last[sam_CIGAR] = str(lb)+"H" + last[sam_CIGAR]
temp[sam_CIGAR] = temp[sam_CIGAR] + str(la)+"H"
if fr:
temp[sam_FLAG] = str(itempflag ^ 0x10) # move it on opposite strand (it flips the bit)
last[sam_QNAME] = last[sam_QNAME][:-1]
temp[sam_QNAME] = temp[sam_QNAME][:-1]
#last[sam_QNAME] = last[sam_QNAME].partition("__")[0]
#temp[sam_QNAME] = temp[sam_QNAME].partition("__")[0]
data.append("\t".join(last))
data.append("\t".join(temp))
if len(data) > limit:
fou.writelines(data)
data = []
last = ['_','1']
if data:
fou.writelines(data)
fin.close()
fou.close()
0
Example 123
Project: fusioncatcher Source File: sort_ttdb.py
def sort_columns(input_filename = '-',
output_filename = '-',
columns=None,
header=False,
ignore_case=False,
unique = False,
tmp_dir=None,
buffer_size = '80%',
parallel = multiprocessing.cpu_count(),
compress_program = None):
"""
It sorts the input file (text tab separated file) based on the specified columns.
It works like SELECT * ORDER BY in SQL.
"""
import locale
locale.setlocale(locale.LC_ALL, 'C')
# check options suppported by SORT command
sh1 = give_me_temp_filename(tmp_dir)
sh2 = give_me_temp_filename(tmp_dir)
# check options suppported by SORT command
sort_parallel = False
r = os.system("sort --help | grep 'parallel' > '%s'" % (sh1,))
if (not r) and (not empty(sh1)) and len(file(sh1,'r').readlines()) == 1:
sort_parallel = True
delete_file(sh1)
# check options suppported by SORT command
sort_buffer = False
r = os.system("sort --help | grep 'buffer-size' > '%s'" % (sh1,))
if (not r) and (not empty(sh1)) and len(file(sh1,'r').readlines()) == 1:
sort_buffer = True
delete_file(sh1)
# check options suppported by SORT command
sort_compress = False
if compress_program:
r = os.system("sort --help | grep 'compress-program' > '%s' ; %s --help 2>/dev/null | grep -i 'compress' > '%s'" % (sh1,compress_program,sh2))
if (not r) and ((not empty(sh1)) and len(file(sh1,'r').readlines()) == 1 and
(not empty(sh2)) and len(file(sh2,'r').readlines()) >= 1):
sort_compress = True
delete_file(sh1)
delete_file(sh2)
# treat the case when the input file is coming from the standard input
fin = input_filename.strip('"').strip("'")
if fin == '-':
fin = give_me_temp_filename(tmp_dir)
fod = open(fin,'w')
fid = sys.stdin
while True:
lines = fid.readlines(10**8)
if not lines:
break
fod.writelines(lines)
fod.close()
fon = output_filename.strip('"').strip("'")
if fon == '-':
fon = give_me_temp_filename(tmp_dir)
if header:
header_saved = file(fin,'r').readline()
file(output_filename,'w').write(header_saved)
else:
file(output_filename,'w').write('')
# process the type of the column, numeric, or string
first_line = file(fin,'r').readline()
if first_line:
nc=len(file(fin,'r').readline().rstrip('\r\n').split('\t'))#read first line in order to find out the number of columns
if columns:
columns=columns.strip().lower()
if columns=='d':
columns=','.join([str(i+1)+'d' for i in range(nc)])
elif columns=='n':
columns=','.join([str(i+1)+'n' for i in range(nc)])
elif columns=='nd' or columns=='dn':
columns=','.join([str(i+1)+'nd' for i in range(nc)])
else:
columns=','.join([str(i+1) for i in range(nc)])
# extra parameters
extra = ""
if sort_buffer and buffer_size and buffer_size != 'no' and buffer_size != 'none':
extra = extra + ' --buffer-size=' + str(buffer_size) + ' '
if sort_parallel and parallel and parallel > 1:
extra = extra + ' --parallel=' + str(parallel) + ' '
if sort_compress and compress_program and compress_program.lower() != 'no' and compress_program.lower() != 'none':
extra = extra + ' --compress-program='+compress_program + ' '
# processing the input columns
columns = ['-k '+el+','+el.replace('n','').replace('r','') for el in columns.replace('d','r').split(',')]
comd = "-s -t '\t' "+" ".join(columns)
if ignore_case:
comd = "-f "+comd
if unique:
comd = "-u "+comd
if tmp_dir:
comd = "-T '"+tmp_dir+"' "+comd
if header:
comd = "LC_ALL=C sed 1d '" + fin + "' | LC_ALL=C sort " + extra + comd + " >> '" + output_filename + "'"
else:
comd = "LC_ALL=C sort " + extra + comd + " '" + fin + "' >> '" + output_filename + "'"
r = os.system(comd)
if r != 0:
print >>sys.stderr, "ERROR (sort_ttdb.py) while running:"
print >>sys.stderr, comd
sys.exit(1)
if input_filename == '-':
os.remove(fin)
if output_filename == '-':
fod = sys.stdout
fid = open(fon,'r')
while True:
gc.disable()
lines = fid.readlines(10**8)
gc.enable()
if not lines:
break
fod.writelines(lines)
fid.close()
os.remove(fon)
0
Example 124
def run_job_flow(branding, json_config, force, no_browser=False,
aws_exe='aws', profile='default', region=None):
""" Submits job flow to EMR. Should eventually monitor it.
branding: text file with branding to print to screen before running
job. This is where the name of a software package or ASCII art
can go.
json_config: JSON configuration file. Google Getting Started with
Amazon Elastic MapReduce for formatting information.
force: True iff all existing directories should be erased when
writing intermediates.
no_browser: True iff webpage with status of job shouldn't automatically
be opened in default browser.
aws_exe: path to AWS CLI executable
profile: name of AWS CLI profile from which to grab AWS access key ID
and secret access key.
region: Amazon data center in which to run job flow. If None, uses
region from AWS CLI.
No return value.
"""
iface = dp_iface.DooplicityInterface(
branding=branding,
opener='Started job flow submission script on {time}.'
)
try:
'''Should ultimately check to see which regions are available to user
w/ API.'''
aws_ansible = ab.AWSAnsible(profile=profile, region=region,
valid_regions=_aws_regions)
s3_ansible = ab.S3Ansible(aws_exe=aws_exe, profile=profile,
iface=iface)
# Serialize JSON configuration
if json_config is not None:
with open(json_config) as json_stream:
full_payload = json.load(json_stream)
else:
full_payload = json.load(sys.stdin)
try:
job_flow = full_payload['Steps']
except Exception:
raise RuntimeError(
'Input JSON not in proper format. Ensure that the JSON '
'object has a Steps key.'
)
step_count = len(job_flow)
steps = OrderedDict()
# Check steps for requred data
required_data = set(['input', 'output', 'mapper', 'reducer'])
bad_output_data = []
try:
for step in job_flow:
if 'hadoop-streaming' \
not in step['HadoopJarStep']['Jar'].lower():
'''Don't check command-line parameters if not a
Hadoop Streaming step.'''
continue
step_args = {}
for j in xrange(0, len(step['HadoopJarStep']['Args']), 2):
arg_name = step['HadoopJarStep']['Args'][j][1:].strip()
if arg_name == 'input':
try:
step_args['input'] = ','.join(
[step['HadoopJarStep']['Args'][j+1],
step_args['input']]
)
except KeyError:
step_args['input'] \
= step['HadoopJarStep']['Args'][j+1]
elif arg_name in required_data:
step_args[step['HadoopJarStep']['Args'][j][1:]] \
= step['HadoopJarStep']['Args'][j+1].strip()
steps[step['Name']] = step_args
except (KeyError, IndexError):
iface.fail(
'JSON file not in proper format. Ensure '
'that each StepConfig object has a HadoopJarStep '
'object with an Args array and a Name string.'
)
raise
iface.step('Read job flow from input JSON.')
iface.status('Checking that output directories on S3 are writable...')
'''Check steps for required Hadoop Streaming command-line parameters
and for whether outputs are writable.'''
missing_data = defaultdict(list)
identity_steps = []
identity_mappers \
= set(['cat', 'org.apache.hadoop.mapred.lib.IdentityMapper'])
identity_reducers \
= set(['cat', 'org.apache.hadoop.mapred.lib.IdentityReducer'])
for step in steps:
step_data = steps[step]
for required_parameter in required_data:
if required_parameter not in step_data:
missing_data[step].append('-' + required_parameter)
elif not force and required_parameter == 'output' \
and ab.Url(step_data['output']).is_s3 \
and s3_ansible.is_dir(step_data['output']):
bad_output_data.append(step)
if 'mapper' in steps[step] and 'reducer' in step_data \
and step_data['mapper'] in identity_mappers \
and step_data['reducer'] in identity_reducers:
identity_steps.append(step)
errors = []
if missing_data:
errors.extend(['Step "%s" is missing required parameter(s) "%s".' %
(step, ', '.join(missing_data[step]))
for step in missing_data])
if bad_output_data:
errors.extend(['Output "directory" "%s" of step "%s" already '
'exists, and --force was not invoked to permit '
'overwriting it.'
% (steps[step]['output'], step)
for step in bad_output_data])
if identity_steps:
errors.extend([('Step "%s" has both an identity mapper and an '
'identity reducer, which is redundant. Remove the '
'step before trying again.') % step])
if errors:
raise RuntimeError('\n'.join([('%d) ' % (i+1)) + error
for i, error in enumerate(errors)]))
iface.step('Verified that output directories on S3 are writable.')
'''Remove intermediate directories and create buckets if they
don't already exist so EMR doesn't choke when writing output. The
idea here is to make a bucket feel like an ordinary directory to
the user.'''
buckets = set()
for step in steps:
step_data = steps[step]
if ab.Url(step_data['output']).is_s3:
s3_ansible.remove_dir(steps[step]['output'])
buckets.add(ab.bucket_from_url(step_data['output']))
for bucket in buckets:
try:
s3_ansible.create_bucket(bucket, region=aws_ansible.region)
except Exception as e:
raise RuntimeError(('Bucket %s already exists on S3. Change '
'affected output directories in job flow '
'and try again. The more distinctive the '
'name chosen, the less likely it\'s '
'already a bucket on S3. It may be '
'easier to create a bucket first using '
'the web interface and use its name + a '
'subdirectory as the output directory.')
% bucket)
iface.step('Set up output directories on S3.')
iface.status('Submitting job flow...')
try:
job_flow_response = aws_ansible.post_request(full_payload)
except urllib2.HTTPError as e:
if 'bad request' in str(e).lower():
raise RuntimeError('(' + str(e) + ');'
+ (' ensure that IAM roles are '
'configured properly. This '
'may require talking to your AWS account '
'admin. See '
'http://docs.aws.amazon.com/ElasticMapReduce/'
'latest/DeveloperGuide/emr-'
'iam-roles-defaultroles.html for more '
'information. In most cases, the solution '
'to is to run "aws emr '
'create-default-roles", and try again.'))
else:
raise
json_response = json.load(job_flow_response)
if 'JobFlowId' not in json_response:
raise RuntimeError('Job flow submission failed. Server returned '
'the following response: %s.'
% json.dumps(json_response, sort_keys=True,
indent=4, separators=(',', ': ')))
else:
job_flow_id = json_response['JobFlowId']
iface.step(('Submitted job flow.\n'
'cuem*Job flow ID is %s .*****\n' % job_flow_id) +
('*****Submission can be monitored at %s .*****' %
_emr_url.format(region=aws_ansible.region,
job_flow_id=job_flow_id)))
if not no_browser:
webbrowser.open(_emr_url.format(region=aws_ansible.region,
job_flow_id=job_flow_id),
new=2) # Open in new tab
iface.step('Opening URL in default browser, if possible.')
iface.done(closer=('Finished job flow submission script '
'on {date}. Run time was {length} seconds.')
)
except (Exception, GeneratorExit):
# GeneratorExit added just in case this happens on modifying code
iface.fail()
raise
except (KeyboardInterrupt, SystemExit):
if 'job_flow_id' not in locals():
iface.fail(opener='*****Terminated without submitting job.*****',
middler=('End time was {date}. '
'Script was running for {length} seconds.'))
else:
iface.fail(opener='*****Terminated after submitting job.*****',
middler=('End time was {date}. '
'Script was running for {length} seconds.'))
raise
0
Example 125
Project: rail Source File: align_reads.py
def go(input_stream=sys.stdin, output_stream=sys.stdout, bowtie2_exe='bowtie2',
bowtie_index_base='genome', bowtie2_index_base='genome2',
manifest_file='manifest', bowtie2_args=None, bin_size=10000, verbose=False,
exon_differentials=True, exon_intervals=False, report_multiplier=1.2,
min_exon_size=8, search_filter=1, min_readlet_size=15, max_readlet_size=25,
readlet_interval=12, capping_multiplier=1.5, drop_deletions=False,
gzip_level=3, scratch=None, index_count=1, output_bam_by_chr=False,
tie_margin=0, no_realign=False, no_polyA=False):
""" Runs Rail-RNA-align_reads.
A single pass of Bowtie is run to find end-to-end alignments. Unmapped
reads are saved for readletizing to determine junctions in sucessive
reduce steps as well as for realignment in a later map step.
Input (read from stdin)
----------------------------
Tab-delimited input tuple columns in a mix of any of the following
three formats:
Format 1 (single-end, 3-column):
1. Nucleotide sequence or its reversed complement, whichever is first
in alphabetical order
2. 1 if sequence was reverse-complemented else 0
3. Name
4. Quality sequence or its reverse, whichever corresponds to field 1
Format 2 (paired, 2 lines, 3 columns each)
(so this is the same as single-end)
1. Nucleotide sequence for mate 1 or its reversed complement,
whichever is first in alphabetical order
2. 1 if sequence was reverse-complemented else 0
3. Name for mate 1
4. Quality sequence for mate 1 or its reverse, whichever corresponds
to field 1
(new line)
1. Nucleotide sequence for mate 2 or its reversed complement,
whichever is first in alphabetical order
2. 1 if sequence was reverse complemented else 0
3. Name for mate 2
4. Quality sequence for mate 2 or its reverse, whichever corresponds
to field 1
Input is partitioned and sorted by field 1, the read sequence.
Hadoop output (written to stdout)
----------------------------
A given RNAME sequence is partitioned into intervals ("bins") of some
user-specified length (see partition.py).
Exonic chunks (aka ECs; three formats, any or all of which may be
emitted):
Format 1 (exon_ival); tab-delimited output tuple columns:
1. Reference name (RNAME in SAM format) + ';' + bin number
2. Sample index
3. EC start (inclusive) on forward strand
4. EC end (exclusive) on forward strand
Format 2 (exon_diff); tab-delimited output tuple columns:
1. Reference name (RNAME in SAM format) + ';' + bin number
2. max(EC start, bin start) (inclusive) on forward strand IFF diff is
positive and EC end (exclusive) on forward strand IFF diff is
negative
3. Sample index
4. '1' if alignment from which diff originates is "unique" according to
--tie-margin criterion; else '0'
5. +1 or -1 * count, the number of instances of a read sequence for
which to print exonic chunks
Note that only unique alignments are currently output as ivals and/or
diffs.
Format 3 (sam); tab-delimited output tuple columns:
Standard SAM output except fields are in different order, and the first
field corresponds to sample label. (Fields are reordered to facilitate
partitioning by sample name/RNAME and sorting by POS.) Each line
corresponds to a spliced alignment. The order of the fields is as
follows.
1. Sample index if outputting BAMs by sample OR
sample-rname index if outputting BAMs by chr
2. (Number string representing RNAME; see BowtieIndexReference
class in bowtie_index for conversion information) OR
'0' if outputting BAMs by chr
3. POS
4. QNAME
5. FLAG
6. MAPQ
7. CIGAR
8. RNEXT
9. PNEXT
10. TLEN
11. SEQ
12. QUAL
... + optional fields
Insertions/deletions (indel_bed)
tab-delimited output tuple columns:
1. 'I' or 'D' insertion or deletion line
2. Number string representing RNAME
3. Start position (Last base before insertion or
first base of deletion)
4. End position (Last base before insertion or last base of deletion
(exclusive))
5. Inserted sequence for insertions or deleted sequence for deletions
6. Sample index
----Next fields are for junctions only; they are '\x1c' for indels----
7. '\x1c'
8. '\x1c'
--------------------------------------------------------------------
9. Number of instances of insertion or deletion in sample; this is
always +1 * count before bed_pre combiner/reducer
Read whose primary alignment is not end-to-end
Tab-delimited output tuple columns (unmapped):
1. Transcriptome Bowtie 2 index group number
2. SEQ
3. 1 if SEQ is reverse-complemented, else 0
4. QNAME
5. QUAL
Tab-delimited output tuple columns (readletized):
1. Readlet sequence or its reversed complement, whichever is first in
alphabetical order
2. read sequence ID + ('-' if readlet
sequence is reverse-complemented; else '+') + '\x1e' + displacement
of readlet's 5' end from read's 5' end + '\x1e' + displacement of
readlet's 3' end from read's 3' end (+, for EXACTLY one readlet of
a read sequence, '\x1e' + read sequence + '\x1e' +
(an '\x1f'-separated list A of unique sample labels with read
sequences that match the original read sequence) + '\x1e' +
(an '\x1f'-separated list of unique sample labels B with read
sequences that match the reversed complement of the original read
sequence)) + '\x1e' + (an '\x1f'-separated list of the number of
instances of the read sequence for each respective sample in list
A) + '\x1e' + (an '\x1f'-separated list of the number of instances
of the read sequence's reversed complement for each respective
sample in list B). Here, a read sequence ID takes the form X:Y,
where X is the "mapred_task_partition" environment variable -- a
unique index for a task within a job -- and Y is the index of the
read sequence relative to the beginning of the input stream.
Tab-delimited tuple columns (postponed_sam):
Standard 11+ -column raw SAM output
Single column (unique):
1. A unique read sequence
Two columns, exactly one line (dummy); ensures creation of junction
index:
1. character "-"
2. the word "dummy"
ALL OUTPUT COORDINATES ARE 1-INDEXED.
input_stream: where to find input reads.
output_stream: where to emit exonic chunks and junctions.
bowtie2_exe: filename of Bowtie2 executable; include path if not in
$PATH.
bowtie_index_base: the basename of the Bowtie1 index files associated
with the reference.
bowtie2_index_base: the basename of the Bowtie2 index files associated
with the reference.
manifest_file: filename of manifest
bowtie2_args: string containing precisely extra command-line arguments
to pass to first-pass Bowtie2.
bin_size: genome is partitioned in units of bin_size for later load
balancing.
verbose: True iff more informative messages should be written to
stderr.
exon_differentials: True iff EC differentials are to be emitted.
exon_intervals: True iff EC intervals are to be emitted.
report_multiplier: if verbose is True, the line number of an alignment
or read written to stderr increases exponentially with base
report_multiplier.
min_exon_size: minimum exon size searched for in junction_search.py
later in pipeline; used to determine how large a soft clip on one
side of a read is necessary to pass it on to junction search
pipeline
search_filter: how large a soft clip on one side of a read is necessary
to pass it on to junction search pipeline
min_readlet_size: "capping" readlets (that is, readlets that terminate
at a given end of the read) are never smaller than this value
max_readlet_size: size of every noncapping readlet
readlet_interval: number of bases separating successive readlets along
the read
capping_multiplier: successive capping readlets on a given end of a
read are increased in size exponentially with base
capping_multiplier
drop_deletions: True iff deletions should be dropped from coverage
vector
gzip_level: compression level to use for temporary files
scratch: scratch directory for storing temporary files or None if
securely created temporary directory
index_count: number of transcriptome Bowtie 2 indexes to which to
assign unmapped reads for later realignment
output_bam_by_chr: True iff final output BAMs will be by chromosome
tie_margin: allowed score difference per 100 bases among ties in
max score. For example, 150 and 144 are tied alignment scores
for a 100-bp read when --tie-margin is 6.
no_realign: True iff job flow does not need more than readlets: this
usually means only a transcript index is being constructed
no_polyA: kill noncapping readlets that are all As and write as
unmapped all reads with polyA prefixes whose suffixes are <
min_exon_size
No return value.
"""
global _input_line_count
reference_index = bowtie_index.BowtieIndexReference(bowtie_index_base)
manifest_object = manifest.LabelsAndIndices(manifest_file)
alignment_printer = AlignmentPrinter(
manifest_object,
reference_index,
bin_size=bin_size,
output_stream=output_stream,
exon_ivals=exon_intervals,
exon_diffs=exon_differentials,
drop_deletions=drop_deletions,
output_bam_by_chr=output_bam_by_chr,
tie_margin=tie_margin
)
# Get task partition to pass to align_reads_delegate.py
try:
task_partition = os.environ['mapred_task_partition']
except KeyError:
# Hadoop 2.x?
try:
task_partition = os.environ['mapreduce_task_partition']
except KeyError:
# A unit test is probably being run
task_partition = '0'
temp_dir = make_temp_dir(scratch)
register_cleanup(tempdel.remove_temporary_directories, [temp_dir])
align_file = os.path.join(temp_dir, 'first_pass_reads.temp.gz')
other_reads_file = os.path.join(temp_dir, 'other_reads.temp.gz')
second_pass_file = os.path.join(temp_dir, 'second_pass_reads.temp.gz')
k_value, _, _ = bowtie.parsed_bowtie_args(bowtie2_args)
nothing_doing = True
# Required length of prefix after poly(A) is trimmed
remaining_seq_size = max(min_exon_size - 1, 1)
with xopen(True, align_file, 'w', gzip_level) as align_stream, \
xopen(True, other_reads_file, 'w', gzip_level) as other_stream:
for seq_number, ((seq,), xpartition) in enumerate(
xstream(sys.stdin, 1)
):
seq_length = len(seq)
if no_polyA and (
all(seq[i] == 'A'
for i in xrange(seq_length - remaining_seq_size))
or all(seq[i] == 'T'
for i in xrange(remaining_seq_size, seq_length))
or all(seq[i] == 'A'
for i in xrange(remaining_seq_size, seq_length))
or all(seq[i] == 'T'
for i in xrange(seq_length - remaining_seq_size))
):
if not no_realign:
'''If a sequence is too short without its poly(A) tail,
make all reads with that sequence unmapped. Technically,
this also kills poly(A)s at 5' ends, but we probably
couldn't align those sequences anyway.'''
reversed_complement_seq = seq[::-1].translate(
_reversed_complement_translation_table
)
for is_reversed, name, qual in xpartition:
if is_reversed == '0':
alignment_printer.print_unmapped_read(
name,
seq,
qual
)
else:
alignment_printer.print_unmapped_read(
name,
reversed_complement_seq,
qual[::-1]
)
continue
nothing_doing = False
'''Select highest-quality read with alphabetically last qname
for first-pass alignment.'''
best_name, best_mean_qual, best_qual_index, i = None, None, 0, 0
others_to_print = dlist()
for is_reversed, name, qual in xpartition:
_input_line_count += 1
others_to_print.append(
'\t'.join([
str(seq_number), is_reversed, name, qual
])
)
mean_qual = (
float(sum([ord(score) for score in qual])) / len(qual)
)
if (mean_qual > best_mean_qual
or mean_qual == best_mean_qual and name > best_name):
best_qual_index = i
best_mean_qual = mean_qual
best_name = name
to_align = '\t'.join([
'%s\x1d%s' % (is_reversed, name),
seq, qual
])
i += 1
assert i >= 1
if i == 1:
print >>other_stream, str(seq_number)
else:
for j, other_to_print in enumerate(others_to_print):
if j != best_qual_index:
print >>other_stream, other_to_print
print >>align_stream, to_align
# Print dummy line
print 'dummy\t-\tdummy'
sys.stdout.flush() # this is REALLY important b/c called script will stdout
if nothing_doing:
# No input
sys.exit(0)
input_command = 'gzip -cd %s' % align_file
bowtie_command = ' '.join([bowtie2_exe,
bowtie2_args if bowtie2_args is not None else '',
' --sam-no-qname-trunc --local -t --no-hd --mm -x',
bowtie2_index_base, '--12 -'])
delegate_command = ''.join(
[sys.executable, ' ', os.path.realpath(__file__)[:-3],
('_delegate.py --task-partition {task_partition} '
'--other-reads {other_reads} --second-pass-reads '
'{second_pass_reads} --min-readlet-size '
'{min_readlet_size} {drop_deletions} '
'--max-readlet-size {max_readlet_size} '
'--readlet-interval {readlet_interval} '
'--capping-multiplier {capping_multiplier:1.12f} '
'{verbose} --report-multiplier {report_multiplier:1.12f} '
'--k-value {k_value} '
'--bowtie-idx {bowtie_index_base} '
'--partition-length {bin_size} '
'--manifest {manifest_file} '
'{exon_differentials} {exon_intervals} '
'--gzip-level {gzip_level} '
'--search-filter {search_filter} '
'--index-count {index_count} '
'--tie-margin {tie_margin} '
'{no_realign} '
'{no_polyA} '
'{output_bam_by_chr}').format(
task_partition=task_partition,
other_reads=other_reads_file,
second_pass_reads=second_pass_file,
min_readlet_size=min_readlet_size,
drop_deletions=('--drop-deletions' if drop_deletions
else ''),
max_readlet_size=max_readlet_size,
readlet_interval=readlet_interval,
capping_multiplier=capping_multiplier,
verbose=('--verbose' if verbose else ''),
report_multiplier=report_multiplier,
k_value=k_value,
bowtie_index_base=bowtie_index_base,
bin_size=bin_size,
manifest_file=manifest_file,
exon_differentials=('--exon-differentials'
if exon_differentials else ''),
exon_intervals=('--exon-intervals'
if exon_intervals else ''),
gzip_level=gzip_level,
search_filter=search_filter,
index_count=index_count,
tie_margin=tie_margin,
no_realign=('--no-realign' if no_realign else ''),
no_polyA=('--no-polyA' if no_polyA else ''),
output_bam_by_chr=('--output-bam-by-chr'
if output_bam_by_chr
else '')
)]
)
full_command = ' | '.join([input_command,
bowtie_command, delegate_command])
print >>sys.stderr, \
'Starting first-pass Bowtie 2 with command: ' + full_command
bowtie_process = subprocess.Popen(' '.join(
['set -exo pipefail;', full_command]
),
bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, shell=True,
executable='/bin/bash')
return_code = bowtie_process.wait()
if return_code:
raise RuntimeError('Error occurred while reading first-pass Bowtie 2 '
'output; exitlevel was %d.' % return_code)
os.remove(align_file)
os.remove(other_reads_file)
if not no_realign:
input_command = 'gzip -cd %s' % second_pass_file
bowtie_command = ' '.join([bowtie2_exe,
bowtie2_args if bowtie2_args is not None else '',
' --sam-no-qname-trunc --local -t --no-hd --mm -x',
bowtie2_index_base, '--12 -'])
delegate_command = ''.join(
[sys.executable, ' ', os.path.realpath(__file__)[:-3],
('_delegate.py --task-partition {task_partition} '
'--min-readlet-size {min_readlet_size} '
'{drop_deletions} '
'--max-readlet-size {max_readlet_size} '
'--readlet-interval {readlet_interval} '
'--capping-multiplier {capping_multiplier:012f} '
'{verbose} '
'--report-multiplier {report_multiplier:012f} '
'--k-value {k_value} '
'--bowtie-idx {bowtie_index_base} '
'--partition-length {bin_size} '
'--manifest {manifest_file} '
'{exon_differentials} {exon_intervals} '
'--gzip-level {gzip_level} '
'--search-filter {search_filter} '
'--index-count {index_count} '
'--tie-margin {tie_margin} '
'{output_bam_by_chr}').format(
task_partition=task_partition,
min_readlet_size=min_readlet_size,
drop_deletions=('--drop-deletions'
if drop_deletions else ''),
readlet_interval=readlet_interval,
max_readlet_size=max_readlet_size,
capping_multiplier=capping_multiplier,
verbose=('--verbose' if verbose else ''),
report_multiplier=report_multiplier,
k_value=k_value,
bowtie_index_base=bowtie_index_base,
bin_size=bin_size,
manifest_file=manifest_file,
exon_differentials=('--exon-differentials'
if exon_differentials else ''),
exon_intervals=('--exon-intervals'
if exon_intervals else ''),
gzip_level=gzip_level,
search_filter=search_filter,
index_count=index_count,
tie_margin=tie_margin,
output_bam_by_chr=('--output-bam-by-chr'
if output_bam_by_chr
else '')
)]
)
full_command = ' | '.join([input_command,
bowtie_command, delegate_command])
print >>sys.stderr, \
'Starting second-pass Bowtie 2 with command: ' + full_command
bowtie_process = subprocess.Popen(' '.join(
['set -exo pipefail;', full_command]
),
bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, shell=True,
executable='/bin/bash')
return_code = bowtie_process.wait()
if return_code:
raise RuntimeError('Error occurred while reading second-pass '
'Bowtie 2 output; exitlevel was %d.'
% return_code)
sys.stdout.flush()
0
Example 126
def _run(self, prefix_char, args):
from rez.vendor import argparse
parser = argparse.ArgumentParser(prog=self.tool_name,
prefix_chars=prefix_char)
def _add_argument(*nargs, **kwargs):
nargs_ = []
for narg in nargs:
nargs_.append(narg.replace('=', prefix_char))
parser.add_argument(*nargs_, **kwargs)
_add_argument(
"=a", "==about", action="store_true",
help="print information about the tool")
_add_argument(
"=i", "==interactive", action="store_true",
help="launch an interactive shell within the tool's configured "
"environment")
_add_argument(
"=p", "==patch", type=str, nargs='*', metavar="PKG",
help="run the tool in a patched environment")
_add_argument(
"==versions", action="store_true",
help="list versions of package providing this tool")
_add_argument(
"==command", type=str, nargs='+', metavar=("COMMAND", "ARG"),
help="read commands from string, rather than executing the tool")
_add_argument(
"==stdin", action="store_true",
help="read commands from standard input, rather than executing the tool")
_add_argument(
"==strict", action="store_true",
help="strict patching. Ignored if ++patch is not present")
_add_argument(
"==nl", "==no-local", dest="no_local", action="store_true",
help="don't load local packages when patching")
_add_argument(
"==peek", action="store_true",
help="diff against the tool's context and a re-resolved copy - "
"this shows how 'stale' the context is")
_add_argument(
"==verbose", action="count", default=0,
help="verbose mode, repeat for more verbosity")
_add_argument(
"==quiet", action="store_true",
help="hide welcome message when entering interactive mode")
_add_argument(
"==no-rez-args", dest="no_rez_args", action="store_true",
help="pass all args to the tool, even if they start with '%s'" % prefix_char)
opts, tool_args = parser.parse_known_args(args)
if opts.no_rez_args:
args = list(args)
args.remove("==no-rez-args".replace('=', prefix_char))
tool_args = args
opts = parser.parse_args([])
# print info
if opts.about:
return self.print_about()
elif opts.versions:
return self.print_package_versions()
elif opts.peek:
return self.peek()
# patching
context = self.context
if opts.patch is not None:
new_request = opts.patch
request = context.get_patched_request(new_request, strict=opts.strict)
config.remove_override("quiet")
pkg_paths = (config.nonlocal_packages_path
if opts.no_local else None)
context = ResolvedContext(request,
package_paths=pkg_paths,
verbosity=opts.verbose)
# reapply quiet mode (see cli.forward)
if "REZ_QUIET" not in os.environ:
config.override("quiet", True)
if opts.stdin:
# generally shells will behave as though the '-s' flag was not present
# when no stdin is available. So here we replicate this behaviour.
import select
if not select.select([sys.stdin], [], [], 0.0)[0]:
opts.stdin = False
# construct command
cmd = None
if opts.command:
cmd = opts.command
elif opts.interactive:
label = self.context_name
if opts.patch:
label += '*'
config.override("prompt", "%s>" % label)
cmd = None
else:
cmd = [self.tool_name] + tool_args
retcode, _, _ = context.execute_shell(command=cmd,
stdin=opts.stdin,
quiet=opts.quiet,
block=True)
return retcode
0
Example 127
def _main(self):
"""
get's automatically called from Memacs class
read the lines from git-rev-list file,parse and write them to org file
"""
# read file
if self._args.gitrevfile:
logging.debug("using as %s input_stream",
self._args.gitrevfile)
input_stream = codecs.open(self._args.gitrevfile,
encoding=self._args.encoding)
else:
logging.debug("using sys.stdin as input_stream")
input_stream = codecs.getreader(self._args.encoding)(sys.stdin)
# now go through the file
# Logic (see example commit below)
# first we are in an header and not in an body
# every newline toggles output
# if we are in body then add the body to commit class
# if we are in header then add the header to commit class
#
# commit 6fb35035c5fa7ead66901073413a42742a323e89
# tree 7027c628031b3ad07ad5401991f5a12aead8237a
# parent 05ba138e6aa1481db2c815ddd2acb52d3597852f
# author Armin Wieser <[email protected]> 1324422878 +0100
# committer Armin Wieser <[email protected]> 1324422878 +0100
#
# PEP8
# Signed-off-by: Armin Wieser <[email protected]>
was_in_body = False
commit = Commit()
commits = []
line = self.get_line_from_stream(input_stream)
while line:
line = line.rstrip() # removing \n
logging.debug("got line: %s", line)
if line.strip() == "" or len(line) != len(line.lstrip()):
commit.add_body(line)
was_in_body = True
else:
if was_in_body:
commits.append(commit)
commit = Commit()
commit.add_header(line)
was_in_body = False
line = self.get_line_from_stream(input_stream)
# adding last commit
if not commit.is_empty():
commits.append(commit)
logging.debug("got %d commits", len(commits))
if len(commits) == 0:
logging.error("Is there an error? Because i found no commits.")
# time to write all commits to org-file
for commit in commits:
output, properties, note, author, timestamp = commit.get_output()
if not(self._args.grepuser) or \
(self._args.grepuser and self._args.grepuser == author):
# only write to stream if
# * grepuser is not set or
# * grepuser is set and we got an entry with the right author
self._writer.write_org_subitem(output=output,
timestamp=timestamp,
properties=properties,
note=note)
if self._args.gitrevfile:
input_stream.close()
0
Example 128
Project: pyon Source File: pycc.py
def main(opts, *args, **kwargs):
"""
Processes arguments and starts the capability container.
"""
def prepare_logging():
# Load logging override config if provided. Supports variants literal and path.
logging_config_override = None
if opts.logcfg:
if '{' in opts.logcfg:
# Variant 1: Value is dict of config values
try:
eval_value = ast.literal_eval(opts.logcfg)
logging_config_override = eval_value
except ValueError:
raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
else:
# Variant 2: Value is path to YAML file containing config values
logutil.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
def prepare_container():
"""
Walks through pyon initialization in a deterministic way and initializes Container.
In particular make sure configuration is loaded in correct order and
pycc startup arguments are considered.
"""
import threading
threading.current_thread().name = "CC-Main"
# SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
import pyon
from pyon.core import bootstrap, config
# Set global testing flag to False. We are running as capability container, because
# we started through the pycc program.
bootstrap.testing = False
# Set sysname if provided in startup argument
if opts.sysname:
bootstrap.set_sys_name(opts.sysname)
# Trigger any initializing default logic in get_sys_name
bootstrap.get_sys_name()
command_line_config = kwargs
# This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
bootstrap_config = None
# This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
pyon_config = config.read_standard_configuration() # Initial pyon.yml + pyon.local.yml
# Load config override if provided. Supports variants literal and list of paths
config_override = None
if opts.config:
if '{' in opts.config:
# Variant 1: Dict of config values
try:
eval_value = ast.literal_eval(opts.config)
config_override = eval_value
except ValueError:
raise Exception("Value error in config arg '%s'" % opts.config)
else:
# Variant 2: List of paths
from pyon.util.config import Config
config_override = Config([opts.config]).data
# Determine bootstrap_config
if opts.config_from_directory:
# Load minimal bootstrap config if option "config_from_directory"
bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
else:
# Otherwise: Set to standard set of local config files plus command line overrides
bootstrap_config = deepcopy(pyon_config)
config.apply_configuration(bootstrap_config, config_override)
config.apply_configuration(bootstrap_config, command_line_config)
# Override sysname from config file or command line
if not opts.sysname and bootstrap_config.get_safe("system.name", None):
new_sysname = bootstrap_config.get_safe("system.name")
bootstrap.set_sys_name(new_sysname)
# Delete sysname datastores if option "force_clean" is set
if opts.force_clean:
from pyon.datastore import clear_couch_util
print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
pyon_config.container.filesystem.force_clean=True
from pyon.core.interfaces.interfaces import InterfaceAdmin
iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)
# If auto_bootstrap, load config and interfaces into directory
# Note: this is idempotent and will not alter anything if this is not the first container to run
if bootstrap_config.system.auto_bootstrap:
print "pycc: auto_bootstrap=True."
stored_config = deepcopy(pyon_config)
config.apply_configuration(stored_config, config_override)
config.apply_configuration(stored_config, command_line_config)
iadm.create_core_datastores()
iadm.store_config(stored_config)
# Determine the final pyon_config
# - Start from standard config already set (pyon.yml + local YML files)
# - Optionally load config from directory
if opts.config_from_directory:
config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
# - Apply container profile specific config
config.apply_profile_configuration(pyon_config, bootstrap_config)
# - Reapply pyon.local.yml here again for good measure
config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
# - Last apply any separate command line config overrides
config.apply_configuration(pyon_config, config_override)
config.apply_configuration(pyon_config, command_line_config)
# Also set the immediate flag, but only if specified - it is an override
if opts.immediate:
from pyon.util.containers import dict_merge
dict_merge(pyon_config, {'system':{'immediate':True}}, True)
# Bootstrap pyon's core. Load configuration etc.
bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)
# Delete any queues/exchanges owned by sysname if option "broker_clean" is set
if opts.broker_clean:
print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()
# build connect str
connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % (pyon_config.get_safe('server.amqp_priv.host', pyon_config.get_safe('server.amqp.host', 'localhost')),
pyon_config.get_safe('container.exchange.management.port', '55672'),
pyon_config.get_safe('container.exchange.management.username', 'guest'),
pyon_config.get_safe('container.exchange.management.password', 'guest'),
'/')
from putil.rabbithelper import clean_by_sysname
deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name())
print " exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges))
print " queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues))
if opts.force_clean:
path = os.path.join(pyon_config.get_safe('container.filesystem.root', '/tmp/ion'), bootstrap.get_sys_name())
print "force_clean: Removing", path
FileSystem._clean(pyon_config)
# Auto-bootstrap interfaces
if bootstrap_config.system.auto_bootstrap:
iadm.store_interfaces(idempotent=True)
iadm.close()
if opts.no_container:
print "pycc: no_container=True. Stopping here."
return None
# Create the container instance
from pyon.container.cc import Container
container = Container(*args, **command_line_config)
return container
def start_container(container):
"""
Start container and all internal managers. Returns when ready.
"""
container.start()
def do_work(container):
"""
Performs initial startup actions with the container as requested in arguments.
Then remains in container shell or infinite wait until container stops.
Returns when container should stop. Raises an exception if anything failed.
"""
if opts.proc:
# Run a one-off process (with the -x argument)
mod, proc = opts.proc.rsplit('.', 1)
print "pycc: Starting process %s" % opts.proc
container.spawn_process(proc, mod, proc, config={'process':{'type':'immediate'}})
# And end
return
if opts.rel:
# Start a rel file
start_ok = container.start_rel_from_url(opts.rel)
if not start_ok:
raise Exception("Cannot start deploy file '%s'" % opts.rel)
if opts.mx:
from pyon.public import CFG
port = CFG.get_safe('container.flask_webapp.port',8080)
container.spawn_process("ContainerUI", "ion.core.containerui", "ContainerUI")
print "pycc: Container UI started ... listening on http://localhost:%s" % port
if opts.signalparent:
import signal
print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % (os.getppid(), os.getpid())
os.kill(os.getppid(), signal.SIGUSR1)
def is_parent_gone():
while os.getppid() != 1:
gevent.sleep(1)
print 'pycc: Now I am an orphan ... notifying serve_forever to stop'
os.kill(os.getpid(), signal.SIGINT)
import gevent
ipg = gevent.spawn(is_parent_gone)
container.gl_parent_watch = ipg
if opts.enable_gbmonitor:
from pyon.util.gevent_block_plugin import get_gevent_monitor_block
get_gevent_monitor_block().start()
if not opts.noshell and not opts.daemon:
# Keep container running while there is an interactive shell
from pyon.container.shell_api import get_shell_api
setup_ipython_shell(get_shell_api(container))
elif not opts.nomanhole:
from pyon.container.shell_api import get_shell_api
setup_ipython_embed(get_shell_api(container))
else:
container.serve_forever()
def stop_container(container):
try:
if container:
container.stop()
return True
except Exception as ex:
# We want to make sure to get out here alive
print "pycc: CONTAINER STOP ERROR"
traceback.print_exc()
return False
def _exists_ipython_dir():
# Fix OOIION-1124:
# When multiple containers are started in parallel, all start an embedded IPython shell/manhole.
# There exists a race condition between the IPython creating the default $HOME/.python dir
# leading to an error.
homedir = os.path.expanduser('~')
homedir = os.path.realpath(homedir)
home_ipdir = os.path.join(homedir, ".ipython")
ipdir = os.path.normpath(os.path.expanduser(home_ipdir))
return os.path.exists(ipdir)
def setup_ipython_shell(shell_api=None):
if not _exists_ipython_dir():
log.warn("IPython profile dir not found. Attempting to avoid race condition")
import gevent
import random
gevent.sleep(random.random() * 3.0) # Introduce a random delay to make conflict less likely
ipy_config = _setup_ipython_config()
# monkeypatch the ipython inputhook to be gevent-friendly
import gevent # should be auto-monkey-patched by pyon already.
import select
def stdin_ready():
infds, outfds, erfds = select.select([sys.stdin], [], [], 0)
if infds:
return True
else:
return False
def inputhook_gevent():
try:
while not stdin_ready():
gevent.sleep(0.05)
except KeyboardInterrupt:
pass
return 0
# install the gevent inputhook
from IPython.lib.inputhook import inputhook_manager
inputhook_manager.set_inputhook(inputhook_gevent)
inputhook_manager._current_gui = 'gevent'
# First import the embeddable shell class
from IPython.frontend.terminal.embed import InteractiveShellEmbed
from mock import patch
# Update namespace of interactive shell
# TODO: Cleanup namespace even further
if shell_api is not None:
locals().update(shell_api)
# Now create an instance of the embeddable shell. The first argument is a
# string with options exactly as you would type them if you were starting
# IPython at the system command line. Any parameters you want to define for
# configuration can thus be specified here.
with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
for tries in range(3):
try:
ipshell = InteractiveShellEmbed(config=ipy_config,
banner1 = """ ____ ____ _____ __ __ ____________
/ __ \/ __ \/ _/ | / /__ / /_ / ____/ ____/
/ / / / / / // // |/ / _ \/ __/ / / / /
/ /_/ / /_/ // // /| / __/ /_ / /___/ /___
\____/\____/___/_/ |_/\___/\__/ \____/\____/""",
exit_msg = 'Leaving OOINet CC shell, shutting down container.')
ipshell('Pyon (PID: %s) - OOINet CC interactive IPython shell. Type ionhelp() for help' % os.getpid())
break
except Exception as ex:
log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
def setup_ipython_embed(shell_api=None):
if not _exists_ipython_dir():
log.warn("IPython profile dir not found. Attempting to avoid race condition")
import gevent
import random
gevent.sleep(random.random() * 3.0) # Introduce a random delay to make conflict less likely
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile#, shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
for tries in range(3):
try:
embed_kernel(local_ns=shell_api, config=ipy_config) # blocks until INT
break
except Exception as ex:
log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
# @TODO: race condition here versus ipython, this will leave junk in tmp dir
#try:
# shutil.rmtree(temp_dir)
#except shutil.Error:
# pass
def _setup_ipython_config():
from IPython.config.loader import Config
ipy_config = Config()
ipy_config.KernelApp.connection_file = os.path.join(os.path.abspath(os.curdir), "manhole-%s.json" % os.getpid())
ipy_config.PromptManager.in_template = '><> '
ipy_config.PromptManager.in2_template = '... '
ipy_config.PromptManager.out_template = '--> '
ipy_config.InteractiveShellEmbed.confirm_exit = False
#ipy_config.Application.log_level = 10 # uncomment for debug level ipython logging
return ipy_config
# main() -----> ENTER
# ----------------------------------------------------------------------------------
# Container life cycle
prepare_logging()
container = None
try:
container = prepare_container()
if container is None:
sys.exit(0)
start_container(container)
except Exception as ex:
# print "pycc: ===== CONTAINER START ERROR -- FAIL ====="
# traceback.print_exc()
log.error('container start error', exc_info=True)
stop_container(container)
sys.exit(1)
try:
do_work(container)
except Exception as ex:
stop_container(container)
# print "pycc: ===== CONTAINER PROCESS START ERROR -- ABORTING ====="
# print ex
log.error('container process interruption', exc_info=True)
sys.exit(1)
# Assumption: stop is so robust, it does not fail even if it was only partially started
stop_ok = stop_container(container)
if not stop_ok:
sys.exit(1)
0
Example 129
@classmethod
def getConfig(cls):
"""Generate config from command line arguments and config file."""
# example_config = {
# "daemon": True,
# "run_once": False,
# "debug": False,
# "watchlist": [
# {"servers": [ {"host": "localhost", "port": 514} ],
# "watchfiles": [
# {"tag": "anaconda",
# "log_type": "anaconda",
# "files": ["/tmp/anaconda.log",
# "/mnt/sysimage/root/install.log"]
# }
# ]
# }
# ]
# }
default_config = {"daemon": True,
"run_once": False,
"debug": False,
"hostname": cls._getHostname(),
"watchlist": []
}
# First use default config as running config.
config = dict(default_config)
# Get command line options and validate it.
cmdline = cls.cmdlineParse()[0]
# Check config file source and read it.
if cmdline.config_file or cmdline.stdin_config:
try:
if cmdline.stdin_config is True:
fo = sys.stdin
else:
fo = open(cmdline.config_file, 'r')
parsed_config = json.load(fo)
if cmdline.debug:
print(parsed_config)
except IOError: # Raised if IO operations failed.
main_logger.error("Can not read config file %s\n" %
cmdline.config_file)
exit(1)
except ValueError as e: # Raised if json parsing failed.
main_logger.error("Can not parse config file. %s\n" %
e.message)
exit(1)
# Validate config from config file.
cls.configValidate(parsed_config)
# Copy gathered config from config file to running config
# structure.
for key, value in parsed_config.items():
config[key] = value
else:
# If no config file specified use watchlist setting from
# command line.
watchlist = {"servers": [{"host": cmdline.host,
"port": cmdline.port}],
"watchfiles": [{"tag": cmdline.tag,
"log_type": cmdline.log_type,
"files": cmdline.watchfiles}]}
config['watchlist'].append(watchlist)
# Apply behavioural command line options to running config.
if cmdline.no_daemon:
config["daemon"] = False
if cmdline.run_once:
config["run_once"] = True
if cmdline.debug:
config["debug"] = True
return config
0
Example 130
Project: python-cinderclient Source File: shell.py
def main(self, argv):
# Parse args once to find version and debug settings
parser = self.get_base_parser()
(options, args) = parser.parse_known_args(argv)
self.setup_debugging(options.debug)
api_version_input = True
self.options = options
do_help = ('help' in argv) or (
'--help' in argv) or ('-h' in argv) or not argv
if not options.os_volume_api_version:
api_version = api_versions.get_api_version(
DEFAULT_MAJOR_OS_VOLUME_API_VERSION)
else:
api_version = api_versions.get_api_version(
options.os_volume_api_version)
# build available subcommands based on version
major_version_string = "%s" % api_version.ver_major
self.extensions = client.discover_extensions(major_version_string)
self._run_extension_hooks('__pre_parse_args__')
subcommand_parser = self.get_subcommand_parser(api_version,
do_help, args)
self.parser = subcommand_parser
if options.help or not argv:
subcommand_parser.print_help()
return 0
argv = self._delimit_metadata_args(argv)
args = subcommand_parser.parse_args(argv)
self._run_extension_hooks('__post_parse_args__', args)
# Short-circuit and deal with help right away.
if args.func == self.do_help:
self.do_help(args)
return 0
elif args.func == self.do_bash_completion:
self.do_bash_completion(args)
return 0
(os_username, os_password, os_tenant_name, os_auth_url,
os_region_name, os_tenant_id, endpoint_type,
service_type, service_name, volume_service_name, bypass_url,
cacert, os_auth_system) = (
args.os_username, args.os_password,
args.os_tenant_name, args.os_auth_url,
args.os_region_name, args.os_tenant_id,
args.os_endpoint_type,
args.service_type, args.service_name,
args.volume_service_name,
args.bypass_url, args.os_cacert,
args.os_auth_system)
if os_auth_system and os_auth_system != "keystone":
auth_plugin = cinderclient.auth_plugin.load_plugin(os_auth_system)
else:
auth_plugin = None
if not service_type:
service_type = client.SERVICE_TYPES[major_version_string]
# FIXME(usrleon): Here should be restrict for project id same as
# for os_username or os_password but for compatibility it is not.
# V3 stuff
project_info_provided = ((self.options.os_tenant_name or
self.options.os_tenant_id) or
(self.options.os_project_name and
(self.options.os_project_domain_name or
self.options.os_project_domain_id)) or
self.options.os_project_id)
if not utils.isunauthenticated(args.func):
if auth_plugin:
auth_plugin.parse_opts(args)
if not auth_plugin or not auth_plugin.opts:
if not os_username:
raise exc.CommandError("You must provide a user name "
"through --os-username or "
"env[OS_USERNAME].")
if not os_password:
# No password, If we've got a tty, try prompting for it
if hasattr(sys.stdin, 'isatty') and sys.stdin.isatty():
# Check for Ctl-D
try:
os_password = getpass.getpass('OS Password: ')
# Initialize options.os_password with password
# input from tty. It is used in _get_keystone_session.
options.os_password = os_password
except EOFError:
pass
# No password because we didn't have a tty or the
# user Ctl-D when prompted.
if not os_password:
raise exc.CommandError("You must provide a password "
"through --os-password, "
"env[OS_PASSWORD] "
"or, prompted response.")
if not project_info_provided:
raise exc.CommandError(_(
"You must provide a tenant_name, tenant_id, "
"project_id or project_name (with "
"project_domain_name or project_domain_id) via "
" --os-tenant-name (env[OS_TENANT_NAME]),"
" --os-tenant-id (env[OS_TENANT_ID]),"
" --os-project-id (env[OS_PROJECT_ID])"
" --os-project-name (env[OS_PROJECT_NAME]),"
" --os-project-domain-id "
"(env[OS_PROJECT_DOMAIN_ID])"
" --os-project-domain-name "
"(env[OS_PROJECT_DOMAIN_NAME])"
))
if not os_auth_url:
if os_auth_system and os_auth_system != 'keystone':
os_auth_url = auth_plugin.get_auth_url()
if not os_auth_url:
raise exc.CommandError(
"You must provide an authentication URL "
"through --os-auth-url or env[OS_AUTH_URL].")
if not project_info_provided:
raise exc.CommandError(_(
"You must provide a tenant_name, tenant_id, "
"project_id or project_name (with "
"project_domain_name or project_domain_id) via "
" --os-tenant-name (env[OS_TENANT_NAME]),"
" --os-tenant-id (env[OS_TENANT_ID]),"
" --os-project-id (env[OS_PROJECT_ID])"
" --os-project-name (env[OS_PROJECT_NAME]),"
" --os-project-domain-id "
"(env[OS_PROJECT_DOMAIN_ID])"
" --os-project-domain-name "
"(env[OS_PROJECT_DOMAIN_NAME])"
))
if not os_auth_url:
raise exc.CommandError(
"You must provide an authentication URL "
"through --os-auth-url or env[OS_AUTH_URL].")
auth_session = None
if not auth_plugin:
auth_session = self._get_keystone_session()
insecure = self.options.insecure
self.cs = client.Client(
api_version, os_username,
os_password, os_tenant_name, os_auth_url,
region_name=os_region_name,
tenant_id=os_tenant_id,
endpoint_type=endpoint_type,
extensions=self.extensions,
service_type=service_type,
service_name=service_name,
volume_service_name=volume_service_name,
bypass_url=bypass_url,
retries=options.retries,
http_log_debug=args.debug,
insecure=insecure,
cacert=cacert, auth_system=os_auth_system,
auth_plugin=auth_plugin,
session=auth_session,
logger=self.ks_logger if auth_session else self.client_logger)
try:
if not utils.isunauthenticated(args.func):
self.cs.authenticate()
except exc.Unauthorized:
raise exc.CommandError("OpenStack credentials are not valid.")
except exc.AuthorizationFailure:
raise exc.CommandError("Unable to authorize user.")
endpoint_api_version = None
# Try to get the API version from the endpoint URL. If that fails fall
# back to trying to use what the user specified via
# --os-volume-api-version or with the OS_VOLUME_API_VERSION environment
# variable. Fail safe is to use the default API setting.
try:
endpoint_api_version = \
self.cs.get_volume_api_version_from_endpoint()
except exc.UnsupportedVersion:
endpoint_api_version = options.os_volume_api_version
if api_version_input:
logger.warning("Cannot determine the API version from "
"the endpoint URL. Falling back to the "
"user-specified version: %s" %
endpoint_api_version)
else:
logger.warning("Cannot determine the API version from the "
"endpoint URL or user input. Falling back "
"to the default API version: %s" %
endpoint_api_version)
profile = osprofiler_profiler and options.profile
if profile:
osprofiler_profiler.init(options.profile)
args.func(self.cs, args)
if profile:
trace_id = osprofiler_profiler.get().get_base_id()
print("Trace ID: %s" % trace_id)
print("To display trace use next command:\n"
"osprofiler trace show --html %s " % trace_id)
0
Example 131
Project: python-tackerclient Source File: shell.py
def authenticate_user(self):
"""Authentication validation.
Make sure the user has provided all of the authentication
info we need.
"""
if self.options.os_auth_strategy == 'keystone':
if self.options.os_token or self.options.os_url:
# Token flow auth takes priority
if not self.options.os_token:
raise exc.CommandError(
_("You must provide a token via"
" either --os-token or env[OS_TOKEN]"
" when providing a service URL"))
if not self.options.os_url:
raise exc.CommandError(
_("You must provide a service URL via"
" either --os-url or env[OS_URL]"
" when providing a token"))
else:
# Validate password flow auth
project_info = (self.options.os_tenant_name or
self.options.os_tenant_id or
(self.options.os_project_name and
(self.options.os_project_domain_name or
self.options.os_project_domain_id)) or
self.options.os_project_id)
if (not self.options.os_username
and not self.options.os_user_id):
raise exc.CommandError(
_("You must provide a username or user ID via"
" --os-username, env[OS_USERNAME] or"
" --os-user-id, env[OS_USER_ID]"))
if not self.options.os_password:
# No password, If we've got a tty, try prompting for it
if hasattr(sys.stdin, 'isatty') and sys.stdin.isatty():
# Check for Ctl-D
try:
self.options.os_password = getpass.getpass(
'OS Password: ')
except EOFError:
pass
# No password because we didn't have a tty or the
# user Ctl-D when prompted.
if not self.options.os_password:
raise exc.CommandError(
_("You must provide a password via"
" either --os-password or env[OS_PASSWORD]"))
if (not project_info):
# tenent is deprecated in Keystone v3. Use the latest
# terminology instead.
raise exc.CommandError(
_("You must provide a project_id or project_name ("
"with project_domain_name or project_domain_id) "
"via "
" --os-project-id (env[OS_PROJECT_ID])"
" --os-project-name (env[OS_PROJECT_NAME]),"
" --os-project-domain-id "
"(env[OS_PROJECT_DOMAIN_ID])"
" --os-project-domain-name "
"(env[OS_PROJECT_DOMAIN_NAME])"))
if not self.options.os_auth_url:
raise exc.CommandError(
_("You must provide an auth url via"
" either --os-auth-url or via env[OS_AUTH_URL]"))
auth_session = self._get_keystone_session()
auth = auth_session.auth
else: # not keystone
if not self.options.os_url:
raise exc.CommandError(
_("You must provide a service URL via"
" either --os-url or env[OS_URL]"))
auth_session = None
auth = None
self.client_manager = clientmanager.ClientManager(
token=self.options.os_token,
url=self.options.os_url,
auth_url=self.options.os_auth_url,
tenant_name=self.options.os_tenant_name,
tenant_id=self.options.os_tenant_id,
username=self.options.os_username,
user_id=self.options.os_user_id,
password=self.options.os_password,
region_name=self.options.os_region_name,
api_version=self.api_version,
auth_strategy=self.options.os_auth_strategy,
# FIXME (bklei) honor deprecated service_type and
# endpoint type until they are removed
service_type=self.options.os_service_type or
self.options.service_type,
endpoint_type=self.options.os_endpoint_type or self.endpoint_type,
insecure=self.options.insecure,
ca_cert=self.options.os_cacert,
timeout=self.options.http_timeout,
retries=self.options.retries,
raise_errors=False,
session=auth_session,
auth=auth,
log_credentials=True)
return
0
Example 132
Project: bindep Source File: main.py
def main(depends=None):
usage = "Usage: %prog [options] [profile]"
parser = optparse.OptionParser(
usage=usage, version="%%prog %s" % bindep.version)
parser.add_option(
"-b", "--brief", action="store_true", dest="brief",
help="List only missing packages one per line.")
parser.add_option(
"-f", "--file", action="store", type="string", dest="filename",
default="",
help="Package list file (default: bindep.txt or "
"other-requirements.txt).")
parser.add_option(
"--profiles", action="store_true",
help="List the platform and configuration profiles.")
opts, args = parser.parse_args()
if depends is None:
if opts.filename == "-":
fd = sys.stdin
elif opts.filename:
try:
fd = open(opts.filename, 'rt')
except IOError:
logging.error('Error reading file %s.' % opts.filename)
return 1
else:
if (os.path.isfile('bindep.txt') and
os.path.isfile('other-requirements.txt')):
logging.error('Both bindep.txt and other-requirements.txt '
'files exist, choose one.')
return 1
if os.path.isfile('bindep.txt'):
try:
fd = open('bindep.txt', 'rt')
except IOError:
logging.error('Error reading file bindep.txt.')
return 1
elif os.path.isfile('other-requirements.txt'):
try:
fd = open('other-requirements.txt', 'rt')
except IOError:
logging.error('Error reading file other-requirements.txt.')
return 1
else:
logging.error('Neither file bindep.txt nor file '
'other-requirements.txt exist.')
return 1
depends = bindep.depends.Depends(fd.read())
if opts.profiles:
logging.info("Platform profiles:")
for profile in depends.platform_profiles():
logging.info("%s", profile)
logging.info("")
logging.info("Configuration profiles:")
for profile in depends.profiles():
logging.info("%s", profile)
else:
if args:
profiles = args
else:
profiles = ["default"]
profiles = profiles + depends.platform_profiles()
rules = depends.active_rules(profiles)
errors = depends.check_rules(rules)
for error in errors:
if error[0] == 'missing':
if opts.brief:
logging.info("%s", "\n".join(error[1]))
else:
logging.info("Missing packages:")
logging.info(" %s", " ".join(error[1]))
if error[0] == 'badversion':
if not opts.brief:
logging.info("Bad versions of installed packages:")
for pkg, constraint, version in error[1]:
logging.info(
" %s version %s does not match %s",
pkg, version, constraint)
if errors:
return 1
return 0
0
Example 133
Project: pyector Source File: Ector.py
def main():
# Each decay divided by 2
UttererNode.__decay = 5
SentimentNode.__decay = 5
ExpressionNode.__decay = 20
SentenceNode.__decay = 25
TokenNode.__decay = 20
from optparse import OptionParser
usage = "usage: %prog [-p username][-n botname=Ector][-v|-q][-l logfilepath=ector.log][-s|-g][-h]"
parser = OptionParser(usage=usage, version="%prog 0.3")
parser.add_option("-p", "--person", dest="username", default="User",
help="set the name of the utterer")
parser.add_option("-n", "--name", dest="botname", default="Ector",
help="set the name of the bot")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=True,
help="say all that you can say")
parser.add_option("-q", "--quiet", action="store_false", dest="verbose",
help="shut up!")
parser.add_option("-l", "--log", dest="logname", default="ector.log",
help="log the dialogue in log file")
parser.add_option("-s", "--sentence", action="store_true", dest="sentence", default=False,
help="set sentence reply mode on")
parser.add_option("-g", "--generate", action="store_false", dest="sentence",
help="set generate reply mode on")
parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False,
help="set debug mode on")
(options, args) = parser.parse_args()
license = None
stdin = sys.stdin
stdout = sys.stdout
username = options.username
botname = options.botname.capitalize()
logfilename = options.logname
version = "0.4"
sentence_mode = options.sentence
generate_mode = not sentence_mode # sentence and generate modes are antagonist
verbose = options.verbose
debug = options.debug
# Quiet mode is above sentence or generate modes
if not verbose:
sentence_mode = False
generate_mode = False
ector = Ector(botname, username)
previousSentenceNode = None
nodes = None
print """pyECTOR version %s, Copyright (C) 2008 Francois PARMENTIER
pyECTOR comes with ABSOLUTELY NO WARRANTY; for details type `@show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `@show c' for details.
@help gives a basic help on pyECTOR commands.""" % (version)
ector_path = os.path.dirname(sys.argv[0])
license_path = os.path.abspath(ector_path + "/../LICENSE")
while True:
if stdin.closed:
break
stdout.write(username + ">")
entry = stdin.readline().strip()
# No Warranty
if entry[:7] == "@show w":
if not license:
f = open(license_path)
license = f.readlines()
f.close()
for i in range(257, 278):
stdout.write(license[i])
# Conditions
elif entry[:7] == "@show c":
if not license:
f = open(license_path)
license = f.readlines()
f.close()
for i in range(57, 256):
stdout.write(license[i])
elif entry[:6] == "@usage":
print usage.replace("%prog", "Ectory.py")
elif entry[:7] == "@status":
ector.showStatus()
elif entry[:8] == "@person ":
username = entry[8:].strip()
ector.setUser(username)
elif entry[:6] == "@name ":
botname = entry[6:].strip()
ector.setName(botname)
elif entry[:8] == "@version":
print "pyECTOR version %s" % (version)
elif entry[:6] == "@write":
ector.dump()
elif entry[:5] == "@quit" or entry[:5] == "@exit" or entry[:4] == "@bye":
return 0
elif entry[:10] == "@shownodes":
ector.cn.showNodes()
elif entry[:10] == "@showlinks":
ector.showLinks()
elif entry == "@showstate":
ector.showState(username)
elif entry == "@cleanstate":
ector.cleanState()
elif entry.startswith("@log "):
logfilename = entry[5:]
print "Log file: %s" % (logfilename)
elif entry == "@log" or entry == "@logoff":
print "Log off (%s)" % logfilename
logfilename = ''
elif entry.lower() == "@sentence on":
sentence_mode = True
generate_mode = False # sentence and generate modes are not compatible
print "Sentence reply mode ON"
elif entry.lower() == "@sentence off":
sentence_mode = False
print "Sentence reply mode OFF"
elif entry.lower() == "@sentence":
print "Sentence reply mode", sentence_mode and "ON" or "OFF"
elif entry.lower() == "@generate on":
sentence_mode = False
generate_mode = True # sentence and generate modes are not compatible
print "Generate reply mode ON"
elif entry.lower() == "@generate off":
generate_mode = False
print "Sentence reply mode OFF"
elif entry.lower() == "@generate":
print "Generate reply mode", generate_mode and "ON" or "OFF"
elif entry.lower() == "@debug on":
debug = True
print "Debug mode ON"
elif entry.lower() == "@debug off":
debug = False
print "Debug mode OFF"
elif entry.lower() == "@debug":
print "Debug mode", debug and "ON" or "OFF"
# Help
elif entry[:5] == "@help":
print """You can just start typing phrases.
But there are some commands you can use:
- @usage : print the options of the Ector.py command
- @quit : quit
- @exit : quit
- @bye : quit
- @person : change the utterer name (like -p)
- @name : change the bot's name (like -n)
- @version : give the current version
- @write : save Ector's Concept Network and state
- @shownodes : show the nodes of the Concept Network
- @showlinks : show the links of the Concept Network
- @showstate : show the state of the nodes
- @cleanstate: clean the state from the non-activated nodes
- @log [file]: log the entries in the file (no file turns off the logging)
- @status : show the status of Ector (Concept Network, states)
- @sentence [ON|OFF]: set the sentence reply mode
- @generate [ON|OFF]: set the generate reply mode
- @debug [ON|OFF]: set the debug mode on or off"""
elif entry.startswith("@"):
print "There is no command", entry
elif entry:
entry = unicode(entry, ENCODING)
lastSentenceNode = ector.addEntry(entry)
if previousSentenceNode:
ector.cn.addLink(previousSentenceNode, lastSentenceNode)
elif sentence_mode:
# First sentence of a dialogue
lastSentenceNode.beg += 1
if nodes and lastSentenceNode:
# Make a link from the nodes of the generated
# sentence to the next entry.
# BEWARE: may make a link co-occurrence greater than the
# sentence node occurrence.
for node in nodes:
ector.cn.addLink(node, lastSentenceNode)
previousSentenceNode = lastSentenceNode
# if log is activated, log the entry.
if logfilename:
logEntry(logfilename, username, entry)
# Propagate activation
ector.cleanState()
ector.propagate(2)
# ector.showState(username)
# Get the reply
reply = None
if sentence_mode:
# Get one of the most activated sentences
replyNode = ector.getActivatedSentenceNode()
reply = replyNode.getSymbol()
reply = reply.replace("@bot@", username)
reply = reply.replace("@user@", botname)
previousSentenceNode = replyNode
elif generate_mode:
(reply, nodes) = ector.generateSentence(debug)
reply = reply.replace("@bot@", username)
reply = reply.replace("@user@", botname)
previousSentenceNode = None
if reply:
print "%s>" % (ector.botname), reply.encode(ENCODING)
if logfilename:
logEntry(logfilename, botname, reply)
else:
if debug:
print "No entry given."