sys.stdin

Here are the examples of the python api sys.stdin taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

133 Examples 7

Example 101

Project: qqqfome Source File: entry.py
def main():
    parser = argparse.ArgumentParser(prog='qqqfome',
                                     description='Thank-you-follow-me cli.')

    parser.add_argument('-v', '--verbose', dest='verbose',
                        action='count', default=0,
                        help=s.cmd_help_print_info)
    parser.add_argument('-c', '--cookies', dest='cookies', metavar='FILE',
                        help=s.cmd_help_cookies,
                        type=str)
    parser.add_argument('-p', '--pid-file', dest='pid_file', metavar='FILE',
                        help=s.cmd_help_pid_file,
                        type=str, default='{0}.pid')
    parser.add_argument('-l', '--log-file', dest='log_file', metavar='FILE',
                        help=s.cmd_help_log_file,
                        type=str, default='{0}.log')
    parser.add_argument('-t', '--time', dest='time', metavar='INTERVAL',
                        help=s.cmd_help_time,
                        type=int, default=90)

    group = parser.add_mutually_exclusive_group()
    group.add_argument('-m', '--message', dest='message',
                       help=s.cmd_help_message,
                       type=str, default=s.default_message)
    group.add_argument('-M', '--message-file', dest='message_file',
                       metavar='FILE',
                       help=s.cmd_help_message,
                       type=str)

    parser.add_argument('-s', '--stop-at', dest='stop_at', metavar='NUM',
                        help=s.cmd_help_stop_at, type=int, default=10)
    parser.add_argument('-d', '--daemon', dest='daemon', action='store_true',
                        default=False, help='work in daemon mode')
    parser.add_argument('command', help=s.cmd_help_command, type=str,
                        choices=['init', 'start', 'stop'])
    parser.add_argument('file', help=s.cmd_help_file, type=str,
                        action=SetDefaultPID, nargs='?')

    args = parser.parse_args()

    # Logger settings
    level = logging.ERROR
    if args.verbose == 1:
        level = logging.INFO
    if args.verbose >= 2:
        level = logging.DEBUG

    ch = logging.StreamHandler()
    ch.setLevel(level)
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    db.set_logger_level(level)
    db.set_logger_handle(ch)

    L.setLevel(level)
    L.addHandler(ch)

    L.debug(args)

    if args.command == "init":
        init_db(args.cookies)
    elif args.file is None:
            parser.error(s.cmd_help_no_file_error.format(args.command))

    if args.command == 'start':
        if args.message_file is not None:
            with open(args.message_file, 'r') as f:
                args.message = f.read()
        if args.daemon:
            p = backend.BackendCode(args.pid_file)
            try:
                p.start(args.file, args.message, args.time, args.log_file)
            except OSError:     # daemon mode not support the system
                p.run(args.file, args.message, args.time, args.log_file)
        else:
            p = backend.BackendCode(args.pid_file, stdin=sys.stdin,
                                    stdout=sys.stdout, stderr=sys.stderr)
            p.run(args.file, args.message, args.time, args.log_file)
    elif args.command == 'stop':
        try:
            p = backend.BackendCode(args.pid_file)
            p.stop()
        except FileNotFoundError as e:
            L.error(e)

Example 102

Project: calibre Source File: standalone.py
Function: manage_users
def manage_users(path=None):
    from calibre.srv.users import UserManager
    m = UserManager(path)
    enc = getattr(sys.stdin, 'encoding', preferred_encoding) or preferred_encoding

    def choice(question, choices, default=None, banner=''):
        prints(banner)
        for i, choice in enumerate(choices):
            prints('%d)' % (i+1), choice)
        print()
        while True:
            prompt = question + ' [1-%d]: ' % len(choices)
            if default is not None:
                prompt = question + ' [1-%d %s: %d]' % (len(choices), _('default'), default+1)
            reply = raw_input(prompt)
            if not reply and default is not None:
                reply = str(default + 1)
            if not reply:
                raise SystemExit(0)
            reply = reply.strip()
            try:
                num = int(reply) - 1
                if not (0 <= num < len(choices)):
                    raise Exception('bad num')
                return num
            except Exception:
                prints(_('%s is not a valid choice, try again') % reply)

    def get_valid(prompt, invalidq=lambda x: None):
        while True:
            ans = raw_input(prompt + ': ').strip().decode(enc)
            fail_message = invalidq(ans)
            if fail_message is None:
                return ans
            prints(fail_message)

    def get_valid_user():
        prints(_('Existing user names:'))
        users = sorted(m.all_user_names)
        if not users:
            raise SystemExit(_('There are no users, you must first add an user'))
        prints(', '.join(users))

        def validate(username):
            if not m.has_user(username):
                return _('The username %s does not exist') % username
        return get_valid(_('Enter the username'), validate)

    def get_pass(username):
        while True:
            from getpass import getpass
            one = getpass(_('Enter the new password for %s: ') % username).decode(enc)
            if not one:
                prints(_('Empty passwords are not allowed'))
                continue
            two = getpass(_('Re-enter the new password for %s, to verify: ') % username).decode(enc)
            if one != two:
                prints(_('Passwords do not match'))
                continue
            msg = m.validate_password(one)
            if msg is None:
                return one
            prints(msg)

    def add_user():
        username = get_valid(_('Enter the username'), m.validate_username)
        pw = get_pass(username)
        m.add_user(username, pw)
        prints(_('User %s added successfully!') % username)

    def remove_user():
        un = get_valid_user()
        if raw_input((_('Are you sure you want to remove the user %s?') % un) + ' [y/n]: ').decode(enc) != 'y':
            raise SystemExit(0)
        m.remove_user(un)
        prints(_('User %s successfully removed!') % un)

    def edit_user():
        username = get_valid_user()
        pw = get_pass(username)
        m.change_password(username, pw)
        prints(_('Password for %s successfully changed!') % username)

    def show_password():
        username = get_valid_user()
        pw = m.get(username)
        prints(_('Password for {0} is: {1}').format(username, pw))

    {0:add_user, 1:edit_user, 2:remove_user, 3:show_password}[choice(_('What do you want to do?'), [
        _('Add a new user'), _('Edit an existing user'), _('Remove a user'), _('Show the password for a user')])]()

Example 103

Project: WeasyPrint Source File: __main__.py
Function: main
def main(argv=None, stdout=None, stdin=None):
    """The ``weasyprint`` program takes at least two arguments:

    .. code-block:: sh

        weasyprint [options] <input> <output>

    The input is a filename or URL to an HTML docuement, or ``-`` to read
    HTML from stdin. The output is a filename, or ``-`` to write to stdout.

    Options can be mixed anywhere before, between or after the input and
    output:

    .. option:: -e <input_encoding>, --encoding <input_encoding>

        Force the input character encoding (eg. ``-e utf8``).

    .. option:: -f <output_format>, --format <output_format>

        Choose the output file format among PDF and PNG (eg. ``-f png``).
        Required if the output is not a ``.pdf`` or ``.png`` filename.

    .. option:: -s <filename_or_URL>, --stylesheet <filename_or_URL>

        Filename or URL of a user CSS stylesheet (see
        :ref:`stylesheet-origins`\.) to add to the docuement.
        (eg. ``-s print.css``). Multiple stylesheets are allowed.

    .. option:: -r <dpi>, --resolution <dpi>

        For PNG output only. Set the resolution in PNG pixel per CSS inch.
        Defaults to 96, which means that PNG pixels match CSS pixels.

    .. option:: --base-url <URL>

        Set the base for relative URLs in the HTML input.
        Defaults to the input’s own URL, or the current directory for stdin.

    .. option:: -m <type>, --media-type <type>

        Set the media type to use for ``@media``. Defaults to ``print``.

    .. option:: -a <file>, --attachment <file>

        Adds an attachment to the docuement which is included in the PDF output.
        This option can be added multiple times to attach more files.

    .. option:: -p, --presentational-hints

        Follow HTML presentational hints.

    .. option:: --version

        Show the version number. Other options and arguments are ignored.

    .. option:: -h, --help

        Show the command-line usage. Other options and arguments are ignored.

    """
    parser = argparse.ArgumentParser(
        prog='weasyprint', description='Renders web pages to PDF or PNG.')
    parser.add_argument('--version', action='version',
                        version='WeasyPrint version %s' % VERSION,
                        help="Print WeasyPrint's version number and exit.")
    parser.add_argument('-e', '--encoding',
                        help='Character encoding of the input')
    parser.add_argument('-f', '--format', choices=['pdf', 'png'],
                        help='Output format. Can be ommited if `output` '
                             'ends with a .pdf or .png extension.')
    parser.add_argument('-s', '--stylesheet', action='append',
                        help='URL or filename for a user CSS stylesheet. '
                             'May be given multiple times.')
    parser.add_argument('-m', '--media-type', default='print',
                        help='Media type to use for @media, defaults to print')
    parser.add_argument('-r', '--resolution', type=float,
                        help='PNG only: the resolution in pixel per CSS inch. '
                             'Defaults to 96, one PNG pixel per CSS pixel.')
    parser.add_argument('--base-url',
                        help='Base for relative URLs in the HTML input. '
                             "Defaults to the input's own filename or URL "
                             'or the current directory for stdin.')
    parser.add_argument('-a', '--attachment', action='append',
                        help='URL or filename of a file '
                             'to attach to the PDF docuement')
    parser.add_argument('-p', '--presentational-hints', action='store_true',
                        help='Follow HTML presentational hints.')
    parser.add_argument(
        'input', help='URL or filename of the HTML input, or - for stdin')
    parser.add_argument(
        'output', help='Filename where output is written, or - for stdout')

    args = parser.parse_args(argv)

    if args.format is None:
        output_lower = args.output.lower()
        if output_lower.endswith('.pdf'):
            format_ = 'pdf'
        elif output_lower.endswith('.png'):
            format_ = 'png'
        else:
            parser.error(
                'Either sepecify a format with -f or choose an '
                'output filename that ends in .pdf or .png')
    else:
        format_ = args.format.lower()

    if args.input == '-':
        if stdin is None:
            stdin = sys.stdin
        # stdin.buffer on Py3, stdin on Py2
        source = getattr(stdin, 'buffer', stdin)
        if not args.base_url:
            args.base_url = '.'  # current directory
    else:
        source = args.input

    if args.output == '-':
        if stdout is None:
            stdout = sys.stdout
        # stdout.buffer on Py3, stdout on Py2
        output = getattr(stdout, 'buffer', stdout)
    else:
        output = args.output

    kwargs = {
        'stylesheets': args.stylesheet,
        'presentational_hints': args.presentational_hints}
    if args.resolution:
        if format_ == 'png':
            kwargs['resolution'] = args.resolution
        else:
            parser.error('--resolution only applies for the PNG format.')

    if args.attachment:
        if format_ == 'pdf':
            kwargs['attachments'] = args.attachment
        else:
            parser.error('--attachment only applies for the PDF format.')

    html = HTML(source, base_url=args.base_url, encoding=args.encoding,
                media_type=args.media_type)
    getattr(html, 'write_' + format_)(output, **kwargs)

Example 104

Project: bitmask_client Source File: daemon.py
Function: open
    def open(self):
        """ Become a daemon process.

            :return: ``None``.

            Open the daemon context, turning the current program into a daemon
            process. This performs the following steps:

            * If this instance's `is_open` property is true, return
              immediately. This makes it safe to call `open` multiple times on
              an instance.

            * If the `prevent_core` attribute is true, set the resource limits
              for the process to prevent any core dump from the process.

            * If the `chroot_directory` attribute is not ``None``, set the
              effective root directory of the process to that directory (via
              `os.chroot`).

              This allows running the daemon process inside a “chroot gaol”
              as a means of limiting the system's exposure to rogue behaviour
              by the process. Note that the specified directory needs to
              already be set up for this purpose.

            * Set the process UID and GID to the `uid` and `gid` attribute
              values.

            * Close all open file descriptors. This excludes those listed in
              the `files_preserve` attribute, and those that correspond to the
              `stdin`, `stdout`, or `stderr` attributes.

            * Change current working directory to the path specified by the
              `working_directory` attribute.

            * Reset the file access creation mask to the value specified by
              the `umask` attribute.

            * If the `detach_process` option is true, detach the current
              process into its own process group, and disassociate from any
              controlling terminal.

            * Set signal handlers as specified by the `signal_map` attribute.

            * If any of the attributes `stdin`, `stdout`, `stderr` are not
              ``None``, bind the system streams `sys.stdin`, `sys.stdout`,
              and/or `sys.stderr` to the files represented by the
              corresponding attributes. Where the attribute has a file
              descriptor, the descriptor is duplicated (instead of re-binding
              the name).

            * If the `pidfile` attribute is not ``None``, enter its context
              manager.

            * Mark this instance as open (for the purpose of future `open` and
              `close` calls).

            * Register the `close` method to be called during Python's exit
              processing.

            When the function returns, the running program is a daemon
            process.

            """
        if self.is_open:
            return

        if self.chroot_directory is not None:
            change_root_directory(self.chroot_directory)

        if self.prevent_core:
            prevent_core_dump()

        change_file_creation_mask(self.umask)
        change_working_directory(self.working_directory)
        change_process_owner(self.uid, self.gid)

        if self.detach_process:
            detach_process_context()

        signal_handler_map = self._make_signal_handler_map()
        set_signal_handlers(signal_handler_map)

        exclude_fds = self._get_exclude_file_descriptors()
        close_all_open_files(exclude=exclude_fds)

        redirect_stream(sys.stdin, self.stdin)
        redirect_stream(sys.stdout, self.stdout)
        redirect_stream(sys.stderr, self.stderr)

        if self.pidfile is not None:
            self.pidfile.__enter__()

        self._is_open = True

        register_atexit_function(self.close)

Example 105

Project: textkit Source File: tokens_to_json.py
@click.command()
@click.argument('token_docs', type=click.Path(exists=True), nargs=-1)
@click.option('--ids', type=click.Path(),
              help="File with one id per token docuement, each separated " +
              "by a new line. Ids file is used to set the id attribute in " +
              "the output JSON.")
@click.option('--names', type=click.Path(),
              help="File with one name per token docuement, each separated " +
              "by a new line. Names file is used to set the name attribute " +
              "in the output JSON.")
@click.option('--field', default='tokens', help="Attribute name where " +
              "tokens will be stored in the docuement object.",
              show_default=True)
@click.option('--split/--no-split', default=False, help="If enabled, " +
              "textkit will attempt to split input columns when " +
              "packaging. This is useful when packaging multiple column " +
              "output like counts.",
              show_default=True)
@click.option('-s', '--sep', default=',', help="Separator character between " +
              "columns. Only used if split-columns flag is used.",
              show_default=True)
def tokens2json(ids, names, field, split, sep, token_docs):
    '''Convert a set of token docuements into a
    JSON array of docuement objects.'''

    docs = []

    names = read_names(names)
    ids = read_names(ids)

    for idx, path in enumerate(token_docs):
        if path == '-':
            tokens_doc = sys.stdin
        else:
            tokens_doc = open(path, 'r')
        if split:
            content = read_csv(tokens_doc, sep)
            content = coerce_types(content)
        else:
            content = read_tokens(tokens_doc)

        # ordered so that these attributes stay at the top
        doc = OrderedDict()

        if idx < len(ids) - 1:
            doc['id'] = ids[idx]
        else:
            doc['id'] = path

        if idx < len(names) - 1:
            doc['name'] = names[idx]
        else:
            doc['name'] = path

        doc[field] = content
        docs.append(doc)
        tokens_doc.close()

    out_content = json.dumps(docs, indent=2)
    output(out_content)

Example 106

Project: Duplex-Sequencing Source File: muts_by_read_position.py
Function: main
def main():
    #Read in command-line arguments
    parser = ArgumentParser()
    parser.add_argument('-i',
                        '--infile', 
                        action ='store', 
                        dest = 'inFile', 
                        help = 'An input pileup file. If None, defaults to stdin. [None]', 
                        default = None
                        )
    parser.add_argument('-o', 
                        '--outfile', 
                        action = 'store', 
                        dest = 'outFile', 
                        help = 'A filename for the output file.  [None]', 
                        default = None
                        )
    parser.add_argument('-l',
                        '--rlength',
                        type=int,
                        action = 'store',
                        dest = 'rlength',
                        default = '84',
                        help = 'The length of a single read'
                        )
    parser.add_argument('-C',
                        '--max_clonality',
                        type = float,
                        action = 'store',
                        dest = 'max_clonality',
                        help = 'Maximum clonality to allow when considering a position [0.1]',
                        default = 0.1,
                        )
    o = parser.parse_args()
    
    # If an imput file is given, use it; otherwise, use stdin
    if o.inFile != None:
        f = open(o.inFile, 'r')
    else:
        f = sys.stdin

    counter = myCounts(o.rlength)
    linenum = 0
    lineskips = 0

    for line in f:
        linebin = linePrep(line, o.max_clonality)
        linenum += 1
        readNum = 0
        skips = 0
        if linenum % 10000 == 0:
            print('%s lines processed' % linenum)
        try:
            while readNum < len(linebin):
                # Check what the identity of a charecter is
                if linebin[readNum] in ('M', 'R', 'U', 'm', 'r', 'u'):
                    # Start a new read
                    counter.newRead(linebin[readNum]) 
                elif linebin[readNum] == 'E':
                    # Mark a read to be closed later
                    skips += 1
                    counter.reads[readNum - skips].closeMe = True
                elif linebin[readNum] in ('A', 'G', 'C', 'T', 'a', 'g', 'c', 't'):
                    # Count a mutation
                    counter.reads[readNum-skips].addMut()
                elif linebin[readNum] in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
                    # Count an indel
                    counter.reads[readNum-1-skips].addIndel()
                    # Scroll the read with the indel forward the length of the indel, and mark indel length charecters to be skipped
                    tst = 0
                    indelLength = ''
                    while readNum + tst < len(linebin) and linebin[readNum+tst] in ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'):
                        indelLength += linebin[readNum + tst]
                        skips += 1
                        tst += 1
                    for x in xrange(int(indelLength)):
                        counter.reads[readNum-1-skips+tst].advance()
                elif linebin[readNum] == 'D':
                    counter.reads[readNum-skips-1].addIndel()
                    skips += 1
                elif linebin[readNum] == 'd':
                    counter.reads[readNum-skips].skipMe = True
                elif linebin[readNum] in ('N', 'n'):
                    # Count an N
                    counter.reads[readNum-skips].addN()
                elif linebin[readNum] in ('.', ','):
                    pass
                else:
                    raise ValueError("Not a valid pileup value at %s: %s" % (readNum, linebin[readNum]))
                readNum += 1
        except Exception:
            print('%s[%s]: %s' % (linenum, readNum, linebin))
            raise
        # Close any reads marked for closing
        counter.closeReads()
        # Advance all reads
        counter.advanceReads()
    
    # Generate and save the graphs.
    counter.totals()
    myX = range(1, o.rlength + 1)
    ax1 = pylab.subplot(3, 1, 1)
    pylab.plot(myX, counter.counts[0, :], 'b', linewidth = 2)
    pylab.title('Mutations by Position: %s' % o.inFile)
    pylab.ylabel("% Mutations")
    pylab.setp(ax1.get_xticklabels(), visible=False)
    ax2 = pylab.subplot(3, 1, 2)
    pylab.plot(myX, counter.counts[1, :], 'r', linewidth = 2)
    pylab.title('Indels by Position: %s' % o.inFile)
    pylab.ylabel("% Indels")
    pylab.setp(ax2.get_xticklabels(), visible=False)
    ax3 = pylab.subplot(3, 1, 3)
    pylab.plot(myX, counter.counts[2, :], 'k', linewidth = 2)
    pylab.title('Ns by Position: %s' % o.inFile)
    pylab.ylabel("% Ns")
    pylab.xlabel("Read Position")

    pylab.savefig(o.outFile)
    
    # Generate and save the data file
    outWrite = counter.counts.transpose()
    outFile = open(o.outFile + '.dat', 'w')
    for n in range(o.rlength):
        outStr = ""
        for m in range(3):
            outStr += ' %s' % outWrite[n, m]
        outFile.write(outStr + '\n')
    outFile.close()
    if o.inFile != None:
        f.close()

Example 107

Project: codespell Source File: _codespell.py
def parse_file(filename, colors, summary):
    lines = None
    changed = False
    global misspellings
    global options
    global encodings
    global quiet_level

    encoding = encodings[0]  # if not defined, use UTF-8

    if filename == '-':
        f = sys.stdin
        lines = f.readlines()
    else:
        # ignore binary files
        if not os.path.isfile(filename):
            return 0
        text = is_text_file(filename)
        if not text:
            if not quiet_level & QuietLevels.BINARY_FILE:
                print("WARNING: Binary file: %s " % filename, file=sys.stderr)
            return 0
        try:
            lines, encoding = file_opener.open(filename)
        except Exception:
            return 0

    bad_count = 0
    rx = re.compile(r"[\w\-']+")
    for i, line in enumerate(lines):
        if line in exclude_lines:
            continue

        fixed_words = set()
        asked_for = set()

        for word in rx.findall(line):
            lword = word.lower()
            if lword in misspellings:
                fix = misspellings[lword].fix
                fixword = fix_case(word, misspellings[lword].data)

                if options.interactive and lword not in asked_for:
                    fix, fixword = ask_for_word_fix(lines[i], word,
                                                    misspellings[lword],
                                                    options.interactive)
                    asked_for.add(lword)

                if summary and fix:
                    summary.update(lword)

                if word in fixed_words:  # can skip because of re.sub below
                    continue

                if options.write_changes and fix:
                    changed = True
                    lines[i] = re.sub(r'\b%s\b' % word, fixword, lines[i])
                    fixed_words.add(word)
                    continue

                # otherwise warning was explicitly set by interactive mode
                if (options.interactive & 2 and not fix and not
                        misspellings[lword].reason):
                    continue

                cfilename = "%s%s%s" % (colors.FILE, filename, colors.DISABLE)
                cline = "%s%d%s" % (colors.FILE, i, colors.DISABLE)
                cwrongword = "%s%s%s" % (colors.WWORD, word, colors.DISABLE)
                crightword = "%s%s%s" % (colors.FWORD, fixword, colors.DISABLE)

                if misspellings[lword].reason:
                    if quiet_level & QuietLevels.DISABLED_FIXES:
                        continue

                    creason = "  | %s%s%s" % (colors.FILE,
                                              misspellings[lword].reason,
                                              colors.DISABLE)
                else:
                    if quiet_level & QuietLevels.NON_AUTOMATIC_FIXES:
                        continue

                    creason = ''

                # If we get to this point (uncorrected error) we should change
                # our bad_count and thus return value
                bad_count += 1

                if filename != '-':
                    print("%(FILENAME)s:%(LINE)s: %(WRONGWORD)s "
                          " ==> %(RIGHTWORD)s%(REASON)s"
                          % {'FILENAME': cfilename, 'LINE': cline,
                             'WRONGWORD': cwrongword,
                             'RIGHTWORD': crightword, 'REASON': creason})
                else:
                    print('%(LINE)s: %(STRLINE)s\n\t%(WRONGWORD)s '
                          '==> %(RIGHTWORD)s%(REASON)s'
                          % {'LINE': cline, 'STRLINE': line.strip(),
                             'WRONGWORD': cwrongword,
                             'RIGHTWORD': crightword, 'REASON': creason})

    if changed:
        if filename == '-':
            print("---")
            for line in lines:
                print(line, end='')
        else:
            if not quiet_level & QuietLevels.FIXES:
                print("%sFIXED:%s %s"
                      % (colors.FWORD, colors.DISABLE, filename),
                      file=sys.stderr)
            with codecs.open(filename, 'w', encoding=encoding) as f:
                f.writelines(lines)
    return bad_count

Example 108

Project: python-ftfy Source File: cli.py
def main():
    """
    Run ftfy as a command-line utility.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description="ftfy (fixes text for you), version %s" % __version__
    )
    parser.add_argument('filename', default='-', nargs='?',
                        help='The file whose Unicode is to be fixed. Defaults '
                             'to -, meaning standard input.')
    parser.add_argument('-o', '--output', type=str, default='-',
                        help='The file to output to. Defaults to -, meaning '
                             'standard output.')
    parser.add_argument('-g', '--guess', action='store_true',
                        help="Ask ftfy to guess the encoding of your input. "
                             "This is risky. Overrides -e.")
    parser.add_argument('-e', '--encoding', type=str, default='utf-8',
                        help='The encoding of the input. Defaults to UTF-8.')
    parser.add_argument('-n', '--normalization', type=str, default='NFC',
                        help='The normalization of Unicode to apply. '
                             'Defaults to NFC. Can be "none".')
    parser.add_argument('--preserve-entities', action='store_true',
                        help="Leave HTML entities as they are. The default "
                             "is to decode them, as long as no HTML tags "
                             "have appeared in the file.")

    args = parser.parse_args()

    encoding = args.encoding
    if args.guess:
        encoding = None

    if args.filename == '-':
        # Get a standard input stream made of bytes, so we can decode it as
        # whatever encoding is necessary.
        if PYTHON2:
            file = sys.stdin
        else:
            file = sys.stdin.buffer
    else:
        file = open(args.filename, 'rb')

    if args.output == '-':
        encode_output = PYTHON2
        outfile = sys.stdout
    else:
        encode_output = False
        outfile = io.open(args.output, 'w', encoding='utf-8')

    normalization = args.normalization
    if normalization.lower() == 'none':
        normalization = None

    if args.preserve_entities:
        fix_entities = False
    else:
        fix_entities = 'auto'

    try:
        for line in fix_file(file, encoding=encoding,
                             fix_entities=fix_entities,
                             normalization=normalization):
            if encode_output:
                outfile.write(line.encode('utf-8'))
            else:
                try:
                    outfile.write(line)
                except UnicodeEncodeError:
                    if sys.platform == 'win32':
                        sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS)
                    else:
                        sys.stderr.write(ENCODE_ERROR_TEXT_UNIX)
                    sys.exit(1)
    except UnicodeDecodeError as err:
        sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err))
        sys.exit(1)

Example 109

Project: proofofexistence Source File: genwallet.py
def main():
    parser = argparse.ArgumentParser(description="Generate a private wallet key.")

    parser.add_argument('-a', "--address", help='show as Bitcoin address', action='store_true')
    parser.add_argument('-i', "--info", help='show metadata', action='store_true')
    parser.add_argument('-j', "--json", help='output metadata as JSON', action='store_true')
    parser.add_argument('-w', "--wif", help='show as Bitcoin WIF', action='store_true')
    parser.add_argument('-f', "--wallet-key-file", help='initial wallet key', type=argparse.FileType('r'))
    parser.add_argument('-k', "--wallet-key", help='initial wallet key')
    parser.add_argument('-g', "--gpg", help='use gpg --gen-random to get additional entropy', action='store_true')
    parser.add_argument('-u', "--dev-random", help='use /dev/random to get additional entropy', action='store_true')
    parser.add_argument('-n', "--uncompressed", help='show in uncompressed form', action='store_true')
    parser.add_argument('-p', help='generate wallet key from passphrase. NOT RECOMMENDED', metavar='passphrase')
    parser.add_argument('-s', "--subkey", help='subkey path (example: 0p/2/1)')
    parser.add_argument('-t', help='generate test key', action="store_true")
    parser.add_argument('inputfile', help='source of entropy. stdin by default', type=argparse.FileType(mode='r+b'), nargs='?')
    args = parser.parse_args()

    # args.inputfile doesn't like binary when "-" is passed in. Deal with this.
    if args.inputfile == sys.stdin:
        args.inputfile = sys.stdin.buffer

    entropy = bytearray()
    if args.gpg:
        entropy.extend(gpg_entropy())
    if args.dev_random:
        entropy.extend(dev_random_entropy())
    if args.inputfile:
        entropy.extend(args.inputfile.read())
    if args.p:
        entropy.extend(args.p.encode("utf8"))
    if len(entropy) == 0 and not args.wallet_key and not args.wallet_key_file:
        parser.error("you must specify at least one source of entropy")
    if args.wallet_key and len(entropy) > 0:
        parser.error("don't specify both entropy and a wallet key")
    if args.wallet_key_file:
        wallet = Wallet.from_wallet_key(args.wallet_key_file.readline()[:-1])
    elif args.wallet_key:
        wallet = Wallet.from_wallet_key(args.wallet_key)
    else:
        wallet = Wallet.from_master_secret(bytes(entropy), is_test=args.t)
    try:
        if args.subkey:
            wallet = wallet.subkey_for_path(args.subkey)
        if wallet.child_number >= 0x80000000:
            wc = wallet.child_number - 0x80000000
            child_index = "%dp (%d)" % (wc, wallet.child_number)
        else:
            child_index = "%d" % wallet.child_number
        if args.json:
            d = dict(
                wallet_key=wallet.wallet_key(as_private=wallet.is_private),
                public_pair_x=wallet.public_pair[0],
                public_pair_y=wallet.public_pair[1],
                tree_depth=wallet.depth,
                fingerprint=b2h(wallet.fingerprint()),
                parent_fingerprint=b2h(wallet.parent_fingerprint),
                child_index=child_index,
                chain_code=b2h(wallet.chain_code),
                bitcoin_addr=wallet.bitcoin_address(),
                bitcoin_addr_uncompressed=wallet.bitcoin_address(compressed=False),
                network="test" if wallet.is_test else "main",
            )
            if wallet.is_private:
                d.update(dict(
                    key="private",
                    secret_exponent=wallet.secret_exponent,
                    WIF=wallet.wif(),
                    WIF_uncompressed=wallet.wif(compressed=False)
                ))
            else:
                d.update(dict(key="public"))
            print json.dumps(d, indent=3)
        elif args.info:
            print(wallet.wallet_key(as_private=wallet.is_private))
            if wallet.is_test:
                print("test network")
            else:
                print("main network")
            if wallet.is_private:
                print("private key")
                print("secret exponent: %d" % wallet.secret_exponent)
            else:
                print("public key only")
            print("public pair x:   %d\npublic pair y:   %d" % wallet.public_pair)
            print("tree depth:      %d" % wallet.depth)
            print("fingerprint:     %s" % b2h(wallet.fingerprint()))
            print("parent f'print:  %s" % b2h(wallet.parent_fingerprint))
            print("child index:     %s" % child_index)
            print("chain code:      %s" % b2h(wallet.chain_code))
            if wallet.is_private:
                print("WIF:             %s" % wallet.wif())
                print("  uncompressed:  %s" % wallet.wif(compressed=False))
            print("Bitcoin address: %s" % wallet.bitcoin_address())
            print("  uncompressed:  %s" % wallet.bitcoin_address(compressed=False))
        elif args.address:
            print(wallet.bitcoin_address(compressed=not args.uncompressed))
        elif args.wif:
            print(wallet.wif(compressed=not args.uncompressed))
        else:
            print(wallet.wallet_key(as_private=wallet.is_private))
    except PublicPrivateMismatchError as ex:
        print(ex.args[0])

Example 110

Project: cutadapt Source File: seqio.py
def _seqopen1(file, colorspace=False, fileformat=None, mode='r', qualities=None):
	"""
	Open a single sequence file. See description above.
	"""
	if mode == 'r':
		fastq_handler = ColorspaceFastqReader if colorspace else FastqReader
		fasta_handler = ColorspaceFastaReader if colorspace else FastaReader
	elif mode == 'w':
		fastq_handler = ColorspaceFastqWriter if colorspace else FastqWriter
		fasta_handler = ColorspaceFastaWriter if colorspace else FastaWriter
	else:
		raise ValueError("Mode must be 'r' or 'w'")

	if fileformat:  # Explict file format given
		fileformat = fileformat.lower()
		if fileformat == 'fasta':
			return fasta_handler(file)
		elif fileformat == 'fastq':
			return fastq_handler(file)
		elif fileformat == 'sra-fastq' and colorspace:
			if mode == 'w':
				raise NotImplementedError('Writing to sra-fastq not supported')
			return SRAColorspaceFastqReader(file)
		else:
			raise UnknownFileType("File format {0!r} is unknown (expected "
				"'sra-fastq' (only for colorspace), 'fasta' or 'fastq').".format(fileformat))

	# Detect file format
	name = None
	if file == "-":
		file = sys.stdin if mode == 'r' else sys.stdout
	elif isinstance(file, basestring):
		name = file
	elif hasattr(file, "name"):  # seems to be an open file-like object
		name = file.name

	if name:
		for ext in ('.gz', '.xz', '.bz2'):
			if name.endswith(ext):
				name = name[:-len(ext)]
				break
		name, ext = splitext(name)
		ext = ext.lower()
		if ext in ['.fasta', '.fa', '.fna', '.csfasta', '.csfa']:
			format = 'fasta'
		elif ext in ['.fastq', '.fq'] or (ext == '.txt' and name.endswith('_sequence')):
			format = 'fastq'
		elif mode == 'w' and qualities is True:
			# Format not recognized, but know we want to write reads with qualities
			format = 'fastq'
		elif mode == 'w' and qualities is False:
			# Same, but we know that we want to write reads without qualities
			format = 'fasta'
		else:
			raise UnknownFileType("Could not determine whether file {0!r} is FASTA "
				"or FASTQ: file name extension {1!r} not recognized".format(file, ext))
		if format == 'fastq' and qualities is False:
			raise ValueError("Output format cannot be FASTQ since no quality "
				"values are available.")
		if format == 'fastq':
			return fastq_handler(file)
		else:
			return fasta_handler(file)

	if mode == 'w':
		if qualities is True:
			return fastq_handler(file)
		elif qualities is False:
			return fasta_handler(file)
		raise UnknownFileType('Cannot determine whether to write in FASTA or '
			'FASTQ format')
	# No name available. Try to autodetect type by reading from the file.
	for line in file:
		if line.startswith('#'):
			# Skip comment lines (needed for csfasta)
			continue
		if line.startswith('>'):
			return fasta_handler(FileWithPrependedLine(file, line))
		if line.startswith('@'):
			return fastq_handler(FileWithPrependedLine(file, line))
	raise UnknownFileType("File is neither FASTQ nor FASTA.")

Example 111

Project: django-twitter-stream Source File: stream.py
    def handle(self, keys_name=settings.DEFAULT_KEYS_NAME, *args, **options):

        # The suggested time between hearbeats
        poll_interval = float(options.get('poll_interval', settings.POLL_INTERVAL))
        prevent_exit = options.get('prevent_exit', settings.PREVENT_EXIT)
        to_file = options.get('to_file', None)
        from_file = options.get('from_file', None)
        from_file_long = options.get('from_file_long', None)
        rate_limit = options.get('rate_limit', 50)
        limit = options.get('limit', None)

        if from_file and from_file_long:
            logger.error("Cannot use both --from-file and --from-file-long")
            exit(1)

        # First expire any old stream process records that have failed
        # to report in for a while
        timeout_seconds = 3 * poll_interval
        models.StreamProcess.expire_timed_out()

        # Create the stream process for tracking ourselves
        stream_process = models.StreamProcess.create(
            timeout_seconds=timeout_seconds
        )

        listener = utils.QueueStreamListener(to_file=to_file)

        if from_file:
            checker = utils.FakeTermChecker(queue_listener=listener,
                                            stream_process=stream_process)
        else:
            checker = utils.FeelsTermChecker(queue_listener=listener,
                                             stream_process=stream_process)

        def stop(signum, frame):
            """
            Register stream's death and exit.
            """

            if stream_process:
                stream_process.status = models.StreamProcess.STREAM_STATUS_STOPPED
                stream_process.heartbeat()

            # Let the tweet listener know it should be quitting asap
            listener.set_terminate()

            logger.error("Terminating")

            raise SystemExit()

        # Installs signal handlers for handling SIGINT and SIGTERM
        # gracefully.
        signal.signal(signal.SIGINT, stop)
        signal.signal(signal.SIGTERM, stop)

        keys = None
        if not from_file:
            # Only need keys if we are connecting to twitter
            while not keys:
                try:
                    keys = models.ApiKey.get_keys(keys_name)
                except ObjectDoesNotExist:
                    if keys_name:
                        logger.error("Keys for '%s' do not exist in the database. Waiting...", keys_name)
                    else:
                        logger.warn("No keys in the database. Waiting...")

                time.sleep(5)
                stream_process.status = models.StreamProcess.STREAM_STATUS_WAITING
                stream_process.heartbeat()

        try:
            if keys:
                logger.info("Connecting to Twitter with keys for %s/%s", keys.user_name, keys.app_name)
                stream_process.keys = keys
                stream_process.save()

                # Only need auth if we have keys (i.e. connecting to twitter)
                auth = tweepy.OAuthHandler(keys.api_key, keys.api_secret)
                auth.set_access_token(keys.access_token, keys.access_token_secret)

                # Start and maintain the streaming connection...
                stream = twitter_monitor.DynamicTwitterStream(auth, listener, checker)

            elif from_file or from_file_long:

                read_pretty = False
                if from_file_long:
                    from_file = from_file
                    read_pretty = True

                if from_file == '-':
                    from_file = sys.stdin
                    logger.info("Reading tweets from stdin")
                else:
                    if read_pretty:
                        logger.info("Reading tweets from JSON file %s (pretty-printed)", from_file)
                    else:
                        logger.info("Reading tweets from JSON file %s", from_file)

                stream = utils.FakeTwitterStream(from_file, pretty=read_pretty,
                                                 listener=listener, term_checker=checker,
                                                 limit=limit, rate_limit=rate_limit)
            else:
                raise Exception("No api keys and we're not streaming from a file.")

            if to_file:
                logger.info("Saving tweets to %s", to_file)

            if prevent_exit:
                while checker.ok():
                    try:
                        stream.start_polling(poll_interval)
                    except Exception as e:
                        checker.error(e)
                        time.sleep(1)  # to avoid craziness
            else:
                stream.start_polling(poll_interval)

            logger.error("Stopping because of excess errors")
            stream_process.status = models.StreamProcess.STREAM_STATUS_STOPPED
            stream_process.heartbeat()

        except Exception as e:
            logger.error(e, exc_info=True)

        finally:
            stop(None, None)

Example 112

Project: cee Source File: CompilerPlugin.py
    def run(self, filename):
        program_output_raw = ""
        message_string = ""

	tmp_fname = "files/output/cee_output_%s" % re.sub('[^0-9a-zA-Z]+', '*', filename)

        output = open(
            tmp_fname,
            "w+"
        )

        args = []
        if self.run_cmd:
            args.append(self.run_cmd)
        args.append(os.path.join(os.getcwd(), filename))

        cookbook = {
            'args': args,
            'stdin': sys.stdin,
            'stdout': output,
            'stderr': output,
            'quota': dict(
                wallclock=100000,
                cpu=50000,
                memory=500000000,
                disk=1048576
            )
        }

        try:
            msb = plugins.MiniSandbox.MiniSandbox(**cookbook)
            msb.run()
        except ValueError as e:
            print(e)
            return "<killed> ( recieved fork attempt )"
            output.flush()
            output.close()
        else:
            # verbose statistics
            program_output_data = msb.probe()

            output.flush()
            output.close()
            output = open(
                tmp_fname,
                "r"
            )

            program_output_raw = output.read()

            temp = program_output_raw.replace("\r", "")
            program_output = temp.split("\n")
            message_string = program_output[0]
            message_string.rstrip()

            message_string = to_bytes(message_string)
            print message_string

            print(program_output_data.get("result", False))

            if program_output_data.get("result", False) == "TL":
                message_string = "<killed> ( timed out )"
            elif program_output_data.get("result", False) == "RF":
                message_string = \
                    "<killed> ( restricted function used: %d(%d) )" % (
                        program_output_data.get("syscall_info")[0],
                        program_output_data.get("syscall_info")[1]
                    )
            elif program_output_data.get("result", False) == "ML":
                message_string = "<killed> ( memory limit exceeded )"
            else:
                if program_output[0]:

                    if len(program_output) > 1:
                        message_string = to_bytes(
                            message_string +
                            " [+%d deleted lines]" % (len(program_output) - 1)
                        )

                    max_msg_len = 400 - len("[+nnn deleted bytes]")
                    if len(message_string) > max_msg_len:
                        message_string = (
                            message_string[:max_msg_len] +
                            (
                                "[+%d deleted bytes]" %
                                (len(message_string) - max_msg_len)
                            )
                        )

                else:
                    message_string = "<no output> ( return value was %d ) " % (
                        program_output_data.get("exitcode", -1)
                    )
            output.close()
            os.remove(tmp_fname)
            return message_string

Example 113

Project: Arelle Source File: CntlrWebMain.py
def startWebserver(_cntlr, options):
    """Called once from main program in CmtlrCmdLine to initiate web server on specified local port.
    To test WebServer run from source in IIS, use an entry like this: c:\python33\python.exe c:\\users\\myname\\mySourceFolder\\arelleCmdLine.py %s
       
    :param options: OptionParser options from parse_args of main argv arguments (the argument *webserver* provides hostname and port), port being used to startup the webserver on localhost.
    :type options: optparse.Values
    """
    global imagesDir, cntlr, optionsPrototype
    cntlr = _cntlr
    imagesDir = cntlr.imagesDir
    optionValuesTypes = _STR_NUM_TYPES + (type(None),)
    optionsPrototype = dict((option,value if isinstance(value,_STR_NUM_TYPES) else None)
                            for option in dir(options)
                            for value in (getattr(options, option),)
                            if isinstance(value,optionValuesTypes) and not option.startswith('_'))
    host, sep, portServer = options.webserver.partition(":")
    port, sep, server = portServer.partition(":")
    # start a Bottle application
    app = Bottle()
    
    GETorPOST = ('GET', 'POST')
    GET = 'GET'
    POST = 'POST'

    # install REST API interfaces
    # if necessary to support CGI hosted servers below root, add <prefix:path> as first part of routes
    # and corresponding arguments to the handler methods
    app.route('/rest/login', GET, login_form)
    app.route('/rest/login', POST, login_submit)
    app.route('/rest/logout', GET, logout)
    app.route('/favicon.ico', GET, arelleIcon)
    app.route('/rest/xbrl/<file:path>/open', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/close', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/validation/xbrl', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/DTS', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/concepts', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/pre', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/cal', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/dim', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/facts', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/factTable', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/roleTypes', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/arcroleTypes', GETorPOST, validation)
    app.route('/rest/xbrl/<file:path>/formulae', GETorPOST, validation)
    app.route('/rest/xbrl/validation', GETorPOST, validation)
    app.route('/rest/xbrl/view', GETorPOST, validation)
    app.route('/rest/xbrl/open', GETorPOST, validation)
    app.route('/rest/xbrl/close', GETorPOST, validation)
    app.route('/images/<imgFile>', GET, image)
    app.route('/rest/xbrl/diff', GET, diff)
    app.route('/rest/configure', GET, configure)
    app.route('/rest/stopWebServer', GET, stopWebServer)
    app.route('/quickbooks/server.asmx', POST, quickbooksServer)
    app.route('/rest/quickbooks/<qbReport>/xbrl-gl/<file:path>', GET, quickbooksGLrequest)
    app.route('/rest/quickbooks/<qbReport>/xbrl-gl/<file:path>/view', GET, quickbooksGLrequest)
    app.route('/rest/quickbooks/<qbReport>/xbrl-gl/view', GET, quickbooksGLrequest)
    app.route('/rest/quickbooks/response', GET, quickbooksGLresponse)
    app.route('/quickbooks/server.html', GET, quickbooksWebPage)
    app.route('/quickbooks/localhost.crt', GET, localhostCertificate)
    app.route('/localhost.crt', GET, localhostCertificate)
    app.route('/help', GET, helpREST)
    app.route('/about', GET, about)
    app.route('/', GET, indexPageREST)
    if server == "cgi":
        # catch a non-REST interface by cgi Interface (may be a cgi app exe module, etc)
        app.route('<cgiAppPath:path>', GETorPOST, cgiInterface)
    if server == "wsgi":
        return app
    elif server == "cgi":
        if sys.stdin is None:
            sys.stdin = open(os.devnull, 'r')
        app.run(server=server)
        sys.exit(0)
    elif server:
        app.run(host=host, port=port or 80, server=server)
    else:
        app.run(host=host, port=port or 80)

Example 114

Project: genmod Source File: analyze.py
@click.command()
@click.argument('variant_file',
                    nargs=1,
                    type=click.Path(exists=True),
                    metavar='<vcf_file> or "-"'
)
@click.option('-t' ,'--family_type', 
                type=click.Choice(['ped', 'alt', 'cmms', 'mip']), 
                default='ped',
                help="""If the analysis use one of the known setups, 
                      please specify which one."""
)
# @click.option('-c', '--config_file',
#                     type=click.Path(exists=True),
#                     help="""Specify the path to a config file."""
# )
@click.option('--frequency_treshold', '-freq',
                    default=0.02, 
                    nargs=1,
                    help="""Specify maf treshold for variants to be considered.
                            Default 0.02"""
)
@click.option('--frequency_keyword', '-freqkey',
                    default='1000G_freq', 
                    nargs=1,
                    help="""Specify keyword for frequency in vcf. 
                            Default 1000G_freq"""
)
@click.option('--cadd_treshold', '-cadd',
                    default=12.0, 
                    nargs=1,
                    help="""Specify the cadd treshold for variants to be 
                            considered. Default 12.0"""
)
@click.option('--cadd_keyword', '-caddkey',
                    default='CADD', 
                    nargs=1,
                    help="""Specify keyword for CADD scores in vcf. 
                            Default CADD"""
)
@click.option('--coverage', '-cov',
                    default=7, 
                    nargs=1,
                    help="""Specify minimum read depth in all individuals for
                           variant to be considered. Default 7"""
)
@click.option('--gq_treshold', '-gq',
                    default=20, 
                    nargs=1,
                    help="""Specify genotype quality treshold for variants 
                            to be considered. Default 20."""
)
# @click.option('-p', '--patterns',
#                     type=click.Choice(['AR', 'AD', 'X']),
#                     multiple=True,
#                     help='Specify the inheritance patterns. Default is all patterns'
# )
@click.option('-o', '--outdir',
                    type=click.Path(exists=True),
                    default=os.getcwd(),
                    help="""Specify the path to a directory where results 
                            should be stored. Default is ./"""
)
@click.option('-s', '--silent', 
                is_flag=True,
                help='Do not output variants.'
)
@click.option('-exclude', '--exclude_problematic', 
                is_flag=True,
                help="""Exclude problematic genes. This flag is preferable 
                        if analysis of only one individual."""
)
@click.option('-v', '--verbose', 
                is_flag=True,
                help='Increase output verbosity.'
)
def analyze(variant_file, family_type, frequency_treshold, frequency_keyword, 
            cadd_treshold, cadd_keyword, coverage, gq_treshold, outdir, silent,
            exclude_problematic, verbose):
    """Analyze the annotated variants in a VCF file. 
        
        If there are multiple families in the ped one analysis per family will
        be done. The variants are analyzed in five different categories based 
        on what inheritance patterns that are followed.
        The differen analysies are: 
        
                AR compound\n
                AR humozygote\n
                Dominant\n
                X linked\n
                Dominant dn\n
        
        Which variants to be considered are specified in the command line. 
        Defaults are (based on a rare disease assumption):
        
            MAF < 0.02\n
            CADD score > 12\n
            Coverage in all individuals > 7\n
            Call quality > 20\n
        
        The highest scoring variants of each category is printed to screen.
        The full list of each category is printed to new vcf files in a 
        directory specified by the user. Default current dir.
        File names are the same like the input vcf with the name of the 
        analysis appended.
    
    """
    
    start_time_analysis = datetime.now()
    
    # configs = ConfigObj(config_file)        
    # prefered_models = make_models([])
    
    inheritance_keyword = 'GeneticModels'
    families = check_families(variant_file)
    file_name = os.path.splitext(os.path.split(variant_file)[-1])[0]
    
    
    # if config_file:
    #     frequency_treshold = float(configs.get('frequency', {}).get('rare', frequency_treshold))
    #     freq_keyword = configs.get('frequency', {}).get('keyword', freq_keyword)
    #     inheritance_patterns = [pattern for pattern in configs.get('inheritance', {}).get('patterns',[])]
    #     inheritance_keyword = configs.get('inheritance', {}).get('keyword',inheritance_keyword)
    #     prefered_models = make_models(inheritance_patterns)
    
    if variant_file == '-':
        variant_parser = VCFParser(fsock = sys.stdin)
    else:
        variant_parser = VCFParser(infile = variant_file)
    
    for family_id in families:
        print('Analysis for family: %s' % family_id)
    
        head = variant_parser.metadata
            
        dominant_dict = {}
        humozygote_dict = {}
        compound_dict = {}
        x_linked_dict = {}
        dominant_dn_dict = {}
        
        
        get_interesting_variants(variant_parser,
                                    family_id,
                                    dominant_dict, 
                                    humozygote_dict, 
                                    compound_dict, 
                                    x_linked_dict, 
                                    dominant_dn_dict, 
                                    frequency_treshold, 
                                    frequency_keyword, 
                                    cadd_treshold, 
                                    cadd_keyword,
                                    gq_treshold,
                                    coverage, 
                                    exclude_problematic)
        
        remove_inacurate_compounds(compound_dict, family_id)
        
        if len(dominant_dict) > 0:
            dominant_file = os.path.join(
                                    outdir, 
                                    file_name+'_dominant_analysis.vcf'
                                    )
            
            print_headers(head, dominant_file)
            
            print_results(
                      dominant_dict, 
                      dominant_file, 
                      family_id, 
                      variant_parser.header, 
                      cadd_keyword, 
                      frequency_keyword, 
                      mode='dominant'
                      )
                    
        if len(humozygote_dict) > 0:
            humozygote_file = os.path.join(
                                      outdir, 
                                      file_name+'_humozygote_analysis.vcf'
                                      )
            print_headers(head, humozygote_file)
            
            print_results(
                      humozygote_dict, 
                      humozygote_file, 
                      family_id, 
                      variant_parser.header, 
                      cadd_keyword, 
                      frequency_keyword, 
                      mode='humozygote'
                      )
            
        if len(compound_dict) > 0:
            compound_file = os.path.join(
                      outdir, 
                      file_name+'_compound_analysis.vcf'
                      )
            print_headers(head, compound_file)
            
            print_results(
                      compound_dict, 
                      compound_file, 
                      family_id, 
                      variant_parser.header, 
                      cadd_keyword, 
                      frequency_keyword, 
                      mode='compound'
                      )
        
        if len(x_linked_dict) > 0:
            xlinked_file = os.path.join(
                                  outdir, 
                                  file_name+'_x_linked_analysis.vcf'
                                  )
            print_headers(head, xlinked_file)
            
            print_results(
                      x_linked_dict, 
                      xlinked_file, 
                      family_id, 
                      variant_parser.header, 
                      cadd_keyword, 
                      frequency_keyword, 
                      mode='xlinked'
                      )
        
        if len(dominant_dn_dict) > 0:
            dominant_dn_file = os.path.join(
                      outdir, 
                      file_name+'_ad_denovo_analysis.vcf'
                      )
            print_headers(head, dominant_dn_file)
        
            print_results(
                      dominant_dn_dict, 
                      dominant_dn_file, 
                      family_id, 
                      variant_parser.header, 
                      cadd_keyword, 
                      frequency_keyword, 
                      mode='denovo'
                      )
        
        print('')
        
        print('Number of interesting Dominant variants: %s' % 
                len(dominant_dict))
        print('Number of interesting Homozygote variants: %s' %
                len(humozygote_dict))
        print('Number of interesting Compound variants: %s' %
                len(compound_dict))
        print('Number of interesting X-linked variants: %s' %
                len(x_linked_dict))
        print('Number of interesting Autosomal Dominant de novo variants: %s' %
                len(dominant_dn_dict))
        
        # pp(compound_dict)
        
        print('Time for analysis: %s' % str(datetime.now() - start_time_analysis))

Example 115

Project: genmod Source File: summarize_variants.py
@click.command()
@variant_file
@family_file
# @click.option('-c', '--config_file',
#                     type=click.Path(exists=True),
#                     help="""Specify the path to a config file."""
# )
@click.option('--frequency_keyword', '-freqkey',
                    default='1000G_freq', 
                    nargs=1,
                    help='Specify keyword for frequency in vcf. Default 1000G_freq'
)
@click.option('--frequency_treshold', '-freq',
                    default=0.05,
                    nargs=1,
                    help='Specify the ferquency treshold for variants to be considered. Default=0.05'
)
@click.option('--cadd_keyword', '-caddkey',
                    default='CADD', 
                    nargs=1,
                    help='Specify keyword for CADD scores in vcf. Default CADD'
)
@click.option('--gq_treshold', '-gq',
                    default=20.0,
                    nargs=1,
                    help='Specify the genotype quality treshold for variants to be considered. Default=50'
)
@click.option('--read_depth_treshold', '-depth',
                    default=10.0,
                    nargs=1,
                    help="""Specify the genotype quality treshold for variants to be considered. Default=10.
                            The read deth is taken from AD, so it is the sum of the quality reads from reference and alternative alleles."""
)
@click.option('--cadd_treshold', '-cadd',
                    default=12.0, 
                    nargs=1,
                    help='Specify the cadd treshold for variants to be considered. Default 12.0'
)

# @click.option('-p', '--patterns',
#                     type=click.Choice(['AR', 'AD', 'X']),
#                     multiple=True,
#                     help='Specify the inheritance patterns. Default is all patterns'
# )
# @click.option('-o', '--outfile',
#                     type=click.Path(exists=False),
#                     help='Specify the path to a file where results should be stored.'
# )
# @click.option('-v', '--verbose',
#                 is_flag=True,
#                 help='Increase output verbosity.'
# )
def summarize(variant_file, family_file, frequency_treshold, frequency_keyword,
                cadd_treshold, cadd_keyword, gq_treshold, read_depth_treshold):
    """
    Summarize the the variants in a vcf.
    
    There will be one result line per individual.
    
    - How many variants found\n
    - How many variants did not satisfy the base call 
        quality treshold. (Default 20)\n
    - How many variants where not covered in all individuals. 
        (Default depth 10)\n
    - How many variants followed each model in each family:\n
            - AR_hom\n
            - AR_comp\n
            - AR_hom_dn\n
            - AR_comp_dn\n
            - AD\n
            - AD_dn\n
            - XD\n
            - XD_dn\n
            - XR\n
            - XR_dn\n
        - How many rare variants (Default maf < 0.02)\n
        - How many high scored cadd. (Default cadd = 0)\n
        - How many rare + high score cadd\n
        - How many no cadd score\n
        - How many indels\n
        - How many indels without cadd score\n
    
    """
    logger = logging.getLogger(__name__)
    logger = logging.getLogger("genmod.commands.summarize_variants")
    
    head = HeaderParser()
    
    nr_of_variants = 0
    
    header = ['sample_id', 'nr_of_variants']
    
    samples = {}
    
    logger.debug("Setting up a variant parser")
    if variant_file == '-':
        variant_parser = VCFParser(
            fsock = sys.stdin,
            check_info=False
            )
    else:
        variant_parser = VCFParser(
            infile = variant_file,
            check_info=False
            )
    logger.debug("Variant parser setup")
    
    head = variant_parser.metadata
    
    for sample_id in head.individuals:
        samples[sample_id] = {}
        samples[sample_id]["nr_of_variants"] = 0
    
    
    for variant in variant_parser:
        for sample_id in samples:
            samples[sample_id]["nr_of_variants"] += 1
            print(variant['genotypes'][sample_id].depth_of_coverage)
    
    print(json.dumps(samples))

Example 116

Project: vcf_parser Source File: command_line.py
@click.command()
@click.argument('variant_file', 
                    nargs=1, 
                    type=click.Path(),
                    metavar='<vcf_file> or -'
)
@click.option('-v', '--vep', 
                    is_flag=True,
                    help='If variants are annotated with the Variant Effect Predictor.'
)
@click.option('-s', '--split', 
                    is_flag=True,
                    help='Split the variants with multiallelic calls.'
)
@click.option('-o', '--outfile', 
                    type=click.Path(exists=False),
                    help='Path to a outfile.'
)
@click.option('-a',"--allele_symbol",
                default='0',
                help="The symbol that should be used when representing "\
                "unobserved alleles. Default is '0'"
)
@click.option('-v', '--verbose',
                is_flag=True,
                help='Increase output verbosity.'
)
@click.option('-i', '--check_info', 
                is_flag=True,
                help='Skip to check if info fields are correct for all variants.'
)
@click.option('--silent', 
                is_flag=True,
                help='Do not print vcf data.'
)
@click.option('--version',
                is_flag=True,
                callback=print_version,
                expose_value=False,
                is_eager=True
)
@click.option('-l', '--logfile',
                    type=click.Path(exists=False),
                    help="Path to log file. If none logging is "\
                          "printed to stderr."
)
@click.option('--loglevel',
                    type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 
                                        'CRITICAL']),
                    help="Set the level of log output."
)
def cli(variant_file, vep, split, outfile, verbose, silent, check_info,
        allele_symbol, logfile, loglevel):
    """
    Tool for parsing vcf files.
    
    Prints the vcf file to output. 
    If --split/-s is used all multiallelic calls will be splitted and printed 
    as single variant calls.
    For more information, please see github.com/moonso/vcf_parser.
    """
    from vcf_parser import logger, init_log

    if not loglevel:
        if verbose:
            loglevel = 'INFO'

    init_log(logger, logfile, loglevel)
    nr_of_variants = 0
    start = datetime.now()
    
    # with open(variant_file, 'r', encoding="utf-8") as f:
    #     for line in f:
    #         if not line.startswith('#'):
    #             nr_of_variants += 1
    
    if variant_file == '-':
        logger.info("Start parsing variants from stdin")
        my_parser = VCFParser(
            fsock=sys.stdin, 
            split_variants=split,
            check_info=check_info, 
            allele_symbol=allele_symbol
        )
    else:
        logger.info("Start parsing variants from file {0}".format(variant_file))
        my_parser = VCFParser(
            infile = variant_file,
            split_variants=split, 
            check_info=check_info, 
            allele_symbol=allele_symbol
        )

    if outfile:
        f = open(outfile, 'w', encoding='utf-8')
        logger.info("Printing vcf to file {0}".format(outfile))
    
    if not silent:
        logger.info("Printing vcf to stdout")
    else:
        logger.info("Skip printing since silent is active")
    
    for line in my_parser.metadata.print_header():
        if outfile:
            f.write(line+'\n')
        else:
            if not silent:
                print(line)
    try:
        for variant in my_parser:
            variant_line = '\t'.join([variant[head] for head in my_parser.header])
            if outfile:
                f.write(variant_line + '\n')
            else:
                if not silent:
                    print(variant_line)
            nr_of_variants += 1
    except SyntaxError as e:
        print(e)

    logger.info('Number of variants: {0}'.format(nr_of_variants))
    logger.info('Time to parse file: {0}'.format(str(datetime.now() - start)))

Example 117

Project: fjord Source File: parse.py
def parse():
    optParser = getOptParser()
    opts,args = optParser.parse_args()
    encoding = 'utf8'

    try:
        f = args[-1]
        # Try opening from the internet
        if f.startswith('http://'):
            try:
                import urllib.request, urllib.parse, urllib.error, cgi
                f = urllib.request.urlopen(f)
                contentType = f.headers.get('content-type')
                if contentType:
                    (mediaType, params) = cgi.parse_header(contentType)
                    encoding = params.get('charset')
            except:
                pass
        elif f == '-':
            f = sys.stdin
            if sys.version_info[0] >= 3:
                encoding = None
        else:
            try:
                # Try opening from file system
                f = open(f, 'rb')
            except IOError as e:
                sys.stderr.write('Unable to open file: %s\n' % e)
                sys.exit(1)
    except IndexError:
        sys.stderr.write('No filename provided. Use -h for help\n')
        sys.exit(1)

    treebuilder = treebuilders.getTreeBuilder(opts.treebuilder)

    if opts.sanitize:
        tokenizer = sanitizer.HTMLSanitizer
    else:
        tokenizer = HTMLTokenizer

    p = html5parser.HTMLParser(tree=treebuilder, tokenizer=tokenizer, debug=opts.log)

    if opts.fragment:
        parseMethod = p.parseFragment
    else:
        parseMethod = p.parse

    if opts.profile:
        import cProfile
        import pstats
        cProfile.runctx('run(parseMethod, f, encoding)', None,
                        {'run': run,
                         'parseMethod': parseMethod,
                         'f': f,
                         'encoding': encoding},
                        'stats.prof')
        # XXX - We should use a temp file here
        stats = pstats.Stats('stats.prof')
        stats.strip_dirs()
        stats.sort_stats('time')
        stats.print_stats()
    elif opts.time:
        import time
        t0 = time.time()
        docuement = run(parseMethod, f, encoding)
        t1 = time.time()
        if docuement:
            printOutput(p, docuement, opts)
            t2 = time.time()
            sys.stderr.write('\n\nRun took: %fs (plus %fs to print the output)'%(t1-t0, t2-t1))
        else:
            sys.stderr.write('\n\nRun took: %fs'%(t1-t0))
    else:
        docuement = run(parseMethod, f, encoding)
        if docuement:
            printOutput(p, docuement, opts)

Example 118

Project: socorro Source File: mboxutils.py
def getmbox(name):
    """Return an mbox iterator given a file/directory/folder name."""

    if name == "-":
        return [get_message(sys.stdin)]

    if name.startswith("+"):
        # MH folder name: +folder, +f1,f2,f2, or +ALL
        name = name[1:]
        import mhlib
        mh = mhlib.MH()
        if name == "ALL":
            names = mh.listfolders()
        elif ',' in name:
            names = name.split(',')
        else:
            names = [name]
        mboxes = []
        mhpath = mh.getpath()
        for name in names:
            filename = os.path.join(mhpath, name)
            mbox = mailbox.MHMailbox(filename, get_message)
            mboxes.append(mbox)
        if len(mboxes) == 1:
            return iter(mboxes[0])
        else:
            return _cat(mboxes)

    elif name.startswith(":"):
        # IMAP mailbox name:
        #   :username:password@server:folder1,...folderN
        #   :username:password@server:port:folder1,...folderN
        #   :username:password@server:ALL
        #   :username:password@server:port:ALL
        parts = re.compile(
':(?P<user>[^@:]+):(?P<pwd>[^@]+)@(?P<server>[^:]+(:[0-9]+)?):(?P<name>[^:]+)'
        ).match(name).groupdict()
        
        from scripts.sb_imapfilter import IMAPSession, IMAPFolder
        from spambayes import Stats, message
        from spambayes.Options import options
        
        session = IMAPSession(parts['server'])
        session.login(parts['user'], parts['pwd'])
        folder_list = session.folder_list()
        
        if name == "ALL":
            names = folder_list
        else:
            names = parts['name'].split(',')

        message_db = message.Message().message_info_db
        stats = Stats.Stats(options, message_db)
        mboxes = [IMAPFolder(n, session, stats) for n in names]
        
        if len(mboxes) == 1:
            return full_messages(mboxes[0])
        else:
            return _cat([full_messages(x) for x in mboxes])
        
    if os.path.isdir(name):
        # XXX Bogus: use a Maildir if /cur is a subdirectory, else a MHMailbox
        # if the pathname contains /Mail/, else a DirOfTxtFileMailbox.
        if os.path.exists(os.path.join(name, 'cur')):
            mbox = mailbox.Maildir(name, get_message)
        elif name.find("/Mail/") >= 0:
            mbox = mailbox.MHMailbox(name, get_message)
        else:
            mbox = DirOfTxtFileMailbox(name, get_message)
    else:
        fp = open(name, "rb")
        mbox = mailbox.PortableUnixMailbox(fp, get_message)
    return iter(mbox)

Example 119

Project: iot-utilities Source File: compileall.py
Function: main
def main():
    """Script main program."""
    import argparse

    parser = argparse.ArgumentParser(
        description='Utilities to support installing Python libraries.')
    parser.add_argument('-l', action='store_const', const=0,
                        default=10, dest='maxlevels',
                        help="don't recurse into subdirectories")
    parser.add_argument('-r', type=int, dest='recursion',
                        help=('control the maximum recursion level. '
                              'if `-l` and `-r` options are specified, '
                              'then `-r` takes precedence.'))
    parser.add_argument('-f', action='store_true', dest='force',
                        help='force rebuild even if timestamps are up to date')
    parser.add_argument('-q', action='count', dest='quiet', default=0,
                        help='output only error messages; -qq will suppress '
                             'the error messages as well.')
    parser.add_argument('-b', action='store_true', dest='legacy',
                        help='use legacy (pre-PEP3147) compiled file locations')
    parser.add_argument('-d', metavar='DESTDIR',  dest='ddir', default=None,
                        help=('directory to prepend to file paths for use in '
                              'compile-time tracebacks and in runtime '
                              'tracebacks in cases where the source file is '
                              'unavailable'))
    parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
                        help=('skip files matching the regular expression; '
                              'the regexp is searched for in the full path '
                              'of each file considered for compilation'))
    parser.add_argument('-i', metavar='FILE', dest='flist',
                        help=('add all the files and directories listed in '
                              'FILE to the list considered for compilation; '
                              'if "-", names are read from stdin'))
    parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
                        help=('zero or more file and directory names '
                              'to compile; if no arguments given, defaults '
                              'to the equivalent of -l sys.path'))
    parser.add_argument('-j', '--workers', default=1,
                        type=int, help='Run compileall concurrently')

    args = parser.parse_args()
    compile_dests = args.compile_dest

    if (args.ddir and (len(compile_dests) != 1
            or not os.path.isdir(compile_dests[0]))):
        parser.exit('-d destdir requires exactly one directory argument')
    if args.rx:
        import re
        args.rx = re.compile(args.rx)


    if args.recursion is not None:
        maxlevels = args.recursion
    else:
        maxlevels = args.maxlevels

    # if flist is provided then load it
    if args.flist:
        try:
            with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
                for line in f:
                    compile_dests.append(line.strip())
        except OSError:
            if args.quiet < 2:
                print("Error reading file list {}".format(args.flist))
            return False

    if args.workers is not None:
        args.workers = args.workers or None

    success = True
    try:
        if compile_dests:
            for dest in compile_dests:
                if os.path.isfile(dest):
                    if not compile_file(dest, args.ddir, args.force, args.rx,
                                        args.quiet, args.legacy):
                        success = False
                else:
                    if not compile_dir(dest, maxlevels, args.ddir,
                                       args.force, args.rx, args.quiet,
                                       args.legacy, workers=args.workers):
                        success = False
            return success
        else:
            return compile_path(legacy=args.legacy, force=args.force,
                                quiet=args.quiet)
    except KeyboardInterrupt:
        if args.quiet < 2:
            print("\n[interrupted]")
        return False
    return True

Example 120

Project: pombola Source File: document_parser.py
def main(args):
  
  parseType = args[1]
  year = args[2]
  questions = []
  answers   = []
  p, a, ap = defaultdict(int), defaultdict(int), defaultdict(int)
  mps = {}
  summaries = []
  
  for f in sys.stdin:
    fin = open(f.strip(), 'r')
    content = fin.read()
    fin.close()
    basename =os.path.basename(f)

    #parse appropriately
    if parseType == 'questions':
      q,a = qa(content);
    
      [questions.append(item) for item in q if item.get('timestamp_question') != None and str(item.get('timestamp_question').year) == year]
      [answers.append(item)   for item in a if item.get('timestamp_answer') != None and str(item.get('timestamp_answer').year) == year]

    elif parseType == 'rollcall':
      #outfile = open('/Users/idesouza/Hutspace/output.txt','w')
      timestamp, present, absent, with_permission = roll_call(content)
      
      if str(timestamp.year) != year:
        continue

      total = len(present) + len(absent) + len(with_permission)
      summaries.append({ 'timestamp': timestamp, 'text': '%s --- Present: %s Absent with Permission: %s Absent: %s Total: %s'%(timestamp, len(present),len(with_permission), len(absent), total)})

      
      #print '------- %s -------\n'% os.path.splitext(basename)[0]
      #if len(present) + len(with_permission) + len(absent) < 275:
      #  inspect = "INSPECT"
      #else :
      #  inspect = ""
        
      #outfile.write('------- %s ------- %s\n'% (os.path.splitext(basename)[0], inspect ))
      #outfile.write('\tPresent: %s Absent with Permission: %s Absent: %s\n'%(len(present),len(with_permission), len(absent)))
      
      for line in present:  
        p[line['mp']] += 1
        mps[line['mp']] = line['constituency']
      for line in absent:  
        a[line['mp']] += 1
        mps[line['mp']] = line['constituency']
      for line in with_permission:  
        ap[line['mp']] += 1
        mps[line['mp']] = line['constituency']

    else:
      pass
    
    client = MongoClient('mongodb://localhost:27017/')
    db = client['odekro']

    if parseType == 'questions':
      collection = db.collection['questions']
      
      #insert/update questions
      for item in questions:
        collection.update({ 'ref': item['ref'] }, { '$set': item }, True) 

      #match answers to questions
      for item in answers:
          collection.update({ 'ref' :   item['ref'] }, { '$set': item }, True);

    elif parseType == 'rollcall':
      collection = db.collection['rollcall']
      collection.update( {'timestamp': timestamp}, {'timestamp': timestamp, 'present' : present, 'absent': absent, 'permission': with_permission} ,True)

    else:
      pass
  #loop done

  #output what we want
  if parseType == 'rollcall':
    for summary in sorted(summaries, key=lambda k: k['timestamp']) :
      print summary['text']

    print 'mp\tconstituency\t#present\t#absent with permission\t#absent'
    for mp in mps.keys():
      print '%s\t%s\t%s\t%s\t%s' %(mp, mps[mp], p[mp], ap[mp], a[mp])
  
  if parseType == 'questions':
     collection = db.collection['questions']
     cursor = collection.aggregate( [ 
                              { '$group' : { '_id': '$source', 'total' : {"$sum" : 1}}},
                            ])
     print "MP\t# Questions Asked"
     for doc in cursor['result']:
        print '%s\t%s'%(doc['_id'], doc['total'])
  
  

  #outfile.close()
  return 0

Example 121

Project: mysql-utilities Source File: daemon.py
    def daemonize(self):
        """Creates the daemon.

        It will fork a child process and then exit parent. By performing a
        double fork, set the current process's user id, change the current
        working directory, set the current numeric umask, redirect standard
        streams and write the pid to a file.
        """
        def redirect_stream(system_stream, target_stream):
            """Redirect a system stream to a specified file.
            """
            if target_stream is None:
                target_f = os.open(os.devnull, os.O_RDWR)
            else:
                target_f = target_stream.fileno()
            os.dup2(target_f, system_stream.fileno())

        def fork_then_exit_parent(error_message):
            """Fork a child process, then exit the parent process.
            """
            try:
                pid = os.fork()
                if pid > 0:
                    os._exit(0)  # pylint: disable=W0212
            except OSError as err:
                msg = "{0}: [{1}] {2}".format(error_message, err.errno,
                                              err.strerror)
                self._report(msg, logging.CRITICAL)
                raise UtilDaemonError(msg)

        # Fork
        fork_then_exit_parent("Failed first fork.")

        try:
            os.setsid()
            os.chdir(self.chdir)
            os.umask(self.umask)
        except Exception as err:
            msg = "Unable to change directory ({0})".format(err)
            self._report(msg, logging.CRITICAL)
            raise UtilDaemonError(msg)

        # Double fork
        fork_then_exit_parent("Failed second fork.")

        # Redirect streams
        redirect_stream(sys.stdin, self.stdin)
        redirect_stream(sys.stdout, self.stdout)
        redirect_stream(sys.stderr, self.stderr)

        # write pidfile
        atexit.register(self.delete_pidfile)
        pid = str(os.getpid())
        try:
            with open(self.pidfile, "w") as f:
                f.write("{0}\n".format(pid))
        except IOError as err:
            msg = "Unable to write pidfile: {0}".format(err.strerror)
            self._report(msg, logging.CRITICAL)
            raise UtilDaemonError(msg)

Example 122

Project: fusioncatcher Source File: merge-sam.py
Function: merge_sam
def merge_sam(file_in, file_ou, fr = False, mismatches = 10000, mismatches20 = 10000, short = 20):
    # It merges consecutive lines which represent one read

    fin = None
    if file_in == '-':
        fin = sys.stdin
    else:
        fin = open(file_in,'r')

    fou = None
    if file_ou == '-':
        fou = sys.stdout
    else:
        fou = open(file_ou,'w')

    data = []
    size = 10**6
    last = ['_','1']
    limit = 10**5
    while True:
        lines = fin.readlines(size)
        if not lines:
            break

        for line in lines:
            if line.startswith('@'):
                data.append(line)
            else:
                temp = line.split("\t")
                ilastflag = int(last[sam_FLAG])
                itempflag = int(temp[sam_FLAG])
                proper_pair = ( itempflag & 0x02 ) and ( itempflag & 0x01 )
                if not proper_pair:
                    last = ['_','1']
                    continue
                # mismatches
                tag_nm_i = [e.partition("NM:i:")[2] for e in temp[sam_TAG:] if e.startswith('NM:i:')] # NM is mismatches per reads
                tag_nm_i = int(tag_nm_i[0]) if tag_nm_i else 0
                if (tag_nm_i > mismatches) or (len(temp[sam_SEQ]) < short+1 and tag_nm_i > mismatches20):
                    last = ['_','1']
                    continue
                        
                sa = False if ilastflag & 0x10 else True
                sb = False if itempflag & 0x10 else True
                if last[sam_QNAME][:-1] != temp[sam_QNAME][:-1] or (ilastflag & itempflag & 0xC0) or (fr and sa == sb) or (fr == False and sa != sb ):# test that they form a pair #or last[sam_QNAME][-1:] == temp[sam_QNAME][-1:]:
                    last = temp
                else:
                    #if temp[sam_QNAME][-1:].endswith("a"):
                    #    (last,temp) = (temp,last)
                    if itempflag & 0x40: # test if it is first in pair
                        (last,temp) = (temp,last) # last is always first in pair
                        (ilastflag,itempflag) = (itempflag,ilastflag) # last is always first in pair
                        (sa,sb) = (sb,sa)


                    la = len(last[sam_SEQ])
                    lb = len(temp[sam_SEQ])

                    if sa: #sa == "+": # forward strand
                        last[sam_CIGAR] = last[sam_CIGAR] + str(lb)+"H"
                        temp[sam_CIGAR] = str(la)+"H" + temp[sam_CIGAR]
                    else:
                        last[sam_CIGAR] = str(lb)+"H" + last[sam_CIGAR]
                        temp[sam_CIGAR] = temp[sam_CIGAR] + str(la)+"H"
                    if fr:
                       temp[sam_FLAG] = str(itempflag ^ 0x10) # move it on opposite strand (it flips the bit)

                    last[sam_QNAME] = last[sam_QNAME][:-1]
                    temp[sam_QNAME] = temp[sam_QNAME][:-1]

                    #last[sam_QNAME] = last[sam_QNAME].partition("__")[0]
                    #temp[sam_QNAME] = temp[sam_QNAME].partition("__")[0]


                    data.append("\t".join(last))
                    data.append("\t".join(temp))

                    if len(data) > limit:
                        fou.writelines(data)
                        data = []
                    last = ['_','1']
    if data:
        fou.writelines(data)
    fin.close()
    fou.close()

Example 123

Project: fusioncatcher Source File: sort_ttdb.py
def sort_columns(input_filename = '-',
                 output_filename = '-',
                 columns=None,
                 header=False,
                 ignore_case=False,
                 unique = False,
                 tmp_dir=None,
                 buffer_size = '80%',
                 parallel = multiprocessing.cpu_count(),
                 compress_program = None):
    """
    It sorts the input file (text tab separated file) based on the specified columns.
    It works like SELECT * ORDER BY in SQL.
    """
    import locale

    locale.setlocale(locale.LC_ALL, 'C')

    # check options suppported by SORT command
    sh1 = give_me_temp_filename(tmp_dir)
    sh2 = give_me_temp_filename(tmp_dir)
    # check options suppported by SORT command
    sort_parallel = False
    r = os.system("sort --help | grep 'parallel' > '%s'" % (sh1,))
    if (not r) and (not empty(sh1)) and len(file(sh1,'r').readlines()) == 1:
        sort_parallel = True
    delete_file(sh1)
    # check options suppported by SORT command
    sort_buffer = False
    r = os.system("sort --help | grep 'buffer-size' > '%s'" % (sh1,))
    if (not r) and (not empty(sh1)) and len(file(sh1,'r').readlines()) == 1:
        sort_buffer = True
    delete_file(sh1)
    # check options suppported by SORT command
    sort_compress = False
    if compress_program:
        r = os.system("sort --help | grep 'compress-program' > '%s' ; %s --help 2>/dev/null | grep -i 'compress' > '%s'" % (sh1,compress_program,sh2))
        if (not r) and ((not empty(sh1)) and len(file(sh1,'r').readlines()) == 1 and
            (not empty(sh2)) and len(file(sh2,'r').readlines()) >= 1):
            sort_compress = True
        delete_file(sh1)
        delete_file(sh2)

    # treat the case when the input file is coming from the standard input
    fin = input_filename.strip('"').strip("'")
    if fin == '-':
        fin = give_me_temp_filename(tmp_dir)
        fod = open(fin,'w')
        fid = sys.stdin
        while True:
            lines = fid.readlines(10**8)
            if not lines:
                break
            fod.writelines(lines)
        fod.close()

    fon = output_filename.strip('"').strip("'")
    if fon == '-':
        fon = give_me_temp_filename(tmp_dir)

    if header:
        header_saved = file(fin,'r').readline()
        file(output_filename,'w').write(header_saved)
    else:
        file(output_filename,'w').write('')

    # process the type of the column, numeric, or string
    first_line = file(fin,'r').readline()
    if first_line:
        nc=len(file(fin,'r').readline().rstrip('\r\n').split('\t'))#read first line in order to find out the number of columns
        if columns:
            columns=columns.strip().lower()
            if columns=='d':
                columns=','.join([str(i+1)+'d' for i in range(nc)])
            elif columns=='n':
                columns=','.join([str(i+1)+'n' for i in range(nc)])
            elif columns=='nd' or columns=='dn':
                columns=','.join([str(i+1)+'nd' for i in range(nc)])
        else:
            columns=','.join([str(i+1) for i in range(nc)])

        # extra parameters
        extra = ""

        if sort_buffer and buffer_size and buffer_size != 'no' and buffer_size != 'none':
                extra = extra + ' --buffer-size=' + str(buffer_size) + ' '
        if sort_parallel and parallel and parallel > 1:
                extra = extra + ' --parallel=' + str(parallel) + ' '
        if sort_compress and compress_program and compress_program.lower() != 'no' and compress_program.lower() != 'none':
                extra = extra + ' --compress-program='+compress_program + ' '

        # processing the input columns
        columns = ['-k '+el+','+el.replace('n','').replace('r','') for el in columns.replace('d','r').split(',')]
        comd = "-s -t '\t' "+" ".join(columns)
        if ignore_case:
            comd = "-f "+comd
        if unique:
            comd = "-u "+comd
        if tmp_dir:
            comd = "-T '"+tmp_dir+"' "+comd
        if header:
            comd = "LC_ALL=C sed 1d '" + fin + "' | LC_ALL=C sort " + extra + comd + " >> '" + output_filename + "'"
        else:
            comd = "LC_ALL=C sort " + extra + comd + " '" + fin + "' >> '" + output_filename + "'"
        r = os.system(comd)
        if r != 0:
            print >>sys.stderr, "ERROR (sort_ttdb.py) while running:"
            print >>sys.stderr, comd
            sys.exit(1)

    if input_filename == '-':
        os.remove(fin)

    if output_filename == '-':
        fod = sys.stdout
        fid = open(fon,'r')
        while True:
            gc.disable()
            lines = fid.readlines(10**8)
            gc.enable()
            if not lines:
                break
            fod.writelines(lines)
        fid.close()
        os.remove(fon)

Example 124

Project: rail Source File: emr_runner.py
Function: run_job_flow
def run_job_flow(branding, json_config, force, no_browser=False,
                    aws_exe='aws', profile='default', region=None):
    """ Submits job flow to EMR. Should eventually monitor it.

        branding: text file with branding to print to screen before running
            job. This is where the name of a software package or ASCII art 
            can go.
        json_config: JSON configuration file. Google Getting Started with
            Amazon Elastic MapReduce for formatting information.
        force: True iff all existing directories should be erased when
            writing intermediates.
        no_browser: True iff webpage with status of job shouldn't automatically
            be opened in default browser.
        aws_exe: path to AWS CLI executable
        profile: name of AWS CLI profile from which to grab AWS access key ID
            and secret access key.
        region: Amazon data center in which to run job flow. If None, uses
            region from AWS CLI.

        No return value.
    """
    iface = dp_iface.DooplicityInterface(
                        branding=branding,
                        opener='Started job flow submission script on {time}.'
                    )
    try:
        '''Should ultimately check to see which regions are available to user
        w/ API.'''
        aws_ansible = ab.AWSAnsible(profile=profile, region=region,
                                    valid_regions=_aws_regions)
        s3_ansible = ab.S3Ansible(aws_exe=aws_exe, profile=profile,
                                    iface=iface)
        # Serialize JSON configuration
        if json_config is not None:
            with open(json_config) as json_stream:
                full_payload = json.load(json_stream)
        else:
            full_payload = json.load(sys.stdin)
        try:
            job_flow = full_payload['Steps']
        except Exception:
            raise RuntimeError(
                    'Input JSON not in proper format. Ensure that the JSON '
                    'object has a Steps key.'
                )
        step_count = len(job_flow)
        steps = OrderedDict()
        # Check steps for requred data
        required_data = set(['input', 'output', 'mapper', 'reducer'])
        bad_output_data = []
        try:
            for step in job_flow:
                if 'hadoop-streaming' \
                    not in step['HadoopJarStep']['Jar'].lower():
                    '''Don't check command-line parameters if not a 
                    Hadoop Streaming step.'''
                    continue
                step_args = {}
                for j in xrange(0, len(step['HadoopJarStep']['Args']), 2):
                    arg_name = step['HadoopJarStep']['Args'][j][1:].strip()
                    if arg_name == 'input':
                        try:
                            step_args['input'] = ','.join(
                                    [step['HadoopJarStep']['Args'][j+1],
                                     step_args['input']]
                                )
                        except KeyError:
                            step_args['input'] \
                                = step['HadoopJarStep']['Args'][j+1]
                    elif arg_name in required_data:
                        step_args[step['HadoopJarStep']['Args'][j][1:]] \
                            = step['HadoopJarStep']['Args'][j+1].strip()
                steps[step['Name']] = step_args
        except (KeyError, IndexError):
            iface.fail(
                    'JSON file not in proper format. Ensure '
                    'that each StepConfig object has a HadoopJarStep '
                    'object with an Args array and a Name string.'
                )
            raise
        iface.step('Read job flow from input JSON.')
        iface.status('Checking that output directories on S3 are writable...')
        '''Check steps for required Hadoop Streaming command-line parameters
        and for whether outputs are writable.'''
        missing_data = defaultdict(list)
        identity_steps = []
        identity_mappers \
            = set(['cat', 'org.apache.hadoop.mapred.lib.IdentityMapper'])
        identity_reducers \
            = set(['cat', 'org.apache.hadoop.mapred.lib.IdentityReducer'])
        for step in steps:
            step_data = steps[step]
            for required_parameter in required_data:
                if required_parameter not in step_data:
                    missing_data[step].append('-' + required_parameter)
                elif not force and required_parameter == 'output' \
                    and ab.Url(step_data['output']).is_s3 \
                         and s3_ansible.is_dir(step_data['output']):
                    bad_output_data.append(step)
            if 'mapper' in steps[step] and 'reducer' in step_data \
                and step_data['mapper'] in identity_mappers \
                and step_data['reducer'] in identity_reducers:
                identity_steps.append(step)
        errors = []
        if missing_data:
            errors.extend(['Step "%s" is missing required parameter(s) "%s".' % 
                                (step, ', '.join(missing_data[step]))
                                for step in missing_data])
        if bad_output_data:
            errors.extend(['Output "directory" "%s" of step "%s" already '
                           'exists, and --force was not invoked to permit '
                           'overwriting it.' 
                           % (steps[step]['output'], step)
                           for step in bad_output_data])
        if identity_steps:
            errors.extend([('Step "%s" has both an identity mapper and an '
                            'identity reducer, which is redundant. Remove the '
                            'step before trying again.') % step])
        if errors:
            raise RuntimeError('\n'.join([('%d) ' % (i+1)) + error
                                    for i, error in enumerate(errors)]))
        iface.step('Verified that output directories on S3 are writable.')
        '''Remove intermediate directories and create buckets if they
        don't already exist so EMR doesn't choke when writing output. The
        idea here is to make a bucket feel like an ordinary directory to
        the user.'''
        buckets = set()
        for step in steps:
            step_data = steps[step]
            if ab.Url(step_data['output']).is_s3:
                s3_ansible.remove_dir(steps[step]['output'])
                buckets.add(ab.bucket_from_url(step_data['output']))
        for bucket in buckets:
            try:
                s3_ansible.create_bucket(bucket, region=aws_ansible.region)
            except Exception as e:
                raise RuntimeError(('Bucket %s already exists on S3. Change '
                                    'affected output directories in job flow '
                                    'and try again. The more distinctive the '
                                    'name chosen, the less likely it\'s '
                                    'already a bucket on S3. It may be '
                                    'easier to create a bucket first using '
                                    'the web interface and use its name + a '
                                    'subdirectory as the output directory.')
                                    % bucket)
        iface.step('Set up output directories on S3.')
        iface.status('Submitting job flow...')
        try:
            job_flow_response = aws_ansible.post_request(full_payload)
        except urllib2.HTTPError as e:
            if 'bad request' in str(e).lower():
                raise RuntimeError('(' + str(e) + ');'
                                + (' ensure that IAM roles are '
                                'configured properly. This '
                                'may require talking to your AWS account '
                                'admin. See '
                                'http://docs.aws.amazon.com/ElasticMapReduce/'
                                'latest/DeveloperGuide/emr-'
                                'iam-roles-defaultroles.html for more '
                                'information. In most cases, the solution '
                                'to is to run "aws emr '
                                'create-default-roles", and try again.'))
            else:
                raise
        json_response = json.load(job_flow_response)
        if 'JobFlowId' not in json_response:
            raise RuntimeError('Job flow submission failed. Server returned '
                               'the following response: %s.'
                                % json.dumps(json_response, sort_keys=True,
                                    indent=4, separators=(',', ': ')))
        else:
            job_flow_id = json_response['JobFlowId']
            iface.step(('Submitted job flow.\n'
                        'cuem*Job flow ID is %s .*****\n' % job_flow_id) + 
                        ('*****Submission can be monitored at %s .*****' %
                            _emr_url.format(region=aws_ansible.region,
                                            job_flow_id=job_flow_id)))
            if not no_browser:
                webbrowser.open(_emr_url.format(region=aws_ansible.region,
                                                job_flow_id=job_flow_id),
                                    new=2) # Open in new tab
                iface.step('Opening URL in default browser, if possible.')
        iface.done(closer=('Finished job flow submission script '
                           'on {date}. Run time was {length} seconds.')
                )
    except (Exception, GeneratorExit):
         # GeneratorExit added just in case this happens on modifying code
        iface.fail()
        raise
    except (KeyboardInterrupt, SystemExit):
        if 'job_flow_id' not in locals():
            iface.fail(opener='*****Terminated without submitting job.*****',
                        middler=('End time was {date}. '
                                 'Script was running for {length} seconds.'))
        else:
            iface.fail(opener='*****Terminated after submitting job.*****',
                        middler=('End time was {date}. '
                                 'Script was running for {length} seconds.'))
        raise

Example 125

Project: rail Source File: align_reads.py
def go(input_stream=sys.stdin, output_stream=sys.stdout, bowtie2_exe='bowtie2',
    bowtie_index_base='genome', bowtie2_index_base='genome2', 
    manifest_file='manifest', bowtie2_args=None, bin_size=10000, verbose=False,
    exon_differentials=True, exon_intervals=False, report_multiplier=1.2,
    min_exon_size=8, search_filter=1, min_readlet_size=15, max_readlet_size=25,
    readlet_interval=12, capping_multiplier=1.5, drop_deletions=False,
    gzip_level=3, scratch=None, index_count=1, output_bam_by_chr=False,
    tie_margin=0, no_realign=False, no_polyA=False):
    """ Runs Rail-RNA-align_reads.

        A single pass of Bowtie is run to find end-to-end alignments. Unmapped
        reads are saved for readletizing to determine junctions in sucessive
        reduce steps as well as for realignment in a later map step.

        Input (read from stdin)
        ----------------------------
        Tab-delimited input tuple columns in a mix of any of the following
        three formats:
        Format 1 (single-end, 3-column):
          1. Nucleotide sequence or its reversed complement, whichever is first
            in alphabetical order
          2. 1 if sequence was reverse-complemented else 0
          3. Name
          4. Quality sequence or its reverse, whichever corresponds to field 1

        Format 2 (paired, 2 lines, 3 columns each)
        (so this is the same as single-end)
          1. Nucleotide sequence for mate 1 or its reversed complement,
            whichever is first in alphabetical order
          2. 1 if sequence was reverse-complemented else 0
          3. Name for mate 1
          4. Quality sequence for mate 1 or its reverse, whichever corresponds
            to field 1
            
            (new line)

          1. Nucleotide sequence for mate 2 or its reversed complement,
            whichever is first in alphabetical order
          2. 1 if sequence was reverse complemented else 0
          3. Name for mate 2
          4. Quality sequence for mate 2 or its reverse, whichever corresponds
            to field 1

        Input is partitioned and sorted by field 1, the read sequence.

        Hadoop output (written to stdout)
        ----------------------------
        A given RNAME sequence is partitioned into intervals ("bins") of some 
        user-specified length (see partition.py).

        Exonic chunks (aka ECs; three formats, any or all of which may be
        emitted):

        Format 1 (exon_ival); tab-delimited output tuple columns:
        1. Reference name (RNAME in SAM format) + ';' + bin number
        2. Sample index
        3. EC start (inclusive) on forward strand
        4. EC end (exclusive) on forward strand

        Format 2 (exon_diff); tab-delimited output tuple columns:
        1. Reference name (RNAME in SAM format) + ';' + bin number
        2. max(EC start, bin start) (inclusive) on forward strand IFF diff is
            positive and EC end (exclusive) on forward strand IFF diff is
            negative
        3. Sample index
        4. '1' if alignment from which diff originates is "unique" according to
            --tie-margin criterion; else '0'
        5. +1 or -1 * count, the number of instances of a read sequence for
            which to print exonic chunks

        Note that only unique alignments are currently output as ivals and/or
        diffs.

        Format 3 (sam); tab-delimited output tuple columns:
        Standard SAM output except fields are in different order, and the first
        field corresponds to sample label. (Fields are reordered to facilitate
        partitioning by sample name/RNAME and sorting by POS.) Each line
        corresponds to a spliced alignment. The order of the fields is as
        follows.
        1. Sample index if outputting BAMs by sample OR
                sample-rname index if outputting BAMs by chr
        2. (Number string representing RNAME; see BowtieIndexReference
            class in bowtie_index for conversion information) OR
            '0' if outputting BAMs by chr
        3. POS
        4. QNAME
        5. FLAG
        6. MAPQ
        7. CIGAR
        8. RNEXT
        9. PNEXT
        10. TLEN
        11. SEQ
        12. QUAL
        ... + optional fields

        Insertions/deletions (indel_bed)

        tab-delimited output tuple columns:
        1. 'I' or 'D' insertion or deletion line
        2. Number string representing RNAME
        3. Start position (Last base before insertion or 
            first base of deletion)
        4. End position (Last base before insertion or last base of deletion 
                            (exclusive))
        5. Inserted sequence for insertions or deleted sequence for deletions
        6. Sample index
        ----Next fields are for junctions only; they are '\x1c' for indels----
        7. '\x1c'
        8. '\x1c'
        --------------------------------------------------------------------
        9. Number of instances of insertion or deletion in sample; this is
            always +1 * count before bed_pre combiner/reducer

        Read whose primary alignment is not end-to-end

        Tab-delimited output tuple columns (unmapped):
        1. Transcriptome Bowtie 2 index group number
        2. SEQ
        3. 1 if SEQ is reverse-complemented, else 0
        4. QNAME
        5. QUAL

        Tab-delimited output tuple columns (readletized):
        1. Readlet sequence or its reversed complement, whichever is first in
            alphabetical order
        2. read sequence ID + ('-' if readlet
            sequence is reverse-complemented; else '+') + '\x1e' + displacement
            of readlet's 5' end from read's 5' end + '\x1e' + displacement of
            readlet's 3' end from read's 3' end (+, for EXACTLY one readlet of
            a read sequence, '\x1e' + read sequence + '\x1e' +
            (an '\x1f'-separated list A of unique sample labels with read
            sequences that match the original read sequence) + '\x1e' +
            (an '\x1f'-separated list  of unique sample labels B with read
            sequences that match the reversed complement of the original read
            sequence)) + '\x1e' + (an '\x1f'-separated list of the number of
            instances of the read sequence for each respective sample in list
            A) + '\x1e' + (an '\x1f'-separated list of the number of instances
            of the read sequence's reversed complement for each respective
            sample in list B). Here, a read sequence ID takes the form X:Y,
            where X is the "mapred_task_partition" environment variable -- a
            unique index for a task within a job -- and Y is the index of the
            read sequence relative to the beginning of the input stream.

        Tab-delimited tuple columns (postponed_sam):
        Standard 11+ -column raw SAM output

        Single column (unique):
        1. A unique read sequence

        Two columns, exactly one line (dummy); ensures creation of junction
            index:
        1. character "-"
        2. the word "dummy"

        ALL OUTPUT COORDINATES ARE 1-INDEXED.

        input_stream: where to find input reads.
        output_stream: where to emit exonic chunks and junctions.
        bowtie2_exe: filename of Bowtie2 executable; include path if not in
            $PATH.
        bowtie_index_base: the basename of the Bowtie1 index files associated
            with the reference.
        bowtie2_index_base: the basename of the Bowtie2 index files associated
            with the reference.
        manifest_file: filename of manifest
        bowtie2_args: string containing precisely extra command-line arguments
            to pass to first-pass Bowtie2.
        bin_size: genome is partitioned in units of bin_size for later load
            balancing.
        verbose: True iff more informative messages should be written to
            stderr.
        exon_differentials: True iff EC differentials are to be emitted.
        exon_intervals: True iff EC intervals are to be emitted.
        report_multiplier: if verbose is True, the line number of an alignment
            or read written to stderr increases exponentially with base
            report_multiplier.
        min_exon_size: minimum exon size searched for in junction_search.py
            later in pipeline; used to determine how large a soft clip on one
            side of a read is necessary to pass it on to junction search
            pipeline
        search_filter: how large a soft clip on one side of a read is necessary
            to pass it on to junction search pipeline
        min_readlet_size: "capping" readlets (that is, readlets that terminate
            at a given end of the read) are never smaller than this value
        max_readlet_size: size of every noncapping readlet
        readlet_interval: number of bases separating successive readlets along
            the read
        capping_multiplier: successive capping readlets on a given end of a
            read are increased in size exponentially with base
            capping_multiplier
        drop_deletions: True iff deletions should be dropped from coverage
            vector
        gzip_level: compression level to use for temporary files
        scratch: scratch directory for storing temporary files or None if 
            securely created temporary directory
        index_count: number of transcriptome Bowtie 2 indexes to which to
            assign unmapped reads for later realignment
        output_bam_by_chr: True iff final output BAMs will be by chromosome
        tie_margin: allowed score difference per 100 bases among ties in
            max score. For example, 150 and 144 are tied alignment scores
            for a 100-bp read when --tie-margin is 6.
        no_realign: True iff job flow does not need more than readlets: this
            usually means only a transcript index is being constructed
        no_polyA: kill noncapping readlets that are all As and write as
            unmapped all reads with polyA prefixes whose suffixes are <
            min_exon_size

        No return value.
    """
    global _input_line_count
    reference_index = bowtie_index.BowtieIndexReference(bowtie_index_base)
    manifest_object = manifest.LabelsAndIndices(manifest_file)
    alignment_printer = AlignmentPrinter(
            manifest_object,
            reference_index,
            bin_size=bin_size,
            output_stream=output_stream,
            exon_ivals=exon_intervals,
            exon_diffs=exon_differentials,
            drop_deletions=drop_deletions,
            output_bam_by_chr=output_bam_by_chr,
            tie_margin=tie_margin
        )
    # Get task partition to pass to align_reads_delegate.py
    try:
        task_partition = os.environ['mapred_task_partition']
    except KeyError:
        # Hadoop 2.x?
        try:
            task_partition = os.environ['mapreduce_task_partition']
        except KeyError:
            # A unit test is probably being run
            task_partition = '0'
    temp_dir = make_temp_dir(scratch)
    register_cleanup(tempdel.remove_temporary_directories, [temp_dir])
    align_file = os.path.join(temp_dir, 'first_pass_reads.temp.gz')
    other_reads_file = os.path.join(temp_dir, 'other_reads.temp.gz')
    second_pass_file = os.path.join(temp_dir, 'second_pass_reads.temp.gz')
    k_value, _, _ = bowtie.parsed_bowtie_args(bowtie2_args)
    nothing_doing = True
    # Required length of prefix after poly(A) is trimmed
    remaining_seq_size = max(min_exon_size - 1, 1)
    with xopen(True, align_file, 'w', gzip_level) as align_stream, \
        xopen(True, other_reads_file, 'w', gzip_level) as other_stream:
        for seq_number, ((seq,), xpartition) in enumerate(
                                                        xstream(sys.stdin, 1)
                                                    ):
            seq_length = len(seq)
            if no_polyA and (
                    all(seq[i] == 'A' 
                         for i in xrange(seq_length - remaining_seq_size))
                    or all(seq[i] == 'T' 
                         for i in xrange(remaining_seq_size, seq_length))
                    or all(seq[i] == 'A' 
                         for i in xrange(remaining_seq_size, seq_length))
                    or all(seq[i] == 'T' 
                         for i in xrange(seq_length - remaining_seq_size))
                ):
                if not no_realign:
                    '''If a sequence is too short without its poly(A) tail,
                    make all reads with that sequence unmapped. Technically,
                    this also kills poly(A)s at 5' ends, but we probably
                    couldn't align those sequences anyway.'''
                    reversed_complement_seq = seq[::-1].translate(
                                        _reversed_complement_translation_table
                                    )
                    for is_reversed, name, qual in xpartition:
                        if is_reversed == '0':
                            alignment_printer.print_unmapped_read(
                                                    name,
                                                    seq,
                                                    qual
                                                )
                        else:
                            alignment_printer.print_unmapped_read(
                                                    name,
                                                    reversed_complement_seq,
                                                    qual[::-1]
                                                )
                continue
            nothing_doing = False
            '''Select highest-quality read with alphabetically last qname
            for first-pass alignment.'''
            best_name, best_mean_qual, best_qual_index, i = None, None, 0, 0
            others_to_print = dlist()
            for is_reversed, name, qual in xpartition:
                _input_line_count += 1
                others_to_print.append(
                        '\t'.join([
                            str(seq_number), is_reversed, name, qual
                        ])
                    )
                mean_qual = (
                        float(sum([ord(score) for score in qual])) / len(qual)
                    )
                if (mean_qual > best_mean_qual
                        or mean_qual == best_mean_qual and name > best_name):
                    best_qual_index = i
                    best_mean_qual = mean_qual
                    best_name = name
                    to_align = '\t'.join([
                                        '%s\x1d%s' % (is_reversed, name),
                                        seq, qual
                                    ])
                i += 1
            assert i >= 1
            if i == 1:
                print >>other_stream, str(seq_number)
            else:
                for j, other_to_print in enumerate(others_to_print):
                    if j != best_qual_index:
                        print >>other_stream, other_to_print
            print >>align_stream, to_align
    # Print dummy line
    print 'dummy\t-\tdummy'
    sys.stdout.flush() # this is REALLY important b/c called script will stdout
    if nothing_doing:
        # No input
        sys.exit(0)
    input_command = 'gzip -cd %s' % align_file
    bowtie_command = ' '.join([bowtie2_exe,
        bowtie2_args if bowtie2_args is not None else '',
        ' --sam-no-qname-trunc --local -t --no-hd --mm -x',
        bowtie2_index_base, '--12 -'])
    delegate_command = ''.join(
                [sys.executable, ' ', os.path.realpath(__file__)[:-3],
                    ('_delegate.py --task-partition {task_partition} '
                     '--other-reads {other_reads} --second-pass-reads '
                     '{second_pass_reads} --min-readlet-size '
                     '{min_readlet_size} {drop_deletions} '
                     '--max-readlet-size {max_readlet_size} '
                     '--readlet-interval {readlet_interval} '
                     '--capping-multiplier {capping_multiplier:1.12f} '
                     '{verbose} --report-multiplier {report_multiplier:1.12f} '
                     '--k-value {k_value} '
                     '--bowtie-idx {bowtie_index_base} '
                     '--partition-length {bin_size} '
                     '--manifest {manifest_file} '
                     '{exon_differentials} {exon_intervals} '
                     '--gzip-level {gzip_level} '
                     '--search-filter {search_filter} '
                     '--index-count {index_count} '
                     '--tie-margin {tie_margin} '
                     '{no_realign} '
                     '{no_polyA} '
                     '{output_bam_by_chr}').format(
                        task_partition=task_partition,
                        other_reads=other_reads_file,
                        second_pass_reads=second_pass_file,
                        min_readlet_size=min_readlet_size,
                        drop_deletions=('--drop-deletions' if drop_deletions
                                            else ''),
                        max_readlet_size=max_readlet_size,
                        readlet_interval=readlet_interval,
                        capping_multiplier=capping_multiplier,
                        verbose=('--verbose' if verbose else ''),
                        report_multiplier=report_multiplier,
                        k_value=k_value,
                        bowtie_index_base=bowtie_index_base,
                        bin_size=bin_size,
                        manifest_file=manifest_file,
                        exon_differentials=('--exon-differentials'
                                            if exon_differentials else ''),
                        exon_intervals=('--exon-intervals'
                                        if exon_intervals else ''),
                        gzip_level=gzip_level,
                        search_filter=search_filter,
                        index_count=index_count,
                        tie_margin=tie_margin,
                        no_realign=('--no-realign' if no_realign else ''),
                        no_polyA=('--no-polyA' if no_polyA else ''),
                        output_bam_by_chr=('--output-bam-by-chr'
                                            if output_bam_by_chr
                                            else '')
                     )]
            )
    full_command = ' | '.join([input_command, 
                                bowtie_command, delegate_command])
    print >>sys.stderr, \
        'Starting first-pass Bowtie 2 with command: ' + full_command
    bowtie_process = subprocess.Popen(' '.join(
                    ['set -exo pipefail;', full_command]
                ),
            bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, shell=True,
            executable='/bin/bash')
    return_code = bowtie_process.wait()
    if return_code:
        raise RuntimeError('Error occurred while reading first-pass Bowtie 2 '
                           'output; exitlevel was %d.' % return_code)
    os.remove(align_file)
    os.remove(other_reads_file)
    if not no_realign:
        input_command = 'gzip -cd %s' % second_pass_file
        bowtie_command = ' '.join([bowtie2_exe,
            bowtie2_args if bowtie2_args is not None else '',
            ' --sam-no-qname-trunc --local -t --no-hd --mm -x',
            bowtie2_index_base, '--12 -'])
        delegate_command = ''.join(
                    [sys.executable, ' ', os.path.realpath(__file__)[:-3],
                        ('_delegate.py --task-partition {task_partition} '
                         '--min-readlet-size {min_readlet_size} '
                         '{drop_deletions} '
                         '--max-readlet-size {max_readlet_size} '
                         '--readlet-interval {readlet_interval} '
                         '--capping-multiplier {capping_multiplier:012f} '
                         '{verbose} '
                         '--report-multiplier {report_multiplier:012f} '
                         '--k-value {k_value} '
                         '--bowtie-idx {bowtie_index_base} '
                         '--partition-length {bin_size} '
                         '--manifest {manifest_file} '
                         '{exon_differentials} {exon_intervals} '
                         '--gzip-level {gzip_level} '
                         '--search-filter {search_filter} ' 
                         '--index-count {index_count} '
                         '--tie-margin {tie_margin} '
                         '{output_bam_by_chr}').format(
                            task_partition=task_partition,
                            min_readlet_size=min_readlet_size,
                            drop_deletions=('--drop-deletions'
                                                if drop_deletions else ''),
                            readlet_interval=readlet_interval,
                            max_readlet_size=max_readlet_size,
                            capping_multiplier=capping_multiplier,
                            verbose=('--verbose' if verbose else ''),
                            report_multiplier=report_multiplier,
                            k_value=k_value,
                            bowtie_index_base=bowtie_index_base,
                            bin_size=bin_size,
                            manifest_file=manifest_file,
                            exon_differentials=('--exon-differentials'
                                                if exon_differentials else ''),
                            exon_intervals=('--exon-intervals'
                                            if exon_intervals else ''),
                            gzip_level=gzip_level,
                            search_filter=search_filter,
                            index_count=index_count,
                            tie_margin=tie_margin,
                            output_bam_by_chr=('--output-bam-by-chr'
                                                if output_bam_by_chr
                                                else '')
                         )]
                )
        full_command = ' | '.join([input_command, 
                                    bowtie_command, delegate_command])
        print >>sys.stderr, \
            'Starting second-pass Bowtie 2 with command: ' + full_command
        bowtie_process = subprocess.Popen(' '.join(
                        ['set -exo pipefail;', full_command]
                    ),
                bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, shell=True,
                executable='/bin/bash')
        return_code = bowtie_process.wait()
        if return_code:
            raise RuntimeError('Error occurred while reading second-pass '
                               'Bowtie 2 output; exitlevel was %d.'
                                % return_code)
    sys.stdout.flush()

Example 126

Project: rez Source File: wrapper.py
Function: run
    def _run(self, prefix_char, args):
        from rez.vendor import argparse

        parser = argparse.ArgumentParser(prog=self.tool_name,
                                         prefix_chars=prefix_char)

        def _add_argument(*nargs, **kwargs):
            nargs_ = []
            for narg in nargs:
                nargs_.append(narg.replace('=', prefix_char))
            parser.add_argument(*nargs_, **kwargs)

        _add_argument(
            "=a", "==about", action="store_true",
            help="print information about the tool")
        _add_argument(
            "=i", "==interactive", action="store_true",
            help="launch an interactive shell within the tool's configured "
            "environment")
        _add_argument(
            "=p", "==patch", type=str, nargs='*', metavar="PKG",
            help="run the tool in a patched environment")
        _add_argument(
            "==versions", action="store_true",
            help="list versions of package providing this tool")
        _add_argument(
            "==command", type=str, nargs='+', metavar=("COMMAND", "ARG"),
            help="read commands from string, rather than executing the tool")
        _add_argument(
            "==stdin", action="store_true",
            help="read commands from standard input, rather than executing the tool")
        _add_argument(
            "==strict", action="store_true",
            help="strict patching. Ignored if ++patch is not present")
        _add_argument(
            "==nl", "==no-local", dest="no_local", action="store_true",
            help="don't load local packages when patching")
        _add_argument(
            "==peek", action="store_true",
            help="diff against the tool's context and a re-resolved copy - "
            "this shows how 'stale' the context is")
        _add_argument(
            "==verbose", action="count", default=0,
            help="verbose mode, repeat for more verbosity")
        _add_argument(
            "==quiet", action="store_true",
            help="hide welcome message when entering interactive mode")
        _add_argument(
            "==no-rez-args", dest="no_rez_args", action="store_true",
            help="pass all args to the tool, even if they start with '%s'" % prefix_char)

        opts, tool_args = parser.parse_known_args(args)

        if opts.no_rez_args:
            args = list(args)
            args.remove("==no-rez-args".replace('=', prefix_char))
            tool_args = args
            opts = parser.parse_args([])

        # print info
        if opts.about:
            return self.print_about()
        elif opts.versions:
            return self.print_package_versions()
        elif opts.peek:
            return self.peek()

        # patching
        context = self.context
        if opts.patch is not None:
            new_request = opts.patch
            request = context.get_patched_request(new_request, strict=opts.strict)
            config.remove_override("quiet")
            pkg_paths = (config.nonlocal_packages_path
                         if opts.no_local else None)

            context = ResolvedContext(request,
                                      package_paths=pkg_paths,
                                      verbosity=opts.verbose)

            # reapply quiet mode (see cli.forward)
            if "REZ_QUIET" not in os.environ:
                config.override("quiet", True)

        if opts.stdin:
            # generally shells will behave as though the '-s' flag was not present
            # when no stdin is available. So here we replicate this behaviour.
            import select
            if not select.select([sys.stdin], [], [], 0.0)[0]:
                opts.stdin = False

        # construct command
        cmd = None
        if opts.command:
            cmd = opts.command
        elif opts.interactive:
            label = self.context_name
            if opts.patch:
                label += '*'
            config.override("prompt", "%s>" % label)
            cmd = None
        else:
            cmd = [self.tool_name] + tool_args

        retcode, _, _ = context.execute_shell(command=cmd,
                                              stdin=opts.stdin,
                                              quiet=opts.quiet,
                                              block=True)
        return retcode

Example 127

Project: Memacs Source File: git.py
Function: main
    def _main(self):
        """
        get's automatically called from Memacs class
        read the lines from git-rev-list file,parse and write them to org file
        """

        # read file
        if self._args.gitrevfile:
            logging.debug("using as %s input_stream",
                          self._args.gitrevfile)
            input_stream = codecs.open(self._args.gitrevfile,
                                       encoding=self._args.encoding)
        else:
            logging.debug("using sys.stdin as input_stream")
            input_stream = codecs.getreader(self._args.encoding)(sys.stdin)

        # now go through the file
        # Logic (see example commit below)
        # first we are in an header and not in an body
        # every newline toggles output
        # if we are in body then add the body to commit class
        # if we are in header then add the header to commit class
        #
        # commit 6fb35035c5fa7ead66901073413a42742a323e89
        # tree 7027c628031b3ad07ad5401991f5a12aead8237a
        # parent 05ba138e6aa1481db2c815ddd2acb52d3597852f
        # author Armin Wieser <[email protected]> 1324422878 +0100
        # committer Armin Wieser <[email protected]> 1324422878 +0100
        #
        #     PEP8
        #     Signed-off-by: Armin Wieser <[email protected]>

        was_in_body = False
        commit = Commit()
        commits = []

        line = self.get_line_from_stream(input_stream)

        while line:
            line = line.rstrip()  # removing \n
            logging.debug("got line: %s", line)
            if line.strip() == "" or len(line) != len(line.lstrip()):
                commit.add_body(line)
                was_in_body = True
            else:
                if was_in_body:
                    commits.append(commit)
                    commit = Commit()
                commit.add_header(line)
                was_in_body = False

            line = self.get_line_from_stream(input_stream)

        # adding last commit
        if not commit.is_empty():
            commits.append(commit)

        logging.debug("got %d commits", len(commits))
        if len(commits) == 0:
            logging.error("Is there an error? Because i found no commits.")

        # time to write all commits to org-file
        for commit in commits:
            output, properties, note, author, timestamp = commit.get_output()

            if not(self._args.grepuser) or \
            (self._args.grepuser and self._args.grepuser == author):
                # only write to stream if
                # * grepuser is not set or
                # * grepuser is set and we got an entry with the right author
                self._writer.write_org_subitem(output=output,
                                               timestamp=timestamp,
                                               properties=properties,
                                               note=note)

        if self._args.gitrevfile:
            input_stream.close()

Example 128

Project: pyon Source File: pycc.py
def main(opts, *args, **kwargs):
    """
    Processes arguments and starts the capability container.
    """
    def prepare_logging():
        # Load logging override config if provided. Supports variants literal and path.
        logging_config_override = None
        if opts.logcfg:
            if '{' in opts.logcfg:
                # Variant 1: Value is dict of config values
                try:
                    eval_value = ast.literal_eval(opts.logcfg)
                    logging_config_override = eval_value
                except ValueError:
                    raise Exception("Value error in logcfg arg '%s'" % opts.logcfg)
            else:
                # Variant 2: Value is path to YAML file containing config values
                logutil.DEFAULT_LOGGING_PATHS.append(opts.logcfg)
        logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)

    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading
        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()      # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util
            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
            clear_couch_util.clear_couch(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
            pyon_config.container.filesystem.force_clean=True

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
        config.apply_configuration(pyon_config, config_override)
        config.apply_configuration(pyon_config, command_line_config)

        # Also set the immediate flag, but only if specified - it is an override
        if opts.immediate:
            from pyon.util.containers import dict_merge
            dict_merge(pyon_config, {'system':{'immediate':True}}, True)

        # Bootstrap pyon's core. Load configuration etc.
        bootstrap.bootstrap_pyon(pyon_cfg=pyon_config)

        # Delete any queues/exchanges owned by sysname if option "broker_clean" is set
        if opts.broker_clean:
            print "pycc: broker_clean=True, sysname:", bootstrap.get_sys_name()

            # build connect str
            connect_str = "-q -H %s -P %s -u %s -p %s -V %s" % (pyon_config.get_safe('server.amqp_priv.host', pyon_config.get_safe('server.amqp.host', 'localhost')),
                                                                   pyon_config.get_safe('container.exchange.management.port', '55672'),
                                                                   pyon_config.get_safe('container.exchange.management.username', 'guest'),
                                                                   pyon_config.get_safe('container.exchange.management.password', 'guest'),
                                                                   '/')

            from putil.rabbithelper import clean_by_sysname
            deleted_exchanges, deleted_queues = clean_by_sysname(connect_str, bootstrap.get_sys_name())
            print "      exchanges deleted (%s): %s" % (len(deleted_exchanges), ",".join(deleted_exchanges))
            print "         queues deleted (%s): %s" % (len(deleted_queues), ",".join(deleted_queues))

        if opts.force_clean:
            path = os.path.join(pyon_config.get_safe('container.filesystem.root', '/tmp/ion'), bootstrap.get_sys_name())
            print "force_clean: Removing", path
            FileSystem._clean(pyon_config)

        # Auto-bootstrap interfaces
        if bootstrap_config.system.auto_bootstrap:
            iadm.store_interfaces(idempotent=True)

        iadm.close()

        if opts.no_container:
            print "pycc: no_container=True. Stopping here."
            return None

        # Create the container instance
        from pyon.container.cc import Container
        container = Container(*args, **command_line_config)

        return container

    def start_container(container):
        """
        Start container and all internal managers. Returns when ready.
        """
        container.start()

    def do_work(container):
        """
        Performs initial startup actions with the container as requested in arguments.
        Then remains in container shell or infinite wait until container stops.
        Returns when container should stop. Raises an exception if anything failed.
        """
        if opts.proc:
            # Run a one-off process (with the -x argument)
            mod, proc = opts.proc.rsplit('.', 1)
            print "pycc: Starting process %s" % opts.proc
            container.spawn_process(proc, mod, proc, config={'process':{'type':'immediate'}})
            # And end
            return

        if opts.rel:
            # Start a rel file
            start_ok = container.start_rel_from_url(opts.rel)
            if not start_ok:
                raise Exception("Cannot start deploy file '%s'" % opts.rel)

        if opts.mx:
            from pyon.public import CFG
            port = CFG.get_safe('container.flask_webapp.port',8080)
            container.spawn_process("ContainerUI", "ion.core.containerui", "ContainerUI")
            print "pycc: Container UI started ... listening on http://localhost:%s" % port

        if opts.signalparent:
            import signal
            print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % (os.getppid(), os.getpid())
            os.kill(os.getppid(), signal.SIGUSR1)

            def is_parent_gone():
                while os.getppid() != 1:
                    gevent.sleep(1)
                print 'pycc: Now I am an orphan ... notifying serve_forever to stop'
                os.kill(os.getpid(), signal.SIGINT)
            import gevent
            ipg = gevent.spawn(is_parent_gone)

            container.gl_parent_watch = ipg

        if opts.enable_gbmonitor:
            from pyon.util.gevent_block_plugin import get_gevent_monitor_block
            get_gevent_monitor_block().start()

        if not opts.noshell and not opts.daemon:
            # Keep container running while there is an interactive shell
            from pyon.container.shell_api import get_shell_api
            setup_ipython_shell(get_shell_api(container))
        elif not opts.nomanhole:
            from pyon.container.shell_api import get_shell_api
            setup_ipython_embed(get_shell_api(container))
        else:
            container.serve_forever()

    def stop_container(container):
        try:
            if container:
                container.stop()
            return True
        except Exception as ex:
            # We want to make sure to get out here alive
            print "pycc: CONTAINER STOP ERROR"
            traceback.print_exc()
            return False

    def _exists_ipython_dir():
        # Fix OOIION-1124:
        # When multiple containers are started in parallel, all start an embedded IPython shell/manhole.
        # There exists a race condition between the IPython creating the default $HOME/.python dir
        # leading to an error.
        homedir = os.path.expanduser('~')
        homedir = os.path.realpath(homedir)
        home_ipdir = os.path.join(homedir, ".ipython")
        ipdir = os.path.normpath(os.path.expanduser(home_ipdir))
        return os.path.exists(ipdir)

    def setup_ipython_shell(shell_api=None):
        if not _exists_ipython_dir():
            log.warn("IPython profile dir not found. Attempting to avoid race condition")
            import gevent
            import random
            gevent.sleep(random.random() * 3.0)  # Introduce a random delay to make conflict less likely

        ipy_config = _setup_ipython_config()

        # monkeypatch the ipython inputhook to be gevent-friendly
        import gevent   # should be auto-monkey-patched by pyon already.
        import select

        def stdin_ready():
            infds, outfds, erfds = select.select([sys.stdin], [], [], 0)
            if infds:
                return True
            else:
                return False

        def inputhook_gevent():
            try:
                while not stdin_ready():
                    gevent.sleep(0.05)
            except KeyboardInterrupt:
                pass

            return 0

        # install the gevent inputhook
        from IPython.lib.inputhook import inputhook_manager
        inputhook_manager.set_inputhook(inputhook_gevent)
        inputhook_manager._current_gui = 'gevent'

        # First import the embeddable shell class
        from IPython.frontend.terminal.embed import InteractiveShellEmbed
        from mock import patch

        # Update namespace of interactive shell
        # TODO: Cleanup namespace even further
        if shell_api is not None:
            locals().update(shell_api)

        # Now create an instance of the embeddable shell. The first argument is a
        # string with options exactly as you would type them if you were starting
        # IPython at the system command line. Any parameters you want to define for
        # configuration can thus be specified here.
        with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
            for tries in range(3):
                try:
                    ipshell = InteractiveShellEmbed(config=ipy_config,
                        banner1 = """           ____  ____  _____   __     __     ____________
          / __ \/ __ \/  _/ | / /__  / /_   / ____/ ____/
         / / / / / / // //  |/ / _ \/ __/  / /   / /
        / /_/ / /_/ // // /|  /  __/ /_   / /___/ /___
        \____/\____/___/_/ |_/\___/\__/   \____/\____/""",
                        exit_msg = 'Leaving OOINet CC shell, shutting down container.')

                    ipshell('Pyon (PID: %s) - OOINet CC interactive IPython shell. Type ionhelp() for help' % os.getpid())
                    break
                except Exception as ex:
                    log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
                    import gevent
                    import random
                    gevent.sleep(random.random() * 0.5)

    def setup_ipython_embed(shell_api=None):
        if not _exists_ipython_dir():
            log.warn("IPython profile dir not found. Attempting to avoid race condition")
            import gevent
            import random
            gevent.sleep(random.random() * 3.0)  # Introduce a random delay to make conflict less likely

        from gevent_zeromq import monkey_patch
        monkey_patch()

        # patch in device:
        # gevent-zeromq does not support devices, which block in the C layer.
        # we need to support the "heartbeat" which is a simple bounceback, so we
        # simulate it using the following method.
        import zmq
        orig_device = zmq.device

        def device_patch(dev_type, insock, outsock, *args):
            if dev_type == zmq.FORWARDER:
                while True:
                    m = insock.recv()
                    outsock.send(m)
            else:
                orig_device.device(dev_type, insock, outsock, *args)

        zmq.device = device_patch

        # patch in auto-completion support
        # added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
        # we effectively make the same patches via monkeypatching
        from IPython.core.interactiveshell import InteractiveShell
        from IPython.zmq.ipkernel import IPKernelApp
        old_start = IPKernelApp.start
        old_set_completer_frame = InteractiveShell.set_completer_frame

        def new_start(appself):
            # restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
            bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
            appself.shell.set_completer_frame = bound_scf
            appself.shell.set_completer_frame()
            old_start(appself)

        IPKernelApp.start = new_start

        from IPython import embed_kernel
        ipy_config = _setup_ipython_config()

        # set specific manhole options
        import tempfile#, shutil
        from mock import patch
        temp_dir = tempfile.mkdtemp()
        ipy_config.Application.ipython_dir = temp_dir

        with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
            for tries in range(3):
                try:
                    embed_kernel(local_ns=shell_api, config=ipy_config)      # blocks until INT
                    break
                except Exception as ex:
                    log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
                    import gevent
                    import random
                    gevent.sleep(random.random() * 0.5)

        # @TODO: race condition here versus ipython, this will leave junk in tmp dir
        #try:
        #    shutil.rmtree(temp_dir)
        #except shutil.Error:
        #    pass

    def _setup_ipython_config():
        from IPython.config.loader import Config
        ipy_config = Config()
        ipy_config.KernelApp.connection_file = os.path.join(os.path.abspath(os.curdir), "manhole-%s.json" % os.getpid())
        ipy_config.PromptManager.in_template = '><> '
        ipy_config.PromptManager.in2_template = '... '
        ipy_config.PromptManager.out_template = '--> '
        ipy_config.InteractiveShellEmbed.confirm_exit = False
        #ipy_config.Application.log_level = 10      # uncomment for debug level ipython logging

        return ipy_config

    # main() -----> ENTER
    # ----------------------------------------------------------------------------------
    # Container life cycle

    prepare_logging()
    container = None
    try:
        container = prepare_container()
        if container is None:
            sys.exit(0)

        start_container(container)
    except Exception as ex:
#        print "pycc: ===== CONTAINER START ERROR -- FAIL ====="
#        traceback.print_exc()
        log.error('container start error', exc_info=True)
        stop_container(container)
        sys.exit(1)

    try:
        do_work(container)
    except Exception as ex:
        stop_container(container)
#        print "pycc: ===== CONTAINER PROCESS START ERROR -- ABORTING ====="
#        print ex
        log.error('container process interruption', exc_info=True)

        sys.exit(1)

    # Assumption: stop is so robust, it does not fail even if it was only partially started
    stop_ok = stop_container(container)
    if not stop_ok:
        sys.exit(1)

Example 129

Project: fuel-agent Source File: send2syslog.py
Function: get_config
    @classmethod
    def getConfig(cls):
        """Generate config from command line arguments and config file."""

        # example_config = {
        #       "daemon": True,
        #       "run_once": False,
        #       "debug": False,
        #       "watchlist": [
        #           {"servers": [ {"host": "localhost", "port": 514} ],
        #            "watchfiles": [
        #               {"tag": "anaconda",
        #                "log_type": "anaconda",
        #                "files": ["/tmp/anaconda.log",
        #                   "/mnt/sysimage/root/install.log"]
        #                }
        #               ]
        #            }
        #           ]
        #       }

        default_config = {"daemon": True,
                          "run_once": False,
                          "debug": False,
                          "hostname": cls._getHostname(),
                          "watchlist": []
                          }
        # First use default config as running config.
        config = dict(default_config)
        # Get command line options and validate it.
        cmdline = cls.cmdlineParse()[0]
        # Check config file source and read it.
        if cmdline.config_file or cmdline.stdin_config:
            try:
                if cmdline.stdin_config is True:
                    fo = sys.stdin
                else:
                    fo = open(cmdline.config_file, 'r')
                parsed_config = json.load(fo)
                if cmdline.debug:
                    print(parsed_config)
            except IOError:  # Raised if IO operations failed.
                main_logger.error("Can not read config file %s\n" %
                                  cmdline.config_file)
                exit(1)
            except ValueError as e:  # Raised if json parsing failed.
                main_logger.error("Can not parse config file. %s\n" %
                                  e.message)
                exit(1)
            #  Validate config from config file.
            cls.configValidate(parsed_config)
            # Copy gathered config from config file to running config
            # structure.
            for key, value in parsed_config.items():
                config[key] = value
        else:
            # If no config file specified use watchlist setting from
            # command line.
            watchlist = {"servers": [{"host": cmdline.host,
                                      "port": cmdline.port}],
                         "watchfiles": [{"tag": cmdline.tag,
                                         "log_type": cmdline.log_type,
                                         "files": cmdline.watchfiles}]}
            config['watchlist'].append(watchlist)

        # Apply behavioural command line options to running config.
        if cmdline.no_daemon:
            config["daemon"] = False
        if cmdline.run_once:
            config["run_once"] = True
        if cmdline.debug:
            config["debug"] = True
        return config

Example 130

Project: python-cinderclient Source File: shell.py
    def main(self, argv):
        # Parse args once to find version and debug settings
        parser = self.get_base_parser()
        (options, args) = parser.parse_known_args(argv)
        self.setup_debugging(options.debug)
        api_version_input = True
        self.options = options

        do_help = ('help' in argv) or (
            '--help' in argv) or ('-h' in argv) or not argv

        if not options.os_volume_api_version:
            api_version = api_versions.get_api_version(
                DEFAULT_MAJOR_OS_VOLUME_API_VERSION)
        else:
            api_version = api_versions.get_api_version(
                options.os_volume_api_version)

        # build available subcommands based on version
        major_version_string = "%s" % api_version.ver_major
        self.extensions = client.discover_extensions(major_version_string)
        self._run_extension_hooks('__pre_parse_args__')

        subcommand_parser = self.get_subcommand_parser(api_version,
                                                       do_help, args)
        self.parser = subcommand_parser

        if options.help or not argv:
            subcommand_parser.print_help()
            return 0

        argv = self._delimit_metadata_args(argv)
        args = subcommand_parser.parse_args(argv)
        self._run_extension_hooks('__post_parse_args__', args)

        # Short-circuit and deal with help right away.
        if args.func == self.do_help:
            self.do_help(args)
            return 0
        elif args.func == self.do_bash_completion:
            self.do_bash_completion(args)
            return 0

        (os_username, os_password, os_tenant_name, os_auth_url,
         os_region_name, os_tenant_id, endpoint_type,
         service_type, service_name, volume_service_name, bypass_url,
         cacert, os_auth_system) = (
             args.os_username, args.os_password,
             args.os_tenant_name, args.os_auth_url,
             args.os_region_name, args.os_tenant_id,
             args.os_endpoint_type,
             args.service_type, args.service_name,
             args.volume_service_name,
             args.bypass_url, args.os_cacert,
             args.os_auth_system)
        if os_auth_system and os_auth_system != "keystone":
            auth_plugin = cinderclient.auth_plugin.load_plugin(os_auth_system)
        else:
            auth_plugin = None

        if not service_type:
            service_type = client.SERVICE_TYPES[major_version_string]

        # FIXME(usrleon): Here should be restrict for project id same as
        # for os_username or os_password but for compatibility it is not.

        # V3 stuff
        project_info_provided = ((self.options.os_tenant_name or
                                  self.options.os_tenant_id) or
                                 (self.options.os_project_name and
                                  (self.options.os_project_domain_name or
                                   self.options.os_project_domain_id)) or
                                 self.options.os_project_id)

        if not utils.isunauthenticated(args.func):
            if auth_plugin:
                auth_plugin.parse_opts(args)

            if not auth_plugin or not auth_plugin.opts:
                if not os_username:
                    raise exc.CommandError("You must provide a user name "
                                           "through --os-username or "
                                           "env[OS_USERNAME].")

            if not os_password:
                # No password, If we've got a tty, try prompting for it
                if hasattr(sys.stdin, 'isatty') and sys.stdin.isatty():
                    # Check for Ctl-D
                    try:
                        os_password = getpass.getpass('OS Password: ')
                        # Initialize options.os_password with password
                        # input from tty. It is used in _get_keystone_session.
                        options.os_password = os_password
                    except EOFError:
                        pass
                # No password because we didn't have a tty or the
                # user Ctl-D when prompted.
                if not os_password:
                    raise exc.CommandError("You must provide a password "
                                           "through --os-password, "
                                           "env[OS_PASSWORD] "
                                           "or, prompted response.")

            if not project_info_provided:
                raise exc.CommandError(_(
                    "You must provide a tenant_name, tenant_id, "
                    "project_id or project_name (with "
                    "project_domain_name or project_domain_id) via "
                    "  --os-tenant-name (env[OS_TENANT_NAME]),"
                    "  --os-tenant-id (env[OS_TENANT_ID]),"
                    "  --os-project-id (env[OS_PROJECT_ID])"
                    "  --os-project-name (env[OS_PROJECT_NAME]),"
                    "  --os-project-domain-id "
                    "(env[OS_PROJECT_DOMAIN_ID])"
                    "  --os-project-domain-name "
                    "(env[OS_PROJECT_DOMAIN_NAME])"
                ))

            if not os_auth_url:
                if os_auth_system and os_auth_system != 'keystone':
                    os_auth_url = auth_plugin.get_auth_url()

            if not os_auth_url:
                raise exc.CommandError(
                    "You must provide an authentication URL "
                    "through --os-auth-url or env[OS_AUTH_URL].")

        if not project_info_provided:
            raise exc.CommandError(_(
                "You must provide a tenant_name, tenant_id, "
                "project_id or project_name (with "
                "project_domain_name or project_domain_id) via "
                "  --os-tenant-name (env[OS_TENANT_NAME]),"
                "  --os-tenant-id (env[OS_TENANT_ID]),"
                "  --os-project-id (env[OS_PROJECT_ID])"
                "  --os-project-name (env[OS_PROJECT_NAME]),"
                "  --os-project-domain-id "
                "(env[OS_PROJECT_DOMAIN_ID])"
                "  --os-project-domain-name "
                "(env[OS_PROJECT_DOMAIN_NAME])"
            ))

        if not os_auth_url:
            raise exc.CommandError(
                "You must provide an authentication URL "
                "through --os-auth-url or env[OS_AUTH_URL].")

        auth_session = None
        if not auth_plugin:
            auth_session = self._get_keystone_session()

        insecure = self.options.insecure

        self.cs = client.Client(
            api_version, os_username,
            os_password, os_tenant_name, os_auth_url,
            region_name=os_region_name,
            tenant_id=os_tenant_id,
            endpoint_type=endpoint_type,
            extensions=self.extensions,
            service_type=service_type,
            service_name=service_name,
            volume_service_name=volume_service_name,
            bypass_url=bypass_url,
            retries=options.retries,
            http_log_debug=args.debug,
            insecure=insecure,
            cacert=cacert, auth_system=os_auth_system,
            auth_plugin=auth_plugin,
            session=auth_session,
            logger=self.ks_logger if auth_session else self.client_logger)

        try:
            if not utils.isunauthenticated(args.func):
                self.cs.authenticate()
        except exc.Unauthorized:
            raise exc.CommandError("OpenStack credentials are not valid.")
        except exc.AuthorizationFailure:
            raise exc.CommandError("Unable to authorize user.")

        endpoint_api_version = None
        # Try to get the API version from the endpoint URL.  If that fails fall
        # back to trying to use what the user specified via
        # --os-volume-api-version or with the OS_VOLUME_API_VERSION environment
        # variable.  Fail safe is to use the default API setting.
        try:
            endpoint_api_version = \
                self.cs.get_volume_api_version_from_endpoint()
        except exc.UnsupportedVersion:
            endpoint_api_version = options.os_volume_api_version
            if api_version_input:
                logger.warning("Cannot determine the API version from "
                               "the endpoint URL. Falling back to the "
                               "user-specified version: %s" %
                               endpoint_api_version)
            else:
                logger.warning("Cannot determine the API version from the "
                               "endpoint URL or user input. Falling back "
                               "to the default API version: %s" %
                               endpoint_api_version)

        profile = osprofiler_profiler and options.profile
        if profile:
            osprofiler_profiler.init(options.profile)

        args.func(self.cs, args)

        if profile:
            trace_id = osprofiler_profiler.get().get_base_id()
            print("Trace ID: %s" % trace_id)
            print("To display trace use next command:\n"
                  "osprofiler trace show --html %s " % trace_id)

Example 131

Project: python-tackerclient Source File: shell.py
    def authenticate_user(self):
        """Authentication validation.

        Make sure the user has provided all of the authentication
        info we need.
        """
        if self.options.os_auth_strategy == 'keystone':
            if self.options.os_token or self.options.os_url:
                # Token flow auth takes priority
                if not self.options.os_token:
                    raise exc.CommandError(
                        _("You must provide a token via"
                          " either --os-token or env[OS_TOKEN]"
                          " when providing a service URL"))

                if not self.options.os_url:
                    raise exc.CommandError(
                        _("You must provide a service URL via"
                          " either --os-url or env[OS_URL]"
                          " when providing a token"))

            else:
                # Validate password flow auth
                project_info = (self.options.os_tenant_name or
                                self.options.os_tenant_id or
                                (self.options.os_project_name and
                                    (self.options.os_project_domain_name or
                                     self.options.os_project_domain_id)) or
                                self.options.os_project_id)

                if (not self.options.os_username
                        and not self.options.os_user_id):
                    raise exc.CommandError(
                        _("You must provide a username or user ID via"
                          "  --os-username, env[OS_USERNAME] or"
                          "  --os-user-id, env[OS_USER_ID]"))

                if not self.options.os_password:
                    # No password, If we've got a tty, try prompting for it
                    if hasattr(sys.stdin, 'isatty') and sys.stdin.isatty():
                        # Check for Ctl-D
                        try:
                            self.options.os_password = getpass.getpass(
                                'OS Password: ')
                        except EOFError:
                            pass
                    # No password because we didn't have a tty or the
                    # user Ctl-D when prompted.
                    if not self.options.os_password:
                        raise exc.CommandError(
                            _("You must provide a password via"
                              " either --os-password or env[OS_PASSWORD]"))

                if (not project_info):
                    # tenent is deprecated in Keystone v3. Use the latest
                    # terminology instead.
                    raise exc.CommandError(
                        _("You must provide a project_id or project_name ("
                          "with project_domain_name or project_domain_id) "
                          "via "
                          "  --os-project-id (env[OS_PROJECT_ID])"
                          "  --os-project-name (env[OS_PROJECT_NAME]),"
                          "  --os-project-domain-id "
                          "(env[OS_PROJECT_DOMAIN_ID])"
                          "  --os-project-domain-name "
                          "(env[OS_PROJECT_DOMAIN_NAME])"))

                if not self.options.os_auth_url:
                    raise exc.CommandError(
                        _("You must provide an auth url via"
                          " either --os-auth-url or via env[OS_AUTH_URL]"))
            auth_session = self._get_keystone_session()
            auth = auth_session.auth
        else:   # not keystone
            if not self.options.os_url:
                raise exc.CommandError(
                    _("You must provide a service URL via"
                      " either --os-url or env[OS_URL]"))
            auth_session = None
            auth = None

        self.client_manager = clientmanager.ClientManager(
            token=self.options.os_token,
            url=self.options.os_url,
            auth_url=self.options.os_auth_url,
            tenant_name=self.options.os_tenant_name,
            tenant_id=self.options.os_tenant_id,
            username=self.options.os_username,
            user_id=self.options.os_user_id,
            password=self.options.os_password,
            region_name=self.options.os_region_name,
            api_version=self.api_version,
            auth_strategy=self.options.os_auth_strategy,
            # FIXME (bklei) honor deprecated service_type and
            # endpoint type until they are removed
            service_type=self.options.os_service_type or
            self.options.service_type,
            endpoint_type=self.options.os_endpoint_type or self.endpoint_type,
            insecure=self.options.insecure,
            ca_cert=self.options.os_cacert,
            timeout=self.options.http_timeout,
            retries=self.options.retries,
            raise_errors=False,
            session=auth_session,
            auth=auth,
            log_credentials=True)
        return

Example 132

Project: bindep Source File: main.py
def main(depends=None):
    usage = "Usage: %prog [options] [profile]"
    parser = optparse.OptionParser(
        usage=usage, version="%%prog %s" % bindep.version)
    parser.add_option(
        "-b", "--brief", action="store_true", dest="brief",
        help="List only missing packages one per line.")
    parser.add_option(
        "-f", "--file", action="store", type="string", dest="filename",
        default="",
        help="Package list file (default: bindep.txt or "
             "other-requirements.txt).")
    parser.add_option(
        "--profiles", action="store_true",
        help="List the platform and configuration profiles.")
    opts, args = parser.parse_args()
    if depends is None:
        if opts.filename == "-":
            fd = sys.stdin
        elif opts.filename:
            try:
                fd = open(opts.filename, 'rt')
            except IOError:
                logging.error('Error reading file %s.' % opts.filename)
                return 1
        else:
            if (os.path.isfile('bindep.txt') and
                os.path.isfile('other-requirements.txt')):
                logging.error('Both bindep.txt and other-requirements.txt '
                              'files exist, choose one.')
                return 1
            if os.path.isfile('bindep.txt'):
                try:
                    fd = open('bindep.txt', 'rt')
                except IOError:
                    logging.error('Error reading file bindep.txt.')
                    return 1
            elif os.path.isfile('other-requirements.txt'):
                try:
                    fd = open('other-requirements.txt', 'rt')
                except IOError:
                    logging.error('Error reading file other-requirements.txt.')
                    return 1
            else:
                logging.error('Neither file bindep.txt nor file '
                              'other-requirements.txt exist.')
                return 1

        depends = bindep.depends.Depends(fd.read())
    if opts.profiles:
        logging.info("Platform profiles:")
        for profile in depends.platform_profiles():
            logging.info("%s", profile)
        logging.info("")
        logging.info("Configuration profiles:")
        for profile in depends.profiles():
            logging.info("%s", profile)
    else:
        if args:
            profiles = args
        else:
            profiles = ["default"]
        profiles = profiles + depends.platform_profiles()
        rules = depends.active_rules(profiles)
        errors = depends.check_rules(rules)
        for error in errors:
            if error[0] == 'missing':
                if opts.brief:
                    logging.info("%s", "\n".join(error[1]))
                else:
                    logging.info("Missing packages:")
                    logging.info("    %s", " ".join(error[1]))
            if error[0] == 'badversion':
                if not opts.brief:
                    logging.info("Bad versions of installed packages:")
                    for pkg, constraint, version in error[1]:
                        logging.info(
                            "    %s version %s does not match %s",
                            pkg, version, constraint)
        if errors:
            return 1
    return 0

Example 133

Project: pyector Source File: Ector.py
def main():
    # Each decay divided by 2
    UttererNode.__decay = 5
    SentimentNode.__decay = 5
    ExpressionNode.__decay = 20
    SentenceNode.__decay = 25
    TokenNode.__decay = 20
    from optparse import OptionParser

    usage = "usage: %prog [-p username][-n botname=Ector][-v|-q][-l logfilepath=ector.log][-s|-g][-h]"
    parser = OptionParser(usage=usage, version="%prog 0.3")
    parser.add_option("-p", "--person", dest="username", default="User",
                      help="set the name of the utterer")
    parser.add_option("-n", "--name", dest="botname", default="Ector",
                      help="set the name of the bot")
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=True,
                      help="say all that you can say")
    parser.add_option("-q", "--quiet", action="store_false", dest="verbose",
                      help="shut up!")
    parser.add_option("-l", "--log", dest="logname", default="ector.log",
                      help="log the dialogue in log file")
    parser.add_option("-s", "--sentence", action="store_true", dest="sentence", default=False,
                      help="set sentence reply mode on")
    parser.add_option("-g", "--generate", action="store_false", dest="sentence",
                      help="set generate reply mode on")
    parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False,
                      help="set debug mode on")

    (options, args) = parser.parse_args()

    license = None
    stdin = sys.stdin
    stdout = sys.stdout
    username = options.username
    botname = options.botname.capitalize()
    logfilename = options.logname
    version = "0.4"
    sentence_mode = options.sentence
    generate_mode = not sentence_mode    # sentence and generate modes are antagonist
    verbose = options.verbose
    debug = options.debug

    # Quiet mode is above sentence or generate modes
    if not verbose:
        sentence_mode = False
        generate_mode = False

    ector = Ector(botname, username)

    previousSentenceNode = None
    nodes = None

    print """pyECTOR version %s, Copyright (C) 2008 Francois PARMENTIER
pyECTOR comes with ABSOLUTELY NO WARRANTY; for details type `@show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `@show c' for details.
@help gives a basic help on pyECTOR commands.""" % (version)

    ector_path = os.path.dirname(sys.argv[0])
    license_path = os.path.abspath(ector_path + "/../LICENSE")

    while True:
        if stdin.closed:
            break
        stdout.write(username + ">")
        entry = stdin.readline().strip()

        # No Warranty
        if entry[:7] == "@show w":
            if not license:
                f = open(license_path)
                license = f.readlines()
                f.close()
            for i in range(257, 278):
                stdout.write(license[i])
        # Conditions
        elif entry[:7] == "@show c":
            if not license:
                f = open(license_path)
                license = f.readlines()
                f.close()
            for i in range(57, 256):
                stdout.write(license[i])
        elif entry[:6] == "@usage":
            print usage.replace("%prog", "Ectory.py")
        elif entry[:7] == "@status":
            ector.showStatus()
        elif entry[:8] == "@person ":
            username = entry[8:].strip()
            ector.setUser(username)
        elif entry[:6] == "@name ":
            botname = entry[6:].strip()
            ector.setName(botname)
        elif entry[:8] == "@version":
            print "pyECTOR version %s" % (version)
        elif entry[:6] == "@write":
            ector.dump()
        elif entry[:5] == "@quit" or entry[:5] == "@exit" or entry[:4] == "@bye":
            return 0
        elif entry[:10] == "@shownodes":
            ector.cn.showNodes()
        elif entry[:10] == "@showlinks":
            ector.showLinks()
        elif entry == "@showstate":
            ector.showState(username)
        elif entry == "@cleanstate":
            ector.cleanState()
        elif entry.startswith("@log "):
            logfilename = entry[5:]
            print "Log file: %s" % (logfilename)
        elif entry == "@log" or entry == "@logoff":
            print "Log off (%s)" % logfilename
            logfilename = ''
        elif entry.lower() == "@sentence on":
            sentence_mode = True
            generate_mode = False     # sentence and generate modes are not compatible
            print "Sentence reply mode ON"
        elif entry.lower() == "@sentence off":
            sentence_mode = False
            print "Sentence reply mode OFF"
        elif entry.lower() == "@sentence":
            print "Sentence reply mode", sentence_mode and "ON" or "OFF"
        elif entry.lower() == "@generate on":
            sentence_mode = False
            generate_mode = True     # sentence and generate modes are not compatible
            print "Generate reply mode ON"
        elif entry.lower() == "@generate off":
            generate_mode = False
            print "Sentence reply mode OFF"
        elif entry.lower() == "@generate":
            print "Generate reply mode", generate_mode and "ON" or "OFF"
        elif entry.lower() == "@debug on":
            debug = True
            print "Debug mode ON"
        elif entry.lower() == "@debug off":
            debug = False
            print "Debug mode OFF"
        elif entry.lower() == "@debug":
            print "Debug mode", debug and "ON" or "OFF"
        # Help
        elif entry[:5] == "@help":
            print """You can just start typing phrases.
But there are some commands you can use:
 - @usage     : print the options of the Ector.py command
 - @quit      : quit
 - @exit      : quit
 - @bye       : quit
 - @person    : change the utterer name (like -p)
 - @name      : change the bot's name (like -n)
 - @version   : give the current version
 - @write     : save Ector's Concept Network and state
 - @shownodes : show the nodes of the Concept Network
 - @showlinks : show the links of the Concept Network
 - @showstate : show the state of the nodes
 - @cleanstate: clean the state from the non-activated nodes
 - @log [file]: log the entries in the file (no file turns off the logging)
 - @status    : show the status of Ector (Concept Network, states)
 - @sentence [ON|OFF]: set the sentence reply mode
 - @generate [ON|OFF]: set the generate reply mode
 - @debug [ON|OFF]: set the debug mode on or off"""
        elif entry.startswith("@"):
            print "There is no command", entry
        elif entry:
            entry = unicode(entry, ENCODING)
            lastSentenceNode = ector.addEntry(entry)
            if previousSentenceNode:
                ector.cn.addLink(previousSentenceNode, lastSentenceNode)
            elif sentence_mode:
                # First sentence of a dialogue
                lastSentenceNode.beg += 1

            if nodes and lastSentenceNode:
                # Make a link from the nodes of the generated
                # sentence to the next entry.
                # BEWARE: may make a link co-occurrence greater than the
                # sentence node occurrence.
                for node in nodes:
                    ector.cn.addLink(node, lastSentenceNode)

            previousSentenceNode = lastSentenceNode
            # if log is activated, log the entry.
            if logfilename:
                logEntry(logfilename, username, entry)
            # Propagate activation
            ector.cleanState()
            ector.propagate(2)
#            ector.showState(username)
             # Get the reply
            reply = None
            if sentence_mode:
                # Get one of the most activated sentences
                replyNode = ector.getActivatedSentenceNode()
                reply = replyNode.getSymbol()
                reply = reply.replace("@bot@",  username)
                reply = reply.replace("@user@", botname)
                previousSentenceNode = replyNode
            elif generate_mode:
                (reply, nodes) = ector.generateSentence(debug)
                reply = reply.replace("@bot@",  username)
                reply = reply.replace("@user@", botname)
                previousSentenceNode = None
            if reply:
                print "%s>" % (ector.botname), reply.encode(ENCODING)
                if logfilename:
                    logEntry(logfilename, botname, reply)
        else:
            if debug:
                print "No entry given."
See More Examples - Go to Next Page
Page 1 Page 2 Page 3 Selected