sys.stderr

Here are the examples of the python api sys.stderr taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: quickrelease
Source File: command.py
View license
    def Run(self):
        """
        Launch the external command specified by this 
        L{RunShellCommand<quickrelease.command.RunShellCommand>} object.

        @raise RunShellCommandError: if C{raiseErrors} was set in the 
        constructor and the external command either returns with a failure
        value or times out, a RunShellCommandError will be raised.
        """

        if self._verbose:
            timeoutStr = ""
            if self.timeout is not None and gUsingKillableProcess:
                secondsStr = "seconds"
                if self.timeout == 1:
                    secondsStr = "second"
                timeoutStr = " with timeout %d %s" % (self.timeout, secondsStr)

            print >> sys.stderr, ("RunShellCommand(): Running %s in directory "
             "%s%s." % (str(self), self.workdir, timeoutStr))

            # Make sure all the output streams are flushed before we start; this
            # really only ever seems to have caused a problem on Win32
            sys.stderr.flush()
            sys.stdout.flush()

        commandLaunched = False
        try:
            logDescs = []

            if self._logfile:
                if self._appendLogfile:
                    logHandle = _OpenLog(self._logfile, 'a')
                else:
                    logHandle = _OpenLog(self._logfile, 'w') 

                logDescs.append(_LogHandleDesc(logHandle, _PIPE_STDOUT))

                if self._combineOutput:
                    logDescs.append(_LogHandleDesc(logHandle, _PIPE_STDERR))

            if not self._combineOutput and self._errorLogfile is not None:
                if self._appendErrorLogfile:
                    errorLogHandle = _OpenLog(self._errorLogfile, 'a')
                else:
                    errorLogHandle = _OpenLog(self._errorLogfile, 'w')

                logDescs.append(_LogHandleDesc(errorLogHandle, _PIPE_STDERR))

            outputQueue = Queue()

            stdinArg = None
            if self._input is not None:
                stdinArg = PIPE

            self._startTime = time.time()
            process = Popen(self._execArray, stdin=stdinArg, stdout=PIPE,
             stderr=PIPE, cwd=self.workdir, bufsize=0)
            commandLaunched = True

            stdinWriter = None
            if self._stdin is not None:
                #print >> sys.stderr, "Starting stdinWriter"
                stdinWriter = Thread(target=_WriteInput,
                 name="RunShellCommand() stdin writer",
                 args=(self._stdin, process.stdin))
                stdinWriter.start()

            stdoutReader = Thread(target=_EnqueueOutput,
             name="RunShellCommand() stdout reader",
             args=(process.stdout, outputQueue, _PIPE_STDOUT))
            stderrReader = Thread(target=_EnqueueOutput,
             name="RunShellCommand() stderr reader",
             args=(process.stderr, outputQueue, _PIPE_STDERR))
            self._outputMonitor = _OutputQueueReader(queue=outputQueue,
             logHandleDescriptors=logDescs, printOutput=self._printOutput)

            stdoutReader.start()
            stderrReader.start()
            self._outputMonitor.start()

            try:
                # If you're not using killable process, you theoretically have 
                # something else (buildbot) that's implementing a timeout for
                # you; so, all timeouts here are ignored... ...
                if self.timeout is not None and gUsingKillableProcess:
                    process.wait(self.timeout)
                else:
                    process.wait()

            except KeyboardInterrupt, ex:
                process.kill()
                self._processWasKilled = True
                raise ex
        except OSError, ex:
            if ex.errno == errno.ENOENT:
                raise RunShellCommandError(self, "Invalid command or working "
                 "dir: %s, working dir: %s" % (self, self.workdir))
            raise ReleaseFrameworkError("OSError: %s" % str(ex), details=ex)
        #except Exception, ex:
        #    print "EX: %s" % (ex)
        #    raise ex
        finally:
            if commandLaunched:
                procEndTime = time.time()

                #print >> sys.stderr, "Joining stderrReader"
                stderrReader.join()
                #print >> sys.stderr, "Joining stdoutReader"
                stdoutReader.join()
                #print >> sys.stderr, "Joining outputMonitor"
                self._outputMonitor.join()
                #print >> sys.stderr, "Joining q"
                outputQueue.join()
                if stdinWriter is not None:
                    #print >> sys.stderr, "Joining stdinWriter"
                    stdinWriter.join()

                for h in logDescs:
                    h.handle.close()

                # Assume if the runtime was up to/beyond the timeout, that it 
                # was killed, due to timeout.
                if commandLaunched and self.runningtime >= self.timeout:
                    self._processWasKilled = True
                    self._processTimedOut = True

                #om = outputMonitor.collectedOutput[_PIPE_STDOUT]
                #for i in range(om):
                #    print "STDOUT line %d (%d): %s" % (i, om[i].time,
                #     om[i].content)
                #
                #om = outputMonitor.collectedOutput[_PIPE_STDERR]
                # for i in range(om):
                #     print "STDERR line %d (%d): %s" % (i, om[i].time,
                #      om[i].content)

                #self._stdout = outputMonitor.GetOutput(_PIPE_STDOUT)
                #self._rawstdout = outputMonitor.GetOutput(_PIPE_STDOUT,
                # raw=True)
                #self._stderr = outputMonitor.GetOutput(_PIPE_STDERR)
                self._endTime = procEndTime
                self._returncode = process.returncode

                if self._input is not None and type(self._input) is str:
                    #print >> sys.stderr, "Closing stdin file."
                    self._stdin.close()

        if self._raiseErrors and self.returncode:
            raise RunShellCommandError(self)

Example 2

Project: madcow
Source File: migrate_madcow_1to2.py
View license
def migrate(fromdir, todir):
    os.makedirs(todir)
    dbdir = os.path.join(todir, 'db')
    if not os.path.exists(dbdir):
        os.makedirs(dbdir)
    data = None
    config = ConfigParser()
    config.read(os.path.join(fromdir, 'include', 'defaults.ini'))
    for filename in os.listdir(fromdir):
        src = os.path.join(fromdir, filename)
        if filename == 'grufti-responses.txt':
            copy(src, os.path.join(todir, 'response.grufti'))
        elif filename == 'data':
            data = src
        elif filename == 'madcow.ini':
            config.read(src)

    # migrate settings
    nicks = set()
    nicks.add(config.get('irc', 'nick'))
    nicks.add(config.get('silcplugin', 'nick'))
    settings.PROTOCOL = config.get('main', 'module')
    settings.IRC_HOST = config.get('irc', 'host')
    settings.IRC_PORT = config.getint('irc', 'port')
    settings.IRC_SSL = config.get('irc', 'ssl') == 'yes'
    settings.IRC_PASSWORD = config.get('irc', 'password')
    settings.IRC_RECONNECT = config.get('irc', 'reconnect') == 'yes'
    settings.IRC_RECONNECT_WAIT = config.getint('irc', 'reconnectWait')
    settings.IRC_REJOIN = config.get('irc', 'rejoin') == 'yes'
    settings.IRC_REJOIN_WAIT = config.getint('irc', 'rejoinWait')
    settings.IRC_REJOIN_MESSAGE = config.get('irc', 'rejoinReply')
    settings.IRC_QUIT_MESSAGE = config.get('irc', 'quitMessage')
    settings.IRC_OPER = config.get('irc', 'oper') == 'yes'
    settings.IRC_OPER_PASS = config.get('irc', 'operPass')
    settings.IRC_OPER_USER = config.get('irc', 'operUser')
    settings.IRC_NICKSERV_PASS = config.get('irc', 'nickServPass')
    settings.IRC_NICKSERV_USER = config.get('irc', 'nickServUser')
    settings.IRC_FORCE_WRAP = config.getint('irc', 'wrapSize')
    settings.IRC_DELAY_LINES = config.getint('irc', 'delay')
    settings.IRC_IDENTIFY_NICKSERV = bool(settings.IRC_NICKSERV_PASS and settings.IRC_NICKSERV_USER)
    settings.IRC_KEEPALIVE = config.get('irc', 'keepalive') == 'yes'
    settings.IRC_KEEPALIVE_FREQ = config.getint('irc', 'keepalive_freq')
    settings.IRC_KEEPALIVE_TIMEOUT = config.getint('irc', 'keepalive_timeout')
    settings.IRC_RECONNECT_MESSAGE = settings.IRC_REJOIN_MESSAGE
    settings.MODULES = []
    for module in os.listdir(os.path.join(madcow.PREFIX, 'modules')):
        if module.endswith('.py') and module != '__init__.py':
            module = module.replace('.py', '')
            try:
                try:
                    test = RENAMED_MODULES[module]
                except KeyError:
                    test = module
                if config.get('modules', test) == 'yes':
                    settings.MODULES.append(module)
                else:
                    print >> sys.stderr, 'WARN: %r module is disabled' % module
            except NoOptionError:
                print >> sys.stderr, 'WARN: unknown module: %r' % module
    if config.get('steam', 'enabled') == 'yes':
        settings.MODULES.append('steam')
    settings.STEAM_GROUP = config.get('steam', 'group')
    settings.STEAM_SHOW_ONLINE = config.get('steam', 'online') == 'yes'
    settings.AIM_AUTOJOIN_CHAT = config.get('aim', 'autojoin') == 'yes'
    settings.AIM_PASSWORD = config.get('aim', 'password')
    settings.AIM_PROFILE = config.get('aim', 'profile')
    settings.AIM_USERNAME = config.get('aim', 'username')
    nicks.add(settings.AIM_USERNAME)
    settings.PRIVATE_MODULES = config.get('modules', 'private').split(',')
    settings.DETACH = config.get('main', 'detach') == 'yes'
    settings.WORKERS = config.getint('main', 'workers')
    settings.YELP_DEFAULT_LOCATION = config.get('yelp', 'default_location')
    settings.TASKS = []
    if config.get('twitter', 'enabled') == 'yes':
        settings.TASKS.append('tweets')
    settings.TWITTER_CONSUMER_KEY = config.get('twitter', 'username')
    settings.TWITTER_CONSUMER_SECRET = config.get('twitter', 'password')
    try:
        settings.TWITTER_TOKEN_KEY = config.get('twitter', 'token_key')
        settings.TWITTER_TOKEN_SECRET = config.get('twitter', 'token_secret')
    except NoOptionError:
        settings.TWITTER_TOKEN_KEY = None
        settings.TWITTER_TOKEN_SECRET = None
    settings.TWITTER_UPDATE_FREQ = config.getint('twitter', 'updatefreq')
    if config.get('ircops', 'enabled') == 'yes':
        settings.TASKS.append('ircops')
    settings.IRC_GIVE_OPS_FREQ = config.getint('ircops', 'updatefreq')
    settings.HTTP_AGENT = config.get('http', 'agent')
    settings.HTTP_COOKIES = config.get('http', 'cookies') == 'yes'
    settings.HTTP_TIMEOUT = config.getint('http', 'timeout')
    settings.PIDFILE = config.get('main', 'pidfile')
    channels = set()
    channels.update(config.get('irc', 'channels').split(','))
    channels.update(config.get('silcplugin', 'channels').split(','))
    channels.add(config.get('gateway', 'channel'))
    channels.add(config.get('twitter', 'channel'))
    channels = [channel for channel in channels if channel]

    channels.remove('#madcow')
    if len(channels) > 1:
        channels = validate_list('channels', channels)
    settings.IRC_CHANNELS = channels
    settings.SILC_CHANNELS = channels
    settings.TWITTER_CHANNELS = 'ALL'
    settings.GATEWAY_CHANNELS = 'ALL'
    settings.LOG_PUBLIC = config.get('main', 'logpublic') == 'yes'
    settings.IGNORE_NICKS = config.get('main', 'ignoreList').split(',')
    settings.LOGGING_LEVEL = config.get('main', 'loglevel')
    settings.LOGGING_ENCODING = settings.ENCODING = config.get('main', 'charset')
    settings.OWNER_NICK = config.get('main', 'owner')
    settings.ALIASES = config.get('main', 'aliases').split(',')
    settings.SILC_DELAY = config.getint('silcplugin', 'delay')
    settings.SILC_HOST = config.get('silcplugin', 'host')
    settings.SILC_PORT = config.getint('silcplugin', 'port')
    settings.SILC_RECONNECT = config.get('silcplugin', 'reconnect') == 'yes'
    settings.SILC_RECONNECT_WAIT = config.getint('silcplugin', 'reconnectWait')
    settings.SMTP_FROM = config.get('smtp', 'sender')
    settings.SMTP_PASS = config.get('smtp', 'password')
    settings.SMTP_SERVER = config.get('smtp', 'server')
    settings.SMTP_USER = config.get('smtp', 'user')
    settings.GATEWAY_ADDR = config.get('gateway', 'bind')
    settings.GATEWAY_ENABLED = config.get('gateway', 'enabled') == 'yes'
    settings.GATEWAY_IMAGE_PATH = config.get('gateway', 'imagePath')
    settings.GATEWAY_IMAGE_URL = config.get('gateway', 'imageURL')
    settings.GATEWAY_PORT = config.getint('gateway', 'port')
    settings.GATEWAY_SAVE_IMAGES = bool(settings.GATEWAY_IMAGE_URL and settings.GATEWAY_IMAGE_PATH)
    settings.ADMIN_ENABLED = config.get('admin', 'enabled') == 'yes'
    settings.ALLOW_REGISTRATION = config.get('admin', 'allowRegistration') == 'yes'
    settings.DEFAULT_FLAGS = config.get('admin', 'defaultFlags')
    settings.DELICIOUS_USERNAME = config.get('delicious', 'username')
    settings.DELICIOUS_PASSWORD = config.get('delicious', 'password')
    nicks = [nick for nick in set(nicks) if nick]
    if len(nicks) > 1:
        print 'Multiple nicks found, pick one to use:'
        for i, nick in enumerate(nicks):
            print '%d %s' % (i + 1, nick)
        while True:
            i = raw_input('>>> ')
            try:
                nick = nicks[int(i) - 1]
                break
            except:
                pass
    else:
        nick = nicks[0]
    settings.BOTNAME = nick

    with open(os.path.join(todir, 'settings.py'), 'wb') as fp:
        for key in sorted(dir(settings)):
            if key.isupper():
                val = getattr(settings, key)
                if val == '':
                    val = None
                out = None
                if isinstance(val, list):
                    val = [_ for _ in val if _]
                    if val:
                        out = dumplist(val, key, 80)
                if not out:
                    out = '%s = %r' % (key, val)
                print >> fp, out
    print 'wrote settings.py'

    dbdir = os.path.join(todir, 'db')
    namespace = config.get('modules', 'dbNamespace')

    def get_dbfile(file):
        valid_namespace = False
        name = None
        try:
            basedir, filename = os.path.split(file)
            filename = filename.replace('.db', '')
            key, this_namespace, name = filename.split('-')
            if this_namespace != namespace:
                raise ValueError
            valid_namespace = True
            for ext in '', '.db':
                test = os.path.join(basedir, filename + ext)
                try:
                    db = dbm.open(test, 'r')
                except:
                    continue
                db.close()
                return test, name, 'dbm'
        except:
            pass
        if valid_namespace and name:
            return file, name, 'txt'

    if data is not None:
        for filename in os.listdir(data):
            src = os.path.join(data, filename)
            if os.path.isfile(src):
                try:
                    src, name, type = get_dbfile(src)
                except ValueError:
                    continue

                if type == 'dbm':
                    dst = os.path.join(dbdir, name)
                    db = dbm.open(src, 'r')
                    try:
                        new = dbm.open(dst, 'c', 0640)
                        try:
                            print 'copying %d items for %s database' % (len(db), name)
                            for key in db.keys():
                                new[key] = db[key]
                        finally:
                            new.close()
                    finally:
                        db.close()
                elif type == 'txt':
                    dst = os.path.join(dbdir, name)
                    shutil.copy(src, dst)
                    print 'copied %s' % name

            elif os.path.isdir(src) and filename == 'megahal':
                seen = set()
                megahal_from = src
                megahal_to = os.path.join(dbdir, filename)
                for basedir, subdirs, filenames in os.walk(megahal_from):
                    for filename in filenames:
                        if filename in BRAIN_FILES:
                            src = os.path.join(basedir, filename)
                            dstpath = src.replace(megahal_from + os.sep, '')
                            dst = os.path.join(megahal_to,  dstpath)
                            dstdir, dstfile = os.path.split(dst)
                            if not os.path.exists(dstdir):
                                os.makedirs(dstdir)
                            shutil.copy(src, dst)
                            print 'copied %s' % dstpath

    # memebot needs migration too since we ditched sqlobject
    try:
        migrate_memebot(config, dbdir, fromdir)
    except Exception, error:
        print >> sys.stderr, "couldn't migrate memebot: %s" % error

    src = os.path.join(fromdir, 'logs')
    if os.path.isdir(src):
        print 'copying logs'
        logdir = os.path.join(todir, 'log')
        os.makedirs(logdir)
        dst = os.path.join(logdir, 'public')
        main_log = []
        name = 'madcow.log'
        count = [0]
        def check(basedir, filenames):
            count[0] += len(filenames)
            if name in filenames:
                main_log.append(os.path.join(basedir, name))
                count[0] -= 1
                return [name]
            return []
        shutil.copytree(src, dst, ignore=check)
        print 'copied %d public logs' % count[0]
        if main_log:
            copy(main_log[0], os.path.join(logdir, name))

Example 3

Project: FIDDLE
Source File: data4trainng.py
View license
def main():
    usage = 'usage: %prog [options] <assembly> <annotation_file> <out_file>'
    parser = OptionParser(usage)
    parser.add_option('-d', dest='rootDir', type='str', default='.', help='Root directory of the project [Default: %default]')
    parser.add_option('-b', dest='chunkSize', default=1000, type='int', help='Align sizes with batch size')
    parser.add_option('-e', dest='width', type='int', default=500, help='Extend all sequences to this length [Default: %default]')
    parser.add_option('-r', dest='stride', default=20, type='int', help='Stride sequences [Default: %default]')

    (options,args) = parser.parse_args()

    if len(args) !=3 :
        print(args)
        print(options)
        print(len(args))
        parser.error('Must provide assembly, annotation file and an output name')
    else:
        assembly = args[0]
        annot_file = args[1]
        out_file = args[2]

    # read in the annotation file
    annot = pd.read_table(annot_file,sep=',')

    # Make directory for the project
    directory = "../../data/hdf5datasets/"
    if not os.path.exists(directory):
        os.makedirs(directory)

    # Parameters
    x1 = 500  #upstream
    x2 = 500  #downstream

    # open the hdf5 file to write
    f = h5py.File(os.path.join(directory,out_file), "w")
    trainSize = np.floor(0.90*len(annot)*(x1+x2-options.width)/options.stride)
    testSize = np.floor(0.05*len(annot)*(x1+x2-options.width)/options.stride)
    # make sure that the sizes are integer multiple of chunk size
    trainSize -= (trainSize % options.chunkSize)
    testSize -= (testSize % options.chunkSize)
    trainSize = int(trainSize)
    testSize = int(testSize)
    print >> sys.stderr, '%d training sequences\n %d test sequences ' % (trainSize,testSize)

    # note that we have 1 channel and 4xoptions.width matrices for dna sequence.
    NStrainData = f.create_dataset("NStrainInp", (trainSize,2,1,options.width))
    MStrainData = f.create_dataset("MStrainInp", (trainSize,2,1,options.width))
    DStrainData = f.create_dataset("DStrainInp", (trainSize,4,1,options.width))
    RStrainData = f.create_dataset("RStrainInp", (trainSize,1,1,options.width))
    TFtrainData = f.create_dataset("TFtrainInp", (trainSize,2,1,options.width))

    trainTarget = f.create_dataset("trainOut", (trainSize,1,1,options.width))

    # note that we have 1 channel and 4xoptions.width matrices for dna sequence.
    NStestData = f.create_dataset("NStestInp", (testSize,2,1,options.width))
    MStestData = f.create_dataset("MStestInp", (testSize,2,1,options.width))
    DStestData = f.create_dataset("DStestInp", (testSize,4,1,options.width))
    RStestData = f.create_dataset("RStestInp", (testSize,1,1,options.width))
    TFtestData = f.create_dataset("TFtestInp", (testSize,2,1,options.width))

    testTarget = f.create_dataset("testOut", (testSize,1,1,options.width))

    info = f.create_dataset("info", (trainSize+testSize,4)) # chromosome no,  strand, index of the annotation, genomic position

    chrRange = annot['chr'].unique()
    # Use 4 channel and 1xoptions.width
    NSdata = np.zeros([options.chunkSize,2,1,options.width])
    MSdata = np.zeros([options.chunkSize,2,1,options.width])
    DSdata = np.zeros([options.chunkSize,4,1,options.width])
    RSdata = np.zeros([options.chunkSize,1,1,options.width])
    TFdata = np.zeros([options.chunkSize,2,1,options.width])
    target = np.zeros([options.chunkSize,1,1,options.width])

    infodata = np.zeros([options.chunkSize,4])

    qq=0;
    cc=0;
    ps =0;
    nestVar = 0;
    debugMode = True

    # if options.species in ['YJ167','YJ168','YJ169','Scer','YSC001']:
    #     assembly = 'sacCer3'
    # elif options.species in ['YJ160']:
    #     assembly = 'Klla'
    # elif options.species in ['YJ177']:
    #     assembly = 'DeHa2'
    # else:
    #     raise('unknown species')

    print assembly
    gdb = genome.db.GenomeDB(path='/Users/umut/Projects/genome/data/share/genome_db',assembly=assembly)

    NSpos = gdb.open_track('NSpos')
    NSneg = gdb.open_track('NSneg')
    MSpos = gdb.open_track('MSpos')
    MSneg = gdb.open_track('MSneg')
    TFpos = gdb.open_track('TFpos')
    TFneg = gdb.open_track('TFneg')
    RS = gdb.open_track('RS')
    TSpos = gdb.open_track('TSpos')
    TSneg = gdb.open_track('TSneg')
    seq = gdb.open_track("seq")

    for chname in chrRange:
        if nestVar:
                break
        cc +=1
        tf = annot.chr==chname
        print 'doing %s' % (chname)

        for i in range(sum(tf)):
            if nestVar:
                break
            tss = annot[tf].tss.iloc[i]

            if annot[tf].strand.iloc[i]=="-":
                xran = range(tss-x2,tss+x1-options.width,options.stride)
            else:
                if tss<1000:
                    continue
                xran = range(tss-x1,tss+x2-options.width,options.stride)

            annotIdx = annot[tf].index[i]

            for pos in xran:
                if nestVar:
                    break
                seqVec = seq.get_seq_str(chname,pos+1,(pos+options.width))
                dsdata = vectorizeSequence(seqVec.lower())

                nsP = NSpos.get_nparray(chname,pos+1,(pos+options.width))
                nsN = NSneg.get_nparray(chname,pos+1,(pos+options.width))
                msP = MSpos.get_nparray(chname,pos+1,(pos+options.width))
                msN = MSneg.get_nparray(chname,pos+1,(pos+options.width))
                tfP = TFpos.get_nparray(chname,pos+1,(pos+options.width))
                tfN = TFneg.get_nparray(chname,pos+1,(pos+options.width))
                rs = RS.get_nparray(chname,pos+1,(pos+options.width))
                tsP = TSpos.get_nparray(chname,pos+1,(pos+options.width))
                tsN = TSneg.get_nparray(chname,pos+1,(pos+options.width))


                if debugMode:
                    if not checkData(np.r_[nsP,nsN,msP,msN,rs,tsP,tsN,tfP,tfN]):
                        print('NaN detected in chr' + chname + ' and at the position:' + str(pos))
                        # print nsmstsrsdata
                        # nestVar = 1;
                        continue

                if annot[tf].strand.iloc[i]=="+":
                    NSdata[qq,0,0,:] = nsP.T
                    NSdata[qq,1,0,:] = nsN.T
                    MSdata[qq,0,0,:] =msP.T
                    MSdata[qq,1,0,:] = msN.T
                    DSdata[qq,:,0,:] = dsdata.T
                    RSdata[qq,0,0,:] = rs.T
                    TFdata[qq,0,0,:] =tfP.T
                    TFdata[qq,1,0,:] = tfN.T
                    if sum(tsP)==0:
                        tsP = tsP + 1/np.float(options.width)
                    else:
                        tsP = tsP/sum(tsP+1e-5)
                    target[qq,0,0,:] =tsP.T
                    infodata[qq,:] = [cc, 1,annotIdx,pos]
                else:
                    NSdata[qq,0,0,:] = np.flipud(nsN).T
                    NSdata[qq,1,0,:] = np.flipud(nsP).T
                    MSdata[qq,0,0,:] = np.flipud(msN).T
                    MSdata[qq,1,0,:] = np.flipud(msP).T
                    RSdata[qq,0,0,:] = np.flipud(rs).T
                    DSdata[qq,:,0,:] = np.flipud(np.fliplr(dsdata)).T
                    TFdata[qq,0,0,:] = np.flipud(tfN).T
                    TFdata[qq,1,0,:] = np.flipud(tfP).T
                    if sum(tsN)==0:
                        tsN = tsN + 1/np.float(options.width)
                    else:
                        tsN = tsN/sum(tsN+1e-5)
                    target[qq,0,0,:] =np.flipud(tsN).T
                    infodata[qq,:] = [cc, -1,annotIdx,pos]

                qq+=1

                if ((ps+options.chunkSize) <= trainSize) and (qq>=options.chunkSize):
                    stp = options.chunkSize
                    NStrainData[range(ps,ps+stp),:,:,:] = NSdata
                    MStrainData[range(ps,ps+stp),:,:,:] = MSdata
                    DStrainData[range(ps,ps+stp),:,:,:] = DSdata
                    RStrainData[range(ps,ps+stp),:,:,:] = RSdata
                    TFtrainData[range(ps,ps+stp),:,:,:] = TFdata
                    trainTarget[range(ps,ps+stp),:,:,:] = target
                    info[range(ps,ps+stp),:] = infodata
                    NSdata = np.zeros([options.chunkSize,2,1,options.width])
                    MSdata = np.zeros([options.chunkSize,2,1,options.width])
                    DSdata = np.zeros([options.chunkSize,4,1,options.width])
                    RSdata = np.zeros([options.chunkSize,1,1,options.width])
                    infodata = np.zeros([options.chunkSize,4])
                    ps+=stp
                    qq=0
                    print >> sys.stderr, '%d  training chunk saved ' % ps
                if (ps >= trainSize) & (ps < (trainSize + testSize)) and (qq>=options.chunkSize):
                    NStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = NSdata
                    MStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = MSdata
                    DStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = DSdata
                    RStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = RSdata
                    TFtestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = TFdata
                    testTarget[range(ps-trainSize,ps-trainSize+stp),:,:,:] = target
                    info[range(ps,ps+stp),:] = infodata
                    rt = ps-trainSize
                    ps+=stp
                    qq=0
                    print >> sys.stderr, '%d  testing chunk saved ' % rt
                if ps >=(trainSize+testSize):
                    nestVar = 1;
                    break

    print ps
    f.close()
    NSpos.close()
    NSneg.close()
    MSpos.close()
    MSneg.close()
    RS.close()
    TFpos.close()
    TFneg.close()
    TSpos.close()
    TSneg.close()
    seq.close()

Example 4

Project: rivulet
Source File: __init__.py
View license
def session_thread(receive_queue, queue_poll_interval, torrents_by_hash, config, session_obj, session):
    # TODO make sure the torrent_handles are hashable
    def alert_to_torrent(alert):
        for torrent_handle, torrent in torrents.items():
            if torrent_handle == alert.handle:
                return torrent
        raise LibtorrentErrorException('Got alert for unknown torrent')

    def send_events(streamer_infos, alert):
        # Timeout
        # TODO: this code is unreachable
        if isinstance(alert, type(None)):
            for torrent in torrents:
                for si in streamer_infos:
                    si.time_waited += queue_poll_interval
                    if si.time_waited >= si.timeout:
                        err = "TorrentStream %d timed out waiting for data" % my_id
                        si.put(('kill', TorrentTimeoutException(err)))
        # Error
        elif alert.category() == lt.alert.category_t.error_notification:
            if isinstance(alert, lt.torrent_alert):
                torrent = alert_to_torrent(alert)
                # kill all streamers for this torrent
                for si in torrent.streamers.items().values():
                    si.put(('kill', LibtorrentErrorException(alert)))
            else:
                raise LibtorrentErrorException(alert)
        # Metadata
        elif isinstance(alert, lt.metadata_received_alert):
            torrent = alert_to_torrent(alert)
            for si in torrent.streamers:
                metadata = get_torrent_info(torrent.torrent_handle)
                print >> sys.stderr, 'XXXXXXXXMETADATA', metadata
                torrents_by_hash[str(metadata.info_hash())] = torrent.torrent_handle
                si.put(('metadata', metadata))
        # Data
        elif isinstance(alert, lt.read_piece_alert):
            print >> sys.stderr, 'received data', alert
            torrent = alert_to_torrent(alert)
            print >> sys.stderr, 'found streamers', torrent.streamers
            # TODO: figure out which streamer requested the piece and send the data only to that streamer
            for si in torrent.streamers:
                print >> sys.stderr, 'found streamer', si
                si.time_waited = 0
                # very important! duplicate the alert! Otherwise accessing the alert from another thread segfaults
                si.put(('data', {'piece': alert.piece, 'size': alert.size, 'buffer': alert.buffer[:]}))
        else:
            print >> sys.stderr, 'unknown alert', alert, type(alert), alert.category()

    root_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", os.path.expanduser(config.get('private_data_path', DEFAULT_PRIVATE_DATA_PATH)))
    resume_data_path = os.path.join(root_path, os.path.expanduser(config.get('resume_data_path', DEFAULT_RESUME_DATA_PATH)))
    mkdir_p(os.path.dirname(resume_data_path))
    session.listen_on(6881, 6891)
    session.set_alert_mask(lt.alert.category_t.status_notification)
    session.add_dht_router("dht.transmissionbt.com", 6881)
    session.add_dht_router("router.bittorrent.com", 6881)
    session.add_dht_router("router.utorrent.com", 6881)
    torrents = {}
    streamer_infos = {}

    # load resume data for each torrent if available
    try:
        with open(resume_data_path, 'r') as f:
            resume_data = json.load(f)
            for info_hash, resume_data in resume_data.items():
                params = session_obj.torrent_params_from_info_hash(info_hash)
                params['resume_data'] = base64.b64decode(resume_data)
                th = session.add_torrent(params)
                th.set_sequential_download(True)
                torrents_by_hash[str(get_torrent_info(th).info_hash())] = th
    except IOError:
        pass

    while True:
        # alert processing loop forever
        alert = session.wait_for_alert(queue_poll_interval)
        if alert:
          print >> sys.stderr, 'alert:', alert, type(alert), alert.category()
          send_events(streamer_infos, alert)
          session.pop_alert()
          #TODO: check if any recipients are done and remove them from the list


        # every X seconds or every alert check for requests from new callers
        if not receive_queue.empty():
            action, caller, data = receive_queue.get()
            print >> sys.stderr, 'action:', action, caller, data
            if action == 'subscribe_streamer':
                torrent_params, file_index_or_name, streamer_info = data
                streamer_infos[caller] = streamer_info

                #TODO check if this is safe to do on another thread so we can get rid of the queue
                torrent_handle = session.add_torrent(torrent_params)
                torrent_handle.set_sequential_download(True)

                # TODO: make this check work even if the metadata is not already available
                # if this torrent already exists, use the existing Torrent object instead of creating a new one
                torrent = None
                for th, t in torrents.items():
                    if torrent_handle == th:
                        torrent = t
                # we have to assume at first that this is a new torrent; if it turns out that it is not,
                # when we receive the alert telling us this, we merge it with the existing torrent
                if torrent is None:
                    torrent = Torrent(torrent_handle)
                torrents[torrent.torrent_handle] = torrent
                # this is done here so that metadata can be delivered to this streamer; the streamer removes itself
                torrent.add_streamer(streamer_info)

                # if the metadata is already available, there will be no event, so we have to do this here:
                torrent_info = None
                try:
                    torrent_info = get_torrent_info(torrent_handle)
                    torrents_by_hash[str(torrent_info.info_hash())] = torrent.torrent_handle
                except:
                    # and if the metadata is not available, it will come via an event
                    pass

                streamer_info.torrent = torrent
                streamer_info.put((torrent, torrent_info))

            elif action == 'unsubscribe_streamer':
                streamer_info = data
                last_user = streamer_info.torrent.remove_streamer(streamer_info)
                if last_user == True:
                    print >> sys.stderr, 'Torrent not needed any more, maybe we should remove it?'
                    del torrents_by_hash[str(get_torrent_info(streamer_info.torrent.torrent_handle).info_hash())]
                streamer_info.torrent = None

            elif action == 'quit':
                print 'shutting down gracefully and saving resume data'
                # go through each torrent and save their resume_data
                outstanding = 0
                for th, t in torrents.items():
                    if not th.is_valid():
                        continue
                    if not th.status().has_metadata:
                        continue
                    if not th.need_save_resume_data():
                        continue
                    th.save_resume_data(3)  # TODO: use the proper named flags
                    outstanding += 1

                save = {}
                try:
                    with open(resume_data_path, 'r') as old_f:
                        save = json.load(old_f)
                except IOError:
                    pass

                while outstanding > 0:
                    alert = session.wait_for_alert(10000) #wait extra long in this case
                    if not alert:
                        # give up if it times out
                        print 'timed out on shutdown'
                        break

                    if isinstance(alert, lt.save_resume_data_failed_alert):
                        print 'failed to save resume data'

                    elif isinstance(alert, lt.save_resume_data_alert):
                        th = alert.handle
                        hsh = str(get_torrent_info(th).info_hash())
                        # Sorry. If you feel like improving this, be my guest, but I didn't want to deal with it
                        save[hsh] = base64.b64encode(lt.bencode(alert.resume_data))

                    session.pop_alert()
                    outstanding -= 1

                print 'dumping resume_data'
                with open(resume_data_path, 'w') as f:
                    json.dump(save, f)
                print 'dumped resume_data'
                return
            else:
                raise Exception('WTF')

Example 5

View license
def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Generate a 'prerelease' script")
    add_argument_config_url(parser)
    add_argument_rosdistro_name(parser)
    add_argument_build_name(parser, 'source')
    add_argument_os_name(parser)
    add_argument_os_code_name(parser)
    add_argument_arch(parser)
    add_argument_output_dir(parser, required=True)

    group = parser.add_argument_group(
        'Repositories in underlay workspace',
        description='The repositories in the underlay workspace will be ' +
                    'built and installed as well as built and tested. ' +
                    'Dependencies will be provided by binary packages.')
    group.add_argument(
        'source_repos',
        nargs='*',
        default=[],
        metavar='REPO_NAME',
        help="A name of a 'repository' from the distribution file")
    group.add_argument(
        '--custom-branch',
        nargs='*',
        type=_repository_name_and_branch,
        default=[],
        metavar='REPO_NAME:BRANCH_OR_TAG_NAME',
        help="A name of a 'repository' from the distribution file followed " +
             'by a colon and a branch / tag name')
    group.add_argument(
        '--custom-repo',
        nargs='*',
        type=_repository_name_and_type_and_url_and_branch,
        default=[],
        metavar='REPO_NAME:REPO_TYPE:REPO_URL:BRANCH_OR_TAG_NAME',
        help='The name, type, url and branch / tag name of a repository')

    add_overlay_arguments(parser)

    args = parser.parse_args(argv)

    print('Fetching buildfarm configuration...')
    config = get_config_index(args.config_url)
    build_files = get_source_build_files(config, args.rosdistro_name)
    build_file = build_files[args.source_build_name]

    print('Fetching rosdistro cache...')
    # Targets defined by source build file are subset of targets
    # defined by release build files. To increase the number of supported
    # pre-release targets, we combine all targets defined by all release
    # build files and use that when configuring the devel job.
    release_build_files = get_release_build_files(config, args.rosdistro_name)
    release_targets_combined = {}
    if release_build_files:
        release_targets_combined[args.os_name] = {}
        for build_name, rel_obj in release_build_files.items():
            if args.os_name not in rel_obj.targets:
                continue
            for dist_name, targets in rel_obj.targets[args.os_name].items():
                if dist_name not in release_targets_combined[args.os_name]:
                    release_targets_combined[args.os_name][dist_name] = {}
                release_targets_combined[args.os_name][dist_name].update(targets)

    index = get_index(config.rosdistro_index_url)
    dist_cache = get_distribution_cache(index, args.rosdistro_name)
    dist_file = dist_cache.distribution_file

    # determine source repositories for underlay workspace
    repositories = {}
    for repo_name in args.source_repos:
        if repo_name in repositories:
            print("The repository '%s' appears multiple times" % repo_name,
                  file=sys.stderr)
            return 1
        try:
            repositories[repo_name] = \
                dist_file.repositories[repo_name].source_repository
        except KeyError:
            print(("The repository '%s' was not found in the distribution " +
                   "file") % repo_name, file=sys.stderr)
            return 1

    for repo_name, custom_version in args.custom_branch:
        if repo_name in repositories:
            print("The repository '%s' appears multiple times" % repo_name,
                  file=sys.stderr)
            return 1
        try:
            source_repo = dist_file.repositories[repo_name].source_repository
        except KeyError:
            print(("The repository '%s' was not found in the distribution " +
                   "file") % repo_name, file=sys.stderr)
            return 1
        source_repo = deepcopy(source_repo)
        source_repo.version = custom_version
        repositories[repo_name] = source_repo

    for repo_name, repo_type, repo_url, version in args.custom_repo:
        if repo_name in repositories and repositories[repo_name]:
            print("custom_repos option overriding '%s' to pull via '%s' "
                  "from '%s' with version '%s'. " %
                  (repo_name, repo_type, repo_url, version),
                  file=sys.stderr)
        source_repo = RepositorySpecification(
            repo_name, {
                'type': repo_type,
                'url': repo_url,
                'version': version,
            })
        repositories[repo_name] = source_repo

    scms = [(repositories[k], 'catkin_workspace/src/%s' % k)
            for k in sorted(repositories.keys())]

    # collect all template snippets of specific types
    class IncludeHook(Hook):

        def __init__(self):
            Hook.__init__(self)
            self.scripts = []

        def beforeInclude(self, *args, **kwargs):
            template_path = kwargs['file'].name
            if template_path.endswith('/snippet/builder_shell.xml.em'):
                self.scripts.append(kwargs['locals']['script'])

    hook = IncludeHook()
    from ros_buildfarm import templates
    templates.template_hooks = [hook]

    # use random source repo to pass to devel job template
    source_repository = deepcopy(list(repositories.values())[0])
    if not source_repository:
        print(("The repository '%s' does not have a source entry in the distribution " +
               'file. We cannot generate a prerelease without a source entry.') % repo_name,
              file=sys.stderr)
        return 1
    source_repository.name = 'prerelease'
    print('Evaluating job templates...')
    configure_devel_job(
        args.config_url, args.rosdistro_name, args.source_build_name,
        None, args.os_name, args.os_code_name, args.arch,
        config=config, build_file=build_file,
        index=index, dist_file=dist_file, dist_cache=dist_cache,
        jenkins=False, views=False,
        source_repository=source_repository,
        build_targets=release_targets_combined)

    templates.template_hooks = None

    # derive scripts for overlay workspace from underlay
    overlay_scripts = []
    for script in hook.scripts:
        # skip cloning of ros_buildfarm repository
        if 'git clone' in script and '.git ros_buildfarm' in script:
            continue
        # skip build-and-install step
        if 'build and install' in script:
            continue

        # add prerelease overlay flag
        run_devel_job = '/run_devel_job.py'
        if run_devel_job in script:
            script = script.replace(
                run_devel_job, run_devel_job + ' --prerelease-overlay')

        # replace mounted workspace volume with overlay and underlay
        # used by:
        # - create_devel_task_generator.py needs to find packages in both
        # the underlay as well as the overlay workspace
        # - catkin_make_isolated_and_test.py needs to source the environment of
        # the underlay before building the overlay
        mount_volume = '-v $WORKSPACE/catkin_workspace:/tmp/catkin_workspace'
        if mount_volume in script:
            script = script.replace(
                mount_volume, mount_volume + ':ro ' + '-v $WORKSPACE/' +
                'catkin_workspace_overlay:/tmp/catkin_workspace_overlay')

        # relocate all docker files
        docker_path = '$WORKSPACE/docker_'
        if docker_path in script:
            script = script.replace(
                docker_path, docker_path + 'overlay_')

        # rename all docker images
        name_suffix = '_prerelease'
        if name_suffix in script:
            script = script.replace(
                name_suffix, name_suffix + '_overlay')

        overlay_scripts.append(script)

    from ros_buildfarm import __file__ as ros_buildfarm_file
    data = deepcopy(args.__dict__)
    data.update({
        'scms': scms,
        'scripts': hook.scripts,
        'overlay_scripts': overlay_scripts,
        'ros_buildfarm_python_path': os.path.dirname(
            os.path.dirname(os.path.abspath(ros_buildfarm_file))),
        'python_executable': sys.executable,
        'prerelease_script_path': os.path.dirname(os.path.abspath(__file__))})

    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    # generate multiple scripts
    for script_name in [
            'prerelease',
            'prerelease_build_overlay',
            'prerelease_build_underlay',
            'prerelease_clone_overlay',
            'prerelease_clone_underlay']:
        content = expand_template(
            'prerelease/%s_script.sh.em' % script_name, data,
            options={BANGPATH_OPT: False})
        script_file = os.path.join(args.output_dir, script_name + '.sh')
        with open(script_file, 'w') as h:
            h.write(content)
        os.chmod(script_file, os.stat(script_file).st_mode | stat.S_IEXEC)

    print('')
    print('Generated prerelease script - to execute it run:')
    if os.path.abspath(args.output_dir) != os.path.abspath(os.curdir):
        print('  cd %s' % args.output_dir)
    print('  ./prerelease.sh')

Example 6

Project: bedup
Source File: __main__.py
View license
def main(argv):
    progname = 'bedup' if is_in_path('bedup') else 'python3 -m bedup'
    io_enc = codecs.lookup(locale.getpreferredencoding()).name
    if io_enc == 'ascii':
        print(
            'bedup will abort because Python was configured to use ASCII '
            'for console I/O.\nSee https://git.io/vnzk6 which '
              'explains how to use a UTF-8 locale.', file=sys.stderr)
        return 1
    parser = argparse.ArgumentParser(prog=progname)
    parser.add_argument(
        '--debug', action='store_true', help=argparse.SUPPRESS)
    commands = parser.add_subparsers(dest='command', metavar='command')

    sp_scan_vol = commands.add_parser(
        'scan', help='Scan', description="""
Scans volumes to keep track of potentially duplicated files.""")
    sp_scan_vol.set_defaults(action=vol_cmd)
    scan_flags(sp_scan_vol)

    # In Python 3.2+ we can add aliases here.
    # Hidden aliases doesn't seem supported though.
    sp_dedup_vol = commands.add_parser(
        'dedup', help='Scan and deduplicate', description="""
Runs scan, then deduplicates identical files.""")
    sp_dedup_vol.set_defaults(action=vol_cmd)
    scan_flags(sp_dedup_vol)
    sp_dedup_vol.add_argument(
        '--defrag', action='store_true',
        help='Defragment files that are going to be deduplicated')

    # An alias so as not to break btrfs-time-machine.
    # help='' is unset, which should make it (mostly) invisible.
    sp_dedup_vol_compat = commands.add_parser(
        'dedup-vol', description="""
A deprecated alias for the 'dedup' command.""")
    sp_dedup_vol_compat.set_defaults(action=vol_cmd)
    scan_flags(sp_dedup_vol_compat)

    sp_reset_vol = commands.add_parser(
        'reset', help='Reset tracking metadata', description="""
Reset tracking data for the listed volumes. Mostly useful for testing.""")
    sp_reset_vol.set_defaults(action=vol_cmd)
    vol_flags(sp_reset_vol)

    sp_show_vols = commands.add_parser(
        'show', help='Show metadata overview', description="""
Shows filesystems and volumes with their tracking status.""")
    sp_show_vols.set_defaults(action=cmd_show_vols)
    sp_show_vols.add_argument('fsuuid_or_device', nargs='?')
    sp_show_vols.add_argument(
        '--show-deleted', dest='show_deleted', action='store_true',
        help='Show volumes that have been deleted')
    sql_flags(sp_show_vols)

    sp_find_new = commands.add_parser(
        'find-new', help='List changed files', description="""
lists changes to volume since generation

This is a reimplementation of btrfs find-new,
modified to include directories as well.""")
    sp_find_new.set_defaults(action=cmd_find_new)
    sp_find_new.add_argument(
        '-0|--zero-terminated', dest='zero_terminated', action='store_true',
        help='Use a NUL character as the line separator')
    sp_find_new.add_argument(
        '--terse', dest='terse', action='store_true', help='Print names only')
    sp_find_new.add_argument('volume', help='Volume to search')
    sp_find_new.add_argument(
        'generation', type=int, nargs='?', default=0,
        help='Only show items modified at generation or a newer transaction')

    sp_forget_fs = commands.add_parser(
        'forget-fs', help='Wipe all metadata', description="""
Wipe all metadata for the listed filesystems.
Useful if the filesystems don't exist anymore.""")
    sp_forget_fs.set_defaults(action=cmd_forget_fs)
    sp_forget_fs.add_argument('uuid', nargs='+', help='Btrfs filesystem uuids')
    sql_flags(sp_forget_fs)

    sp_dedup_files = commands.add_parser(
        'dedup-files', help='Deduplicate listed', description="""
Freezes listed files, checks them for being identical,
and projects the extents of the first file onto the other files.

The effects are visible with filefrag -v (apt:e2fsprogs),
which displays the extent map of files.
        """.strip())
    sp_dedup_files.set_defaults(action=cmd_dedup_files)
    sp_dedup_files.add_argument('source', metavar='SRC', help='Source file')
    sp_dedup_files.add_argument(
        'dests', metavar='DEST', nargs='+', help='Dest files')
    # Don't forget to also set new options in the dedup-vol test in vol_cmd
    sp_dedup_files.add_argument(
        '--defrag', action='store_true',
        help='Defragment the source file first')

    sp_generation = commands.add_parser(
        'generation', help='Display volume generation', description="""
Display the btrfs generation of VOLUME.""")
    sp_generation.set_defaults(action=cmd_generation)
    sp_generation.add_argument('volume', help='Btrfs volume')
    sp_generation.add_argument(
        '--flush', action='store_true', dest='flush',
        help='Flush outstanding data using syncfs before lookup')

    sp_size_lookup = commands.add_parser(
        'size-lookup', help='Look up inodes by size', description="""
List tracked inodes with a given size.""")
    sp_size_lookup.set_defaults(action=cmd_size_lookup)
    sp_size_lookup.add_argument('size', type=int)
    sp_size_lookup.add_argument(
        '-0|--zero-terminated', dest='zero_terminated', action='store_true',
        help='Use a NUL character as the line separator')
    sql_flags(sp_size_lookup)

    sp_shell = commands.add_parser(
        'shell', description="""
Run an interactive shell (useful for prototyping).""")
    sp_shell.set_defaults(action=cmd_shell)
    sql_flags(sp_shell)

    sp_fake_updates = commands.add_parser(
        'fake-updates', description="""
Fake inode updates from the latest dedup events (useful for benchmarking).""")
    sp_fake_updates.set_defaults(action=cmd_fake_updates)
    sp_fake_updates.add_argument('max_events', type=int)
    sql_flags(sp_fake_updates)

    # Give help when no subcommand is given
    if not argv[1:]:
        parser.print_help()
        return

    args = parser.parse_args(argv[1:])

    if args.debug:
        try:
            from ipdb import launch_ipdb_on_exception
        except ImportError:
            sys.stderr.write(
                'Please install bedup[interactive] for this feature\n')
            return 1
        with launch_ipdb_on_exception():
            # Handle all warnings as errors.
            # Overrides the default filter that ignores deprecations
            # and prints the rest.
            warnings.simplefilter('error')
            warnings.filterwarnings('ignore', module='IPython\..*')
            warnings.filterwarnings('ignore', module='alembic\..*')
            return args.action(args)
    else:
        try:
            return args.action(args)
        except IOError as err:
            if err.errno == errno.EPERM:
                sys.stderr.write(
                    "You need to run this command as root.\n")
                return 1
            raise

Example 7

Project: p2pool-n
Source File: main.py
View license
@defer.inlineCallbacks
def main(args, net, datadir_path, merged_urls, worker_endpoint):
    try:
        print 'p2pool (version %s)' % (p2pool.__version__,)
        print
        
        @defer.inlineCallbacks
        def connect_p2p():
            # connect to bitcoind over bitcoin-p2p
            print '''Testing bitcoind P2P connection to '%s:%s'...''' % (args.bitcoind_address, args.bitcoind_p2p_port)
            factory = bitcoin_p2p.ClientFactory(net.PARENT)
            reactor.connectTCP(args.bitcoind_address, args.bitcoind_p2p_port, factory)
            def long():
                print '''    ...taking a while. Common reasons for this include all of bitcoind's connection slots being used...'''
            long_dc = reactor.callLater(5, long)
            yield factory.getProtocol() # waits until handshake is successful
            if not long_dc.called: long_dc.cancel()
            print '    ...success!'
            print
            defer.returnValue(factory)
        
        if args.testnet: # establish p2p connection first if testnet so bitcoind can work without connections
            factory = yield connect_p2p()
        
        # connect to bitcoind over JSON-RPC and do initial getmemorypool
        url = '%s://%s:%i/' % ('https' if args.bitcoind_rpc_ssl else 'http', args.bitcoind_address, args.bitcoind_rpc_port)
        print '''Testing bitcoind RPC connection to '%s' with username '%s'...''' % (url, args.bitcoind_rpc_username)
        bitcoind = jsonrpc.HTTPProxy(url, dict(Authorization='Basic ' + base64.b64encode(args.bitcoind_rpc_username + ':' + args.bitcoind_rpc_password)), timeout=30)
        yield helper.check(bitcoind, net)
        temp_work = yield helper.getwork(bitcoind)
        
        bitcoind_getinfo_var = variable.Variable(None)
        @defer.inlineCallbacks
        def poll_warnings():
            bitcoind_getinfo_var.set((yield deferral.retry('Error while calling getinfo:')(bitcoind.rpc_getinfo)()))
        yield poll_warnings()
        deferral.RobustLoopingCall(poll_warnings).start(20*60)
        
        print '    ...success!'
        print '    Current block hash: %x' % (temp_work['previous_block'],)
        print '    Current block height: %i' % (temp_work['height'] - 1,)
        print
        
        if not args.testnet:
            factory = yield connect_p2p()
        
        print 'Determining payout address...'
        if args.pubkey_hash is None:
            address_path = os.path.join(datadir_path, 'cached_payout_address')
            
            if os.path.exists(address_path):
                with open(address_path, 'rb') as f:
                    address = f.read().strip('\r\n')
                print '    Loaded cached address: %s...' % (address,)
            else:
                address = None
            
            if address is not None:
                res = yield deferral.retry('Error validating cached address:', 5)(lambda: bitcoind.rpc_validateaddress(address))()
                if not res['isvalid'] or not res['ismine']:
                    print '    Cached address is either invalid or not controlled by local bitcoind!'
                    address = None
            
            if address is None:
                print '    Getting payout address from bitcoind...'
                address = yield deferral.retry('Error getting payout address from bitcoind:', 5)(lambda: bitcoind.rpc_getaccountaddress('p2pool'))()
            
            with open(address_path, 'wb') as f:
                f.write(address)
            
            my_pubkey_hash = bitcoin_data.address_to_pubkey_hash(address, net.PARENT)
        else:
            my_pubkey_hash = args.pubkey_hash
        print '    ...success! Payout address:', bitcoin_data.pubkey_hash_to_address(my_pubkey_hash, net.PARENT)
        print
        
        print "Loading shares..."
        shares = {}
        known_verified = set()
        def share_cb(share):
            share.time_seen = 0 # XXX
            shares[share.hash] = share
            if len(shares) % 1000 == 0 and shares:
                print "    %i" % (len(shares),)
        ss = p2pool_data.ShareStore(os.path.join(datadir_path, 'shares.'), net, share_cb, known_verified.add)
        print "    ...done loading %i shares (%i verified)!" % (len(shares), len(known_verified))
        print
        
        
        print 'Initializing work...'
        
        node = p2pool_node.Node(factory, bitcoind, shares.values(), known_verified, net)
        yield node.start()
        
        for share_hash in shares:
            if share_hash not in node.tracker.items:
                ss.forget_share(share_hash)
        for share_hash in known_verified:
            if share_hash not in node.tracker.verified.items:
                ss.forget_verified_share(share_hash)
        node.tracker.removed.watch(lambda share: ss.forget_share(share.hash))
        node.tracker.verified.removed.watch(lambda share: ss.forget_verified_share(share.hash))
        
        def save_shares():
            for share in node.tracker.get_chain(node.best_share_var.value, min(node.tracker.get_height(node.best_share_var.value), 2*net.CHAIN_LENGTH)):
                ss.add_share(share)
                if share.hash in node.tracker.verified.items:
                    ss.add_verified_hash(share.hash)
        deferral.RobustLoopingCall(save_shares).start(60)
        
        print '    ...success!'
        print
        
        
        print 'Joining p2pool network using port %i...' % (args.p2pool_port,)
        
        @defer.inlineCallbacks
        def parse(host):
            port = net.P2P_PORT
            if ':' in host:
                host, port_str = host.split(':')
                port = int(port_str)
            defer.returnValue(((yield reactor.resolve(host)), port))
        
        addrs = {}
        if os.path.exists(os.path.join(datadir_path, 'addrs')):
            try:
                with open(os.path.join(datadir_path, 'addrs'), 'rb') as f:
                    addrs.update(dict((tuple(k), v) for k, v in json.loads(f.read())))
            except:
                print >>sys.stderr, 'error parsing addrs'
        for addr_df in map(parse, net.BOOTSTRAP_ADDRS):
            try:
                addr = yield addr_df
                if addr not in addrs:
                    addrs[addr] = (0, time.time(), time.time())
            except:
                log.err()
        
        connect_addrs = set()
        for addr_df in map(parse, args.p2pool_nodes):
            try:
                connect_addrs.add((yield addr_df))
            except:
                log.err()
        
        node.p2p_node = p2pool_node.P2PNode(node,
            port=args.p2pool_port,
            max_incoming_conns=args.p2pool_conns,
            addr_store=addrs,
            connect_addrs=connect_addrs,
            desired_outgoing_conns=args.p2pool_outgoing_conns,
            advertise_ip=args.advertise_ip,
        )
        node.p2p_node.start()
        
        def save_addrs():
            with open(os.path.join(datadir_path, 'addrs'), 'wb') as f:
                f.write(json.dumps(node.p2p_node.addr_store.items()))
        deferral.RobustLoopingCall(save_addrs).start(60)
        
        print '    ...success!'
        print
        
        if args.upnp:
            @defer.inlineCallbacks
            def upnp_thread():
                while True:
                    try:
                        is_lan, lan_ip = yield ipdiscover.get_local_ip()
                        if is_lan:
                            pm = yield portmapper.get_port_mapper()
                            yield pm._upnp.add_port_mapping(lan_ip, args.p2pool_port, args.p2pool_port, 'p2pool', 'TCP')
                    except defer.TimeoutError:
                        pass
                    except:
                        if p2pool.DEBUG:
                            log.err(None, 'UPnP error:')
                    yield deferral.sleep(random.expovariate(1/120))
            upnp_thread()
        
        # start listening for workers with a JSON-RPC server
        
        print 'Listening for workers on %r port %i...' % (worker_endpoint[0], worker_endpoint[1])
        
        wb = work.WorkerBridge(node, my_pubkey_hash, args.donation_percentage, merged_urls, args.worker_fee)
        web_root = web.get_web_root(wb, datadir_path, bitcoind_getinfo_var)
        caching_wb = worker_interface.CachingWorkerBridge(wb)
        worker_interface.WorkerInterface(caching_wb).attach_to(web_root, get_handler=lambda request: request.redirect('static/'))
        web_serverfactory = server.Site(web_root)
        
        
        serverfactory = switchprotocol.FirstByteSwitchFactory({'{': stratum.StratumServerFactory(caching_wb)}, web_serverfactory)
        deferral.retry('Error binding to worker port:', traceback=False)(reactor.listenTCP)(worker_endpoint[1], serverfactory, interface=worker_endpoint[0])
        
        with open(os.path.join(os.path.join(datadir_path, 'ready_flag')), 'wb') as f:
            pass
        
        print '    ...success!'
        print
        
        
        # done!
        print 'Started successfully!'
        print 'Go to http://127.0.0.1:%i/ to view graphs and statistics!' % (worker_endpoint[1],)
        if args.donation_percentage > 1.1:
            print '''Donating %.1f%% of work towards P2Pool's development. Thanks for the tip!''' % (args.donation_percentage,)
        elif args.donation_percentage < .9:
            print '''Donating %.1f%% of work towards P2Pool's development. Please donate to encourage further development of P2Pool!''' % (args.donation_percentage,)
        else:
            print '''Donating %.1f%% of work towards P2Pool's development. Thank you!''' % (args.donation_percentage,)
            print 'You can increase this amount with --give-author argument! (or decrease it, if you must)'
        print
        
        
        if hasattr(signal, 'SIGALRM'):
            signal.signal(signal.SIGALRM, lambda signum, frame: reactor.callFromThread(
                sys.stderr.write, 'Watchdog timer went off at:\n' + ''.join(traceback.format_stack())
            ))
            signal.siginterrupt(signal.SIGALRM, False)
            deferral.RobustLoopingCall(signal.alarm, 30).start(1)
        
        if args.irc_announce:
            from twisted.words.protocols import irc
            class IRCClient(irc.IRCClient):
                nickname = 'p2pool%02i' % (random.randrange(100),)
                channel = net.ANNOUNCE_CHANNEL
                def lineReceived(self, line):
                    if p2pool.DEBUG:
                        print repr(line)
                    irc.IRCClient.lineReceived(self, line)
                def signedOn(self):
                    self.in_channel = False
                    irc.IRCClient.signedOn(self)
                    self.factory.resetDelay()
                    self.join(self.channel)
                    @defer.inlineCallbacks
                    def new_share(share):
                        if not self.in_channel:
                            return
                        if share.pow_hash <= share.header['bits'].target and abs(share.timestamp - time.time()) < 10*60:
                            yield deferral.sleep(random.expovariate(1/60))
                            message = '\x02%s BLOCK FOUND by %s! %s%064x' % (net.NAME.upper(), bitcoin_data.script2_to_address(share.new_script, net.PARENT), net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header_hash)
                            if all('%x' % (share.header_hash,) not in old_message for old_message in self.recent_messages):
                                self.say(self.channel, message)
                                self._remember_message(message)
                    self.watch_id = node.tracker.verified.added.watch(new_share)
                    self.recent_messages = []
                def joined(self, channel):
                    self.in_channel = True
                def left(self, channel):
                    self.in_channel = False
                def _remember_message(self, message):
                    self.recent_messages.append(message)
                    while len(self.recent_messages) > 100:
                        self.recent_messages.pop(0)
                def privmsg(self, user, channel, message):
                    if channel == self.channel:
                        self._remember_message(message)
                def connectionLost(self, reason):
                    node.tracker.verified.added.unwatch(self.watch_id)
                    print 'IRC connection lost:', reason.getErrorMessage()
            class IRCClientFactory(protocol.ReconnectingClientFactory):
                protocol = IRCClient

            reactor.connectTCP("irc.freenode.net", 6667, IRCClientFactory(), bindAddress=(worker_endpoint[0], 0))
        
        @defer.inlineCallbacks
        def status_thread():
            last_str = None
            last_time = 0
            while True:
                yield deferral.sleep(3)
                try:
                    height = node.tracker.get_height(node.best_share_var.value)
                    this_str = 'P2Pool: %i shares in chain (%i verified/%i total) Peers: %i (%i incoming)' % (
                        height,
                        len(node.tracker.verified.items),
                        len(node.tracker.items),
                        len(node.p2p_node.peers),
                        sum(1 for peer in node.p2p_node.peers.itervalues() if peer.incoming),
                    ) + (' FDs: %i R/%i W' % (len(reactor.getReaders()), len(reactor.getWriters())) if p2pool.DEBUG else '')
                    
                    datums, dt = wb.local_rate_monitor.get_datums_in_last()
                    my_att_s = sum(datum['work']/dt for datum in datums)
                    my_shares_per_s = sum(datum['work']/dt/bitcoin_data.target_to_average_attempts(datum['share_target']) for datum in datums)
                    this_str += '\n Local: %sH/s in last %s Local dead on arrival: %s Expected time to share: %s' % (
                        math.format(int(my_att_s)),
                        math.format_dt(dt),
                        math.format_binomial_conf(sum(1 for datum in datums if datum['dead']), len(datums), 0.95),
                        math.format_dt(1/my_shares_per_s) if my_shares_per_s else '???',
                    )
                    
                    if height > 2:
                        (stale_orphan_shares, stale_doa_shares), shares, _ = wb.get_stale_counts()
                        stale_prop = p2pool_data.get_average_stale_prop(node.tracker, node.best_share_var.value, min(60*60//net.SHARE_PERIOD, height))
                        real_att_s = p2pool_data.get_pool_attempts_per_second(node.tracker, node.best_share_var.value, min(height - 1, 60*60//net.SHARE_PERIOD)) / (1 - stale_prop)
                        
                        this_str += '\n Shares: %i (%i orphan, %i dead) Stale rate: %s Efficiency: %s Current payout: %.4f %s' % (
                            shares, stale_orphan_shares, stale_doa_shares,
                            math.format_binomial_conf(stale_orphan_shares + stale_doa_shares, shares, 0.95),
                            math.format_binomial_conf(stale_orphan_shares + stale_doa_shares, shares, 0.95, lambda x: (1 - x)/(1 - stale_prop)),
                            node.get_current_txouts().get(bitcoin_data.pubkey_hash_to_script2(my_pubkey_hash), 0)*1e-8, net.PARENT.SYMBOL,
                        )
                        this_str += '\n Pool: %sH/s Stale rate: %.1f%% Expected time to block: %s' % (
                            math.format(int(real_att_s)),
                            100*stale_prop,
                            math.format_dt(2**256 / node.bitcoind_work.value['bits'].target / real_att_s),
                        )
                        
                        for warning in p2pool_data.get_warnings(node.tracker, node.best_share_var.value, net, bitcoind_getinfo_var.value, node.bitcoind_work.value):
                            print >>sys.stderr, '#'*40
                            print >>sys.stderr, '>>> Warning: ' + warning
                            print >>sys.stderr, '#'*40
                        
                        if gc.garbage:
                            print '%i pieces of uncollectable cyclic garbage! Types: %r' % (len(gc.garbage), map(type, gc.garbage))
                    
                    if this_str != last_str or time.time() > last_time + 15:
                        print this_str
                        last_str = this_str
                        last_time = time.time()
                except:
                    log.err()
        status_thread()
    except:
        reactor.stop()
        log.err(None, 'Fatal error:')

Example 8

Project: dx-toolkit
Source File: __init__.py
View license
def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
                  timeout=DEFAULT_TIMEOUT,
                  use_compression=None, jsonify_data=True, want_full_response=False,
                  decode_response_body=True, prepend_srv=True, session_handler=None,
                  max_retries=DEFAULT_RETRIES, always_retry=False,
                  **kwargs):
    '''
    :param resource: API server route, e.g. "/record/new". If *prepend_srv* is False, a fully qualified URL is expected. If this argument is a callable, it will be called just before each request attempt, and expected to return a tuple (URL, headers). Headers returned by the callback are updated with *headers* (including headers set by this method).
    :type resource: string
    :param data: Content of the request body
    :type data: list or dict, if *jsonify_data* is True; or string or file-like object, otherwise
    :param headers: Names and values of HTTP headers to submit with the request (in addition to those needed for authentication, compression, or other options specified with the call).
    :type headers: dict
    :param auth:
        Controls the ``Authentication`` header or other means of authentication supplied with the request. If ``True``
        (default), a token is obtained from the ``DX_SECURITY_CONTEXT``. If the value evaluates to false, no action is
        taken to prepare authentication for the request. Otherwise, the value is assumed to be callable, and called with
        three arguments (method, url, headers) and expected to prepare the authentication headers by reference.
    :type auth: tuple, object, True (default), or None
    :param timeout: HTTP request timeout, in seconds
    :type timeout: float
    :param config: *config* value to pass through to :meth:`requests.request`
    :type config: dict
    :param use_compression: Deprecated
    :type use_compression: string or None
    :param jsonify_data: If True, *data* is converted from a Python list or dict to a JSON string
    :type jsonify_data: boolean
    :param want_full_response: If True, the full :class:`requests.Response` object is returned (otherwise, only the content of the response body is returned)
    :type want_full_response: boolean
    :param decode_response_body: If True (and *want_full_response* is False), the response body is decoded and, if it is a JSON string, deserialized. Otherwise, the response body is uncompressed if transport compression is on, and returned raw.
    :type decode_response_body: boolean
    :param prepend_srv: If True, prepends the API server location to the URL
    :type prepend_srv: boolean
    :param session_handler: Deprecated.
    :param max_retries: Maximum number of retries to perform for a request. A "failed" request is retried if any of the following is true:

                        - A response is received from the server, and the content length received does not match the "Content-Length" header.
                        - A response is received from the server, and the response has an HTTP status code in 5xx range.
                        - A response is received from the server, the "Content-Length" header is not set, and the response JSON cannot be parsed.
                        - No response is received from the server, and either *always_retry* is True or the request *method* is "GET".

    :type max_retries: int
    :param always_retry: If True, indicates that it is safe to retry a request on failure

                        - Note: It is not guaranteed that the request will *always* be retried on failure; rather, this is an indication to the function that it would be safe to do so.

    :type always_retry: boolean
    :returns: Response from API server in the format indicated by *want_full_response* and *decode_response_body*.
    :raises: :exc:`exceptions.DXAPIError` or a subclass if the server returned a non-200 status code; :exc:`requests.exceptions.HTTPError` if an invalid response was received from the server; or :exc:`requests.exceptions.ConnectionError` if a connection cannot be established.

    Wrapper around :meth:`requests.request()` that makes an HTTP
    request, inserting authentication headers and (by default)
    converting *data* to JSON.

    .. note:: Bindings methods that make API calls make the underlying
       HTTP request(s) using :func:`DXHTTPRequest`, and most of them
       will pass any unrecognized keyword arguments you have supplied
       through to :func:`DXHTTPRequest`.

    '''
    if headers is None:
        headers = {}

    global _UPGRADE_NOTIFY

    seq_num = _get_sequence_number()

    url = APISERVER + resource if prepend_srv else resource
    method = method.upper()  # Convert method name to uppercase, to ease string comparisons later

    if auth is True:
        auth = AUTH_HELPER

    if auth:
        auth(_RequestForAuth(method, url, headers))

    pool_args = {arg: kwargs.pop(arg, None) for arg in ("verify", "cert_file", "key_file")}
    test_retry = kwargs.pop("_test_retry_http_request", False)

    if _DEBUG >= 2:
        if isinstance(data, basestring) or isinstance(data, mmap.mmap):
            if len(data) == 0:
                formatted_data = '""'
            else:
                formatted_data = "<file data>"
        else:
            try:
                if _DEBUG >= 3:
                    formatted_data = json.dumps(data, indent=2)
                else:
                    formatted_data = json.dumps(data)
            except (UnicodeDecodeError, TypeError):
                formatted_data = "<binary data>"

    if jsonify_data:
        data = json.dumps(data)
        if 'Content-Type' not in headers and method == 'POST':
            headers['Content-Type'] = 'application/json'

    # If the input is a buffer, its data gets consumed by
    # requests.request (moving the read position). Record the initial
    # buffer position so that we can return to it if the request fails
    # and needs to be retried.
    rewind_input_buffer_offset = None
    if hasattr(data, 'seek') and hasattr(data, 'tell'):
        rewind_input_buffer_offset = data.tell()

    try_index = 0
    retried_responses = []
    while True:
        success, time_started = True, None
        response = None
        req_id = None
        try:
            time_started = time.time()
            _method, _url, _headers = _process_method_url_headers(method, url, headers)

            if _DEBUG >= 2:
                maybe_headers = ''
                if 'Range' in _headers:
                    maybe_headers = " " + json.dumps({"Range": _headers["Range"]})
                print("%s [%f] %s %s%s => %s\n" % (YELLOW(BOLD(">%d" % seq_num)),
                                                   time_started,
                                                   BLUE(method),
                                                   _url,
                                                   maybe_headers,
                                                   formatted_data),
                      file=sys.stderr,
                      end="")
            elif _DEBUG > 0:
                from repr import Repr
                print("%s [%f] %s %s => %s\n" % (YELLOW(BOLD(">%d" % seq_num)),
                                                 time_started,
                                                 BLUE(method),
                                                 _url,
                                                 Repr().repr(data)),
                      file=sys.stderr,
                      end="")

            body = _maybe_trucate_request(_url, try_index, data)

            # throws BadStatusLine if the server returns nothing
            try:
                pool_manager = _get_pool_manager(**pool_args)

                def unicode2str(s):
                    if isinstance(s, unicode):
                        return s.encode('ascii')
                    else:
                        return s

                _headers['User-Agent'] = USER_AGENT
                _headers['DNAnexus-API'] = API_VERSION

                # Converted Unicode headers to ASCII and throw an error if not possible
                _headers = {unicode2str(k): unicode2str(v) for k, v in _headers.items()}
                response = pool_manager.request(_method, _url, headers=_headers, body=body,
                                                timeout=timeout, retries=False, **kwargs)
            except urllib3.exceptions.ClosedPoolError:
                # If another thread closed the pool before the request was
                # started, will throw ClosedPoolError
                raise exceptions.UrllibInternalError("ClosedPoolError")

            _raise_error_for_testing(try_index, method)
            req_id = response.headers.get("x-request-id", "unavailable")

            if _UPGRADE_NOTIFY and response.headers.get('x-upgrade-info', '').startswith('A recommended update is available') and '_ARGCOMPLETE' not in os.environ:
                logger.info(response.headers['x-upgrade-info'])
                try:
                    with file(_UPGRADE_NOTIFY, 'a'):
                        os.utime(_UPGRADE_NOTIFY, None)
                except:
                    pass
                _UPGRADE_NOTIFY = False

            # If an HTTP code that is not in the 200 series is received and the content is JSON, parse it and throw the
            # appropriate error.  Otherwise, raise the usual exception.
            if response.status // 100 != 2:
                # response.headers key lookup is case-insensitive
                if response.headers.get('content-type', '').startswith('application/json'):
                    try:
                        content = response.data.decode('utf-8')
                    except AttributeError:
                        raise exceptions.UrllibInternalError("Content is none", response.status)
                    try:
                        content = json.loads(content)
                    except ValueError:
                        # The JSON is not parsable, but we should be able to retry.
                        raise exceptions.BadJSONInReply("Invalid JSON received from server", response.status)
                    try:
                        error_class = getattr(exceptions, content["error"]["type"], exceptions.DXAPIError)
                    except (KeyError, AttributeError, TypeError):
                        raise exceptions.HTTPError(response.status, content)
                    raise error_class(content, response.status, time_started, req_id)
                else:
                    try:
                        content = response.data.decode('utf-8')
                    except AttributeError:
                        raise exceptions.UrllibInternalError("Content is none", response.status)
                    raise exceptions.HTTPError("{} {} [Time={} RequestID={}]\n{}".format(response.status,
                                                                                         response.reason,
                                                                                         time_started,
                                                                                         req_id,
                                                                                         content))

            if want_full_response:
                return response
            else:
                if 'content-length' in response.headers:
                    if int(response.headers['content-length']) != len(response.data):
                        range_str = (' (%s)' % (headers['Range'],)) if 'Range' in headers else ''
                        raise exceptions.ContentLengthError(
                            "Received response with content-length header set to %s but content length is %d%s. " +
                            "[Time=%f RequestID=%s]" %
                            (response.headers['content-length'], len(response.data), range_str, time_started, req_id)
                        )

                content = response.data

                content_to_print = "(%d bytes)" % len(content) if len(content) > 0 else ''

                if decode_response_body:
                    content = content.decode('utf-8')
                    if response.headers.get('content-type', '').startswith('application/json'):
                        try:
                            content = json.loads(content)
                        except ValueError:
                            # The JSON is not parsable, but we should be able to retry.
                            raise exceptions.BadJSONInReply("Invalid JSON received from server", response.status)
                        if _DEBUG >= 3:
                            content_to_print = "\n  " + json.dumps(content, indent=2).replace("\n", "\n  ")
                        elif _DEBUG == 2:
                            content_to_print = json.dumps(content)
                        elif _DEBUG > 0:
                            content_to_print = Repr().repr(content)

                if _DEBUG > 0:
                    t = int((time.time() - time_started) * 1000)
                    req_id = response.headers.get('x-request-id') or "--"
                    code_format = GREEN if (200 <= response.status < 300) else RED
                    print("  " + YELLOW(BOLD("<%d" % seq_num)),
                          "[%f]" % time_started,
                          BLUE(method),
                          req_id,
                          _url,
                          "<=",
                          code_format(str(response.status)),
                          WHITE(BOLD("(%dms)" % t)),
                          content_to_print,
                          file=sys.stderr)

                if test_retry:
                    retried_responses.append(content)
                    if len(retried_responses) == 1:
                        continue
                    else:
                        _set_retry_response(retried_responses[0])
                        return retried_responses[1]

                return content
            raise AssertionError('Should never reach this line: expected a result to have been returned by now')
        except Exception as e:
            # Avoid reusing connections in the pool, since they may be
            # in an inconsistent state (observed as "ResponseNotReady"
            # errors).
            _get_pool_manager(**pool_args).clear()
            success = False
            exception_msg = _extract_msg_from_last_exception()
            if isinstance(e, _expected_exceptions):
                if response is not None and response.status == 503:
                    seconds_to_wait = _extract_retry_after_timeout(response)
                    logger.warn("%s %s: %s. Request Time=[%f] RequestID=[%s] Waiting %d seconds due to server unavailability...",
                                method, url, exception_msg, time_started, req_id, seconds_to_wait)
                    time.sleep(seconds_to_wait)
                    # Note, we escape the "except" block here without
                    # incrementing try_index because 503 responses with
                    # Retry-After should not count against the number of
                    # permitted retries.
                    continue

                # Total number of allowed tries is the initial try + up to
                # (max_retries) subsequent retries.
                total_allowed_tries = max_retries + 1
                ok_to_retry = False
                is_retryable = always_retry or (method == 'GET') or _is_retryable_exception(e)
                # Because try_index is not incremented until we escape this
                # iteration of the loop, try_index is equal to the number of
                # tries that have failed so far, minus one. Test whether we
                # have exhausted all retries.
                #
                # BadStatusLine ---  server did not return anything
                # BadJSONInReply --- server returned JSON that didn't parse properly
                if try_index + 1 < total_allowed_tries:
                    if response is None or \
                       isinstance(e, (exceptions.ContentLengthError, BadStatusLine, exceptions.BadJSONInReply, \
                                      urllib3.exceptions.ProtocolError, exceptions.UrllibInternalError)):
                        ok_to_retry = is_retryable
                    else:
                        ok_to_retry = 500 <= response.status < 600

                    # The server has closed the connection prematurely
                    if response is not None and \
                       response.status == 400 and is_retryable and method == 'PUT' and \
                       isinstance(e, requests.exceptions.HTTPError):
                        if '<Code>RequestTimeout</Code>' in exception_msg:
                            logger.info("Retrying 400 HTTP error, due to slow data transfer. " +
                                        "Request Time=[%f] RequestID=[%s]", time_started, req_id)
                        else:
                            logger.info("400 HTTP error, of unknown origin, exception_msg=[%s]. " +
                                        "Request Time=[%f] RequestID=[%s]", exception_msg, time_started, req_id)
                        ok_to_retry = True

                if ok_to_retry:
                    if rewind_input_buffer_offset is not None:
                        data.seek(rewind_input_buffer_offset)
                    delay = min(2 ** try_index, DEFAULT_TIMEOUT)
                    range_str = (' (range=%s)' % (headers['Range'],)) if 'Range' in headers else ''
                    logger.warn("[%s] %s %s: %s. Waiting %d seconds before retry %d of %d... %s",
                                time.ctime(), method, url, exception_msg, delay, try_index + 1, max_retries, range_str)
                    time.sleep(delay)
                    try_index += 1
                    continue

            # All retries have been exhausted OR the error is deemed not
            # retryable. Print the latest error and propagate it back to the caller.
            if not isinstance(e, exceptions.DXAPIError):
                logger.error("[%s] %s %s: %s.", time.ctime(), method, url, exception_msg)

            # Retries have been exhausted, and we are unable to get a full
            # buffer from the data source. Raise a special exception.
            if isinstance(e, urllib3.exceptions.ProtocolError) and \
               'Connection broken: IncompleteRead' in exception_msg:
                raise exceptions.DXIncompleteReadsError(exception_msg)
            raise
        finally:
            if success and try_index > 0:
                logger.info("[%s] %s %s: Recovered after %d retries", time.ctime(), method, url, try_index)

        raise AssertionError('Should never reach this line: should have attempted a retry or reraised by now')
    raise AssertionError('Should never reach this line: should never break out of loop')

Example 9

Project: WASP
Source File: combined_test.py
View license
def main():
    options = parse_options()

    if options.pc_file:
        pc_matrix = load_covariates(options.pc_file)
        num_pcs = options.num_pcs
    else:
        pc_matrix = []
        num_pcs = 0

    if options.out_file.endswith(".gz"):
        outfile = gzip.open(options.out_file, "wb")
    else:
        outfile = open(options.out_file, 'w')

    if options.benchmark:
        if options.benchmark == "-":
            bench_file = sys.stderr
        else:
            bench_file = open(options.benchmark, "w")

        bench_file.write("TEST.TYPE TIME\n")

    write_header(outfile)

    # read list of input files (one for each individual)
    infiles = open_input_files(options.infile_list)

    # read dispersion parameters for each individual
    bnb_sigmas = read_bnb_sigmas(options, infiles)
    as_sigmas = read_as_sigmas(options, infiles)


    # add first row of each input file to snpinfo list
    snpinfo = []
    for f in infiles:
        snpinfo.append(f.readline().strip().split())

    row_count = 0
    finished=False

    options.dup_snp_warn = True
    
    while not finished:
        try:
            test_snps = []
            # parse test SNP and associated info from input file row
            for i in range(len(infiles)):
                test_snps.append(parse_test_snp(snpinfo[i], options))

            # rescale totals to put values into reasonable range for
            # alpha and beta parameter estimation
            rescale_totals(test_snps)
            

            # how many allele-specific reads are there across all
            # linked SNPs and and individuals?
            ref_as_counts = sum([np.sum(x.AS_target_ref) for x in test_snps])
            alt_as_counts = sum([np.sum(x.AS_target_alt) for x in test_snps])
            tot_as_counts = ref_as_counts + alt_as_counts
            
            all_counts = sum([test_snps[i].counts for i in range(len(test_snps))])

            if tot_as_counts < options.min_as_counts:
                if options.verbose:
                    sys.stderr.write("-----\nskipping SNP %s because "
                                     "total AS counts %d <= %d\n" %
                                     (test_snps[0].name, tot_as_counts, options.min_as_counts))

                # skip, not enough allele-specific counts
                for i in range(len(infiles)):
                    line = infiles[i].readline().strip()
                    if line:
                        snpinfo[i] = line.split()
                    else:
                        # out of lines from at least one file, assume we are finished
                        finished = True
                continue

            if options.verbose:
                sys.stderr.write("-----\ntesting SNP %s\n" % test_snps[0].name)

            row_count+=1
            old_genos = [test_snps[y].geno_hap1 + test_snps[y].geno_hap2
                         for y in range(len(test_snps))]

            if options.shuffle:
                # permute genotypes
                perm = range(len(test_snps))
                random.shuffle(perm)
                geno1temp = [test_snps[y].geno_hap1 for y in perm]
                geno2temp = [test_snps[y].geno_hap2 for y in perm]
                for i in range(len(test_snps)):
                    test_snps[i].geno_hap1 = geno1temp[i]
                    test_snps[i].geno_hap2 = geno2temp[i]


            if options.benchmark:
                # start timing test for NULL model
                bench_file.write("null %s %s %d %d " % (snpinfo[0][0], snpinfo[0][1],
                                                        tot_as_counts, all_counts))
                bench_file.flush()
            t1 = time.time()

            starting_gene = [np.float64(x) for x in [0.1, 0.001]]
            maxlike = 10000000000

            for start in starting_gene:
                starts = [np.float64(0.5), np.float64(start)]

                # regress against the covariates and get residuals
                #fit_cov(test_snps,cov_table)

                # maximize likelihood with alpha = beta (no difference between genotypes)
                res = minimize(ll_one, starts, args=(test_snps, True, #options.is_bnb_only,
                                                     options.is_as_only,
                                                     bnb_sigmas,
                                                     as_sigmas,
                                                     options.read_error_rate,
                                                     [],
                                                     pc_matrix),
                                options={"maxiter" : 50000, "disp" : options.verbose},
                                method=OPTIMIZER)

                new_par = res.x

                new_loglike = ll_one(new_par, test_snps, options.is_bnb_only,
                                     options.is_as_only, bnb_sigmas,
                                     as_sigmas, options.read_error_rate,
                                     [], pc_matrix)
                if new_loglike < maxlike:
                    starting_par = new_par

            pc_coefs=[]
            for pc in range(num_pcs):
                res = minimize(ll_pc, [np.float64(0)],
                               args=(starting_par, test_snps, True, #options.is_bnb_only,
                                     options.is_as_only, bnb_sigmas, as_sigmas,
                                     options.read_error_rate, pc_coefs, pc_matrix),
                               options={"maxiter" : 50000, "disp" : options.verbose},
                               method=OPTIMIZER)

                new_coef = res.x
                pc_coefs = np.concatenate([pc_coefs, new_coef])

            res = minimize(ll_one, starting_par,
                           args=(test_snps, options.is_bnb_only, options.is_as_only, bnb_sigmas,
                                 as_sigmas, options.read_error_rate, pc_coefs, pc_matrix),
                           options={"maxiter" : 50000, "disp" : options.verbose},
                           method=OPTIMIZER)
            best1par = res.x

            time_taken = time.time() - t1
            if options.verbose:
                sys.stderr.write("null model optimization took %.3fs\n" % time_taken)
            if options.benchmark:
                bench_file.write("%.3f\n" % time_taken)
                bench_file.flush()


            loglike1par = ll_one(best1par, test_snps, options.is_bnb_only,
                                 options.is_as_only, bnb_sigmas,
                                 as_sigmas, options.read_error_rate,
                                 pc_coefs, pc_matrix)


            start = [best1par[0], best1par[0], best1par[1]]

            if options.benchmark:
                # start timing test for ALT model
                bench_file.write("alt %s %s %d %d " % (snpinfo[0][0], snpinfo[0][1],
                                                       tot_as_counts, all_counts))
                bench_file.flush()

            t1 = time.time()

            # maximize likelihood with alpha and beta as separate parameters
            res = minimize(ll_two, start,
                           args=(test_snps, options.is_bnb_only, options.is_as_only,
                                 bnb_sigmas, as_sigmas, options.read_error_rate,
                                 pc_coefs, pc_matrix),
                           options={"maxiter" : 50000, "disp" : options.verbose},
                           method=OPTIMIZER)
            best2par = res.x

            time_taken = time.time() - t1
            if options.verbose:
                sys.stderr.write("alternative model optimization took %.3fs\n" % time_taken)
            if options.benchmark:
                bench_file.write("%.3f\n" % time_taken)
                bench_file.flush()

            loglike2par = ll_two(best2par, test_snps, options.is_bnb_only,
                                 options.is_as_only, bnb_sigmas,
                                 as_sigmas, options.read_error_rate,
                                 pc_coefs, pc_matrix)

            write_results(outfile, snpinfo, loglike1par, loglike2par, best2par,
                          tot_as_counts, ref_as_counts, alt_as_counts, all_counts)

        except Exception as e:
            write_empty_result(outfile, snpinfo)
            # an error occured, write to output file, but put 0s for all params and
            sys.stderr.write("An error occurred, writing line with 0s for SNP:\n%s\n" % str(e))
            # continue
            raise

        # read next set of lines from input file
        for i in range(len(infiles)):
            line = infiles[i].readline().strip()
            if line:
                snpinfo[i] = line.split()
            else:
                # out of lines from at least one file, assume we are finished
                finished = True

Example 10

Project: simpleflow
Source File: command.py
View license
@click.option('--heartbeat',
              type=int,
              required=False,
              default=60,
              help='Heartbeat interval in seconds (0 to disable heartbeating).')
@click.option('--nb-workers',
              type=int,
              required=False,
              help='Number of parallel processes handling activity tasks.')
@click.option('--nb-deciders',
              type=int,
              required=False,
              help='Number of parallel processes handling decision tasks.')
@click.option('--input', '-i',
              required=False,
              help='JSON input of the workflow.')
@click.option('--input-file',
              required=False, type=click.File(),
              help='JSON file with the input of the workflow.')
@click.option('--tags',
              type=comma_separated_list,
              required=False,
              help='Tags identifying the workflow execution.')
@click.option('--decision-tasks-timeout',
              required=False,
              help='Decision tasks timeout.')
@click.option('--execution-timeout',
              required=False,
              help='Timeout for the whole workflow execution.')
@click.option('--workflow-id',
              required=False,
              help='ID of the workflow execution.')
@click.option('--domain', '-d',
              envvar='SWF_DOMAIN',
              required=True,
              help='SWF Domain.')
@click.option('--display-status',
              type=bool, required=False,
              help='Display execution status.'
              )
@click.option('--repair',
              type=str, required=False,
              help='Repair a failed workflow execution ("<workflow id>" or "<workflow id> <run id>").'
              )
@click.option('--force-activities',
              type=str, required=False,
              help='Force the re-execution of some activities in when --repair is enabled.'
              )
@click.argument('workflow')
@cli.command('standalone', help='Execute a workflow with a single process.')
@click.pass_context
def standalone(context,
               workflow,
               domain,
               workflow_id,
               execution_timeout,
               tags,
               decision_tasks_timeout,
               input,
               input_file,
               nb_workers,
               nb_deciders,
               heartbeat,
               display_status,
               repair,
               force_activities,
               ):
    """
    This command spawn a decider and an activity worker to execute a workflow
    with a single main process.

    """
    disable_boto_connection_pooling()

    if force_activities and not repair:
        raise ValueError(
            "You should only use --force-activities with --repair."
        )

    if not workflow_id:
        workflow_id = get_workflow(workflow).name

    wf_input = None
    if input or input_file:
        wf_input = get_or_load_input(input_file, input)

    if repair:
        repair_run_id = None
        if " " in repair:
            repair, repair_run_id = repair.split(" ", 1)
        # get the previous execution history, it will serve as "default history"
        # for activities that succeeded in the previous execution
        logger.info(
            'retrieving history of previous execution: domain={} '
            'workflow_id={} run_id={}'.format(domain, repair, repair_run_id)
        )
        previous_history = get_workflow_history(domain, repair, run_id=repair_run_id)
        previous_history.parse()
        # get the previous execution input if none passed
        if not input and not input_file:
            wf_input = previous_history.events[0].input
    else:
        previous_history = None

    task_list = get_task_list(workflow_id)
    logger.info('using task list {}'.format(task_list))
    decider_proc = multiprocessing.Process(
        target=decider.command.start,
        args=(
            [workflow],
            domain,
            task_list,
        ),
        kwargs={
            'nb_processes': nb_deciders,
            'repair_with': previous_history,
            'force_activities': force_activities,
            'is_standalone': True,
        },
    )
    decider_proc.start()

    worker_proc = multiprocessing.Process(
        target=worker.command.start,
        args=(
            domain,
            task_list,
        ),
        kwargs={
            'nb_processes': nb_workers,
            'heartbeat': heartbeat,
        },
    )
    worker_proc.start()

    print('starting workflow {}'.format(workflow), file=sys.stderr)
    ex = start_workflow.callback(
        workflow,
        domain,
        workflow_id,
        task_list,
        execution_timeout,
        tags,
        decision_tasks_timeout,
        json_dumps(wf_input),
        None,
        local=False,
    )
    while True:
        time.sleep(2)
        ex = helpers.get_workflow_execution(
            domain,
            ex.workflow_id,
            ex.run_id,
        )
        if display_status:
            print('status: {}'.format(ex.status), file=sys.stderr)
        if ex.status == ex.STATUS_CLOSED:
            print('execution {} finished'.format(ex.workflow_id), file=sys.stderr)
            break

    os.kill(worker_proc.pid, signal.SIGTERM)
    worker_proc.join()
    os.kill(decider_proc.pid, signal.SIGTERM)
    decider_proc.join()

Example 11

Project: rlpy
Source File: game.py
View license
    def run(self):
        """
        Main control loop for game play.
        """
        self.display.initialize(self.state.data)
        self.numMoves = 0

        # self.display.initialize(self.state.makeObservation(1).data)
        # inform learning agents of the game start
        for i in range(len(self.agents)):
            agent = self.agents[i]
            if not agent:
                self.mute(i)
                # this is a null agent, meaning it failed to load
                # the other team wins
                print >>sys.stderr, "Agent %d failed to load" % i
                self.unmute()
                self._agentCrash(i, quiet=True)
                return
            if ("registerInitialState" in dir(agent)):
                self.mute(i)
                if self.catchExceptions:
                    try:
                        timed_func = TimeoutFunction(
                            agent.registerInitialState,
                            int(self.rules.getMaxStartupTime(i)))
                        try:
                            start_time = time.time()
                            timed_func(self.state.deepCopy())
                            time_taken = time.time() - start_time
                            self.totalAgentTimes[i] += time_taken
                        except TimeoutFunctionException:
                            print >>sys.stderr, "Agent %d ran out of time on startup!" % i
                            self.unmute()
                            self.agentTimeout = True
                            self._agentCrash(i, quiet=True)
                            return
                    except Exception as data:
                        self._agentCrash(i, quiet=False)
                        self.unmute()
                        return
                else:
                    agent.registerInitialState(self.state.deepCopy())
                # TODO: could this exceed the total time
                self.unmute()

        agentIndex = self.startingIndex
        numAgents = len(self.agents)

        while not self.gameOver:
            # Fetch the next agent
            agent = self.agents[agentIndex]
            move_time = 0
            skip_action = False
            # Generate an observation of the state
            if 'observationFunction' in dir(agent):
                self.mute(agentIndex)
                if self.catchExceptions:
                    try:
                        timed_func = TimeoutFunction(
                            agent.observationFunction,
                            int(self.rules.getMoveTimeout(agentIndex)))
                        try:
                            start_time = time.time()
                            observation = timed_func(self.state.deepCopy())
                        except TimeoutFunctionException:
                            skip_action = True
                        move_time += time.time() - start_time
                        self.unmute()
                    except Exception as data:
                        self._agentCrash(agentIndex, quiet=False)
                        self.unmute()
                        return
                else:
                    observation = agent.observationFunction(
                        self.state.deepCopy())
                self.unmute()
            else:
                observation = self.state.deepCopy()

            # Solicit an action
            action = None
            self.mute(agentIndex)
            if self.catchExceptions:
                try:
                    timed_func = TimeoutFunction(
                        agent.getAction,
                        int(self.rules.getMoveTimeout(agentIndex)) - int(move_time))
                    try:
                        start_time = time.time()
                        if skip_action:
                            raise TimeoutFunctionException()
                        action = timed_func(observation)
                    except TimeoutFunctionException:
                        print >>sys.stderr, "Agent %d timed out on a single move!" % agentIndex
                        self.agentTimeout = True
                        self._agentCrash(agentIndex, quiet=True)
                        self.unmute()
                        return

                    move_time += time.time() - start_time

                    if move_time > self.rules.getMoveWarningTime(agentIndex):
                        self.totalAgentTimeWarnings[agentIndex] += 1
                        print >>sys.stderr, "Agent %d took too long to make a move! This is warning %d" % (
                            agentIndex, self.totalAgentTimeWarnings[agentIndex])
                        if self.totalAgentTimeWarnings[agentIndex] > self.rules.getMaxTimeWarnings(agentIndex):
                            print >>sys.stderr, "Agent %d exceeded the maximum number of warnings: %d" % (
                                agentIndex, self.totalAgentTimeWarnings[agentIndex])
                            self.agentTimeout = True
                            self._agentCrash(agentIndex, quiet=True)
                            self.unmute()
                            return

                    self.totalAgentTimes[agentIndex] += move_time
                    # print "Agent: %d, time: %f, total: %f" % (agentIndex,
                    # move_time, self.totalAgentTimes[agentIndex])
                    if self.totalAgentTimes[agentIndex] > self.rules.getMaxTotalTime(agentIndex):
                        print >>sys.stderr, "Agent %d ran out of time! (time: %1.2f)" % (
                            agentIndex, self.totalAgentTimes[agentIndex])
                        self.agentTimeout = True
                        self._agentCrash(agentIndex, quiet=True)
                        self.unmute()
                        return
                    self.unmute()
                except Exception as data:
                    self._agentCrash(agentIndex)
                    self.unmute()
                    return
            else:
                action = agent.getAction(observation)
            self.unmute()

            # Execute the action
            self.moveHistory.append((agentIndex, action))
            if self.catchExceptions:
                try:
                    self.state = self.state.generateSuccessor(
                        agentIndex,
                        action)
                except Exception as data:
                    self.mute(agentIndex)
                    self._agentCrash(agentIndex)
                    self.unmute()
                    return
            else:
                self.state = self.state.generateSuccessor(agentIndex, action)

            # Change the display
            self.display.update(self.state.data)
            ###idx = agentIndex - agentIndex % 2 + 1
            ###self.display.update( self.state.makeObservation(idx).data )

            # Allow for game specific conditions (winning, losing, etc.)
            self.rules.process(self.state, self)
            # Track progress
            if agentIndex == numAgents + 1:
                self.numMoves += 1
            # Next agent
            agentIndex = (agentIndex + 1) % numAgents

            if _BOINC_ENABLED:
                boinc.set_fraction_done(self.getProgress())

        # inform a learning agent of the game result
        for agentIndex, agent in enumerate(self.agents):
            if "final" in dir(agent):
                try:
                    self.mute(agentIndex)
                    agent.final(self.state)
                    self.unmute()
                except Exception as data:
                    if not self.catchExceptions:
                        raise
                    self._agentCrash(agentIndex)
                    self.unmute()
                    return
        self.display.finish()

Example 12

Project: ros_buildfarm
Source File: release_job.py
View license
def configure_release_jobs(
        config_url, rosdistro_name, release_build_name, groovy_script=None,
        dry_run=False, whitelist_package_names=None):
    """
    Configure all Jenkins release jobs.

    L{configure_release_job} will be invoked for every released package and
    target which matches the build file criteria.

    Additionally a job to import Debian packages into the Debian repository is
    created.
    """
    config = get_config_index(config_url)
    build_files = get_release_build_files(config, rosdistro_name)
    build_file = build_files[release_build_name]

    index = get_index(config.rosdistro_index_url)

    # get targets
    platforms = []
    for os_name in build_file.targets.keys():
        for os_code_name in build_file.targets[os_name].keys():
            platforms.append((os_name, os_code_name))
    print('The build file contains the following targets:')
    for os_name, os_code_name in platforms:
        print('  - %s %s: %s' % (os_name, os_code_name, ', '.join(
            build_file.targets[os_name][os_code_name])))

    dist_file = get_distribution_file(index, rosdistro_name, build_file)
    if not dist_file:
        print('No distribution file matches the build file')
        return

    pkg_names = dist_file.release_packages.keys()
    filtered_pkg_names = build_file.filter_packages(pkg_names)
    explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names)
    if explicitly_ignored_pkg_names:
        print(('The following packages are being %s because of ' +
               'white-/blacklisting:') %
              ('ignored' if build_file.skip_ignored_packages else 'disabled'))
        for pkg_name in sorted(explicitly_ignored_pkg_names):
            print('  -', pkg_name)

    dist_cache = get_distribution_cache(index, rosdistro_name)

    if explicitly_ignored_pkg_names:
        # get direct dependencies from distro cache for each package
        direct_dependencies = {}
        for pkg_name in pkg_names:
            direct_dependencies[pkg_name] = _get_direct_dependencies(
                pkg_name, dist_cache, pkg_names) or set([])

        # find recursive downstream deps for all explicitly ignored packages
        ignored_pkg_names = set(explicitly_ignored_pkg_names)
        while True:
            implicitly_ignored_pkg_names = _get_downstream_package_names(
                ignored_pkg_names, direct_dependencies)
            if implicitly_ignored_pkg_names - ignored_pkg_names:
                ignored_pkg_names |= implicitly_ignored_pkg_names
                continue
            break
        implicitly_ignored_pkg_names = \
            ignored_pkg_names - explicitly_ignored_pkg_names

        if implicitly_ignored_pkg_names:
            print(('The following packages are being %s because their ' +
                   'dependencies are being ignored:') % ('ignored'
                  if build_file.skip_ignored_packages else 'disabled'))
            for pkg_name in sorted(implicitly_ignored_pkg_names):
                print('  -', pkg_name)
            filtered_pkg_names = \
                set(filtered_pkg_names) - implicitly_ignored_pkg_names

    # all further configuration will be handled by either the Jenkins API
    # or by a generated groovy script
    jenkins = connect(config.jenkins_url) if groovy_script is None else False

    all_view_configs = {}
    all_job_configs = OrderedDict()

    job_name, job_config = configure_import_package_job(
        config_url, rosdistro_name, release_build_name,
        config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
    if not jenkins:
        all_job_configs[job_name] = job_config

    job_name, job_config = configure_sync_packages_to_main_job(
        config_url, rosdistro_name, release_build_name,
        config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
    if not jenkins:
        all_job_configs[job_name] = job_config

    for os_name, os_code_name in platforms:
        for arch in sorted(build_file.targets[os_name][os_code_name]):
            job_name, job_config = configure_sync_packages_to_testing_job(
                config_url, rosdistro_name, release_build_name,
                os_code_name, arch,
                config=config, build_file=build_file, jenkins=jenkins,
                dry_run=dry_run)
            if not jenkins:
                all_job_configs[job_name] = job_config

    targets = []
    for os_name, os_code_name in platforms:
        targets.append((os_name, os_code_name, 'source'))
        for arch in build_file.targets[os_name][os_code_name]:
            targets.append((os_name, os_code_name, arch))
    views = configure_release_views(
        jenkins, rosdistro_name, release_build_name, targets,
        dry_run=dry_run)
    if not jenkins:
        all_view_configs.update(views)
    groovy_data = {
        'dry_run': dry_run,
        'expected_num_views': len(views),
    }

    # binary jobs must be generated in topological order
    from catkin_pkg.package import parse_package_string
    from ros_buildfarm.common import topological_order_packages
    pkgs = {}
    for pkg_name in pkg_names:
        if pkg_name not in dist_cache.release_package_xmls:
            print("Skipping package '%s': no released package.xml in cache" %
                  (pkg_name), file=sys.stderr)
            continue
        pkg_xml = dist_cache.release_package_xmls[pkg_name]
        pkg = parse_package_string(pkg_xml)
        pkgs[pkg_name] = pkg
    ordered_pkg_tuples = topological_order_packages(pkgs)

    other_build_files = [v for k, v in build_files.items() if k != release_build_name]

    all_source_job_names = []
    all_binary_job_names = []
    for pkg_name in [p.name for _, p in ordered_pkg_tuples]:
        if whitelist_package_names:
            if pkg_name not in whitelist_package_names:
                print("Skipping package '%s' not in the explicitly passed list" %
                      pkg_name, file=sys.stderr)
                continue

        pkg = dist_file.release_packages[pkg_name]
        repo_name = pkg.repository_name
        repo = dist_file.repositories[repo_name]
        is_disabled = pkg_name not in filtered_pkg_names
        if is_disabled and build_file.skip_ignored_packages:
            print("Skipping ignored package '%s' in repository '%s'" %
                  (pkg_name, repo_name), file=sys.stderr)
            continue
        if not repo.release_repository:
            print(("Skipping package '%s' in repository '%s': no release " +
                   "section") % (pkg_name, repo_name), file=sys.stderr)
            continue
        if not repo.release_repository.version:
            print(("Skipping package '%s' in repository '%s': no release " +
                   "version") % (pkg_name, repo_name), file=sys.stderr)
            continue

        for os_name, os_code_name in platforms:
            other_build_files_same_platform = []
            for other_build_file in other_build_files:
                if os_name not in other_build_file.targets:
                    continue
                if os_code_name not in other_build_file.targets[os_name]:
                    continue
                other_build_files_same_platform.append(other_build_file)

            try:
                source_job_names, binary_job_names, job_configs = \
                    configure_release_job(
                        config_url, rosdistro_name, release_build_name,
                        pkg_name, os_name, os_code_name,
                        config=config, build_file=build_file,
                        index=index, dist_file=dist_file,
                        dist_cache=dist_cache,
                        jenkins=jenkins, views=views,
                        generate_import_package_job=False,
                        generate_sync_packages_jobs=False,
                        is_disabled=is_disabled,
                        other_build_files_same_platform=other_build_files_same_platform,
                        groovy_script=groovy_script,
                        dry_run=dry_run)
                all_source_job_names += source_job_names
                all_binary_job_names += binary_job_names
                if groovy_script is not None:
                    print('Configuration for jobs: ' +
                          ', '.join(source_job_names + binary_job_names))
                    for source_job_name in source_job_names:
                        all_job_configs[source_job_name] = job_configs[source_job_name]
                    for binary_job_name in binary_job_names:
                        all_job_configs[binary_job_name] = job_configs[binary_job_name]
            except JobValidationError as e:
                print(e.message, file=sys.stderr)

    groovy_data['expected_num_jobs'] = len(all_job_configs)
    groovy_data['job_prefixes_and_names'] = {}

    # with an explicit list of packages we don't delete obsolete jobs
    if not whitelist_package_names:
        # delete obsolete binary jobs
        for os_name, os_code_name in platforms:
            for arch in build_file.targets[os_name][os_code_name]:
                binary_view = get_release_binary_view_name(
                    rosdistro_name, release_build_name,
                    os_name, os_code_name, arch)
                binary_job_prefix = '%s__' % binary_view

                excluded_job_names = set([
                    j for j in all_binary_job_names
                    if j.startswith(binary_job_prefix)])
                if groovy_script is None:
                    print("Removing obsolete binary jobs with prefix '%s'" %
                          binary_job_prefix)
                    remove_jobs(
                        jenkins, binary_job_prefix, excluded_job_names,
                        dry_run=dry_run)
                else:
                    binary_key = 'binary_%s_%s_%s' % \
                        (os_name, os_code_name, arch)
                    groovy_data['job_prefixes_and_names'][binary_key] = \
                        (binary_job_prefix, excluded_job_names)

        # delete obsolete source jobs
        # requires knowledge about all other release build files
        for os_name, os_code_name in platforms:
            other_source_job_names = []
            # get source job names for all other release build files
            for other_release_build_name in [
                    k for k in build_files.keys() if k != release_build_name]:
                other_build_file = build_files[other_release_build_name]
                other_dist_file = get_distribution_file(
                    index, rosdistro_name, other_build_file)
                if not other_dist_file:
                    continue

                if os_name not in other_build_file.targets or \
                        os_code_name not in other_build_file.targets[os_name]:
                    continue

                if other_build_file.skip_ignored_packages:
                    filtered_pkg_names = other_build_file.filter_packages(
                        pkg_names)
                else:
                    filtered_pkg_names = pkg_names
                for pkg_name in sorted(filtered_pkg_names):
                    pkg = other_dist_file.release_packages[pkg_name]
                    repo_name = pkg.repository_name
                    repo = other_dist_file.repositories[repo_name]
                    if not repo.release_repository:
                        continue
                    if not repo.release_repository.version:
                        continue

                    other_job_name = get_sourcedeb_job_name(
                        rosdistro_name, other_release_build_name,
                        pkg_name, os_name, os_code_name)
                    other_source_job_names.append(other_job_name)

            source_view_prefix = get_release_source_view_name(
                rosdistro_name, os_name, os_code_name)
            source_job_prefix = '%s__' % source_view_prefix
            excluded_job_names = set([
                j for j in (all_source_job_names + other_source_job_names)
                if j.startswith(source_job_prefix)])
            if groovy_script is None:
                print("Removing obsolete source jobs with prefix '%s'" %
                      source_job_prefix)
                remove_jobs(
                    jenkins, source_job_prefix, excluded_job_names,
                    dry_run=dry_run)
            else:
                source_key = 'source_%s_%s' % (os_name, os_code_name)
                groovy_data['job_prefixes_and_names'][source_key] = (
                    source_job_prefix, excluded_job_names)

    if groovy_script is not None:
        print(
            "Writing groovy script '%s' to reconfigure %d views and %d jobs" %
            (groovy_script, len(all_view_configs), len(all_job_configs)))
        content = expand_template(
            'snippet/reconfigure_jobs.groovy.em', groovy_data)
        write_groovy_script_and_configs(
            groovy_script, content, all_job_configs,
            view_configs=all_view_configs)

Example 13

Project: ns-3
Source File: ns3modulescan-modular.py
View license
    def __call__(self, module_parser,
                 pygccxml_definition,
                 global_annotations,
                 parameter_annotations):
        try:
            ns3_header = get_ns3_relative_path(pygccxml_definition.location.file_name)
        except ValueError: # the header is not from ns3
            return # ignore the definition, it's not ns-3 def.

        definition_module = self.headers_map[ns3_header]

        ## Note: we don't include line numbers in the comments because
        ## those numbers are very likely to change frequently, which would
        ## cause needless changes, since the generated python files are
        ## kept under version control.

        #global_annotations['pygen_comment'] = "%s:%i: %s" % \
        #    (ns3_header, pygccxml_definition.location.line, pygccxml_definition)
        global_annotations['pygen_comment'] = "%s (module %r): %s" % \
            (ns3_header, definition_module, pygccxml_definition)


        ## handle ns3::Object::GetObject (left to its own devices,
        ## pybindgen will generate a mangled name containing the template
        ## argument type name).
        if isinstance(pygccxml_definition, member_function_t) \
                and pygccxml_definition.parent.name == 'Object' \
                and pygccxml_definition.name == 'GetObject':
            template_args = templates.args(pygccxml_definition.demangled_name)
            if template_args == ['ns3::Object']:
                global_annotations['template_instance_names'] = 'ns3::Object=>GetObject'

        ## Don't wrap Simulator::Schedule* (manually wrapped)
        if isinstance(pygccxml_definition, member_function_t) \
                and pygccxml_definition.parent.name == 'Simulator' \
                and pygccxml_definition.name.startswith('Schedule'):
            global_annotations['ignore'] = None

        # manually wrapped
        if isinstance(pygccxml_definition, member_function_t) \
                and pygccxml_definition.parent.name == 'Simulator' \
                and pygccxml_definition.name == 'Run':
            global_annotations['ignore'] = True

        ## http://www.gccxml.org/Bug/view.php?id=9915
        if isinstance(pygccxml_definition, calldef_t):
            for arg in pygccxml_definition.arguments:
                if arg.default_value is None:
                    continue
                elif arg.default_value == "ns3::MilliSeconds( )":
                    arg.default_value = "ns3::MilliSeconds(0)"
                elif arg.default_value == "ns3::Seconds( )":
                    arg.default_value = "ns3::Seconds(0)"

        ## classes
        if isinstance(pygccxml_definition, class_t):
            print >> sys.stderr, pygccxml_definition
            # no need for helper classes to allow subclassing in Python, I think...
            #if pygccxml_definition.name.endswith('Helper'):
            #    global_annotations['allow_subclassing'] = 'false'

            #
            # If a class is template instantiation, even if the
            # template was defined in some other module, if a template
            # argument belongs to this module then the template
            # instantiation will belong to this module.
            # 
            if templates.is_instantiation(pygccxml_definition.decl_string):
                cls_name, template_parameters = templates.split(pygccxml_definition.name)
                template_parameters_decls = [find_declaration_from_name(module_parser.global_ns, templ_param)
                                             for templ_param in template_parameters]
                #print >> sys.stderr, "********************", cls_name, repr(template_parameters_decls)
                
                template_parameters_modules = []
                for templ in template_parameters_decls:
                    if not hasattr(templ, 'location'):
                        continue
                    try:
                        h = get_ns3_relative_path(templ.location.file_name)
                    except ValueError:
                        continue
                    template_parameters_modules.append(self.headers_map[h])

                for templ_mod in template_parameters_modules:
                    if templ_mod == self.module:
                        definition_module = templ_mod
                        break
                #print >> sys.stderr, "********************", cls_name, repr(template_parameters_modules)


            if definition_module != self.module:
                global_annotations['import_from_module'] = 'ns.%s' % (definition_module.replace('-', '_'),)

            if pygccxml_definition.decl_string.startswith('::ns3::SimpleRefCount<'):
                global_annotations['incref_method'] = 'Ref'
                global_annotations['decref_method'] = 'Unref'
                global_annotations['peekref_method'] = 'GetReferenceCount'
                global_annotations['automatic_type_narrowing'] = 'true'
                return

            if pygccxml_definition.decl_string.startswith('::ns3::Callback<'):
                # manually handled in ns3modulegen_core_customizations.py
                global_annotations['ignore'] = None
                return

            if pygccxml_definition.decl_string.startswith('::ns3::TracedCallback<'):
                global_annotations['ignore'] = None
                return

            if pygccxml_definition.decl_string.startswith('::ns3::Ptr<'):
                # handled by pybindgen "type transformation"
                global_annotations['ignore'] = None
                return

            # table driven class customization
            try:
                annotations = type_annotations[pygccxml_definition.decl_string]
            except KeyError:
                pass
            else:
                global_annotations.update(annotations)

        ## enums
        if isinstance(pygccxml_definition, enumeration_t):
            if definition_module != self.module:
                global_annotations['import_from_module'] = 'ns.%s' % definition_module

        ## free functions
        if isinstance(pygccxml_definition, free_function_t):

            if definition_module != self.module:
                global_annotations['ignore'] = None
                return

            if pygccxml_definition.name == 'PeekPointer':
                global_annotations['ignore'] = None
                return

        ## table driven methods/constructors/functions customization
        if isinstance(pygccxml_definition, (free_function_t, member_function_t, constructor_t)):
            try:
                annotations = type_annotations[str(pygccxml_definition)]
            except KeyError:
                pass
            else:
                for key,value in annotations.items():
                    if key == 'params':
                        parameter_annotations.update (value)
                        del annotations['params']
                global_annotations.update(annotations)

Example 14

Project: ganeti
Source File: daemon.py
View license
def GenericMain(daemon_name, optionparser,
                check_fn, prepare_fn, exec_fn,
                multithreaded=False, console_logging=False,
                default_ssl_cert=None, default_ssl_key=None,
                warn_breach=False):
  """Shared main function for daemons.

  @type daemon_name: string
  @param daemon_name: daemon name
  @type optionparser: optparse.OptionParser
  @param optionparser: initialized optionparser with daemon-specific options
                       (common -f -d options will be handled by this module)
  @type check_fn: function which accepts (options, args)
  @param check_fn: function that checks start conditions and exits if they're
                   not met
  @type prepare_fn: function which accepts (options, args)
  @param prepare_fn: function that is run before forking, or None;
      it's result will be passed as the third parameter to exec_fn, or
      if None was passed in, we will just pass None to exec_fn
  @type exec_fn: function which accepts (options, args, prepare_results)
  @param exec_fn: function that's executed with the daemon's pid file held, and
                  runs the daemon itself.
  @type multithreaded: bool
  @param multithreaded: Whether the daemon uses threads
  @type console_logging: boolean
  @param console_logging: if True, the daemon will fall back to the system
                          console if logging fails
  @type default_ssl_cert: string
  @param default_ssl_cert: Default SSL certificate path
  @type default_ssl_key: string
  @param default_ssl_key: Default SSL key path
  @type warn_breach: bool
  @param warn_breach: issue a warning at daemon launch time, before
      daemonizing, about the possibility of breaking parameter privacy
      invariants through the otherwise helpful debug logging.

  """
  optionparser.add_option("-f", "--foreground", dest="fork",
                          help="Don't detach from the current terminal",
                          default=True, action="store_false")
  optionparser.add_option("-d", "--debug", dest="debug",
                          help="Enable some debug messages",
                          default=False, action="store_true")
  optionparser.add_option("--syslog", dest="syslog",
                          help="Enable logging to syslog (except debug"
                          " messages); one of 'no', 'yes' or 'only' [%s]" %
                          constants.SYSLOG_USAGE,
                          default=constants.SYSLOG_USAGE,
                          choices=["no", "yes", "only"])

  family = ssconf.SimpleStore().GetPrimaryIPFamily()
  # family will default to AF_INET if there is no ssconf file (e.g. when
  # upgrading a cluster from 2.2 -> 2.3. This is intended, as Ganeti clusters
  # <= 2.2 can not be AF_INET6
  if daemon_name in constants.DAEMONS_PORTS:
    default_bind_address = constants.IP4_ADDRESS_ANY
    if family == netutils.IP6Address.family:
      default_bind_address = constants.IP6_ADDRESS_ANY

    default_port = netutils.GetDaemonPort(daemon_name)

    # For networked daemons we allow choosing the port and bind address
    optionparser.add_option("-p", "--port", dest="port",
                            help="Network port (default: %s)" % default_port,
                            default=default_port, type="int")
    optionparser.add_option("-b", "--bind", dest="bind_address",
                            help=("Bind address (default: '%s')" %
                                  default_bind_address),
                            default=default_bind_address, metavar="ADDRESS")
    optionparser.add_option("-i", "--interface", dest="bind_interface",
                            help=("Bind interface"), metavar="INTERFACE")

  if default_ssl_key is not None and default_ssl_cert is not None:
    optionparser.add_option("--no-ssl", dest="ssl",
                            help="Do not secure HTTP protocol with SSL",
                            default=True, action="store_false")
    optionparser.add_option("-K", "--ssl-key", dest="ssl_key",
                            help=("SSL key path (default: %s)" %
                                  default_ssl_key),
                            default=default_ssl_key, type="string",
                            metavar="SSL_KEY_PATH")
    optionparser.add_option("-C", "--ssl-cert", dest="ssl_cert",
                            help=("SSL certificate path (default: %s)" %
                                  default_ssl_cert),
                            default=default_ssl_cert, type="string",
                            metavar="SSL_CERT_PATH")

  # Disable the use of fork(2) if the daemon uses threads
  if multithreaded:
    utils.DisableFork()

  options, args = optionparser.parse_args()

  if getattr(options, "bind_interface", None) is not None:
    if options.bind_address != default_bind_address:
      msg = ("Can't specify both, bind address (%s) and bind interface (%s)" %
             (options.bind_address, options.bind_interface))
      print >> sys.stderr, msg
      sys.exit(constants.EXIT_FAILURE)
    interface_ip_addresses = \
      netutils.GetInterfaceIpAddresses(options.bind_interface)
    if family == netutils.IP6Address.family:
      if_addresses = interface_ip_addresses[constants.IP6_VERSION]
    else:
      if_addresses = interface_ip_addresses[constants.IP4_VERSION]
    if len(if_addresses) < 1:
      msg = "Failed to find IP for interface %s" % options.bind_interace
      print >> sys.stderr, msg
      sys.exit(constants.EXIT_FAILURE)
    options.bind_address = if_addresses[0]

  if getattr(options, "ssl", False):
    ssl_paths = {
      "certificate": options.ssl_cert,
      "key": options.ssl_key,
      }

    for name, path in ssl_paths.iteritems():
      if not os.path.isfile(path):
        print >> sys.stderr, "SSL %s file '%s' was not found" % (name, path)
        sys.exit(constants.EXIT_FAILURE)

    # TODO: By initiating http.HttpSslParams here we would only read the files
    # once and have a proper validation (isfile returns False on directories)
    # at the same time.

  result, running_uid, expected_uid = _VerifyDaemonUser(daemon_name)
  if not result:
    msg = ("%s started using wrong user ID (%d), expected %d" %
           (daemon_name, running_uid, expected_uid))
    print >> sys.stderr, msg
    sys.exit(constants.EXIT_FAILURE)

  if check_fn is not None:
    check_fn(options, args)

  log_filename = constants.DAEMONS_LOGFILES[daemon_name]

  # node-daemon logging in lib/http/server.py, _HandleServerRequestInner
  if options.debug and warn_breach:
    sys.stderr.write(constants.DEBUG_MODE_CONFIDENTIALITY_WARNING % daemon_name)

  if options.fork:
    # Newer GnuTLS versions (>= 3.3.0) use a library constructor for
    # initialization and open /dev/urandom on library load time, way before we
    # fork(). Closing /dev/urandom causes subsequent ganeti.http.client
    # requests to fail and the process to receive a SIGABRT. As we cannot
    # reliably detect GnuTLS's socket, we work our way around this by keeping
    # all fds referring to /dev/urandom open.
    noclose_fds = []
    for fd in os.listdir("/proc/self/fd"):
      try:
        if os.readlink(os.path.join("/proc/self/fd", fd)) == "/dev/urandom":
          noclose_fds.append(int(fd))
      except EnvironmentError:
        # The fd might have disappeared (although it shouldn't as we're running
        # single-threaded).
        continue

    utils.CloseFDs(noclose_fds=noclose_fds)
    (wpipe, stdio_reopen_fn) = utils.Daemonize(logfile=log_filename)
  else:
    (wpipe, stdio_reopen_fn) = (None, None)

  log_reopen_fn = \
    utils.SetupLogging(log_filename, daemon_name,
                       debug=options.debug,
                       stderr_logging=not options.fork,
                       multithreaded=multithreaded,
                       syslog=options.syslog,
                       console_logging=console_logging)

  # Reopen log file(s) on SIGHUP
  signal.signal(signal.SIGHUP,
                compat.partial(_HandleSigHup, [log_reopen_fn, stdio_reopen_fn]))

  try:
    utils.WritePidFile(utils.DaemonPidFileName(daemon_name))
  except errors.PidFileLockError, err:
    print >> sys.stderr, "Error while locking PID file:\n%s" % err
    sys.exit(constants.EXIT_FAILURE)

  try:
    try:
      logging.info("%s daemon startup", daemon_name)
      if callable(prepare_fn):
        prep_results = prepare_fn(options, args)
      else:
        prep_results = None
    except Exception, err:
      utils.WriteErrorToFD(wpipe, _BeautifyError(err))
      raise

    if wpipe is not None:
      # we're done with the preparation phase, we close the pipe to
      # let the parent know it's safe to exit
      os.close(wpipe)

    exec_fn(options, args, prep_results)
  finally:
    utils.RemoveFile(utils.DaemonPidFileName(daemon_name))

Example 15

Project: cloud-scheduler
Source File: config.py
View license
def setup(path=None):
    """Setup cloudscheduler using config file.
       setup will look for a configuration file specified on the command line,
       or in ~/.cloudscheduler.conf or /etc/cloudscheduler.conf
    """

    global condor_webservice_url
    global condor_collector_url
    global condor_retrieval_method
    global condor_q_command
    global condor_status_command
    global condor_status_master_command
    global condor_hold_command
    global condor_release_command
    global condor_off_command
    global condor_on_command
    global condor_advertise_command
    global ssh_path
    global openssl_path
    global condor_context_file
    global condor_host
    global condor_host_on_vm
    global vm_lifetime
    global cert_file
    global key_file
    global cert_file_on_vm
    global key_file_on_vm
    global ca_root_certs
    global ca_signing_policies
    global cloudscheduler_ssh_key
    global cloud_resource_config
    global image_attach_device
    global scratch_attach_device
    global info_server_port
    global admin_server_port
    global workspace_path
    global persistence_file
    global user_limit_file
    global target_cloud_alias_file
    global job_ban_timeout
    global ban_tracking
    global ban_file
    global ban_min_track
    global ban_failrate_threshold
    global polling_error_threshold
    global condor_register_time_limit
    global graceful_shutdown
    global graceful_shutdown_method
    global retire_before_lifetime
    global retire_before_lifetime_factor
    global cleanup_missing_vms
    global clean_shutdown_idle
    global getclouds
    global scheduling_metric
    global scheduling_algorithm
    global job_distribution_type
    global high_priority_job_support
    global high_priority_job_weight
    global cpu_distribution_weight
    global memory_distribution_weight
    global storage_distribution_weight
    global cleanup_interval
    global vm_poller_interval
    global job_poller_interval
    global machine_poller_interval
    global scheduler_interval
    global job_proxy_refresher_interval
    global job_proxy_renewal_threshold
    global vm_proxy_refresher_interval
    global vm_proxy_renewal_threshold
    global vm_proxy_shutdown_threshold
    global vm_connection_fail_threshold
    global vm_start_running_timeout
    global vm_idle_threshold
    global max_starting_vm
    global max_keepalive
    global proxy_cache_dir
    global myproxy_logon_command
    global override_vmtype
    global vm_reqs_from_condor_reqs
    global adjust_insufficient_resources
    global use_cloud_init
    global default_yaml
    global validate_yaml
    global retire_reallocate

    global default_VMType
    global default_VMNetwork
    global default_VMCPUArch
    global default_VMHypervisor
    global default_VMName
    global default_VMLoc
    global default_VMAMI
    global default_VMMem
    global default_VMCPUCores
    global default_VMStorage
    global default_VMInstanceType
    global default_VMInstanceTypeList
    global default_VMMaximumPrice
    global default_VMProxyNonBoot
    global default_VMUserData
    global default_TargetClouds
    global default_VMAMIConfig
    global default_VMInjectCA
    global default_VMJobPerCore

    global log_level
    global log_location
    global log_location_cloud_admin
    global admin_log_comments
    global log_stdout
    global log_syslog
    global log_max_size
    global log_format

    global use_pyopenssl

    homedir = os.path.expanduser('~')

    # Find config file
    if not path:
        if os.path.exists(homedir + "/.cloudscheduler/cloud_scheduler.conf"):
            path = homedir + "/.cloudscheduler/cloud_scheduler.conf"
        elif os.path.exists("/etc/cloudscheduler/cloud_scheduler.conf"):
            path = "/etc/cloudscheduler/cloud_scheduler.conf"
        elif os.path.exists("/usr/local/share/cloud-scheduler/cloud_scheduler.conf"):
            path = "/usr/local/share/cloud-scheduler/cloud_scheduler.conf"
        else:
            print >> sys.stderr, "Configuration file problem: There doesn't " \
                  "seem to be a configuration file. " \
                  "You can specify one with the --config-file parameter, " \
                  "or put one in ~/.cloudscheduler/cloud_scheduler.conf or "\
                  "/etc/cloudscheduler/cloud_scheduler.conf"
            sys.exit(1)

    # Read config file
    config_file = ConfigParser.ConfigParser()
    try:
        config_file.read(path)
    except IOError:
        print >> sys.stderr, "Configuration file problem: There was a " \
              "problem reading %s. Check that it is readable," \
              "and that it exists. " % path
        raise
    except ConfigParser.ParsingError:
        print >> sys.stderr, "Configuration file problem: Couldn't " \
              "parse your file. Check for spaces before or after variables."
        raise
    except:
        print "Configuration file problem: There is something wrong with " \
              "your config file."
        raise

    if config_file.has_option("global", "condor_retrieval_method"):
        condor_retrieval_method = config_file.get("global",
                                                "condor_retrieval_method")

    if config_file.has_option("global", "condor_q_command"):
        condor_q_command = config_file.get("global",
                                                "condor_q_command")

    if config_file.has_option("global", "condor_off_command"):
        condor_off_command = config_file.get("global",
                                                "condor_off_command")

    if config_file.has_option("global", "condor_on_command"):
        condor_on_command = config_file.get("global",
                                                "condor_on_command")

    if config_file.has_option("global", "ssh_path"):
        ssh_path = config_file.get("global", "ssh_path")

    if config_file.has_option("global", "openssl_path"):
        openssl_path = config_file.get("global", "openssl_path")

    if config_file.has_option("global", "condor_status_command"):
        condor_status_command = config_file.get("global",
                                                "condor_status_command")

    if config_file.has_option("global", "condor_status_master_command"):
        condor_status_master_command = config_file.get("global",
                                                "condor_status_master_command")

    if config_file.has_option("global", "condor_hold_command"):
        condor_hold_command = config_file.get("global",
                                                "condor_hold_command")

    if config_file.has_option("global", "condor_release_command"):
        condor_release_command = config_file.get("global",
                                                "condor_release_command")

    if config_file.has_option("global", "condor_advertise_command"):
        condor_advertise_command = config_file.get("global",
                                                "condor_advertise_command")
    if config_file.has_option("global", "condor_webservice_url"):
        condor_webservice_url = config_file.get("global",
                                                "condor_webservice_url")

    if config_file.has_option("global", "condor_collector_url"):
        condor_collector_url = config_file.get("global",
                                                "condor_collector_url")

    if config_file.has_option("global", "condor_host_on_vm"):
        condor_host_on_vm = config_file.get("global",
                                                "condor_host_on_vm")

    if config_file.has_option("global", "condor_context_file"):
        condor_context_file = config_file.get("global",
                                                "condor_context_file")

    if config_file.has_option("global", "vm_lifetime"):
        try:
            vm_lifetime = config_file.getint("global", "vm_lifetime")
        except ValueError:
            print "Configuration file problem: vm_lifetime must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "cert_file"):
        cert_file = config_file.get("global", "cert_file")

    if config_file.has_option("global", "key_file"):
        key_file = config_file.get("global", "key_file")

    if config_file.has_option("global", "cert_file_on_vm"):
        cert_file_on_vm = config_file.get("global", "cert_file_on_vm")

    if config_file.has_option("global", "key_file_on_vm"):
        key_file_on_vm = config_file.get("global", "key_file_on_vm")

    if config_file.has_option("global", "ca_root_certs"):
        ca_root_certs = config_file.get("global", "ca_root_certs").split(',')

    if config_file.has_option("global", "ca_signing_policies"):
        ca_signing_policies = config_file.get("global", "ca_signing_policies").split(',')

    if config_file.has_option("global", "cloudscheduler_ssh_key"):
        cloudscheduler_ssh_key = config_file.get("global", "cloudscheduler_ssh_key")

    if config_file.has_option("global", "cloud_resource_config"):
        cloud_resource_config = config_file.get("global",
                                                "cloud_resource_config")

    if config_file.has_option("global", "image_attach_device"):
        image_attach_device = config_file.get("global",
                                                "image_attach_device")

    if config_file.has_option("global", "scratch_attach_device"):
        scratch_attach_device = config_file.get("global",
                                                "scratch_attach_device")

    if config_file.has_option("global", "info_server_port"):
        try:
            info_server_port = config_file.getint("global", "info_server_port")
        except ValueError:
            print "Configuration file problem: info_server_port must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "admin_server_port"):
        try:
            info_server_port = config_file.getint("global", "admin_server_port")
        except ValueError:
            print "Configuration file problem: admin_server_port must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "workspace_path"):
        workspace_path = config_file.get("global", "workspace_path")

    if config_file.has_option("global", "persistence_file"):
        persistence_file = config_file.get("global", "persistence_file")

    if config_file.has_option("global", "user_limit_file"):
        user_limit_file = config_file.get("global", "user_limit_file")

    if config_file.has_option("global", "target_cloud_alias_file"):
        target_cloud_alias_file = config_file.get("global", "target_cloud_alias_file")

    if config_file.has_option("global", "job_ban_timeout"):
        try:
            job_ban_timeout = 60 * config_file.getint("global", "job_ban_timeout")
        except ValueError:
            print "Configuration file problem: job_ban_timeout must be an " \
                  "integer value in minutes."
            sys.exit(1)

    if config_file.has_option("global", "ban_file"):
        ban_file = config_file.get("global", "ban_file")

    if config_file.has_option("global", "polling_error_threshold"):
        try:
            polling_error_threshold = config_file.getint("global", "polling_error_threshold")
        except ValueError:
            print "Configuration file problem: polling_error_threshold must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "ban_failrate_threshold"):
        try:
            ban_failrate_threshold = config_file.getfloat("global", "ban_failrate_threshold")
            if ban_failrate_threshold == 0:
                print "Please use a float value (0, 1.0]"
                sys.exit(1)
        except ValueError:
            print "Configuration file problem: ban_failrate_threshold must be an " \
                  "float value."
            sys.exit(1)

    if config_file.has_option("global", "ban_min_track"):
        try:
            ban_min_track = config_file.getint("global", "ban_min_track")
        except ValueError:
            print "Configuration file problem: ban_min_track must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "condor_register_time_limit"):
        try:
            condor_register_time_limit = 60*config_file.getint("global", "condor_register_time_limit")
        except ValueError:
            print "Configuration file problem: condor_register_time_limit must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "ban_tracking"):
        try:
            ban_tracking = config_file.getboolean("global", "ban_tracking")
        except ValueError:
            print "Configuration file problem: ban_tracking must be an " \
                  "boolean value."
            sys.exit(1)

    if config_file.has_option("global", "graceful_shutdown"):
        try:
            graceful_shutdown = config_file.getboolean("global", "graceful_shutdown")
        except ValueError:
            print "Configuration file problem: graceful_shutdown must be an " \
                  "boolean value."
            sys.exit(1)

    if config_file.has_option("global", "graceful_shutdown_method"):
        graceful_shutdown_method = config_file.get("global", "graceful_shutdown_method")

    if config_file.has_option("global", "retire_before_lifetime"):
        try:
            retire_before_lifetime = config_file.getboolean("global", "retire_before_lifetime")
        except ValueError:
            print "Configuration file problem: retire_before_lifetime must be an " \
                  "boolean value."
            sys.exit(1)

    if config_file.has_option("global", "retire_before_lifetime_factor"):
        try:
            retire_before_lifetime_factor = config_file.getfloat("global", "retire_before_lifetime_factor")
            if retire_before_lifetime_factor < 1.0:
                print "Please use a float value (1.0, X] for the retire_before_lifetime_factor"
                sys.exit(1)
        except ValueError:
            print "Configuration file problem: retire_before_lifetime_factor must be a " \
                  "float value."
            sys.exit(1)

    if config_file.has_option("global", "cleanup_missing_vms"):
        try:
            cleanup_missing_vms = config_file.getboolean("global", "cleanup_missing_vms")
        except ValueError:
            print "Configuration file problem: cleanup_missing_vms must be an " \
                  "boolean value."
            sys.exit(1)

    if config_file.has_option("global", "clean_shutdown_idle"):
        try:
            clean_shutdown_idle = config_file.getboolean("global", "clean_shutdown_idle")
        except ValueError:
            print "Configuration file problem: clean_shutdown_idle must be an " \
                  "boolean value."
            sys.exit(1)

    if config_file.has_option("global", "getclouds"):
        try:
            getclouds = config_file.getboolean("global", "getclouds")
        except ValueError:
            print "Configuration file problem: getclouds must be an " \
                  "boolean value."
            sys.exit(1)

    if config_file.has_option("global", "scheduling_metric"):
        scheduling_metric = config_file.get("global", "scheduling_metric")

    if config_file.has_option("global", "job_distribution_type"):
        job_distribution_type = config_file.get("global", "job_distribution_type")

    if config_file.has_option("global", "memory_distribution_weight"):
        try:
            memory_distribution_weight = config_file.getfloat("global", "memory_distribution_weight")
            if ban_failrate_threshold <= 0:
                print "Please use a float value (0, x]"
                sys.exit(1)
        except ValueError:
            print "Configuration file problem: memory_distribution_weight must be an " \
                  "float value."
            sys.exit(1)

    if config_file.has_option("global", "cpu_distribution_weight"):
        try:
            cpu_distribution_weight = config_file.getfloat("global", "cpu_distribution_weight")
            if ban_failrate_threshold <= 0:
                print "Please use a float value (0, x]"
                sys.exit(1)
        except ValueError:
            print "Configuration file problem: cpu_distribution_weight must be an " \
                  "float value."
            sys.exit(1)

    if config_file.has_option("global", "storage_distribution_weight"):
        try:
            storage_distribution_weight = config_file.getfloat("global", "storage_distribution_weight")
            if ban_failrate_threshold <= 0:
                print "Please use a float value (0, x]"
                sys.exit(1)
        except ValueError:
            print "Configuration file problem: storage_distribution_weight must be an " \
                  "float value."
            sys.exit(1)

    if config_file.has_option("global", "scheduling_algorithm"):
        scheduling_algorithm = config_file.get("global", "scheduling_algorithm")

    if config_file.has_option("global", "high_priority_job_support"):
        try:
            high_priority_job_support = config_file.getboolean("global", "high_priority_job_support")
        except ValueError:
            print "Configuration file problem: high_priority_job_support must be an " \
                  "boolean value."
            sys.exit(1)

    if config_file.has_option("global", "high_priority_job_weight"):
        try:
            high_priority_job_weight = config_file.getint("global", "high_priority_job_weight")
        except ValueError:
            print "Configuration file problem: high_priority_job_weight must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "scheduler_interval"):
        try:
            scheduler_interval = config_file.getint("global", "scheduler_interval")
        except ValueError:
            print "Configuration file problem: scheduler_interval must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "vm_poller_interval"):
        try:
            vm_poller_interval = config_file.getint("global", "vm_poller_interval")
        except ValueError:
            print "Configuration file problem: vm_poller_interval must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "job_poller_interval"):
        try:
            job_poller_interval = config_file.getint("global", "job_poller_interval")
        except ValueError:
            print "Configuration file problem: job_poller_interval must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "machine_poller_interval"):
        try:
            machine_poller_interval = config_file.getint("global", "machine_poller_interval")
        except ValueError:
            print "Configuration file problem: machine_poller_interval must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "cleanup_interval"):
        try:
            cleanup_interval = config_file.getint("global", "cleanup_interval")
        except ValueError:
            print "Configuration file problem: cleanup_interval must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "job_proxy_refresher_interval"):
        try:
            job_proxy_refresher_interval = config_file.getint("global", "job_proxy_refresher_interval")
        except ValueError:
            print "Configuration file problem: job_proxy_refresher_interval must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "job_proxy_renewal_threshold"):
        try:
            job_proxy_renewal_threshold = config_file.getint("global", "job_proxy_renewal_threshold")
        except ValueError:
            print "Configuration file problem: job_proxy_renewal_threshold must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "vm_proxy_refresher_interval"):
        try:
            vm_proxy_refresher_interval = config_file.getint("global", "vm_proxy_refresher_interval")
        except ValueError:
            print "Configuration file problem: vm_proxy_refresher_interval must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "vm_proxy_renewal_threshold"):
        try:
            vm_proxy_renewal_threshold = config_file.getint("global", "vm_proxy_renewal_threshold")
        except ValueError:
            print "Configuration file problem: vm_proxy_renewal_threshold must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "vm_proxy_shutdown_threshold"):
        try:
            vm_proxy_shutdown_threshold = config_file.getint("global", "vm_proxy_shutdown_threshold")
        except ValueError:
            print "Configuration file problem: vm_proxy_shutdown_threshold must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "vm_connection_fail_threshold"):
        try:
            vm_connection_fail_threshold = config_file.getint("global", "vm_connection_fail_threshold")
        except ValueError:
            print "Configuration file problem: vm_connection_fail_threshold must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "vm_idle_threshold"):
        try:
            vm_idle_threshold = config_file.getint("global", "vm_idle_threshold")
        except ValueError:
            print "Configuration file problem: vm_idle_threshold must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "vm_start_running_timeout"):
        try:
            vm_start_running_timeout = config_file.getint("global", "vm_start_running_timeout")
        except ValueError:
            print "Configuration file problem: vm_start_running_timeout must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "max_starting_vm"):
        try:
            max_starting_vm = config_file.getint("global", "max_starting_vm")
            if max_starting_vm < -1:
                max_starting_vm = -1
        except ValueError:
            print "Configuration file problem: max_starting_vm must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "max_keepalive"):
        try:
            max_keepalive = config_file.getint("global", "max_keepalive") * 60
            if max_keepalive < 0:
                max_keepalive = 0
        except ValueError:
            print "Configuration file problem: max_keepalive must be an " \
                  "integer value(# of minutes)."
            sys.exit(1)

    if config_file.has_option("global", "max_destroy_threads"):
        try:
            max_destroy_threads = config_file.getint("global", "max_destroy_threads")
            if max_destroy_threads <= 0:
                max_destroy_threads = 1
        except ValueError:
            print "Configuration file problem: max_destroy_threads must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "proxy_cache_dir"):
        proxy_cache_dir = config_file.get("global", "proxy_cache_dir")

    if config_file.has_option("global", "myproxy_logon_command"):
        myproxy_logon_command = config_file.get("global", "myproxy_logon_command")

    if config_file.has_option("global", "override_vmtype"):
        try:
            override_vmtype = config_file.getboolean("global", "override_vmtype")
        except ValueError:
            print "Configuration file problem: override_vmtype must be a" \
                  " Boolean value."

    if config_file.has_option("global", "vm_reqs_from_condor_reqs"):
        try:
            vm_reqs_from_condor_reqs = config_file.getboolean("global", "vm_reqs_from_condor_reqs")
        except ValueError:
            print "Configuration file problem: vm_reqs_from_condor_reqs must be a" \
                  " Boolean value."

    if config_file.has_option("global", "adjust_insufficient_resources"):
        try:
            adjust_insufficient_resources = config_file.getboolean("global", "adjust_insufficient_resources")
        except ValueError:
            print "Configuration file problem: adjust_insufficient_resources must be a" \
                  " Boolean value."

    if config_file.has_option("global", "connection_fail_disable_time"):
        try:
            connection_fail_disable_time = config_file.getint("global", "connection_fail_disable_time")
        except ValueError:
            print "Configuration file problem: connection_fail_disable_time must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("global", "use_cloud_init"):
        try:
            use_cloud_init = config_file.getboolean("global", "use_cloud_init")
        except ValueError:
            print "Configuration file problem: use_cloud_init must be a" \
                  " Boolean value."

    if config_file.has_option("global", "default_yaml"):
        default_yaml = config_file.get("global", "default_yaml")

    if config_file.has_option("global", "validate_yaml"):
        try:
            validate_yaml = config_file.getboolean("global", "validate_yaml")
        except ValueError:
            print "Configuration file problem: validate_yaml must be a" \
                  " Boolean value."

    if config_file.has_option("global", "retire_reallocate"):
        try:
            retire_reallocate = config_file.getboolean("global", "retire_reallocate")
        except ValueError:
            print "Configuration file problem: retire_reallocate must be a" \
                  " Boolean value."


    # Default Logging options
    if config_file.has_option("logging", "log_level"):
        log_level = config_file.get("logging", "log_level")

    if config_file.has_option("logging", "log_location"):
        log_location = os.path.expanduser(config_file.get("logging", "log_location"))

    if config_file.has_option("logging", "log_location_cloud_admin"):
        log_location_cloud_admin = os.path.expanduser(config_file.get("logging", "log_location_cloud_admin"))

    if config_file.has_option("logging", "admin_log_comments"):
        try:
            admin_log_comments = config_file.getboolean("logging", "admin_log_comments")
        except ValueError:
            print "Configuration file problem: admin_log_comments must be a" \
                  " Boolean value."

    if config_file.has_option("logging", "log_stdout"):
        try:
            log_stdout = config_file.getboolean("logging", "log_stdout")
        except ValueError:
            print "Configuration file problem: log_stdout must be a" \
                  " Boolean value."

    if config_file.has_option("logging", "log_syslog"):
        try:
            log_syslog = config_file.getboolean("logging", "log_syslog")
        except ValueError:
            print "Configuration file problem: log_syslog must be a" \
                  " Boolean value."

    if config_file.has_option("logging", "log_max_size"):
        try:
            log_max_size = config_file.getint("logging", "log_max_size")
        except ValueError:
            print "Configuration file problem: log_max_size must be an " \
                  "integer value in bytes."
            sys.exit(1)

    if config_file.has_option("logging", "log_format"):
        log_format = config_file.get("logging", "log_format", raw=True)

    # Default Job options
    if config_file.has_option("job", "default_VMType"):
        default_VMType = config_file.get("job", "default_VMType")

    if config_file.has_option("job", "default_VMNetwork"):
        default_VMNetwork = config_file.get("job", "default_VMNetwork")

    if config_file.has_option("job", "default_VMCPUArch"):
        default_VMCPUArch = config_file.get("job", "default_VMCPUArch")
        
    if config_file.has_option("job", "default_VMHypervisor"):
        default_VMHypervisor = config_file.get("job", "default_VMHypervisor")

    if config_file.has_option("job", "default_VMName"):
        default_VMName = config_file.get("job", "default_VMName")

    if config_file.has_option("job", "default_VMLoc"):
        default_VMLoc = config_file.get("job", "default_VMLoc")

    if config_file.has_option("job", "default_VMAMI"):
        default_VMAMI = config_file.get("job", "default_VMAMI")

    if config_file.has_option("job", "default_VMMem"):
        try:
            default_VMMem = config_file.getint("job", "default_VMMem")
        except ValueError:
            print "Configuration file problem: default_VMMem must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("job", "default_VMCPUCores"):
        try:
            default_VMCPUCores = config_file.getint("job", "default_VMCPUCores")
        except ValueError:
            print "Configuration file problem: default_VMCPUCores must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("job", "default_VMStorage"):
        try:
            default_VMStorage = config_file.getint("job", "default_VMStorage")
        except ValueError:
            print "Configuration file problem: default_VMStorage must be an " \
                  "integer value."
            sys.exit(1)

    if config_file.has_option("job", "default_VMInstanceType"):
        default_VMInstanceType = config_file.get("job", "default_VMInstanceType")

    if config_file.has_option("job", "default_VMInstanceTypeList"):
        default_VMInstanceTypeList = config_file.get("job", "default_VMInstanceTypeList")

    if config_file.has_option("job", "default_VMMaximumPrice"):
        try:
            default_VMMaximumPrice = config_file.getfloat("job", "default_VMMaximumPrice")
        except ValueError:
            print "Configuration file problem: default_VMMaximumPrice must be an " \
                  "floating point value."
            sys.exit(1)

    if config_file.has_option("job", "default_VMProxyNonBoot"):
        try:
            default_VMProxyNonBoot = config_file.getboolean("global", "default_VMProxyNonBoot")
        except ValueError:
            print "Configuration file problem: default_VMProxyNonBoot must be a" \
                  " Boolean value."

    if config_file.has_option("job", "default_VMUserData"):
        default_VMUserData = config_file.get("job", "default_VMUserData").replace(' ', '').strip('"').split(',')
    
    if config_file.has_option("job", "default_TargetClouds"):
        default_TargetClouds = config_file.get("job", "default_TargetClouds")

    if config_file.has_option("job", "default_VMAMIConfig"):
        default_VMAMIConfig = config_file.get("job", "default_VMAMIConfig")

    if config_file.has_option("job", "default_VMInjectCA"):
        try:
            default_VMInjectCA = config_file.getboolean("job", "default_VMInjectCA")
        except ValueError:
            print "Configuration file problem: default_VMInjectCA must be a" \
                  " Boolean value."

    if config_file.has_option("job", "default_VMJobPerCore"):
        try:
            default_VMJobPerCore = config_file.getboolean("job", "default_VMJobPerCore")
        except ValueError:
            print "Configuration file problem: default_VMJobPerCore must be a" \
                  " Boolean value."    

    # Derived options
    if condor_host_on_vm:
        condor_host = condor_host_on_vm
    else:
        condor_host = utilities.get_hostname_from_url(condor_webservice_url)

    if config_file.has_option("global", "use_pyopenssl"):
        use_pyopenssl = config_file.getboolean("global", "use_pyopenssl")

Example 16

View license
def main():
  parser = optparse.OptionParser()
  parser.add_option('-o', dest='output_file', help='write HTML to FILE',
                    default='trace.html', metavar='FILE')
  parser.add_option('-t', '--time', dest='trace_time', type='int',
                    help='trace for N seconds', metavar='N')
  parser.add_option('-b', '--buf-size', dest='trace_buf_size', type='int',
                    help='use a trace buffer size of N KB', metavar='N')
  parser.add_option('-d', '--disk', dest='trace_disk', default=False,
                    action='store_true', help='trace disk I/O (requires root)')
  parser.add_option('-f', '--cpu-freq', dest='trace_cpu_freq', default=False,
                    action='store_true', help='trace CPU frequency changes')
  parser.add_option('-i', '--cpu-idle', dest='trace_cpu_idle', default=False,
                    action='store_true', help='trace CPU idle events')
  parser.add_option('-l', '--cpu-load', dest='trace_cpu_load', default=False,
                    action='store_true', help='trace CPU load')
  parser.add_option('-s', '--no-cpu-sched', dest='trace_cpu_sched', default=True,
                    action='store_false', help='inhibit tracing CPU ' +
                    'scheduler (allows longer trace times by reducing data ' +
                    'rate into buffer)')
  parser.add_option('-u', '--bus-utilization', dest='trace_bus_utilization',
                    default=False, action='store_true',
                    help='trace bus utilization (requires root)')
  parser.add_option('-w', '--workqueue', dest='trace_workqueue', default=False,
                    action='store_true', help='trace the kernel workqueues ' +
                    '(requires root)')
  parser.add_option('--set-tags', dest='set_tags', action='store',
                    help='set the enabled trace tags and exit; set to a ' +
                    'comma separated list of: ' +
                    ', '.join(trace_tag_bits.iterkeys()))
  parser.add_option('--link-assets', dest='link_assets', default=False,
                    action='store_true', help='link to original CSS or JS resources '
                    'instead of embedding them')
  parser.add_option('--from-file', dest='from_file', action='store',
                    help='read the trace from a file rather than running a live trace')
  parser.add_option('--asset-dir', dest='asset_dir', default='trace-viewer',
                    type='string', help='')
  parser.add_option('-e', '--serial', dest='device_serial', type='string',
                    help='adb device serial number')
  options, args = parser.parse_args()

  if options.set_tags:
    flags = 0
    tags = options.set_tags.split(',')
    for tag in tags:
      try:
        flags |= trace_tag_bits[tag]
      except KeyError:
        parser.error('unrecognized tag: %s\nknown tags are: %s' %
                     (tag, ', '.join(trace_tag_bits.iterkeys())))
    atrace_args = ['adb', 'shell', 'setprop', 'debug.atrace.tags.enableflags', hex(flags)]
    add_adb_serial(atrace_args, options.device_serial)
    try:
      subprocess.check_call(atrace_args)
    except subprocess.CalledProcessError, e:
      print >> sys.stderr, 'unable to set tags: %s' % e
    print '\nSet enabled tags to: %s\n' % ', '.join(tags)
    print ('You will likely need to restart the Android framework for this to ' +
          'take effect:\n\n    adb shell stop\n    adb shell ' +
          'start\n')
    return

  atrace_args = ['adb', 'shell', 'atrace', '-z']
  add_adb_serial(atrace_args, options.device_serial)

  if options.trace_disk:
    atrace_args.append('-d')
  if options.trace_cpu_freq:
    atrace_args.append('-f')
  if options.trace_cpu_idle:
    atrace_args.append('-i')
  if options.trace_cpu_load:
    atrace_args.append('-l')
  if options.trace_cpu_sched:
    atrace_args.append('-s')
  if options.trace_bus_utilization:
    atrace_args.append('-u')
  if options.trace_workqueue:
    atrace_args.append('-w')
  if options.trace_time is not None:
    if options.trace_time > 0:
      atrace_args.extend(['-t', str(options.trace_time)])
    else:
      parser.error('the trace time must be a positive number')
  if options.trace_buf_size is not None:
    if options.trace_buf_size > 0:
      atrace_args.extend(['-b', str(options.trace_buf_size)])
    else:
      parser.error('the trace buffer size must be a positive number')

  if options.from_file is not None:
    atrace_args = ['cat', options.from_file]

  script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))

  if options.link_assets:
    src_dir = os.path.join(script_dir, options.asset_dir, 'src')
    build_dir = os.path.join(script_dir, options.asset_dir, 'build')

    js_files, js_flattenizer, css_files, templates = get_assets(src_dir, build_dir)

    css = '\n'.join(linked_css_tag % (os.path.join(src_dir, f)) for f in css_files)
    js = '<script language="javascript">\n%s</script>\n' % js_flattenizer
    js += '\n'.join(linked_js_tag % (os.path.join(src_dir, f)) for f in js_files)

  else:
    css_filename = os.path.join(script_dir, flattened_css_file)
    js_filename = os.path.join(script_dir, flattened_js_file)
    css = compiled_css_tag % (open(css_filename).read())
    js = compiled_js_tag % (open(js_filename).read())
    templates = ''

  html_filename = options.output_file

  trace_started = False
  leftovers = ''
  adb = subprocess.Popen(atrace_args, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  dec = zlib.decompressobj()
  while True:
    ready = select.select([adb.stdout, adb.stderr], [], [adb.stdout, adb.stderr])
    if adb.stderr in ready[0]:
      err = os.read(adb.stderr.fileno(), 4096)
      sys.stderr.write(err)
      sys.stderr.flush()
    if adb.stdout in ready[0]:
      out = leftovers + os.read(adb.stdout.fileno(), 4096)
      if options.from_file is None:
        out = out.replace('\r\n', '\n')
      if out.endswith('\r'):
        out = out[:-1]
        leftovers = '\r'
      else:
        leftovers = ''
      if not trace_started:
        lines = out.splitlines(True)
        out = ''
        for i, line in enumerate(lines):
          if line == 'TRACE:\n':
            sys.stdout.write("downloading trace...")
            sys.stdout.flush()
            out = ''.join(lines[i+1:])
            html_prefix = read_asset(script_dir, 'prefix.html')
            html_file = open(html_filename, 'w')
            html_file.write(html_prefix % (css, js, templates))
            trace_started = True
            break
          elif 'TRACE:'.startswith(line) and i == len(lines) - 1:
            leftovers = line + leftovers
          else:
            sys.stdout.write(line)
            sys.stdout.flush()
      if len(out) > 0:
        out = dec.decompress(out)
      html_out = out.replace('\n', '\\n\\\n')
      if len(html_out) > 0:
        html_file.write(html_out)
    result = adb.poll()
    if result is not None:
      break
  if result != 0:
    print >> sys.stderr, 'adb returned error code %d' % result
  elif trace_started:
    html_out = dec.flush().replace('\n', '\\n\\\n').replace('\r', '')
    if len(html_out) > 0:
      html_file.write(html_out)
    html_suffix = read_asset(script_dir, 'suffix.html')
    html_file.write(html_suffix)
    html_file.close()
    print " done\n\n    wrote file://%s\n" % (os.path.abspath(options.output_file))
  else:
    print >> sys.stderr, ('An error occured while capturing the trace.  Output ' +
      'file was not written.')

Example 17

View license
def main():
  parser = optparse.OptionParser()
  parser.add_option('-o', dest='output_file', help='write HTML to FILE',
                    default='trace.html', metavar='FILE')
  parser.add_option('-t', '--time', dest='trace_time', type='int',
                    help='trace for N seconds', metavar='N')
  parser.add_option('-b', '--buf-size', dest='trace_buf_size', type='int',
                    help='use a trace buffer size of N KB', metavar='N')
  parser.add_option('-d', '--disk', dest='trace_disk', default=False,
                    action='store_true', help='trace disk I/O (requires root)')
  parser.add_option('-f', '--cpu-freq', dest='trace_cpu_freq', default=False,
                    action='store_true', help='trace CPU frequency changes')
  parser.add_option('-i', '--cpu-idle', dest='trace_cpu_idle', default=False,
                    action='store_true', help='trace CPU idle events')
  parser.add_option('-l', '--cpu-load', dest='trace_cpu_load', default=False,
                    action='store_true', help='trace CPU load')
  parser.add_option('-s', '--no-cpu-sched', dest='trace_cpu_sched', default=True,
                    action='store_false', help='inhibit tracing CPU ' +
                    'scheduler (allows longer trace times by reducing data ' +
                    'rate into buffer)')
  parser.add_option('-u', '--bus-utilization', dest='trace_bus_utilization',
                    default=False, action='store_true',
                    help='trace bus utilization (requires root)')
  parser.add_option('-w', '--workqueue', dest='trace_workqueue', default=False,
                    action='store_true', help='trace the kernel workqueues ' +
                    '(requires root)')
  parser.add_option('--set-tags', dest='set_tags', action='store',
                    help='set the enabled trace tags and exit; set to a ' +
                    'comma separated list of: ' +
                    ', '.join(trace_tag_bits.iterkeys()))
  parser.add_option('--link-assets', dest='link_assets', default=False,
                    action='store_true', help='link to original CSS or JS resources '
                    'instead of embedding them')
  parser.add_option('--from-file', dest='from_file', action='store',
                    help='read the trace from a file rather than running a live trace')
  parser.add_option('--asset-dir', dest='asset_dir', default='trace-viewer',
                    type='string', help='')
  parser.add_option('-e', '--serial', dest='device_serial', type='string',
                    help='adb device serial number')
  options, args = parser.parse_args()

  if options.set_tags:
    flags = 0
    tags = options.set_tags.split(',')
    for tag in tags:
      try:
        flags |= trace_tag_bits[tag]
      except KeyError:
        parser.error('unrecognized tag: %s\nknown tags are: %s' %
                     (tag, ', '.join(trace_tag_bits.iterkeys())))
    atrace_args = ['adb', 'shell', 'setprop', 'debug.atrace.tags.enableflags', hex(flags)]
    add_adb_serial(atrace_args, options.device_serial)
    try:
      subprocess.check_call(atrace_args)
    except subprocess.CalledProcessError, e:
      print >> sys.stderr, 'unable to set tags: %s' % e
    print '\nSet enabled tags to: %s\n' % ', '.join(tags)
    print ('You will likely need to restart the Android framework for this to ' +
          'take effect:\n\n    adb shell stop\n    adb shell ' +
          'start\n')
    return

  atrace_args = ['adb', 'shell', 'atrace', '-z']
  add_adb_serial(atrace_args, options.device_serial)

  if options.trace_disk:
    atrace_args.append('-d')
  if options.trace_cpu_freq:
    atrace_args.append('-f')
  if options.trace_cpu_idle:
    atrace_args.append('-i')
  if options.trace_cpu_load:
    atrace_args.append('-l')
  if options.trace_cpu_sched:
    atrace_args.append('-s')
  if options.trace_bus_utilization:
    atrace_args.append('-u')
  if options.trace_workqueue:
    atrace_args.append('-w')
  if options.trace_time is not None:
    if options.trace_time > 0:
      atrace_args.extend(['-t', str(options.trace_time)])
    else:
      parser.error('the trace time must be a positive number')
  if options.trace_buf_size is not None:
    if options.trace_buf_size > 0:
      atrace_args.extend(['-b', str(options.trace_buf_size)])
    else:
      parser.error('the trace buffer size must be a positive number')

  if options.from_file is not None:
    atrace_args = ['cat', options.from_file]

  script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))

  if options.link_assets:
    src_dir = os.path.join(script_dir, options.asset_dir, 'src')
    build_dir = os.path.join(script_dir, options.asset_dir, 'build')

    js_files, js_flattenizer, css_files, templates = get_assets(src_dir, build_dir)

    css = '\n'.join(linked_css_tag % (os.path.join(src_dir, f)) for f in css_files)
    js = '<script language="javascript">\n%s</script>\n' % js_flattenizer
    js += '\n'.join(linked_js_tag % (os.path.join(src_dir, f)) for f in js_files)

  else:
    css_filename = os.path.join(script_dir, flattened_css_file)
    js_filename = os.path.join(script_dir, flattened_js_file)
    css = compiled_css_tag % (open(css_filename).read())
    js = compiled_js_tag % (open(js_filename).read())
    templates = ''

  html_filename = options.output_file

  trace_started = False
  leftovers = ''
  adb = subprocess.Popen(atrace_args, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  dec = zlib.decompressobj()
  while True:
    ready = select.select([adb.stdout, adb.stderr], [], [adb.stdout, adb.stderr])
    if adb.stderr in ready[0]:
      err = os.read(adb.stderr.fileno(), 4096)
      sys.stderr.write(err)
      sys.stderr.flush()
    if adb.stdout in ready[0]:
      out = leftovers + os.read(adb.stdout.fileno(), 4096)
      if options.from_file is None:
        out = out.replace('\r\n', '\n')
      if out.endswith('\r'):
        out = out[:-1]
        leftovers = '\r'
      else:
        leftovers = ''
      if not trace_started:
        lines = out.splitlines(True)
        out = ''
        for i, line in enumerate(lines):
          if line == 'TRACE:\n':
            sys.stdout.write("downloading trace...")
            sys.stdout.flush()
            out = ''.join(lines[i+1:])
            html_prefix = read_asset(script_dir, 'prefix.html')
            html_file = open(html_filename, 'w')
            html_file.write(html_prefix % (css, js, templates))
            trace_started = True
            break
          elif 'TRACE:'.startswith(line) and i == len(lines) - 1:
            leftovers = line + leftovers
          else:
            sys.stdout.write(line)
            sys.stdout.flush()
      if len(out) > 0:
        out = dec.decompress(out)
      html_out = out.replace('\n', '\\n\\\n')
      if len(html_out) > 0:
        html_file.write(html_out)
    result = adb.poll()
    if result is not None:
      break
  if result != 0:
    print >> sys.stderr, 'adb returned error code %d' % result
  elif trace_started:
    html_out = dec.flush().replace('\n', '\\n\\\n').replace('\r', '')
    if len(html_out) > 0:
      html_file.write(html_out)
    html_suffix = read_asset(script_dir, 'suffix.html')
    html_file.write(html_suffix)
    html_file.close()
    print " done\n\n    wrote file://%s\n" % (os.path.abspath(options.output_file))
  else:
    print >> sys.stderr, ('An error occured while capturing the trace.  Output ' +
      'file was not written.')

Example 18

View license
    def convert(self):
        # Based on http://stackoverflow.com/a/8661021
        represent_dict_order = lambda _self, data: _self.represent_mapping('tag:yaml.org,2002:map', data.items())
        yaml.add_representer(OrderedDict, represent_dict_order)

        def project_prefix(value):
            return '{}-{}'.format(self.project, value) if self.project else value

        if self.networks:
            print >> sys.stderr, ('WARNING: unsupported parameter "networks"')

        for volume in self.volumes:
            print >> sys.stderr, ('WARNING: unsupported parameter "volumes"')

        for service in self.filtered_services:
            service_config = self.services[service]
            service = service.replace('_', '-')
            service_result = OrderedDict([
                ('apiVersion', 'v1'),
                ('kind', 'Service'),
                ('metadata', OrderedDict([
                    ('name', project_prefix(service)),
                    ('labels', OrderedDict())
                ])),
                ('spec', OrderedDict([
                    ('selector', OrderedDict())
                ]))
            ])
            deployment_result = OrderedDict([
                ('apiVersion', 'extensions/v1beta1'),
                ('kind', 'Deployment'),
                ('metadata', OrderedDict([
                    ('name', project_prefix(service))
                ])),
                ('spec', OrderedDict([
                    ('replicas', 1),
                    ('template', OrderedDict([
                        ('metadata', OrderedDict([
                            ('labels', OrderedDict())
                        ])),
                        ('spec', OrderedDict([
                            ('containers', [OrderedDict([
                                ('name', project_prefix(service)),
                            ])])
                        ]))
                    ]))
                ]))
            ])

            service_labels = service_result['metadata']['labels']
            service_selector = service_result['spec']['selector']
            deployment_labels = deployment_result['spec']['template']['metadata']['labels']
            deployment_spec = deployment_result['spec']['template']['spec']
            container = deployment_result['spec']['template']['spec']['containers'][0]

            service_labels['service'] = self.project
            service_labels['app'] = service

            for parameter in service_config:
                value = service_config[parameter]

                def restart():
                    deployment_spec['restartPolicy'] = {'always': 'Always'}[value]

                def logging():
                    pass  # unsupported

                def mem_limit():
                    container['resources'] = {'limits': {'memory': value.replace('m', 'Mi').replace('g', 'Gi')}}

                def image():
                    container['image'] = value

                def command():
                    if isinstance(value, list):
                        container['args'] = value
                    else:
                        container['args'] = value.split(' ')

                def expose():
                    service_result['spec']['ports'] = []
                    container['ports'] = []
                    for port in value:
                        port_int = int(port)
                        service_result['spec']['ports'].append(OrderedDict([('port', port_int), ('targetPort', port_int), ('name', str(port_int))]))
                        container['ports'].append({'containerPort': port_int})

                def container_name():
                    service_result['metadata']['name'] = value
                    deployment_result['metadata']['name'] = value
                    container['name'] = value
                    service_labels['app'] = value

                def hostname():
                    pass  # unsupported

                def labels():
                    pass  # TODO

                def mode():
                    pass  # TODO

                def extra_hosts():
                    pass  # unsupported

                def ports():
                    for port in value:
                        pass  # TODO

                def networks():
                    pass  # unsupported

                def volumes():
                    container['volumeMounts'] = []
                    deployment_spec['volumes'] = []
                    for volume in value:
                        splitted_volume = volume.split(':')
                        src = splitted_volume.pop(0)
                        dst = splitted_volume.pop(0)
                        readonly = 0
                        if splitted_volume and splitted_volume[0] == 'ro':
                            readonly = 1
                        if src.startswith('.'):
                            src = src.replace('.', self.compose_base_dir, 1)

                        if src.startswith('/'):
                            volume_name = src.split('/')[-1].replace('.', '')
                            container['volumeMounts'].append(OrderedDict([('name', volume_name), ('mountPath', dst)]))
                            deployment_spec['volumes'].append(OrderedDict([('name', volume_name), ('hostPath', {'path': src})]))
                            # TODO readonly
                        else:
                            volume_name = src.replace('_', '-')
                            container['volumeMounts'].append(OrderedDict([('name', volume_name), ('mountPath', dst)]))
                            deployment_spec['volumes'].append(
                                OrderedDict([('name', volume_name), ('hostPath', {'path': '/volumes/' + project_prefix(volume_name)})]))
                            # TODO readonly

                def environment():
                    if isinstance(value, dict):
                        container['env'] = []
                        for k, v in value.items():
                            container['env'].append(OrderedDict([('name', k), ('value', v)]))
                    else:
                        for env in value:
                            if env.startswith('constraint') or env.startswith('affinity'):
                                if 'nodeSelector' not in deployment_spec:
                                    deployment_spec['nodeSelector'] = OrderedDict()

                                constraint = env.split(':', 2)[1]
                                selector = 'FIX_ME'

                                if constraint.startswith('node.hostname=='):
                                    selector = 'kubernetes.io/hostname'
                                    constraint = constraint.split('==')[1]

                                if constraint.startswith('engine.labels.'):
                                    [selector, constraint] = constraint.split('==')
                                    selector = selector.replace('engine.labels.', '')

                                deployment_spec['nodeSelector'][selector] = constraint
                            else:
                                if 'env' not in container:
                                    container['env'] = []

                                [k, v] = env.split('=')
                                container['env'].append(OrderedDict([('name', k), ('value', v)]))

                def replicas():
                    deployment_result['spec']['replicas'] = int(value)

                def unsupported():
                    print >> sys.stderr, ('WARNING: unsupported parameter {}'.format(parameter))

                locals().get(parameter, unsupported)()

            service_selector.update(service_labels)
            deployment_labels.update(service_labels)

            sys.stdout.write(yaml.dump(service_result, default_flow_style=False))
            print('---')
            sys.stdout.write(yaml.dump(deployment_result, default_flow_style=False))
            print('---')

Example 19

Project: ssf
Source File: survey.py
View license
def series_export_spreadsheet(matrix, matrixAnswers, logo):
    ######################################################################
    #
    # Now take the matrix data type and generate a spreadsheet from it
    #
    ######################################################################
    import math
    try:
        import xlwt
    except ImportError:
        output = s3_rest_controller(module,
                                resourcename,
                                rheader=response.s3.survey_series_rheader)
        return output

    def wrapText(sheet, cell, style):
        row = cell.row
        col = cell.col
        try:
            text = unicode(cell.text)
        except:
            text = cell.text
        width = 16
        # Wrap text and calculate the row width and height
        characters_in_cell = float(width-2)
        twips_per_row = 255 #default row height for 10 point font
        if cell.merged():
            try:
                sheet.write_merge(cell.row,
                                  cell.row + cell.mergeV,
                                  cell.col,
                                  cell.col + cell.mergeH,
                                  text,
                                  style
                                 )
            except Exception as msg:
                print >> sys.stderr, msg
                print >> sys.stderr, "row: %s + vert: %s, col: %s + horiz %s" % (cell.row, cell.mergeV, cell.col, cell.mergeH)
                posn = "%s,%s"%(cell.row, cell.col)
                if matrix.matrix[posn]:
                    print >> sys.stderr, matrix.matrix[posn]
            rows = math.ceil((len(text) / characters_in_cell) / (1 + cell.mergeH))
        else:
            sheet.write(cell.row,
                        cell.col,
                        text,
                        style
                       )
            rows = math.ceil(len(text) / characters_in_cell)
        new_row_height = int(rows * twips_per_row)
        new_col_width = width * COL_WIDTH_MULTIPLIER
        if sheet.row(row).height < new_row_height:
            sheet.row(row).height = new_row_height
        if sheet.col(col).width < new_col_width:
            sheet.col(col).width = new_col_width

    def mergeStyles(listTemplate, styleList):
        """
            Take a list of styles and return a single style object with
            all the differences from a newly created object added to the
            resultant style.
        """
        if len(styleList) == 0:
            finalStyle = xlwt.XFStyle()
        elif len(styleList) == 1:
            finalStyle = listTemplate[styleList[0]]
        else:
            zeroStyle = xlwt.XFStyle()
            finalStyle = xlwt.XFStyle()
            for i in range(0,len(styleList)):
                finalStyle = mergeObjectDiff(finalStyle,
                                             listTemplate[styleList[i]],
                                             zeroStyle)
        return finalStyle

    def mergeObjectDiff(baseObj, newObj, zeroObj):
        """
            function to copy all the elements in newObj that are different from
            the zeroObj and place them in the baseObj
        """
        elementList = newObj.__dict__
        for (element, value) in elementList.items():
            try:
                baseObj.__dict__[element] = mergeObjectDiff(baseObj.__dict__[element],
                                                            value,
                                                            zeroObj.__dict__[element])
            except:
                if zeroObj.__dict__[element] != value:
                    baseObj.__dict__[element] = value
        return baseObj

    COL_WIDTH_MULTIPLIER = 240
    book = xlwt.Workbook(encoding="utf-8")
    output = StringIO()

    protection = xlwt.Protection()
    protection.cell_locked = 1
    noProtection = xlwt.Protection()
    noProtection.cell_locked = 0

    borders = xlwt.Borders()
    borders.left = xlwt.Borders.DOTTED
    borders.right = xlwt.Borders.DOTTED
    borders.top = xlwt.Borders.DOTTED
    borders.bottom = xlwt.Borders.DOTTED

    borderT1 = xlwt.Borders()
    borderT1.top = xlwt.Borders.THIN
    borderT2 = xlwt.Borders()
    borderT2.top = xlwt.Borders.MEDIUM

    borderL1 = xlwt.Borders()
    borderL1.left = xlwt.Borders.THIN
    borderL2 = xlwt.Borders()
    borderL2.left = xlwt.Borders.MEDIUM

    borderR1 = xlwt.Borders()
    borderR1.right = xlwt.Borders.THIN
    borderR2 = xlwt.Borders()
    borderR2.right = xlwt.Borders.MEDIUM

    borderB1 = xlwt.Borders()
    borderB1.bottom = xlwt.Borders.THIN
    borderB2 = xlwt.Borders()
    borderB2.bottom = xlwt.Borders.MEDIUM

    alignBase = xlwt.Alignment()
    alignBase.horz = xlwt.Alignment.HORZ_LEFT
    alignBase.vert = xlwt.Alignment.VERT_TOP

    alignWrap = xlwt.Alignment()
    alignWrap.horz = xlwt.Alignment.HORZ_LEFT
    alignWrap.vert = xlwt.Alignment.VERT_TOP
    alignWrap.wrap = xlwt.Alignment.WRAP_AT_RIGHT

    shadedFill = xlwt.Pattern()
    shadedFill.pattern = xlwt.Pattern.SOLID_PATTERN
    shadedFill.pattern_fore_colour = 0x16 # 25% Grey
    shadedFill.pattern_back_colour = 0x08 # Black

    headingFill = xlwt.Pattern()
    headingFill.pattern = xlwt.Pattern.SOLID_PATTERN
    headingFill.pattern_fore_colour = 0x1F # ice_blue
    headingFill.pattern_back_colour = 0x08 # Black

    styleTitle =  xlwt.XFStyle()
    styleTitle.font.height = 0x0140 # 320 twips, 16 points
    styleTitle.font.bold = True
    styleTitle.alignment = alignBase
    styleHeader = xlwt.XFStyle()
    styleHeader.font.height = 0x00F0 # 240 twips, 12 points
    styleHeader.font.bold = True
    styleHeader.alignment = alignBase
    styleSubHeader = xlwt.XFStyle()
    styleSubHeader.font.bold = True
    styleSubHeader.alignment = alignWrap
    styleSectionHeading = xlwt.XFStyle()
    styleSectionHeading.font.bold = True
    styleSectionHeading.alignment = alignWrap
    styleSectionHeading.pattern = headingFill
    styleHint = xlwt.XFStyle()
    styleHint.protection = protection
    styleHint.font.height = 160 # 160 twips, 8 points
    styleHint.font.italic = True
    styleHint.alignment = alignWrap
    styleText = xlwt.XFStyle()
    styleText.protection = protection
    styleText.alignment = alignWrap
    styleInstructions = xlwt.XFStyle()
    styleInstructions.font.height = 0x00B4 # 180 twips, 9 points
    styleInstructions.font.italic = True
    styleInstructions.protection = protection
    styleInstructions.alignment = alignWrap
    styleBox = xlwt.XFStyle()
    styleBox.borders = borders
    styleBox.protection = noProtection
    styleInput = xlwt.XFStyle()
    styleInput.borders = borders
    styleInput.protection = noProtection
    styleInput.pattern = shadedFill
    boxL1 = xlwt.XFStyle()
    boxL1.borders = borderL1
    boxL2 = xlwt.XFStyle()
    boxL2.borders = borderL2
    boxT1 = xlwt.XFStyle()
    boxT1.borders = borderT1
    boxT2 = xlwt.XFStyle()
    boxT2.borders = borderT2
    boxR1 = xlwt.XFStyle()
    boxR1.borders = borderR1
    boxR2 = xlwt.XFStyle()
    boxR2.borders = borderR2
    boxB1 = xlwt.XFStyle()
    boxB1.borders = borderB1
    boxB2 = xlwt.XFStyle()
    boxB2.borders = borderB2
    styleList = {}
    styleList["styleTitle"] = styleTitle
    styleList["styleHeader"] = styleHeader
    styleList["styleSubHeader"] = styleSubHeader
    styleList["styleSectionHeading"] = styleSectionHeading
    styleList["styleHint"] = styleHint
    styleList["styleText"] = styleText
    styleList["styleInstructions"] = styleInstructions
    styleList["styleInput"] = styleInput
    styleList["boxL1"] = boxL1
    styleList["boxL2"] = boxL2
    styleList["boxT1"] = boxT1
    styleList["boxT2"] = boxT2
    styleList["boxR1"] = boxR1
    styleList["boxR2"] = boxR2
    styleList["boxB1"] = boxB1
    styleList["boxB2"] = boxB2

    sheet1 = book.add_sheet(T("Assessment"))
    sheetA = book.add_sheet(T("Metadata"))
    maxCol = 0
    for cell in matrix.matrix.values():
        if cell.col + cell.mergeH > 255:
            print  >> sys.stderr, "Cell (%s,%s) - (%s,%s) ignored" % (cell.col, cell.row, cell.col + cell.mergeH, cell.row + cell.mergeV)
            continue
        if cell.col + cell.mergeH > maxCol:
            maxCol = cell.col + cell.mergeH
        if cell.joined():
            continue
        style = mergeStyles(styleList, cell.styleList)
        if (style.alignment.wrap == style.alignment.WRAP_AT_RIGHT):
            # get all the styles from the joined cells
            # and merge these styles in.
            joinedStyles = matrix.joinedElementStyles(cell)
            joinedStyle =  mergeStyles(styleList, joinedStyles)
            try:
                wrapText(sheet1, cell, joinedStyle)
            except:
                pass
        else:
            if cell.merged():
                # get all the styles from the joined cells
                # and merge these styles in.
                joinedStyles = matrix.joinedElementStyles(cell)
                joinedStyle =  mergeStyles(styleList, joinedStyles)
                try:
                    sheet1.write_merge(cell.row,
                                       cell.row + cell.mergeV,
                                       cell.col,
                                       cell.col + cell.mergeH,
                                       unicode(cell.text),
                                       joinedStyle
                                       )
                except Exception as msg:
                    print >> sys.stderr, msg
                    print >> sys.stderr, "row: %s + vert: %s, col: %s + horiz %s" % (cell.row, cell.mergeV, cell.col, cell.mergeH)
                    posn = "%s,%s"%(cell.row, cell.col)
                    if matrix.matrix[posn]:
                        print >> sys.stderr, matrix.matrix[posn]
            else:
                sheet1.write(cell.row,
                             cell.col,
                             unicode(cell.text),
                             style
                             )
    cellWidth = 480 # approximately 2 characters
    if maxCol > 255:
        maxCol = 255
    for col in range(maxCol+1):
        sheet1.col(col).width = cellWidth

    sheetA.write(0, 0, "Question Code")
    sheetA.write(0, 1, "Response Count")
    sheetA.write(0, 2, "Values")
    sheetA.write(0, 3, "Cell Address")
    for cell in matrixAnswers.matrix.values():
        style = mergeStyles(styleList, cell.styleList)
        sheetA.write(cell.row,
                     cell.col,
                     unicode(cell.text),
                     style
                    )

    if logo != None:
        sheet1.insert_bitmap(logo, 0, 0)

    sheet1.protect = True
    sheetA.protect = True
    for i in range(26):
        sheetA.col(i).width = 0
    sheetA.write(0,
                 26,
                 unicode(T("Please do not remove this sheet")),
                 styleHeader
                )
    sheetA.col(26).width = 12000
    book.save(output)
    return output

Example 20

Project: camr
Source File: Aligner.py
View license
    def align_single_concept(self,sent,tokens,cur_var,amr,alignment,unmatched_vars,triples,NEXT=False):
        '''align single concept'''
        
        if cur_var in amr.node_to_concepts:
            cur_concept = amr.node_to_concepts[cur_var]
        else:
            cur_concept = cur_var

        if cur_var in alignment and not NEXT and not isinstance(cur_var,(StrLiteral,Quantity,Polarity)) : # already aligned
            return True, sent,tokens

        match = self.concept_patterns.match(cur_concept)
        if match:
            rule_type = match.lastgroup
            span = None
            update = True
            if rule_type == "NameEntity":
                NE_items = [v[0] for k,v in amr[cur_var].items() if isinstance(v[0],StrLiteral)]
                nep = r'%s|%s'%(r'\s'.join(NE_items),r'\s'.join(n[:4] if len(n) > 3 else n for n in NE_items))
                NE_pattern = re.compile(nep,re.IGNORECASE)
                
                start,end = self._search_sent(NE_pattern,sent,tokens)
                assert end-start == len(NE_items)
                span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],NE_items)
                alignment[cur_var].append(span)
                for k,v in amr[cur_var].items():
                    if isinstance(v[0],StrLiteral):
                        self.remove_aligned_concepts(cur_var,k,v[0],unmatched_vars,triples)

            elif rule_type in ["DateEntity", "haveOrgRole91","RateEntity"]:
                EN_items = []
                EN_spans = []
                for k,v in amr[cur_var].items():                    
                    vconcept = amr.node_to_concepts[v[0]] if v[0] in amr.node_to_concepts else v[0]
                    EN_items.append(vconcept)
                    success, sent, tokens = self.align_single_concept(sent,tokens,v[0],amr,alignment,unmatched_vars,triples)

                    sp = alignment[v[0]][-1]
                    sp.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
                    EN_spans.append(sp)
                    self.remove_aligned_concepts(cur_var,k,v[0],unmatched_vars,triples)
                #print NE_spans,alignment
                start = EN_spans[0].start
                end = EN_spans[-1].end
                span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],EN_items)
                span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
                alignment[cur_var].append(span)

            elif rule_type == "QuantityEntity":
                quantity = ''
                unit = ''
                unit_var = None
                q_success = False
                u_success = False
                                
                for k,v in amr[cur_var].items():
                    if k == 'quant':
                        quantity = v[0]
                        q_success, sent, tokens = self.align_single_concept(sent,tokens,quantity,amr,alignment,unmatched_vars,triples)
                    elif k == 'unit':
                        unit_var = v[0]
                        unit = amr.node_to_concepts[v[0]]
                        u_success, sent, tokens = self.align_single_concept(sent,tokens,unit_var,amr,alignment,unmatched_vars,triples)
                    else:
                        pass
                        
                if q_success and u_success:
                    #QTY_pattern = r'(%s|%s)\s+(%s)s?' % (quantity,english_number(int(quantity)),unit)
                    #QTY_items = [quantity,unit]
                    #start,end = self._search_sent(QTY_pattern,QTY_items,sent,tokens)
                    #assert end - start == len(QTY_items)
                    quantity_span = alignment[quantity][-1]
                    unit_span = alignment[unit_var][0]
                    start = quantity_span.start if quantity_span.start < unit_span.end else unit_span.start
                    end = unit_span.end if quantity_span.start < unit_span.end else quantity_span.end
                    while not (end - len(quantity_span.words)-len(unit_span.words) - start < 2): # wrong match more than one quantity to map in sentence
                        alignment[quantity].pop()
                        q_success, sent, tokens = self.align_single_concept(sent,tokens,quantity,amr,alignment,unmatched_vars,triples,NEXT=True) # redo it on updated sentence
                        quantity_span = alignment[quantity][-1]
                        start = quantity_span.start
                    #assert start == end - 2
                    span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],[quantity,unit])
                    self.remove_aligned_concepts(cur_var,'quant',quantity,unmatched_vars,triples)
                    alignment[cur_var].append(span)
                elif q_success and not u_success: # does not have unit or unit cannot be aligned
                    quantity_span =  alignment[quantity][0]
                    start = quantity_span.start
                    end = quantity_span.end
                    span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],[quantity])
                    self.remove_aligned_concepts(cur_var,'quant',quantity,unmatched_vars,triples)
                    alignment[cur_var].append(span)
                    #self.remove_aligned_concepts(unmatched_vars,amr[cur_var].items())
                elif not q_success and u_success:
                    unit_span = alignment[unit_var][0]
                    span = Span(unit_span.start,unit_span.end,Aligner.ENTITY_TAG_TABLE[rule_type],[unit])
                    self.remove_aligned_concepts(cur_var,'unit',unit_var,unmatched_vars,triples)
                    alignment[cur_var].append(span)
                else:
                    rule_type = 'SingleConcept'
            elif rule_type == "Number":
                '''
                aligned = False
                num = [cur_var]
                num.extend(english_number(int(cur_var)).split('|'))
                for i,token in tokens:
                    if token.lower() in num:
                        aligned = True
                        break
                if aligned:
                    span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE[rule_type],[token])
                    alignment[cur_var].append(span)
                else:
                    print >> sys.stderr, 'Variable/Concept %s/%s cannot be aligned'%(cur_var,cur_concept)
                    update = False
                '''
                if re.match('[0-9]+:[0-9]+',cur_concept):
                    num = [('time','(\\s|^)('+cur_concept+')(\\s|&)'),
                           ('english','(\\s|^)('+to_time(cur_concept)+')(\\s|&)')]
                else:
                    num = [('digit','(\\s|^)('+cur_concept+'|'+format_num(cur_concept)+')(\\s|&)'),
                           ('string','(\\s|^)('+english_number(int(cur_concept))+')(\\s|&)'),
                           ('order','(\\s|^)('+to_order(cur_concept)+')(\\s|&)'),
                           ('round','(\\s|^)('+to_round(int(cur_concept))+')(\\s|&)') 
                       ]
                NUM_pattern = self._compile_regex_rule(num)
                #print NUM_pattern.pattern
                try:
                    start,end = self._search_sent(NUM_pattern,sent,tokens)
                    span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],[w for i,w in tokens if i in range(start,end)])
                    alignment[cur_var].append(span)                
                except Exception as e:
                    update = False
                    print >> sys.stderr,e
                    #raw_input('CONTINUE')
            
            elif rule_type == 'multiple':
                op1 = amr[cur_var]['op1'][0]

                success, sent, tokens = self.align_single_concept(sent,tokens,op1,amr,alignment,unmatched_vars,triples)
                if success:
                    span = alignment[op1][0]
                    alignment[cur_var].append(span)                                    
                    self.remove_aligned_concepts(cur_var,'op1',op1,unmatched_vars,triples)  
                else:
                    update = False
                
            elif rule_type in ["person","picture","country","state","city","desert","organization"]:
                if 'name' in amr[cur_var]:
                    k_var = amr[cur_var]['name'][0]
                    success, sent, tokens = self.align_single_concept(sent,tokens,k_var,amr,alignment,unmatched_vars,triples)
                    span = alignment[k_var][0]
                    span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type+'-name'])
                    alignment[cur_var].append(span)
                else:
                    ind,span = self.try_align_as_single_concept(cur_var,cur_concept,amr,alignment,tokens,unmatched_vars,triples)
                    if ind:
                        pass
                    elif 'ARG0-of' in amr[cur_var]:
                        k_var = amr[cur_var]['ARG0-of'][0]
                        success, sent, tokens = self.align_single_concept(sent,tokens,k_var,amr,alignment,unmatched_vars,triples)
                        if success:
                            span = alignment[k_var][0]
                            span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
                            alignment[cur_var].append(span)
                        else:
                            update = False

                    else:
                        update = False
               


            elif rule_type == "NegPolarity":
                aligned = False
                for i,token in tokens:
                    if token.lower() in Aligner.neg_polarity:
                        aligned = True
                        break
                if aligned:
                    span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE[rule_type],[token])
                    alignment[cur_var].append(span)
                else:
                    print >> sys.stderr, 'Variable/Concept %s/%s cannot be aligned'%(cur_var,cur_concept)
                    update = False

            elif rule_type == "thing":
                if 'ARG1-of' in amr[cur_var]:
                    k_var = amr[cur_var]['ARG1-of'][0]
                    success, sent, tokens = self.align_single_concept(sent,tokens,k_var,amr,alignment,unmatched_vars,triples)
                    if success:
                        span = alignment[k_var][0]
                        span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
                        alignment[cur_var].append(span)
                    else:
                        update = False
                else:
                    rule_type = 'SingleConcept'

            elif rule_type == 'OrdinalEntity':
                val = amr[cur_var]['value'][0]
                success, sent, tokens = self.align_single_concept(sent,tokens,val,amr,alignment,unmatched_vars,triples)
                self.remove_aligned_concepts(cur_var,'value',val,unmatched_vars,triples)
                span = alignment[val][0]
                span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
                alignment[cur_var].append(span)

            elif rule_type == 'RelativePosition':
                if 'direction' in amr[cur_var]:
                    dir_var = amr[cur_var]['direction'][0]
                    if amr.node_to_concepts[dir_var] == 'away':
                        aligned = False
                        for i,tok in tokens:
                            if tok.lower() == 'from':
                                aligned = True
                                break
                        if aligned:
                            span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE[rule_type],[tok])
                            alignment[cur_var].append(span)
                            alignment[dir_var].append(span)
                        else:
                            print >> sys.stderr, 'Variable/Concept %s/%s cannot be aligned'%(cur_var,cur_concept)
                            update = False
                    else:
                        rule_type = 'SingleConcept'
                else:
                    rule_type = 'SingleConcept'
                
            elif self.is_ago(cur_var,cur_concept,amr):
                k_var = amr[cur_var]['op1'][0]
                aligned = False
                for i,tok in tokens:
                    if tok.lower() == 'ago':
                        aligned = True
                        break
                if aligned:
                    span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE['ago'],[tok])
                    alignment[cur_var].append(span)
                    alignment[k_var].append(span)
                else:
                    print >> sys.stderr, '(%s/%s) :op1 (%s/%s) cannot be aligned'%(cur_var,cur_concept,k_var,amr.node_to_concepts[k_var])
                    update = False

            elif self.is_why_question(cur_var,amr):
                arg0_var = amr[cur_var]['ARG0'][0]
                aligned = False
                for i,tok in tokens:
                    if tok.lower() == 'why':
                        aligned = True
                        break
                if aligned:
                    span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE['cause'],[tok])
                    alignment[cur_var].append(span)
                    alignment[arg0_var].append(span)
                else:
                    print >> sys.stderr, '(%s/%s) :op1 (%s/%s) cannot be aligned'%(cur_var,cur_concept,arg0_var,amr.node_to_concepts[arg0_var])
                    update = False
            else:
                pass

            if rule_type == "SingleConcept":
                update,span = self.try_align_as_single_concept(cur_var,cur_concept,amr,alignment,tokens,unmatched_vars,triples)
            elif cur_var in alignment:
                pass
            else:
                print >> sys.stderr, 'Can not find type of concept %s / %s'%(cur_var,cur_concept)

            # update
            #print cur_concept,rule_type
            if update:
                tokens = [(i,tk) for i,tk in tokens if i not in range(span.start,span.end)]
                sent = ' '.join(x for i,x in tokens)
                if self.verbose > 2:
                    print >> sys.stderr, "Concept '%s' Matched to span '%s' "%(cur_concept,' '.join(w for i,w in enumerate(sentence.split()) if i+1 in range(span[0],span[1])))
                    print sent
                    print alignment
                    
                    #raw_input('ENTER to continue')
            return update, sent, tokens

Example 21

Project: haproxy-dconv
Source File: haproxy-dconv.py
View license
def convert(pctxt, infile, outfile, base=''):
    global document, keywords, keywordsCount, chapters, keyword_conflicts

    if len(base) > 0 and base[:-1] != '/':
        base += '/'

    hasSummary = False

    data = []
    fd = file(infile,"r")
    for line in fd:
        line.replace("\t", " " * 8)
        line = line.rstrip()
        data.append(line)
    fd.close()

    parsers = init_parsers(pctxt)

    pctxt.context = {
            'headers':  {},
            'document': "",
            'base':     base,
    }

    sections = []
    currentSection = {
            "details": getTitleDetails(""),
            "content": "",
    }

    chapters = {}

    keywords = {}
    keywordsCount = {}

    specialSections = {
            "default": {
                    "hasKeywords": True,
            },
            "4.1": {
                    "hasKeywords": True,
            },
    }

    pctxt.keywords = keywords
    pctxt.keywordsCount = keywordsCount
    pctxt.chapters = chapters

    print >> sys.stderr, "Importing %s..." % infile

    nblines = len(data)
    i = j = 0
    while i < nblines:
        line = data[i].rstrip()
        if i < nblines - 1:
            next = data[i + 1].rstrip()
        else:
            next = ""
        if (line == "Summary" or re.match("^[0-9].*", line)) and (len(next) > 0) and (next[0] == '-') \
                and ("-" * len(line)).startswith(next):  # Fuzzy underline length detection
            sections.append(currentSection)
            currentSection = {
                "details": getTitleDetails(line),
                "content": "",
            }
            j = 0
            i += 1 # Skip underline
            while not data[i + 1].rstrip():
                i += 1 # Skip empty lines

        else:
            if len(line) > 80:
                print >> sys.stderr, "Line `%i' exceeds 80 columns" % (i + 1)

            currentSection["content"] = currentSection["content"] + line + "\n"
            j += 1
            if currentSection["details"]["title"] == "Summary" and line != "":
                hasSummary = True
                # Learn chapters from the summary
                details = getTitleDetails(line)
                if details["chapter"]:
                    chapters[details["chapter"]] = details
        i += 1
    sections.append(currentSection)

    chapterIndexes = sorted(chapters.keys(), key=lambda chapter: map(int, chapter.split('.')))

    document = ""

    # Complete the summary
    for section in sections:
        details = section["details"]
        title = details["title"]
        if title:
            fulltitle = title
            if details["chapter"]:
                #documentAppend("<a name=\"%s\"></a>" % details["chapter"])
                fulltitle = details["chapter"] + ". " + title
                if not details["chapter"] in chapters:
                    print >> sys.stderr, "Adding '%s' to the summary" % details["title"]
                    chapters[details["chapter"]] = details
                    chapterIndexes = sorted(chapters.keys())

    for section in sections:
        details = section["details"]
        pctxt.details = details
        level = details["level"]
        title = details["title"]
        content = section["content"].rstrip()

        print >> sys.stderr, "Parsing chapter %s..." % title

        if (title == "Summary") or (title and not hasSummary):
            summaryTemplate = pctxt.templates.get_template('summary.html')
            documentAppend(summaryTemplate.render(
                pctxt = pctxt,
                chapters = chapters,
                chapterIndexes = chapterIndexes,
            ))
            if title and not hasSummary:
                hasSummary = True
            else:
                continue

        if title:
            documentAppend('<a class="anchor" id="%s" name="%s"></a>' % (details["chapter"], details["chapter"]))
            if level == 1:
                documentAppend("<div class=\"page-header\">", False)
            documentAppend('<h%d id="chapter-%s" data-target="%s"><small><a class="small" href="#%s">%s.</a></small> %s</h%d>' % (level, details["chapter"], details["chapter"], details["chapter"], details["chapter"], cgi.escape(title, True), level))
            if level == 1:
                documentAppend("</div>", False)

        if content:
            if False and title:
                # Display a navigation bar
                documentAppend('<ul class="well pager">')
                documentAppend('<li><a href="#top">Top</a></li>', False)
                index = chapterIndexes.index(details["chapter"])
                if index > 0:
                    documentAppend('<li class="previous"><a href="#%s">Previous</a></li>' % chapterIndexes[index - 1], False)
                if index < len(chapterIndexes) - 1:
                    documentAppend('<li class="next"><a href="#%s">Next</a></li>' % chapterIndexes[index + 1], False)
                documentAppend('</ul>', False)
            content = cgi.escape(content, True)
            content = re.sub(r'section ([0-9]+(.[0-9]+)*)', r'<a href="#\1">section \1</a>', content)

            pctxt.set_content(content)

            if not title:
                lines = pctxt.get_lines()
                pctxt.context['headers'] = {
                    'title':    '',
                    'subtitle': '',
                    'version':  '',
                    'author':   '',
                    'date':     ''
                }
                if re.match("^-+$", pctxt.get_line().strip()):
                    # Try to analyze the header of the file, assuming it follows
                    # those rules :
                    # - it begins with a "separator line" (several '-' chars)
                    # - then the document title
                    # - an optional subtitle
                    # - a new separator line
                    # - the version
                    # - the author
                    # - the date
                    pctxt.next()
                    pctxt.context['headers']['title'] = pctxt.get_line().strip()
                    pctxt.next()
                    subtitle = ""
                    while not re.match("^-+$", pctxt.get_line().strip()):
                        subtitle += " " + pctxt.get_line().strip()
                        pctxt.next()
                    pctxt.context['headers']['subtitle'] += subtitle.strip()
                    if not pctxt.context['headers']['subtitle']:
                        # No subtitle, try to guess one from the title if it
                        # starts with the word "HAProxy"
                        if pctxt.context['headers']['title'].startswith('HAProxy '):
                            pctxt.context['headers']['subtitle'] = pctxt.context['headers']['title'][8:]
                            pctxt.context['headers']['title'] = 'HAProxy'
                    pctxt.next()
                    pctxt.context['headers']['version'] = pctxt.get_line().strip()
                    pctxt.next()
                    pctxt.context['headers']['author'] = pctxt.get_line().strip()
                    pctxt.next()
                    pctxt.context['headers']['date'] = pctxt.get_line().strip()
                    pctxt.next()
                    if HAPROXY_GIT_VERSION:
                        pctxt.context['headers']['version'] = 'version ' + HAPROXY_GIT_VERSION

                    # Skip header lines
                    pctxt.eat_lines()
                    pctxt.eat_empty_lines()

            documentAppend('<div>', False)

            delay = []
            while pctxt.has_more_lines():
                try:
                    specialSection = specialSections[details["chapter"]]
                except:
                    specialSection = specialSections["default"]

                line = pctxt.get_line()
                if i < nblines - 1:
                    nextline = pctxt.get_line(1)
                else:
                    nextline = ""

                oldline = line
                pctxt.stop = False
                for parser in parsers:
                    line = parser.parse(line)
                    if pctxt.stop:
                        break
                if oldline == line:
                    # nothing has changed,
                    # delays the rendering
                    if delay or line != "":
                        delay.append(line)
                    pctxt.next()
                elif pctxt.stop:
                    while delay and delay[-1].strip() == "":
                        del delay[-1]
                    if delay:
                        remove_indent(delay)
                        documentAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
                    delay = []
                    documentAppend(line, False)
                else:
                    while delay and delay[-1].strip() == "":
                        del delay[-1]
                    if delay:
                        remove_indent(delay)
                        documentAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
                    delay = []
                    documentAppend(line, True)
                    pctxt.next()

            while delay and delay[-1].strip() == "":
                del delay[-1]
            if delay:
                remove_indent(delay)
                documentAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
            delay = []
            documentAppend('</div>')

    if not hasSummary:
        summaryTemplate = pctxt.templates.get_template('summary.html')
        print chapters
        document = summaryTemplate.render(
            pctxt = pctxt,
            chapters = chapters,
            chapterIndexes = chapterIndexes,
        ) + document


    # Log warnings for keywords defined in several chapters
    keyword_conflicts = {}
    for keyword in keywords:
        keyword_chapters = list(keywords[keyword])
        keyword_chapters.sort()
        if len(keyword_chapters) > 1:
            print >> sys.stderr, 'Multi section keyword : "%s" in chapters %s' % (keyword, list(keyword_chapters))
            keyword_conflicts[keyword] = keyword_chapters

    keywords = list(keywords)
    keywords.sort()

    createLinks()

    # Add the keywords conflicts to the keywords list to make them available in the search form
    # And remove the original keyword which is now useless
    for keyword in keyword_conflicts:
        sections = keyword_conflicts[keyword]
        offset = keywords.index(keyword)
        for section in sections:
            keywords.insert(offset, "%s (%s)" % (keyword, chapters[section]['title']))
            offset += 1
        keywords.remove(keyword)

    try:
        footerTemplate = pctxt.templates.get_template('footer.html')
        footer = footerTemplate.render(
            pctxt = pctxt,
            headers = pctxt.context['headers'],
            document = document,
            chapters = chapters,
            chapterIndexes = chapterIndexes,
            keywords = keywords,
            keywordsCount = keywordsCount,
            keyword_conflicts = keyword_conflicts,
            version = VERSION,
            date = datetime.datetime.now().strftime("%Y/%m/%d"),
        )
    except TopLevelLookupException:
        footer = ""

    return {
            'pctxt': pctxt,
            'headers': pctxt.context['headers'],
            'base': base,
            'document': document,
            'chapters': chapters,
            'chapterIndexes': chapterIndexes,
            'keywords': keywords,
            'keywordsCount': keywordsCount,
            'keyword_conflicts': keyword_conflicts,
            'version': VERSION,
            'date': datetime.datetime.now().strftime("%Y/%m/%d"),
            'footer': footer
    }

Example 22

Project: deepcca
Source File: SdA_mapping.py
View license
def test_SdA_regress(finetune_lr=0.05, pretraining_epochs=10,
             pretrain_lr=0.1, training_epochs=10000,
             dataset='mnist.pkl.gz', batch_size=20):
    datasets = load_data_half(dataset)

    train_set_x, train_set_y = datasets[0]##
    valid_set_x, valid_set_y = datasets[1]##
    test_set_x, test_set_y = datasets[2]##
    train_set_x=train_set_x.eval()
    train_set_y=train_set_y.eval()
    import theano
    train_set_x_lab=train_set_x[:,:]
    train_set_x_unlab=train_set_x[:,:]
    train_set_y_lab=train_set_y[:,:]
    train_set_y_unlab=train_set_y[:,:]
    train_set_x_lab=theano.shared(numpy.asarray(train_set_x_lab,
                                                dtype=theano.config.floatX),
                                  borrow=True)
    train_set_y_lab=theano.shared(numpy.asarray(train_set_y_lab,
                                                dtype=theano.config.floatX),
                                  borrow=True)
    train_set_x_unlab=theano.shared(numpy.asarray(train_set_x_unlab,
                                                  dtype=theano.config.floatX),
                                    borrow=True)
    train_set_y_unlab=theano.shared(numpy.asarray(train_set_y_unlab,
                                                  dtype=theano.config.floatX),
                                    borrow=True)

    # compute number of minibatches for training, validation and testing
    n_train_batches_l = train_set_y_lab.eval().shape[0]
    n_train_batches_l /= batch_size
    n_train_batches_u = train_set_y_unlab.eval().shape[0]
    n_train_batches_u /= batch_size
    # compute number of minibatches for training, validation and testing
    #n_train_batches = train_set_x.get_value(borrow=True).shape[0]
    #n_train_batches /= batch_size

    # numpy random generator
    # start-snippet-3
    numpy_rng = numpy.random.RandomState(89677)
    print '... building the model'
    # construct the stacked denoising autoencoder class
    #from SdA_orig import SdA as SdA_old
    hidden_layer_size = 100
    SdA_inp = SdA(numpy_rng,
                  n_ins=392,
                  hidden_layers_sizes=[hidden_layer_size]
    )
    SdA_out = SdA(numpy_rng,
                  n_ins=392,
                  hidden_layers_sizes=[hidden_layer_size]
    )
        
    # PRETRAINING THE MODEL #
    if 0 : # pretrain inp ae
        print '... getting the pretraining functions for INPUT AE'
        pretraining_fns = SdA_inp.pretraining_functions(train_set_x=train_set_x_unlab,
                                                    batch_size=batch_size)
    
        print '... pre-training the model'
        start_time = time.clock()
        ## Pre-train layer-wise
        corruption_levels = [.1, .2, .3]
        for i in xrange(SdA_inp.n_layers):
            # go through pretraining epochs
            for epoch in xrange(pretraining_epochs):
                # go through the training set
                c = []
                for batch_index in xrange(n_train_batches_u):
                    c.append(pretraining_fns[i](index=batch_index,
                             corruption=corruption_levels[i],
                             lr=pretrain_lr))
                print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
                print numpy.mean(c)
    
        end_time = time.clock()
    
        print >> sys.stderr, ('The pretraining code for file ' +
                              os.path.split(__file__)[1] +
                              ' ran for %.2fm' % ((end_time - start_time) / 60.))
    if 0 : # pretrain out ae
        print '... getting the pretraining functions for OUTPUT AE'
        pretraining_fns = SdA_out.pretraining_functions(train_set_x=train_set_y_unlab,
                                                    batch_size=batch_size)
    
        print '... pre-training the model'
        start_time = time.clock()
        ## Pre-train layer-wise
        corruption_levels = [.5, .2, .3]
        for i in xrange(SdA_out.n_layers):
            # go through pretraining epochs
            for epoch in xrange(pretraining_epochs):
                # go through the training set
                c = []
                for batch_index in xrange(n_train_batches_u):
                    c.append(pretraining_fns[i](index=batch_index,
                             corruption=corruption_levels[i],
                             lr=pretrain_lr))
                print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
                print numpy.mean(c)
    
        end_time = time.clock()
    
        print >> sys.stderr, ('The pretraining code for file ' +
                              os.path.split(__file__)[1] +
                              ' ran for %.2fm' % ((end_time - start_time) / 60.))
    
        
    if 0: # save aes
        f=open('aes_shallow_sig_nobias.pkl', 'w+')
        import pickle
        pickle.dump(SdA_inp, f)
        pickle.dump(SdA_out, f)
        f.flush()
        f.close() 
    if 0: # load aes
        f=open('aes_shallow_sig_nobias.pkl', 'r')
        import pickle
        SdA_inp=pickle.load(f)
        SdA_out=pickle.load(f)
        f.close()    
   
    if 1: # cca
        from dcca_numpy import netCCA_nobias, netCCA, dCCA
        from mlp_numpy import expit, logistic_prime, linear, linear_prime, relu, relu_prime, tanh, tanh_prime
        train_y1 = train_set_x_lab.eval()
        train_y2 = train_set_y_lab.eval()
        test_y1 = test_set_x.eval()
        test_y2 = test_set_y.eval()

        ##param1=((train_y1.shape[1],0,0),(2038, relu, relu_prime),(50, relu, relu_prime))
        ##param2=((train_y2.shape[1],0,0),(1608, relu, relu_prime),(50, relu, relu_prime))
        param1=((train_y1.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
        param2=((train_y2.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
        W1s = []
        b1s = []
        for i in range(len(SdA_inp.dA_layers)):
            W1s.append( SdA_inp.dA_layers[i].W.T.eval() )
            ##b1s.append( SdA_inp.dA_layers[i].b.eval() )
            ##b1s[-1] = b1s[-1].reshape((b1s[-1].shape[0], 1))
        W2s = []
        b2s = []
        for i in range(len(SdA_out.dA_layers)):
            W2s.append( SdA_out.dA_layers[i].W.T.eval() )
            ##b2s.append( SdA_out.dA_layers[i].b.eval() )
            ##b2s[-1] = b2s[-1].reshape((b2s[-1].shape[0], 1))

        numpy.random.seed(0)
        N1=netCCA_nobias(train_y1,param1, W1s)
        N2=netCCA_nobias(train_y2,param2, W2s)
        N = dCCA(train_y1, train_y2, N1, N2)
        N1.reconstruct(test_set_x.eval()[0,:])
        cnt = 0
        from dcca_numpy import cca_cost, cca, order_cost, cor_cost
        while True:
            X=N1.predict(test_set_x.eval())
            Y=N2.predict(test_set_y.eval())
            _H1 = numpy.dot(X, N.A1)
            _H2 = numpy.dot(Y, N.A2)
            print '****', cnt, cor_cost(_H1, _H2)
            X1_rec = numpy.tanh(X.dot(N1.weights[0]))
            X2_rec = numpy.tanh(Y.dot(N2.weights[0]))
            param=((hidden_layer_size,0,0),(hidden_layer_size, relu, relu_prime))
            from mlp_numpy import NeuralNetwork as NN

            lr=NN(X,Y,param)
            lr.train(X[:,:],Y[:,:],10, 0.005)
            Yh=lr.predict(X[:,:])
            X2_reg = N2.fs[-1](numpy.dot(Yh,N2.weights[0]))

            #X2_reg = N2.fs[-1](numpy.dot(_H1.dot(numpy.linalg.inv(N.A1)),N2.weights[0]))

            print '****', 'mse1:', numpy.mean((X1_rec-test_set_x.eval())**2.0)
            print '****', 'mse2:', numpy.mean((X2_rec-test_set_y.eval())**2.0)
            print '****', 'mse_map:', numpy.mean((X2_reg-test_set_y.eval())**2.0)

            if cnt % 2:
                N.train(5, True, 10000.0)
            else:
                N.train(5, False, 10000.0)

            cnt += 1
            f=open('netcca.pkl', 'w+')
            import pickle
            pickle.dump(N, f)
            pickle.dump(N, f)
            f.flush()
            f.close() 
            if cnt == 200:
                break
        for i in range(len(SdA_inp.dA_layers)):
            SdA_inp.dA_layers[i].W = theano.shared( N1.weights[i].T )
            SdA_inp.dA_layers[i].b = theano.shared( N1.biases[i][:,0] )
        
        for i in range(len(SdA_out.dA_layers)):
            SdA_out.dA_layers[i].W = theano.shared( N2.weights[i].T )
            SdA_out.dA_layers[i].b = theano.shared( N2.weights[i][:,0] )

        
    if 1 : # pretrain middle layer
        print '... pre-training MIDDLE layer'

        h1 = T.matrix('x')  # the data is presented as rasterized images
        h2 = T.matrix('y')  # the labels are presented as 1D vector of
        log_reg = HiddenLayer(numpy_rng, h1, hidden_layer_size, hidden_layer_size)

        if 1: # for middle layer
            learning_rate = 0.01
            fprop_inp = theano.function(
                [],
                SdA_inp.sigmoid_layers[-1].output,
                givens={
                    SdA_inp.sigmoid_layers[0].input: train_set_x_lab
                },
                name='fprop_inp'
            )
            fprop_out = theano.function(
                [],
                SdA_out.sigmoid_layers[-1].output,
                givens={
                    SdA_out.sigmoid_layers[0].input: train_set_y_lab
                },
                name='fprop_out'
            )
            #H11=fprop_inp() 
            #H21=fprop_out()
            ##H1=N1.predict(train_set_x.eval())
            ##H2=N2.predict(train_set_y.eval())
            H1=fprop_inp()
            H2=fprop_out()
            H1=theano.shared(H1)
            H2=theano.shared(H2)
            # compute the gradients with respect to the model parameters
            logreg_cost = log_reg.mse(h2)

            gparams = T.grad(logreg_cost, log_reg.params)
    
            # compute list of fine-tuning updates
            updates = [
                (param, param - gparam * learning_rate)
                for param, gparam in zip(log_reg.params, gparams)
            ]

            train_fn_middle = theano.function(
                inputs=[],
                outputs=logreg_cost,
                updates=updates,
                givens={
                    h1: H1,
                    h2: H2
                },
                name='train_middle'
            )
        epoch = 0
        while epoch < 10:
            print epoch, train_fn_middle()
            epoch += 1
            
    sda = SdA_regress(
        SdA_inp,
        SdA_out,
        log_reg,
        numpy_rng=numpy_rng,
        n_inp=28*28//2,
        hidden_layers_sizes_inp=[hidden_layer_size],
        hidden_layers_sizes_out=[hidden_layer_size],
        n_out=28*28//2
    )
    # end-snippet-3 start-snippet-4
    # end-snippet-4
    
    # FINETUNING THE MODEL #

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_model, test_model = sda.build_finetune_functions(
        datasets=datasets,
        batch_size=batch_size,
        learning_rate=finetune_lr
    )
    
        
    print '... finetunning the model'
    # early-stopping parameters
    patience = 10 * n_train_batches_l  # look as this many examples regardless
    patience_increase = 2.  # wait this much longer when a new best is
                            # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches_l, patience / 2)
                                  # go through this many
                                  # minibatche before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = time.clock()

    done_looping = False
    epoch = 0
    fprop = theano.function(
        [],
        sda.sigmoid_layers[-1].output,
        givens={
            sda.x: test_set_x
        },
        name='fprop'
    )
    while True:
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches_l):
            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches_l + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                validation_losses = validate_model()
                this_validation_loss = numpy.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches_l,
                       this_validation_loss ))

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if (
                        this_validation_loss < best_validation_loss *
                        improvement_threshold
                    ):
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = test_model()
                    test_score = numpy.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches_l,
                           test_score ))

            if patience <= iter:
                done_looping = True
                #break
            if 0: # vis weights
                fprop = theano.function(
                    [],
                    sda.sigmoid_layers[-1].output,
                    givens={
                        sda.x: test_set_x
                    },
                    name='fprop'
                )
                yh=fprop()
                yh=yh
    end_time = time.clock()
    print(
        (
            'Optimization complete with best validation score of %f %%, '
            'on iteration %i, '
            'with test performance %f %%'
        )
        % (best_validation_loss , best_iter + 1, test_score)
    )
    print >> sys.stderr, ('The training code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))

Example 23

Project: cgat
Source File: Experiment.py
View license
def Start(parser=None,
          argv=sys.argv,
          quiet=False,
          no_parsing=False,
          add_csv_options=False,
          add_database_options=False,
          add_pipe_options=True,
          add_cluster_options=False,
          add_output_options=False,
          return_parser=False):
    """set up an experiment.

    The :py:func:`Start` method will set up a file logger and add some
    default and some optional options to the command line parser.  It
    will then parse the command line and set up input/output
    redirection and start a timer for benchmarking purposes.

    The default options added by this method are:

    ``-v/--verbose``
        the :term:`loglevel`

    ``timeit``
        turn on benchmarking information and save to file

    ``timeit-name``
         name to use for timing information,

    ``timeit-header``
         output header for timing information.

    ``seed``
         the random seed. If given, the python random
         number generator will be initialized with this
         seed.

    Optional options added are:

    add_csv_options

       ``dialect``
            csv_dialect. the default is ``excel-tab``, defaulting to
            :term:`tsv` formatted files.

    add_database_options
       ``-C/--connection``
           psql connection string
       ``-U/--user``
           psql user name

    add_cluster_options
       ``--use-cluster``
           use cluster
       ``--cluster-priority``
           cluster priority to request
       ``--cluster-queue-manager``
           cluster queue manager to use
       ``--cluster-queue``
           cluster queue to use
       ``--cluster-num-jobs``
           number of jobs to submit to the cluster at the same time
       ``--cluster-memory-resource``
           name of the cluster memory resource (SGE specific option)
       ``--cluster-memory-default``
           default amount of memory allocated per job.
       ``--cluster-options``
           additional options to the cluster for each job.

    add_output_options
       ``-P/--output-filename-pattern``
            Pattern to use for output filenames.

    Arguments
    ---------

    param parser : :py:class:`E.OptionParser`
       instance with command line options.

    argv : list
        command line options to parse. Defaults to
        :py:data:`sys.argv`

    quiet : bool
        set :term:`loglevel` to 0 - no logging

    no_parsing : bool
        do not parse command line options

    return_parser : bool
        return the parser object, no parsing. Useful for inspecting
        the command line options of a script without running it.

    add_csv_options : bool
        add common options for parsing :term:`tsv` separated files.

    add_database_options : bool
        add common options for connecting to various databases.

    add_pipe_options : bool
        add common options for redirecting input/output

    add_cluster_options : bool
        add common options for scripts submitting jobs to the cluster

    add_output_options : bool
        add commond options for working with multiple output files

    Returns
    -------
    tuple
       (:py:class:`E.OptionParser` object, list of positional
       arguments)

    """

    if not parser:
        parser = OptionParser(
            version="%prog version: $Id$")

    global global_options, global_args, global_starting_time

    # save default values given by user
    user_defaults = copy.copy(parser.defaults)

    global_starting_time = time.time()

    group = OptionGroup(parser, "Script timing options")

    group.add_option("--timeit", dest='timeit_file', type="string",
                     help="store timeing information in file [%default].")
    group.add_option("--timeit-name", dest='timeit_name', type="string",
                     help="name in timing file for this class of jobs "
                     "[%default].")
    group.add_option("--timeit-header", dest='timeit_header',
                     action="store_true",
                     help="add header for timing information [%default].")
    parser.add_option_group(group)

    group = OptionGroup(parser, "Common options")

    group.add_option("--random-seed", dest='random_seed', type="int",
                     help="random seed to initialize number generator "
                     "with [%default].")

    group.add_option("-v", "--verbose", dest="loglevel", type="int",
                     help="loglevel [%default]. The higher, the more output.")

    group.add_option("-?", dest="short_help", action="callback",
                     callback=callbackShortHelp,
                     help="output short help (command line options only.")

    parser.add_option_group(group)

    if quiet:
        parser.set_defaults(loglevel=0)
    else:
        parser.set_defaults(loglevel=1)

    parser.set_defaults(
        timeit_file=None,
        timeit_name='all',
        timeit_header=None,
        random_seed=None,
    )

    if add_csv_options:
        parser.add_option("--csv-dialect", dest="csv_dialect", type="string",
                          help="csv dialect to use [%default].")

        parser.set_defaults(
            csv_dialect="excel-tab",
            csv_lineterminator="\n",
        )

    if add_cluster_options:
        group = OptionGroup(parser, "cluster options")
        group.add_option("--no-cluster", "--local", dest="without_cluster",
                         action="store_true",
                         help="do no use cluster - run locally [%default].")
        group.add_option("--cluster-priority", dest="cluster_priority",
                         type="int",
                         help="set job priority on cluster [%default].")
        group.add_option("--cluster-queue-manager", dest="cluster_queue_manager",
                         type="string",
                         help="set cluster queue manager [%default].")
        group.add_option("--cluster-queue", dest="cluster_queue",
                         type="string",
                         help="set cluster queue [%default].")
        group.add_option("--cluster-memory-resource", dest="cluster_memory_resource",
                         type="string",
                         help="set cluster memory resource [%default].")
        group.add_option("--cluster-memory-default", dest="cluster_memory_default",
                         type="string",
                         help="set cluster memory default [%default].")
        group.add_option("--cluster-num-jobs", dest="cluster_num_jobs",
                         type="int",
                         help="number of jobs to submit to the queue execute "
                         "in parallel [%default].")
        group.add_option("--cluster-parallel",
                         dest="cluster_parallel_environment",
                         type="string",
                         help="name of the parallel environment to use "
                         "[%default].")
        group.add_option("--cluster-options", dest="cluster_options",
                         type="string",
                         help="additional options for cluster jobs, passed "
                         "on to queuing system [%default].")

        parser.set_defaults(without_cluster=False,
                            cluster_queue_manager=None,
                            cluster_queue=None,
                            cluster_memory_resource=None,
                            cluster_memory_default=None,
                            cluster_priority=None,
                            cluster_num_jobs=None,
                            cluster_parallel_environment=None,
                            cluster_options=None)
        parser.add_option_group(group)

    if add_output_options or add_pipe_options:
        group = OptionGroup(parser, "Input/output options")

        if add_output_options:
            group.add_option(
                "-P", "--output-filename-pattern",
                dest="output_filename_pattern", type="string",
                help="OUTPUT filename pattern for various methods "
                "[%default].")

            group.add_option("-F", "--force-output", dest="output_force",
                             action="store_true",
                             help="force over-writing of existing files.")

            parser.set_defaults(output_filename_pattern="%s",
                                output_force=False)

        if add_pipe_options:

            group.add_option("-I", "--stdin", dest="stdin", type="string",
                             help="file to read stdin from [default = stdin].",
                             metavar="FILE")
            group.add_option("-L", "--log", dest="stdlog", type="string",
                             help="file with logging information "
                             "[default = stdout].",
                             metavar="FILE")
            group.add_option("-E", "--error", dest="stderr", type="string",
                             help="file with error information "
                             "[default = stderr].",
                             metavar="FILE")
            group.add_option("-S", "--stdout", dest="stdout", type="string",
                             help="file where output is to go "
                             "[default = stdout].",
                             metavar="FILE")

            parser.set_defaults(stderr=sys.stderr)
            parser.set_defaults(stdout=sys.stdout)
            parser.set_defaults(stdlog=sys.stdout)
            parser.set_defaults(stdin=sys.stdin)

        parser.add_option_group(group)

    if add_database_options:
        group = OptionGroup(parser, "Database connection options")
        group.add_option(
            "--database-backend", dest="database_backend", type="choice",
            choices=("sqlite", "mysql", "postgres"),
            help="database backend [%default].")
        group.add_option(
            "--database-host", dest="database_host", type="string",
            help="database host [%default].")
        group.add_option(
            "--database-name", dest="database_name", type="string",
            help="name of the database [%default].")
        group.add_option(
            "--database-username", dest="database_username", type="string",
            help="database username [%default].")
        group.add_option(
            "--database-password", dest="database_password", type="string",
            help="database password [%default].")
        group.add_option(
            "--database-port", dest="database_port", type="int",
            help="database port [%default].")

        parser.set_defaults(
            database_backend="sqlite",
            database_name="csvdb",
            database_host="",
            database_port=3306,
            database_username="",
            database_password="")
        parser.add_option_group(group)

    # restore user defaults
    parser.defaults.update(user_defaults)

    if return_parser:
        return parser

    if not no_parsing:
        (global_options, global_args) = parser.parse_args(argv[1:])

    if global_options.random_seed is not None:
        random.seed(global_options.random_seed)

    if add_pipe_options:
        if global_options.stdout != sys.stdout:
            global_options.stdout = openFile(global_options.stdout, "w")
        if global_options.stderr != sys.stderr:
            if global_options.stderr == "stderr":
                global_options.stderr = global_options.stderr
            else:
                global_options.stderr = openFile(global_options.stderr, "w")
        if global_options.stdlog != sys.stdout:
            global_options.stdlog = openFile(global_options.stdlog, "a")
        if global_options.stdin != sys.stdin:
            global_options.stdin = openFile(global_options.stdin, "r")
    else:
        global_options.stderr = sys.stderr
        global_options.stdout = sys.stdout
        global_options.stdlog = sys.stdout
        global_options.stdin = sys.stdin

    if global_options.loglevel >= 1:
        global_options.stdlog.write(getHeader() + "\n")
        global_options.stdlog.write(getParams(global_options) + "\n")
        global_options.stdlog.flush()

    # configure logging
    # map from 0-10 to logging scale
    # 0: quiet
    # 1: little verbositiy
    # >1: increased verbosity
    if global_options.loglevel == 0:
        lvl = logging.ERROR
    elif global_options.loglevel == 1:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG

    if global_options.stdout == global_options.stdlog:
        format = '# %(asctime)s %(levelname)s %(message)s'
    else:
        format = '%(asctime)s %(levelname)s %(message)s'

    logging.basicConfig(
        level=lvl,
        format=format,
        stream=global_options.stdlog)

    # set up multi-line logging
    # Note that .handlers is not part of the API, might change
    # Solution is to configure handlers explicitely.
    for handler in logging.getLogger().handlers:
        handler.setFormatter(MultiLineFormatter(format))

    return global_options, global_args

Example 24

Project: colorclass
Source File: example.py
View license
def main():
    """Main function called upon script execution."""
    if OPTIONS.get('--no-colors'):
        disable_all_colors()
    elif OPTIONS.get('--colors'):
        enable_all_colors()

    if is_enabled() and os.name == 'nt':
        Windows.enable(auto_colors=True, reset_atexit=True)
    elif OPTIONS.get('--light-bg'):
        set_light_background()
    elif OPTIONS.get('--dark-bg'):
        set_dark_background()

    # Light or dark colors.
    print('Autocolors for all backgrounds:')
    print(Color('    {autoblack}Black{/black} {autored}Red{/red} {autogreen}Green{/green} '), end='')
    print(Color('{autoyellow}Yellow{/yellow} {autoblue}Blue{/blue} {automagenta}Magenta{/magenta} '), end='')
    print(Color('{autocyan}Cyan{/cyan} {autowhite}White{/white}'))

    print(Color('    {autobgblack}{autoblack}Black{/black}{/bgblack} '), end='')
    print(Color('{autobgblack}{autored}Red{/red}{/bgblack} {autobgblack}{autogreen}Green{/green}{/bgblack} '), end='')
    print(Color('{autobgblack}{autoyellow}Yellow{/yellow}{/bgblack} '), end='')
    print(Color('{autobgblack}{autoblue}Blue{/blue}{/bgblack} '), end='')
    print(Color('{autobgblack}{automagenta}Magenta{/magenta}{/bgblack} '), end='')
    print(Color('{autobgblack}{autocyan}Cyan{/cyan}{/bgblack} {autobgblack}{autowhite}White{/white}{/bgblack}'))

    print(Color('    {autobgred}{autoblack}Black{/black}{/bgred} {autobgred}{autored}Red{/red}{/bgred} '), end='')
    print(Color('{autobgred}{autogreen}Green{/green}{/bgred} {autobgred}{autoyellow}Yellow{/yellow}{/bgred} '), end='')
    print(Color('{autobgred}{autoblue}Blue{/blue}{/bgred} {autobgred}{automagenta}Magenta{/magenta}{/bgred} '), end='')
    print(Color('{autobgred}{autocyan}Cyan{/cyan}{/bgred} {autobgred}{autowhite}White{/white}{/bgred}'))

    print(Color('    {autobggreen}{autoblack}Black{/black}{/bggreen} '), end='')
    print(Color('{autobggreen}{autored}Red{/red}{/bggreen} {autobggreen}{autogreen}Green{/green}{/bggreen} '), end='')
    print(Color('{autobggreen}{autoyellow}Yellow{/yellow}{/bggreen} '), end='')
    print(Color('{autobggreen}{autoblue}Blue{/blue}{/bggreen} '), end='')
    print(Color('{autobggreen}{automagenta}Magenta{/magenta}{/bggreen} '), end='')
    print(Color('{autobggreen}{autocyan}Cyan{/cyan}{/bggreen} {autobggreen}{autowhite}White{/white}{/bggreen}'))

    print(Color('    {autobgyellow}{autoblack}Black{/black}{/bgyellow} '), end='')
    print(Color('{autobgyellow}{autored}Red{/red}{/bgyellow} '), end='')
    print(Color('{autobgyellow}{autogreen}Green{/green}{/bgyellow} '), end='')
    print(Color('{autobgyellow}{autoyellow}Yellow{/yellow}{/bgyellow} '), end='')
    print(Color('{autobgyellow}{autoblue}Blue{/blue}{/bgyellow} '), end='')
    print(Color('{autobgyellow}{automagenta}Magenta{/magenta}{/bgyellow} '), end='')
    print(Color('{autobgyellow}{autocyan}Cyan{/cyan}{/bgyellow} {autobgyellow}{autowhite}White{/white}{/bgyellow}'))

    print(Color('    {autobgblue}{autoblack}Black{/black}{/bgblue} {autobgblue}{autored}Red{/red}{/bgblue} '), end='')
    print(Color('{autobgblue}{autogreen}Green{/green}{/bgblue} '), end='')
    print(Color('{autobgblue}{autoyellow}Yellow{/yellow}{/bgblue} {autobgblue}{autoblue}Blue{/blue}{/bgblue} '), end='')
    print(Color('{autobgblue}{automagenta}Magenta{/magenta}{/bgblue} '), end='')
    print(Color('{autobgblue}{autocyan}Cyan{/cyan}{/bgblue} {autobgblue}{autowhite}White{/white}{/bgblue}'))

    print(Color('    {autobgmagenta}{autoblack}Black{/black}{/bgmagenta} '), end='')
    print(Color('{autobgmagenta}{autored}Red{/red}{/bgmagenta} '), end='')
    print(Color('{autobgmagenta}{autogreen}Green{/green}{/bgmagenta} '), end='')
    print(Color('{autobgmagenta}{autoyellow}Yellow{/yellow}{/bgmagenta} '), end='')
    print(Color('{autobgmagenta}{autoblue}Blue{/blue}{/bgmagenta} '), end='')
    print(Color('{autobgmagenta}{automagenta}Magenta{/magenta}{/bgmagenta} '), end='')
    print(Color('{autobgmagenta}{autocyan}Cyan{/cyan}{/bgmagenta} '), end='')
    print(Color('{autobgmagenta}{autowhite}White{/white}{/bgmagenta}'))

    print(Color('    {autobgcyan}{autoblack}Black{/black}{/bgcyan} {autobgcyan}{autored}Red{/red}{/bgcyan} '), end='')
    print(Color('{autobgcyan}{autogreen}Green{/green}{/bgcyan} '), end='')
    print(Color('{autobgcyan}{autoyellow}Yellow{/yellow}{/bgcyan} {autobgcyan}{autoblue}Blue{/blue}{/bgcyan} '), end='')
    print(Color('{autobgcyan}{automagenta}Magenta{/magenta}{/bgcyan} '), end='')
    print(Color('{autobgcyan}{autocyan}Cyan{/cyan}{/bgcyan} {autobgcyan}{autowhite}White{/white}{/bgcyan}'))

    print(Color('    {autobgwhite}{autoblack}Black{/black}{/bgwhite} '), end='')
    print(Color('{autobgwhite}{autored}Red{/red}{/bgwhite} {autobgwhite}{autogreen}Green{/green}{/bgwhite} '), end='')
    print(Color('{autobgwhite}{autoyellow}Yellow{/yellow}{/bgwhite} '), end='')
    print(Color('{autobgwhite}{autoblue}Blue{/blue}{/bgwhite} '), end='')
    print(Color('{autobgwhite}{automagenta}Magenta{/magenta}{/bgwhite} '), end='')
    print(Color('{autobgwhite}{autocyan}Cyan{/cyan}{/bgwhite} {autobgwhite}{autowhite}White{/white}{/bgwhite}'))
    print()

    # Light colors.
    print('Light colors for dark backgrounds:')
    print(Color('    {hiblack}Black{/black} {hired}Red{/red} {higreen}Green{/green} '), end='')
    print(Color('{hiyellow}Yellow{/yellow} {hiblue}Blue{/blue} {himagenta}Magenta{/magenta} '), end='')
    print(Color('{hicyan}Cyan{/cyan} {hiwhite}White{/white}'))

    print(Color('    {hibgblack}{hiblack}Black{/black}{/bgblack} '), end='')
    print(Color('{hibgblack}{hired}Red{/red}{/bgblack} {hibgblack}{higreen}Green{/green}{/bgblack} '), end='')
    print(Color('{hibgblack}{hiyellow}Yellow{/yellow}{/bgblack} '), end='')
    print(Color('{hibgblack}{hiblue}Blue{/blue}{/bgblack} '), end='')
    print(Color('{hibgblack}{himagenta}Magenta{/magenta}{/bgblack} '), end='')
    print(Color('{hibgblack}{hicyan}Cyan{/cyan}{/bgblack} {hibgblack}{hiwhite}White{/white}{/bgblack}'))

    print(Color('    {hibgred}{hiblack}Black{/black}{/bgred} {hibgred}{hired}Red{/red}{/bgred} '), end='')
    print(Color('{hibgred}{higreen}Green{/green}{/bgred} {hibgred}{hiyellow}Yellow{/yellow}{/bgred} '), end='')
    print(Color('{hibgred}{hiblue}Blue{/blue}{/bgred} {hibgred}{himagenta}Magenta{/magenta}{/bgred} '), end='')
    print(Color('{hibgred}{hicyan}Cyan{/cyan}{/bgred} {hibgred}{hiwhite}White{/white}{/bgred}'))

    print(Color('    {hibggreen}{hiblack}Black{/black}{/bggreen} '), end='')
    print(Color('{hibggreen}{hired}Red{/red}{/bggreen} {hibggreen}{higreen}Green{/green}{/bggreen} '), end='')
    print(Color('{hibggreen}{hiyellow}Yellow{/yellow}{/bggreen} '), end='')
    print(Color('{hibggreen}{hiblue}Blue{/blue}{/bggreen} '), end='')
    print(Color('{hibggreen}{himagenta}Magenta{/magenta}{/bggreen} '), end='')
    print(Color('{hibggreen}{hicyan}Cyan{/cyan}{/bggreen} {hibggreen}{hiwhite}White{/white}{/bggreen}'))

    print(Color('    {hibgyellow}{hiblack}Black{/black}{/bgyellow} '), end='')
    print(Color('{hibgyellow}{hired}Red{/red}{/bgyellow} '), end='')
    print(Color('{hibgyellow}{higreen}Green{/green}{/bgyellow} '), end='')
    print(Color('{hibgyellow}{hiyellow}Yellow{/yellow}{/bgyellow} '), end='')
    print(Color('{hibgyellow}{hiblue}Blue{/blue}{/bgyellow} '), end='')
    print(Color('{hibgyellow}{himagenta}Magenta{/magenta}{/bgyellow} '), end='')
    print(Color('{hibgyellow}{hicyan}Cyan{/cyan}{/bgyellow} {hibgyellow}{hiwhite}White{/white}{/bgyellow}'))

    print(Color('    {hibgblue}{hiblack}Black{/black}{/bgblue} {hibgblue}{hired}Red{/red}{/bgblue} '), end='')
    print(Color('{hibgblue}{higreen}Green{/green}{/bgblue} '), end='')
    print(Color('{hibgblue}{hiyellow}Yellow{/yellow}{/bgblue} {hibgblue}{hiblue}Blue{/blue}{/bgblue} '), end='')
    print(Color('{hibgblue}{himagenta}Magenta{/magenta}{/bgblue} '), end='')
    print(Color('{hibgblue}{hicyan}Cyan{/cyan}{/bgblue} {hibgblue}{hiwhite}White{/white}{/bgblue}'))

    print(Color('    {hibgmagenta}{hiblack}Black{/black}{/bgmagenta} '), end='')
    print(Color('{hibgmagenta}{hired}Red{/red}{/bgmagenta} '), end='')
    print(Color('{hibgmagenta}{higreen}Green{/green}{/bgmagenta} '), end='')
    print(Color('{hibgmagenta}{hiyellow}Yellow{/yellow}{/bgmagenta} '), end='')
    print(Color('{hibgmagenta}{hiblue}Blue{/blue}{/bgmagenta} '), end='')
    print(Color('{hibgmagenta}{himagenta}Magenta{/magenta}{/bgmagenta} '), end='')
    print(Color('{hibgmagenta}{hicyan}Cyan{/cyan}{/bgmagenta} '), end='')
    print(Color('{hibgmagenta}{hiwhite}White{/white}{/bgmagenta}'))

    print(Color('    {hibgcyan}{hiblack}Black{/black}{/bgcyan} {hibgcyan}{hired}Red{/red}{/bgcyan} '), end='')
    print(Color('{hibgcyan}{higreen}Green{/green}{/bgcyan} '), end='')
    print(Color('{hibgcyan}{hiyellow}Yellow{/yellow}{/bgcyan} {hibgcyan}{hiblue}Blue{/blue}{/bgcyan} '), end='')
    print(Color('{hibgcyan}{himagenta}Magenta{/magenta}{/bgcyan} '), end='')
    print(Color('{hibgcyan}{hicyan}Cyan{/cyan}{/bgcyan} {hibgcyan}{hiwhite}White{/white}{/bgcyan}'))

    print(Color('    {hibgwhite}{hiblack}Black{/black}{/bgwhite} '), end='')
    print(Color('{hibgwhite}{hired}Red{/red}{/bgwhite} {hibgwhite}{higreen}Green{/green}{/bgwhite} '), end='')
    print(Color('{hibgwhite}{hiyellow}Yellow{/yellow}{/bgwhite} '), end='')
    print(Color('{hibgwhite}{hiblue}Blue{/blue}{/bgwhite} '), end='')
    print(Color('{hibgwhite}{himagenta}Magenta{/magenta}{/bgwhite} '), end='')
    print(Color('{hibgwhite}{hicyan}Cyan{/cyan}{/bgwhite} {hibgwhite}{hiwhite}White{/white}{/bgwhite}'))
    print()

    # Dark colors.
    print('Dark colors for light backgrounds:')
    print(Color('    {black}Black{/black} {red}Red{/red} {green}Green{/green} {yellow}Yellow{/yellow} '), end='')
    print(Color('{blue}Blue{/blue} {magenta}Magenta{/magenta} {cyan}Cyan{/cyan} {white}White{/white}'))

    print(Color('    {bgblack}{black}Black{/black}{/bgblack} {bgblack}{red}Red{/red}{/bgblack} '), end='')
    print(Color('{bgblack}{green}Green{/green}{/bgblack} {bgblack}{yellow}Yellow{/yellow}{/bgblack} '), end='')
    print(Color('{bgblack}{blue}Blue{/blue}{/bgblack} {bgblack}{magenta}Magenta{/magenta}{/bgblack} '), end='')
    print(Color('{bgblack}{cyan}Cyan{/cyan}{/bgblack} {bgblack}{white}White{/white}{/bgblack}'))

    print(Color('    {bgred}{black}Black{/black}{/bgred} {bgred}{red}Red{/red}{/bgred} '), end='')
    print(Color('{bgred}{green}Green{/green}{/bgred} {bgred}{yellow}Yellow{/yellow}{/bgred} '), end='')
    print(Color('{bgred}{blue}Blue{/blue}{/bgred} {bgred}{magenta}Magenta{/magenta}{/bgred} '), end='')
    print(Color('{bgred}{cyan}Cyan{/cyan}{/bgred} {bgred}{white}White{/white}{/bgred}'))

    print(Color('    {bggreen}{black}Black{/black}{/bggreen} {bggreen}{red}Red{/red}{/bggreen} '), end='')
    print(Color('{bggreen}{green}Green{/green}{/bggreen} {bggreen}{yellow}Yellow{/yellow}{/bggreen} '), end='')
    print(Color('{bggreen}{blue}Blue{/blue}{/bggreen} {bggreen}{magenta}Magenta{/magenta}{/bggreen} '), end='')
    print(Color('{bggreen}{cyan}Cyan{/cyan}{/bggreen} {bggreen}{white}White{/white}{/bggreen}'))

    print(Color('    {bgyellow}{black}Black{/black}{/bgyellow} {bgyellow}{red}Red{/red}{/bgyellow} '), end='')
    print(Color('{bgyellow}{green}Green{/green}{/bgyellow} {bgyellow}{yellow}Yellow{/yellow}{/bgyellow} '), end='')
    print(Color('{bgyellow}{blue}Blue{/blue}{/bgyellow} {bgyellow}{magenta}Magenta{/magenta}{/bgyellow} '), end='')
    print(Color('{bgyellow}{cyan}Cyan{/cyan}{/bgyellow} {bgyellow}{white}White{/white}{/bgyellow}'))

    print(Color('    {bgblue}{black}Black{/black}{/bgblue} {bgblue}{red}Red{/red}{/bgblue} '), end='')
    print(Color('{bgblue}{green}Green{/green}{/bgblue} {bgblue}{yellow}Yellow{/yellow}{/bgblue} '), end='')
    print(Color('{bgblue}{blue}Blue{/blue}{/bgblue} {bgblue}{magenta}Magenta{/magenta}{/bgblue} '), end='')
    print(Color('{bgblue}{cyan}Cyan{/cyan}{/bgblue} {bgblue}{white}White{/white}{/bgblue}'))

    print(Color('    {bgmagenta}{black}Black{/black}{/bgmagenta} {bgmagenta}{red}Red{/red}{/bgmagenta} '), end='')
    print(Color('{bgmagenta}{green}Green{/green}{/bgmagenta} {bgmagenta}{yellow}Yellow{/yellow}{/bgmagenta} '), end='')
    print(Color('{bgmagenta}{blue}Blue{/blue}{/bgmagenta} {bgmagenta}{magenta}Magenta{/magenta}{/bgmagenta} '), end='')
    print(Color('{bgmagenta}{cyan}Cyan{/cyan}{/bgmagenta} {bgmagenta}{white}White{/white}{/bgmagenta}'))

    print(Color('    {bgcyan}{black}Black{/black}{/bgcyan} {bgcyan}{red}Red{/red}{/bgcyan} '), end='')
    print(Color('{bgcyan}{green}Green{/green}{/bgcyan} {bgcyan}{yellow}Yellow{/yellow}{/bgcyan} '), end='')
    print(Color('{bgcyan}{blue}Blue{/blue}{/bgcyan} {bgcyan}{magenta}Magenta{/magenta}{/bgcyan} '), end='')
    print(Color('{bgcyan}{cyan}Cyan{/cyan}{/bgcyan} {bgcyan}{white}White{/white}{/bgcyan}'))

    print(Color('    {bgwhite}{black}Black{/black}{/bgwhite} {bgwhite}{red}Red{/red}{/bgwhite} '), end='')
    print(Color('{bgwhite}{green}Green{/green}{/bgwhite} {bgwhite}{yellow}Yellow{/yellow}{/bgwhite} '), end='')
    print(Color('{bgwhite}{blue}Blue{/blue}{/bgwhite} {bgwhite}{magenta}Magenta{/magenta}{/bgwhite} '), end='')
    print(Color('{bgwhite}{cyan}Cyan{/cyan}{/bgwhite} {bgwhite}{white}White{/white}{/bgwhite}'))

    if OPTIONS['--wait']:
        print('Waiting for {0} to exist within 10 seconds...'.format(OPTIONS['--wait']), file=sys.stderr, end='')
        stop_after = time.time() + 20
        while not os.path.exists(OPTIONS['--wait']) and time.time() < stop_after:
            print('.', file=sys.stderr, end='')
            sys.stderr.flush()
            time.sleep(0.5)
        print(' done')

Example 25

Project: netgrph
Source File: path.py
View license
def get_full_path(src, dst, popt, rtype="NGTREE"):
    """ Gets the full path (switch->rt->VRF->rt->switch)

        Requires NetDB for switchpath
    """

    rtypes = ('CSV', 'TREE', 'JSON', 'YAML', 'NGTREE', 'QTREE')

    if rtype in rtypes:

        if 'onepath' not in popt:
            popt['onepath'] = True
        if 'l2path' not in popt:
            popt['l2path'] = True
        if 'verbose' not in popt:
            popt['verbose'] = False
        if nglib.verbose:
            popt['verbose'] = True
        if 'depth' not in popt:
            popt['depth'] = '20'

        logger.info("Query: Finding Full Path (%s --> %s) for %s",
                    src, dst, nglib.user)

        net1, net2 = src, dst
        n1tree, n2tree = None, None

        # Translate IPs to CIDRs
        n1tree = nglib.query.net.get_net(net1, rtype="NGTREE", verbose=popt['verbose'])
        if n1tree:
            net1 = n1tree['data'][0]['Name']

        n2tree = nglib.query.net.get_net(net2, rtype="NGTREE", verbose=popt['verbose'])
        if n2tree:
            net2 = n2tree['data'][0]['Name']

        if not n1tree or not n2tree:
            errort = nglib.ngtree.get_ngtree("Path Error", tree_type="L3-PATH")
            errort['Error'] = 'Network Lookup Error'
            errort["Lx Path"] = src + " -> " + dst
            return errort

        srctree, dsttree, srcswp, dstswp = None, None, None, None

        if 'vrfcidr' not in n1tree['data'][0]:
            print("Warning: Could not locate", src, file=sys.stderr)
            return
        if 'vrfcidr' not in n2tree['data'][0]:
            print("Warning: Could not locate", dst, file=sys.stderr)
            return          

        # Routing Check
        routing = True
        if n1tree['data'][0]['vrfcidr'] == n2tree['data'][0]['vrfcidr']:
            routing = False

        if nglib.use_netdb:
            srctree = nglib.netdb.ip.get_netdb_ip(src)
            dsttree = nglib.netdb.ip.get_netdb_ip(dst)

        # Find Switched Path from Source to Router
        if srctree:
            router = n1tree['data'][0]['Router']
            if 'StandbyRouter' in n1tree['data'][0]:
                router = router + '|' + n1tree['data'][0]['StandbyRouter']
            if 'Switch' in srctree and srctree['Switch']:
                srcswp = get_switched_path(srctree['Switch'], router, popt)
            else:
                srctree = None
                print("Warning: Could not find source switch data in NetDB:", src, file=sys.stderr)

        # Find Switched Path from Router to Destination
        if dsttree:
            router = n2tree['data'][0]['Router']
            if 'StandbyRouter' in n2tree['data'][0]:
                router = router + '|' + n2tree['data'][0]['StandbyRouter']
            if 'Switch' in dsttree and dsttree['Switch']:
                dstswp = get_switched_path(router, dsttree['Switch'], popt)
            else:
                dsttree = None
                print("Warning: Could not find destination switch data in NetDB", \
                    dst, file=sys.stderr)

            # If only switching, update srcswp to show switched path
            if not routing and srctree and dsttree:
                srcswp = get_switched_path(srctree['Switch'], dsttree['Switch'], popt)

        # Same switch/vlan check
        switching = True
        if srctree and dsttree:
            if srctree['Switch'] == dsttree['Switch'] and \
                srctree['VLAN'] == dsttree['VLAN']:
                switching = False

        ## Create Parent Data Structure
        ngtree = nglib.ngtree.get_ngtree("L2-L4", tree_type="PATHs")

        # Populate Overall Paths
        if n1tree['data'][0]['Name'] != n2tree['data'][0]['Name']:
            ngtree["L3 Path"] = net1 + " -> " + net2
            ngtree["Lx Path"] = src + " -> " + dst
        if srctree and dsttree:
            ngtree["L2 Path"] = srctree['Switch'] + " (" + srctree['SwitchPort'] \
            + ") -> " + dsttree['Switch'] + " (" + dsttree['SwitchPort'] + ")"
        if popt['onepath']:
            ngtree["Traversal Type"] = 'Single Path'
        else:
            ngtree["Traversal Type"] = 'All Paths'

        ## Add the SRC Data
        if len(n1tree['data']) > 1:
            n1tree['data'][1]['_type'] = "SRC"
            if 'SwitchPort' in n1tree['data'][1]:
                n1tree['data'][1]['Name'] = src + ' ' + n1tree['data'][1]['MAC'] \
                    + ' ' + str(n1tree['data'][1]['Switch']) + '(' \
                    + str(n1tree['data'][1]['SwitchPort']) \
                    + ') [vid:' + str(n1tree['data'][1]['VLAN']) + ']'
            nglib.ngtree.add_child_ngtree(ngtree, n1tree['data'][1])

        # Add L2 path if not switching
        if not switching and len(n1tree['data']) > 1:
            nglib.ngtree.add_child_ngtree(n1tree, n2tree['data'][1])
            n1tree['_type'] = "L2-PATH"
            n1tree['Name'] = src + ' -> ' + dst

        # If there's src switched data add it
        if switching and srcswp and srcswp['data']:
            nglib.ngtree.add_child_ngtree(ngtree, srcswp)

        # Add L3 Gateway
        if switching and routing:
            n1tree['data'][0]['_type'] = "L3-GW"
            n1tree['data'][0]['Name'] = n1tree['data'][0]['Name'] \
                + ' ' + n1tree['data'][0]['Router']
            if 'StandbyRouter' in n1tree['data'][0]:
                n1tree['data'][0]['Name'] = n1tree['data'][0]['Name'] \
                    + '|' + n1tree['data'][0]['StandbyRouter']
            nglib.ngtree.add_child_ngtree(ngtree, n1tree['data'][0])

        ## Check for routed paths (inter/intra VRF)
        rtree = get_full_routed_path(src, dst, popt)
        if rtree and 'PATH' in rtree['_type']:

            # Breakdown L4 Path
            if rtree['_type'] == 'L4-PATH':
                ngtree['L4 Path'] = rtree['Name']
                for p in rtree['data']:
                    nglib.ngtree.add_child_ngtree(ngtree, p)
            else:
                ngtree['L4 Path'] = 'VRF:' + n1tree['data'][0]['VRF']
                nglib.ngtree.add_child_ngtree(ngtree, rtree)


        # Add the DST L3-GW Data
        if switching and routing:
            n2tree['data'][0]['_type'] = "L3-GW"
            n2tree['data'][0]['Name'] = n2tree['data'][0]['Name'] \
                + ' ' + n2tree['data'][0]['Router']
            if 'StandbyRouter' in n2tree['data'][0]:
                n2tree['data'][0]['Name'] = n2tree['data'][0]['Name'] \
                    + '|' + n2tree['data'][0]['StandbyRouter']
            nglib.ngtree.add_child_ngtree(ngtree, n2tree['data'][0])

        # Destination Switch Data
        if switching and routing and dstswp and dstswp['data']:
            nglib.ngtree.add_child_ngtree(ngtree, dstswp)

        # NetDB Destination Data
        if len(n2tree['data']) > 1:
            n2tree['data'][1]['_type'] = "DST"
            if 'SwitchPort' in n2tree['data'][1]:
                n2tree['data'][1]['Name'] = dst + ' ' + n2tree['data'][1]['MAC'] \
                    + ' ' + str(n2tree['data'][1]['Switch']) + '(' \
                    + str(n2tree['data'][1]['SwitchPort']) \
                    + ') [vid:' + str(n2tree['data'][1]['VLAN']) + ']'
            nglib.ngtree.add_child_ngtree(ngtree, n2tree['data'][1])

        # Export NGTree
        if 'Lx Path' in ngtree:
            ngtree = nglib.query.exp_ngtree(ngtree, rtype)
            return ngtree
        else:
            raise ResultError("No Path Results", "Could not find a path from %s -> %s" % (src, dst))

Example 26

Project: quora-backup
Source File: converter.py
View license
def cleanup_tree(doc, src, dest):
    for child in src.childNodes:
        if child.nodeType == Node.TEXT_NODE:
            # Text nodes can simply be left as-is
            dest.appendChild(child.cloneNode(False))
            continue
        if child.nodeType != Node.ELEMENT_NODE:
            # ???
            raise ValueError()
        # Otherwise, it's an element node.
        if child.tagName in ['br', 'hr']:
            dest.appendChild(child.cloneNode(False))
        elif child.tagName in ['b', 'i', 'u', 'h2', 'ol', 'ul', 'li', 'blockquote', 'wbr']:
            # This node doesn't need to be modified but its children might.
            new_node = doc.createElement(child.tagName)
            cleanup_tree(doc, child, new_node)
            dest.appendChild(new_node)
        elif child.getAttribute('data-embed') != '':
            # This is a video. We want to copy the data-embed value, which is HTML for an iframe node.
            # So, we have to parse it into a separate document and import the node.
            iframe_html = child.getAttribute('data-embed')
            parser = HTMLParser(tree=treebuilders.getTreeBuilder('dom'))
            iframe_doc = parser.parse(iframe_html)
            try:
                iframe = iframe_doc.documentElement.childNodes[1].firstChild
                if iframe.tagName != 'iframe':
                    raise ValueError()
                new_node = doc.importNode(iframe, False)
                # Quora uses a protocol-relative URL (//youtube.com/...) so let's make sure we rewrite this.
                src = new_node.getAttribute('src')
                if src.startswith('//'):
                    new_node.setAttribute('src', 'http:' + src)
                # The video will look really bad if we don't explicitly set the dimensions.
                new_node.setAttribute('width', '525')
                new_node.setAttribute('height', '295')
                dest.appendChild(new_node)
            except Exception:
                print('[WARNING] Failed to parse video embed code', file=sys.stderr)
                # Bail out by just copying the original HTML
                dest.appendChild(child.cloneNode(True))
        elif 'inline_codeblock' in child.getAttribute('class'):
            # Inline codeblock. Simply replace this with a <code>.
            try:
                # div > pre > span > (text)
                span = child.firstChild.firstChild
                if span.tagName != 'span':
                    raise ValueError()
                code_element = doc.createElement('code')
                code_element.appendChild(doc.createTextNode(get_text_content(span)))
                dest.appendChild(code_element)
            except ValueError:
                print('[WARNING] Failed to parse inline codeblock', file=sys.stderr)
                # Bail out by just copying the original HTML
                dest.appendChild(child.cloneNode(True))
        elif 'ContentFooter' in child.getAttribute('class') or 'hidden' in child.getAttribute('class'):
            # These are nodes we just want to skip.
            continue
        elif child.tagName in ['span', 'div']:
            # don't insert a span or div; just insert its contents
            cleanup_tree(doc, child, dest)
        # The remaining cases are: link, image (incl. math), and block code.
        elif child.tagName == 'a':
            # A link. We only want to copy the href, and pass the rest through.
            new_node = doc.createElement('a')
            href = child.getAttribute('href')
            if href.startswith('/'):
                href = 'http://quora.com' + href
            new_node.setAttribute('href', href)
            dest.appendChild(new_node)
            cleanup_tree(doc, child, new_node)
        elif child.tagName == 'img':
            src = child.getAttribute('master_src')
            if src == '':
                src = child.getAttribute('src')
            new_node = doc.createElement('img')
            new_node.setAttribute('src', src)
            new_node.setAttribute('alt', child.getAttribute('alt'))
            if args.no_download:
                dest.appendChild(new_node)
                continue
            # Save a copy of the image locally.
            # If an error occurs, just leave the src pointing to Quora.
            try:
                m = re.search('/([^/?]+)(\?|$)', src)
                if m is None:
                    raise ValueError()
                filename = m.group(1)
                if not filename.endswith('.png'):
                    filename += '.png'
                try:
                    img_fd = os.open(args.output_dir + '/' + filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
                except OSError as error:
                    if error.errno == errno.EEXIST:
                        log_if_v('Image %s has already been saved; skipping' % filename)
                        new_node.setAttribute('src', filename)
                        continue
                    else:
                        raise
                log_if_v('Downloading image from %s' % src)
                closed = False
                try:
                    img = urllib.request.urlopen(src).read()
                    time.sleep(args.delay)
                    os.write(img_fd, img)
                    os.close(img_fd)
                    closed = True
                except Exception:
                    os.close(img_fd)
                    closed = True
                    try:
                        os.remove(args.output_dir + '/' + filename)
                    except:
                        print('[WARNING] Failed to remove incomplete file %s' % filename, file=sys.stderr)
                    raise
                finally:
                    if not closed:
                        os.close(img_fd)
                    # Don't leave the file there; we will retry it next time.
                # If everything went according to plan, rewrite the src to the local file.
                new_node.setAttribute('src', filename)
            except urllib.error.URLError as error:
                print('[WARNING] Failed to download image from URL %s (%s)' % (src, error.reason), file=sys.stderr)
            except OSError as error:
                print('[WARNING] Failed to save image from URL %s to file %s (%s)' % (src, filename, error.strerror), file=sys.stderr)
            except ValueError:
                print('[WARNING] Failed to determine image name from URL %s' % src, file=sys.stderr)
            finally:
                dest.appendChild(new_node)
        elif 'codeblocktable' in child.getAttribute('class'):
            # Block (not inline) code. This should become <pre><code>...</code></pre>
            try:
                pre_node = doc.createElement('pre')
                # Each div inside is a line.
                code_node = doc.createElement('code')
                divs = child.getElementsByTagName('div')
                lines = []
                for div in divs:
                    # All the code is inside spans.
                    spans = div.getElementsByTagName('span')
                    line = ''.join([get_text_content(span) for span in spans])
                    lines.append(line)
                text_node = doc.createTextNode('\n'.join(lines))
                code_node.appendChild(text_node)
                pre_node.appendChild(code_node)
                dest.appendChild(pre_node)
            except Exception:
                print('[WARNING] Failed to parse code block', file=sys.stderr)
                dest.appendChild(child.cloneNode(True))
        else:
            print('[WARNING] Unrecognized node', file=sys.stderr)
            # Bail out by just copying the original HTML
            dest.appendChild(child.cloneNode(True))

Example 27

Project: ros_buildfarm
Source File: devel_job.py
View license
def configure_devel_jobs(
        config_url, rosdistro_name, source_build_name, groovy_script=None,
        dry_run=False, whitelist_repository_names=None):
    """
    Configure all Jenkins devel jobs.

    L{configure_release_job} will be invoked for source repository and target
    which matches the build file criteria.
    """
    config = get_config_index(config_url)
    build_files = get_source_build_files(config, rosdistro_name)
    build_file = build_files[source_build_name]

    index = get_index(config.rosdistro_index_url)

    dist_cache = None
    if build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)

    # get targets
    targets = []
    for os_name in build_file.targets.keys():
        for os_code_name in build_file.targets[os_name].keys():
            for arch in build_file.targets[os_name][os_code_name]:
                targets.append((os_name, os_code_name, arch))
    print('The build file contains the following targets:')
    for os_name, os_code_name, arch in targets:
        print('  -', os_name, os_code_name, arch)

    dist_file = get_distribution_file(index, rosdistro_name, build_file)
    if not dist_file:
        print('No distribution file matches the build file')
        return

    devel_view_name = get_devel_view_name(
        rosdistro_name, source_build_name, pull_request=False)
    pull_request_view_name = get_devel_view_name(
        rosdistro_name, source_build_name, pull_request=True)

    # all further configuration will be handled by either the Jenkins API
    # or by a generated groovy script
    from ros_buildfarm.jenkins import connect
    jenkins = connect(config.jenkins_url) if groovy_script is None else False

    view_configs = {}
    views = {}
    if build_file.test_commits_force is not False:
        views[devel_view_name] = configure_devel_view(
            jenkins, devel_view_name, dry_run=dry_run)
    if build_file.test_pull_requests_force is not False:
        views[pull_request_view_name] = configure_devel_view(
            jenkins, pull_request_view_name, dry_run=dry_run)
    if not jenkins:
        view_configs.update(views)
    groovy_data = {
        'dry_run': dry_run,
        'expected_num_views': len(view_configs),
    }

    repo_names = dist_file.repositories.keys()
    filtered_repo_names = build_file.filter_repositories(repo_names)

    devel_job_names = []
    pull_request_job_names = []
    job_configs = OrderedDict()
    for repo_name in sorted(repo_names):
        if whitelist_repository_names:
            if repo_name not in whitelist_repository_names:
                print("Skipping repository '%s' not in the explicitly passed list" %
                      repo_name, file=sys.stderr)
                continue

        is_disabled = repo_name not in filtered_repo_names
        if is_disabled and build_file.skip_ignored_repositories:
            print("Skipping ignored repository '%s'" % repo_name,
                  file=sys.stderr)
            continue

        repo = dist_file.repositories[repo_name]
        if not repo.source_repository:
            print("Skipping repository '%s': no source section" % repo_name)
            continue
        if not repo.source_repository.version:
            print("Skipping repository '%s': no source version" % repo_name)
            continue

        job_types = []
        # check for testing commits
        if build_file.test_commits_force is False:
            print(("Skipping repository '%s': 'test_commits' is forced to " +
                   "false in the build file") % repo_name)
        elif repo.source_repository.test_commits is False:
            print(("Skipping repository '%s': 'test_commits' of the " +
                   "repository set to false") % repo_name)
        elif repo.source_repository.test_commits is None and \
                not build_file.test_commits_default:
            print(("Skipping repository '%s': 'test_commits' defaults to " +
                   "false in the build file") % repo_name)
        else:
            job_types.append('commit')

        if not is_disabled:
            # check for testing pull requests
            if build_file.test_pull_requests_force is False:
                # print(("Skipping repository '%s': 'test_pull_requests' " +
                #        "is forced to false in the build file") % repo_name)
                pass
            elif repo.source_repository.test_pull_requests is False:
                # print(("Skipping repository '%s': 'test_pull_requests' of " +
                #        "the repository set to false") % repo_name)
                pass
            elif repo.source_repository.test_pull_requests is None and \
                    not build_file.test_pull_requests_default:
                # print(("Skipping repository '%s': 'test_pull_requests' " +
                #        "defaults to false in the build file") % repo_name)
                pass
            else:
                print("Pull request job for repository '%s'" % repo_name)
                job_types.append('pull_request')

        for job_type in job_types:
            pull_request = job_type == 'pull_request'
            for os_name, os_code_name, arch in targets:
                try:
                    job_name, job_config = configure_devel_job(
                        config_url, rosdistro_name, source_build_name,
                        repo_name, os_name, os_code_name, arch, pull_request,
                        config=config, build_file=build_file,
                        index=index, dist_file=dist_file,
                        dist_cache=dist_cache, jenkins=jenkins, views=views,
                        is_disabled=is_disabled,
                        groovy_script=groovy_script,
                        dry_run=dry_run)
                    if not pull_request:
                        devel_job_names.append(job_name)
                    else:
                        pull_request_job_names.append(job_name)
                    if groovy_script is not None:
                        print("Configuration for job '%s'" % job_name)
                        job_configs[job_name] = job_config
                except JobValidationError as e:
                    print(e.message, file=sys.stderr)

    groovy_data['expected_num_jobs'] = len(job_configs)
    groovy_data['job_prefixes_and_names'] = {}

    devel_job_prefix = '%s__' % devel_view_name
    pull_request_job_prefix = '%s__' % pull_request_view_name
    if not whitelist_repository_names:
        groovy_data['job_prefixes_and_names']['devel'] = \
            (devel_job_prefix, devel_job_names)
        groovy_data['job_prefixes_and_names']['pull_request'] = \
            (pull_request_job_prefix, pull_request_job_names)

        if groovy_script is None:
            # delete obsolete jobs in these views
            from ros_buildfarm.jenkins import remove_jobs
            print('Removing obsolete devel jobs')
            remove_jobs(
                jenkins, devel_job_prefix, devel_job_names, dry_run=dry_run)
            print('Removing obsolete pull request jobs')
            remove_jobs(
                jenkins, pull_request_job_prefix, pull_request_job_names,
                dry_run=dry_run)
    if groovy_script is not None:
        print(
            "Writing groovy script '%s' to reconfigure %d views and %d jobs" %
            (groovy_script, len(view_configs), len(job_configs)))
        content = expand_template(
            'snippet/reconfigure_jobs.groovy.em', groovy_data)
        write_groovy_script_and_configs(
            groovy_script, content, job_configs, view_configs=view_configs)

Example 28

Project: internetarchive
Source File: item.py
View license
    def upload_file(self, body,
                    key=None,
                    metadata=None,
                    headers=None,
                    access_key=None,
                    secret_key=None,
                    queue_derive=None,
                    verbose=None,
                    verify=None,
                    checksum=None,
                    delete=None,
                    retries=None,
                    retries_sleep=None,
                    debug=None,
                    request_kwargs=None):
        """Upload a single file to an item. The item will be created
        if it does not exist.

        :type body: Filepath or file-like object.
        :param body: File or data to be uploaded.

        :type key: str
        :param key: (optional) Remote filename.

        :type metadata: dict
        :param metadata: (optional) Metadata used to create a new item.

        :type headers: dict
        :param headers: (optional) Add additional IA-S3 headers to request.

        :type queue_derive: bool
        :param queue_derive: (optional) Set to False to prevent an item from
                             being derived after upload.

        :type verify: bool
        :param verify: (optional) Verify local MD5 checksum matches the MD5
                       checksum of the file received by IAS3.

        :type checksum: bool
        :param checksum: (optional) Skip based on checksum.

        :type delete: bool
        :param delete: (optional) Delete local file after the upload has been
                       successfully verified.

        :type retries: int
        :param retries: (optional) Number of times to retry the given request
                        if S3 returns a 503 SlowDown error.

        :type retries_sleep: int
        :param retries_sleep: (optional) Amount of time to sleep between
                              ``retries``.

        :type verbose: bool
        :param verbose: (optional) Print progress to stdout.

        :type debug: bool
        :param debug: (optional) Set to True to print headers to stdout, and
                      exit without sending the upload request.

        Usage::

            >>> import internetarchive
            >>> item = internetarchive.Item('identifier')
            >>> item.upload_file('/path/to/image.jpg',
            ...                  key='photos/image1.jpg')
            True
        """
        # Set defaults.
        headers = {} if headers is None else headers
        metadata = {} if metadata is None else metadata
        access_key = self.session.access_key if access_key is None else access_key
        secret_key = self.session.secret_key if secret_key is None else secret_key
        queue_derive = True if queue_derive is None else queue_derive
        verbose = False if verbose is None else verbose
        verify = True if verify is None else verify
        delete = False if delete is None else delete
        # Set checksum after delete.
        checksum = True if delete or checksum is None else checksum
        retries = 0 if retries is None else retries
        retries_sleep = 30 if retries_sleep is None else retries_sleep
        debug = False if debug is None else debug
        request_kwargs = {} if request_kwargs is None else request_kwargs
        md5_sum = None

        if not hasattr(body, 'read'):
            body = open(body, 'rb')

        size = get_file_size(body)

        if not headers.get('x-archive-size-hint'):
            headers['x-archive-size-hint'] = str(size)

        # Build IA-S3 URL.
        key = body.name.split('/')[-1] if key is None else key
        base_url = '{0.session.protocol}//s3.us.archive.org/{0.identifier}'.format(self)
        url = '{0}/{1}'.format(
            base_url, urllib.parse.quote(key.lstrip('/').encode('utf-8')))

        # Skip based on checksum.
        if checksum:
            md5_sum = get_md5(body)
            ia_file = self.get_file(key)
            if (not self.tasks) and (ia_file) and (ia_file.md5 == md5_sum):
                log.info('{f} already exists: {u}'.format(f=key, u=url))
                if verbose:
                    print(' {f} already exists, skipping.'.format(f=key))
                if delete:
                    log.info(
                        '{f} successfully uploaded to '
                        'https://archive.org/download/{i}/{f} '
                        'and verified, deleting '
                        'local copy'.format(i=self.identifier,
                                            f=key))
                    os.remove(body.name)
                # Return an empty response object if checksums match.
                # TODO: Is there a better way to handle this?
                return Response()

        # require the Content-MD5 header when delete is True.
        if verify or delete:
            if not md5_sum:
                md5_sum = get_md5(body)
            headers['Content-MD5'] = md5_sum

        def _build_request():
            body.seek(0, os.SEEK_SET)
            if verbose:
                try:
                    chunk_size = 1048576
                    expected_size = size / chunk_size + 1
                    chunks = chunk_generator(body, chunk_size)
                    progress_generator = progress.bar(
                        chunks,
                        expected_size=expected_size,
                        label=' uploading {f}: '.format(f=key))
                    data = IterableToFileAdapter(progress_generator, size)
                except:
                    print(' uploading {f}'.format(f=key))
                    data = body
            else:
                data = body

            request = S3Request(method='PUT',
                                url=url,
                                headers=headers,
                                data=data,
                                metadata=metadata,
                                access_key=access_key,
                                secret_key=secret_key,
                                queue_derive=queue_derive)
            return request

        if debug:
            return _build_request()
        else:
            try:
                error_msg = ('s3 is overloaded, sleeping for '
                             '{0} seconds and retrying. '
                             '{1} retries left.'.format(retries_sleep, retries))
                while True:
                    if retries > 0:
                        if self.session.s3_is_overloaded(access_key):
                            sleep(retries_sleep)
                            log.info(error_msg)
                            if verbose:
                                print(' warning: {0}'.format(error_msg), file=sys.stderr)
                            retries -= 1
                            continue
                    request = _build_request()
                    prepared_request = request.prepare()
                    response = self.session.send(prepared_request,
                                                 stream=True,
                                                 **request_kwargs)
                    if (response.status_code == 503) and (retries > 0):
                        log.info(error_msg)
                        if verbose:
                            print(' warning: {0}'.format(error_msg), file=sys.stderr)
                        sleep(retries_sleep)
                        retries -= 1
                        continue
                    else:
                        if response.status_code == 503:
                            log.info('maximum retries exceeded, upload failed.')
                        break
                response.raise_for_status()
                log.info('uploaded {f} to {u}'.format(f=key, u=url))
                if delete and response.status_code == 200:
                    log.info(
                        '{f} successfully uploaded to '
                        'https://archive.org/download/{i}/{f} and verified, deleting '
                        'local copy'.format(i=self.identifier,
                                            f=key))
                    os.remove(body.name)
                return response
            except HTTPError as exc:
                msg = get_s3_xml_text(exc.response.content)
                error_msg = (' error uploading {0} to {1}, '
                             '{2}'.format(key, self.identifier, msg))
                log.error(error_msg)
                if verbose:
                    print(' error uploading {0}: {1}'.format(key, msg), file=sys.stderr)
                # Raise HTTPError with error message.
                raise type(exc)(error_msg, response=exc.response, request=exc.request)

Example 29

Project: kgsgo-dataset-preprocessor
Source File: main.py
View license
def main(args=None):
    """Main program.

    Args:
        fixer_pkg: the name of a package where the fixers are located.
        args: optional; a list of command line arguments. If omitted,
              sys.argv[1:] is used.

    Returns a suggested exit status (0, 1, 2).
    """
    
    # Set up option parser
    parser = optparse.OptionParser(usage="futurize [options] file|dir ...")
    parser.add_option("-V", "--version", action="store_true",
                      help="Report the version number of futurize")
    parser.add_option("-a", "--all-imports", action="store_true",
                      help="Add all __future__ and future imports to each module")
    parser.add_option("-1", "--stage1", action="store_true",
                      help="Modernize Python 2 code only; no compatibility with Python 3 (or dependency on ``future``)")
    parser.add_option("-2", "--stage2", action="store_true",
                      help="Take modernized (stage1) code and add a dependency on ``future`` to provide Py3 compatibility.")
    parser.add_option("-0", "--both-stages", action="store_true",
                      help="Apply both stages 1 and 2")
    parser.add_option("-u", "--unicode-literals", action="store_true",
                      help="Add ``from __future__ import unicode_literals`` to implicitly convert all unadorned string literals '' into unicode strings")
    parser.add_option("-f", "--fix", action="append", default=[],
                      help="Each FIX specifies a transformation; default: all.\nEither use '-f division -f metaclass' etc. or use the fully-qualified module name: '-f lib2to3.fixes.fix_types -f libfuturize.fixes.fix_unicode_keep_u'")
    parser.add_option("-j", "--processes", action="store", default=1,
                      type="int", help="Run 2to3 concurrently")
    parser.add_option("-x", "--nofix", action="append", default=[],
                      help="Prevent a fixer from being run.")
    parser.add_option("-l", "--list-fixes", action="store_true",
                      help="List available transformations")
    parser.add_option("-p", "--print-function", action="store_true",
                      help="Modify the grammar so that print() is a function")
    parser.add_option("-v", "--verbose", action="store_true",
                      help="More verbose logging")
    parser.add_option("--no-diffs", action="store_true",
                      help="Don't show diffs of the refactoring")
    parser.add_option("-w", "--write", action="store_true",
                      help="Write back modified files")
    parser.add_option("-n", "--nobackups", action="store_true", default=False,
                      help="Don't write backups for modified files.")
    parser.add_option("-o", "--output-dir", action="store", type="str",
                      default="", help="Put output files in this directory "
                      "instead of overwriting the input files.  Requires -n. "
                      "For Python >= 2.7 only.")
    parser.add_option("-W", "--write-unchanged-files", action="store_true",
                      help="Also write files even if no changes were required"
                      " (useful with --output-dir); implies -w.")
    parser.add_option("--add-suffix", action="store", type="str", default="",
                      help="Append this string to all output filenames."
                      " Requires -n if non-empty. For Python >= 2.7 only."
                      "ex: --add-suffix='3' will generate .py3 files.")

    # Parse command line arguments
    flags = {}
    refactor_stdin = False
    options, args = parser.parse_args(args)

    if options.write_unchanged_files:
        flags["write_unchanged_files"] = True
        if not options.write:
            warn("--write-unchanged-files/-W implies -w.")
        options.write = True
    # If we allowed these, the original files would be renamed to backup names
    # but not replaced.
    if options.output_dir and not options.nobackups:
        parser.error("Can't use --output-dir/-o without -n.")
    if options.add_suffix and not options.nobackups:
        parser.error("Can't use --add-suffix without -n.")

    if not options.write and options.no_diffs:
        warn("not writing files and not printing diffs; that's not very useful")
    if not options.write and options.nobackups:
        parser.error("Can't use -n without -w")
    if "-" in args:
        refactor_stdin = True
        if options.write:
            print("Can't write to stdin.", file=sys.stderr)
            return 2
    # Is this ever necessary?
    if options.print_function:
        flags["print_function"] = True

    # Set up logging handler
    level = logging.DEBUG if options.verbose else logging.INFO
    logging.basicConfig(format='%(name)s: %(message)s', level=level)
    logger = logging.getLogger('libfuturize.main')

    if options.stage1 or options.stage2:
        assert options.both_stages is None
        options.both_stages = False
    else:
        options.both_stages = True

    avail_fixes = set()

    if options.stage1 or options.both_stages:
        avail_fixes.update(lib2to3_fix_names_stage1)
        avail_fixes.update(libfuturize_fix_names_stage1)
    if options.stage2 or options.both_stages:
        avail_fixes.update(lib2to3_fix_names_stage2)
        avail_fixes.update(libfuturize_fix_names_stage2)

    if options.unicode_literals:
        avail_fixes.add('libfuturize.fixes.fix_unicode_literals_import')

    if options.version:
        print(__version__)
        return 0
    if options.list_fixes:
        print("Available transformations for the -f/--fix option:")
        # for fixname in sorted(refactor.get_all_fix_names(fixer_pkg)):
        for fixname in sorted(avail_fixes):
            print(fixname)
        if not args:
            return 0
    if not args:
        print("At least one file or directory argument required.",
              file=sys.stderr)
        print("Use --help to show usage.", file=sys.stderr)
        return 2

    unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix)

    extra_fixes = set()
    if options.all_imports:
        if options.stage1:
            prefix = 'libfuturize.fixes.'
            extra_fixes.add(prefix +
                            'fix_add__future__imports_except_unicode_literals')
        else:
            # In case the user hasn't run stage1 for some reason:
            prefix = 'libpasteurize.fixes.'
            extra_fixes.add(prefix + 'fix_add_all__future__imports')
            extra_fixes.add(prefix + 'fix_add_future_standard_library_import')
            extra_fixes.add(prefix + 'fix_add_all_future_builtins')
    explicit = set()
    if options.fix:
        all_present = False
        for fix in options.fix:
            if fix == 'all':
                all_present = True
            else:
                if ".fix_" in fix:
                    explicit.add(fix)
                else:
                    # Infer the full module name for the fixer.
                    # First ensure that no names clash (e.g.
                    # lib2to3.fixes.fix_blah and libfuturize.fixes.fix_blah):
                    found = [f for f in avail_fixes
                             if f.endswith('fix_{0}'.format(fix))]
                    if len(found) > 1:
                        print("Ambiguous fixer name. Choose a fully qualified "
                              "module name instead from these:\n" +
                              "\n".join("  " + myf for myf in found),
                              file=sys.stderr)
                        return 2
                    elif len(found) == 0:
                        print("Unknown fixer. Use --list-fixes or -l for a list.",
                              file=sys.stderr)
                        return 2
                    explicit.add(found[0])
        if len(explicit & unwanted_fixes) > 0:
            print("Conflicting usage: the following fixers have been "
                  "simultaneously requested and disallowed:\n" +
                  "\n".join("  " + myf for myf in (explicit & unwanted_fixes)),
                  file=sys.stderr)
            return 2
        requested = avail_fixes.union(explicit) if all_present else explicit
    else:
        requested = avail_fixes.union(explicit)
    fixer_names = (requested | extra_fixes) - unwanted_fixes

    input_base_dir = os.path.commonprefix(args)
    if (input_base_dir and not input_base_dir.endswith(os.sep)
        and not os.path.isdir(input_base_dir)):
        # One or more similar names were passed, their directory is the base.
        # os.path.commonprefix() is ignorant of path elements, this corrects
        # for that weird API.
        input_base_dir = os.path.dirname(input_base_dir)
    if options.output_dir:
        input_base_dir = input_base_dir.rstrip(os.sep)
        logger.info('Output in %r will mirror the input directory %r layout.',
                    options.output_dir, input_base_dir)

    # Initialize the refactoring tool
    if future.utils.PY26:
        extra_kwargs = {}
    else:
        extra_kwargs = {
                        'append_suffix': options.add_suffix,
                        'output_dir': options.output_dir,
                        'input_base_dir': input_base_dir,
                       }

    rt = StdoutRefactoringTool(
            sorted(fixer_names), flags, sorted(explicit),
            options.nobackups, not options.no_diffs,
            **extra_kwargs)

    # Refactor all files and directories passed as arguments
    if not rt.errors:
        if refactor_stdin:
            rt.refactor_stdin()
        else:
            try:
                rt.refactor(args, options.write, None,
                            options.processes)
            except refactor.MultiprocessingUnsupported:
                assert options.processes > 1
                print("Sorry, -j isn't " \
                      "supported on this platform.", file=sys.stderr)
                return 1
        rt.summarize()

    # Return error status (0 if rt.errors is zero)
    return int(bool(rt.errors))

Example 30

Project: internetarchive
Source File: item.py
View license
    def upload_file(self, body,
                    key=None,
                    metadata=None,
                    headers=None,
                    access_key=None,
                    secret_key=None,
                    queue_derive=None,
                    verbose=None,
                    verify=None,
                    checksum=None,
                    delete=None,
                    retries=None,
                    retries_sleep=None,
                    debug=None,
                    request_kwargs=None):
        """Upload a single file to an item. The item will be created
        if it does not exist.

        :type body: Filepath or file-like object.
        :param body: File or data to be uploaded.

        :type key: str
        :param key: (optional) Remote filename.

        :type metadata: dict
        :param metadata: (optional) Metadata used to create a new item.

        :type headers: dict
        :param headers: (optional) Add additional IA-S3 headers to request.

        :type queue_derive: bool
        :param queue_derive: (optional) Set to False to prevent an item from
                             being derived after upload.

        :type verify: bool
        :param verify: (optional) Verify local MD5 checksum matches the MD5
                       checksum of the file received by IAS3.

        :type checksum: bool
        :param checksum: (optional) Skip based on checksum.

        :type delete: bool
        :param delete: (optional) Delete local file after the upload has been
                       successfully verified.

        :type retries: int
        :param retries: (optional) Number of times to retry the given request
                        if S3 returns a 503 SlowDown error.

        :type retries_sleep: int
        :param retries_sleep: (optional) Amount of time to sleep between
                              ``retries``.

        :type verbose: bool
        :param verbose: (optional) Print progress to stdout.

        :type debug: bool
        :param debug: (optional) Set to True to print headers to stdout, and
                      exit without sending the upload request.

        Usage::

            >>> import internetarchive
            >>> item = internetarchive.Item('identifier')
            >>> item.upload_file('/path/to/image.jpg',
            ...                  key='photos/image1.jpg')
            True
        """
        # Set defaults.
        headers = {} if headers is None else headers
        metadata = {} if metadata is None else metadata
        access_key = self.session.access_key if access_key is None else access_key
        secret_key = self.session.secret_key if secret_key is None else secret_key
        queue_derive = True if queue_derive is None else queue_derive
        verbose = False if verbose is None else verbose
        verify = True if verify is None else verify
        delete = False if delete is None else delete
        # Set checksum after delete.
        checksum = True if delete or checksum is None else checksum
        retries = 0 if retries is None else retries
        retries_sleep = 30 if retries_sleep is None else retries_sleep
        debug = False if debug is None else debug
        request_kwargs = {} if request_kwargs is None else request_kwargs
        md5_sum = None

        if not hasattr(body, 'read'):
            body = open(body, 'rb')

        size = get_file_size(body)

        if not headers.get('x-archive-size-hint'):
            headers['x-archive-size-hint'] = str(size)

        # Build IA-S3 URL.
        key = body.name.split('/')[-1] if key is None else key
        base_url = '{0.session.protocol}//s3.us.archive.org/{0.identifier}'.format(self)
        url = '{0}/{1}'.format(
            base_url, urllib.parse.quote(key.lstrip('/').encode('utf-8')))

        # Skip based on checksum.
        if checksum:
            md5_sum = get_md5(body)
            ia_file = self.get_file(key)
            if (not self.tasks) and (ia_file) and (ia_file.md5 == md5_sum):
                log.info('{f} already exists: {u}'.format(f=key, u=url))
                if verbose:
                    print(' {f} already exists, skipping.'.format(f=key))
                if delete:
                    log.info(
                        '{f} successfully uploaded to '
                        'https://archive.org/download/{i}/{f} '
                        'and verified, deleting '
                        'local copy'.format(i=self.identifier,
                                            f=key))
                    os.remove(body.name)
                # Return an empty response object if checksums match.
                # TODO: Is there a better way to handle this?
                return Response()

        # require the Content-MD5 header when delete is True.
        if verify or delete:
            if not md5_sum:
                md5_sum = get_md5(body)
            headers['Content-MD5'] = md5_sum

        def _build_request():
            body.seek(0, os.SEEK_SET)
            if verbose:
                try:
                    chunk_size = 1048576
                    expected_size = size / chunk_size + 1
                    chunks = chunk_generator(body, chunk_size)
                    progress_generator = progress.bar(
                        chunks,
                        expected_size=expected_size,
                        label=' uploading {f}: '.format(f=key))
                    data = IterableToFileAdapter(progress_generator, size)
                except:
                    print(' uploading {f}'.format(f=key))
                    data = body
            else:
                data = body

            request = S3Request(method='PUT',
                                url=url,
                                headers=headers,
                                data=data,
                                metadata=metadata,
                                access_key=access_key,
                                secret_key=secret_key,
                                queue_derive=queue_derive)
            return request

        if debug:
            return _build_request()
        else:
            try:
                error_msg = ('s3 is overloaded, sleeping for '
                             '{0} seconds and retrying. '
                             '{1} retries left.'.format(retries_sleep, retries))
                while True:
                    if retries > 0:
                        if self.session.s3_is_overloaded(access_key):
                            sleep(retries_sleep)
                            log.info(error_msg)
                            if verbose:
                                print(' warning: {0}'.format(error_msg), file=sys.stderr)
                            retries -= 1
                            continue
                    request = _build_request()
                    prepared_request = request.prepare()
                    response = self.session.send(prepared_request,
                                                 stream=True,
                                                 **request_kwargs)
                    if (response.status_code == 503) and (retries > 0):
                        log.info(error_msg)
                        if verbose:
                            print(' warning: {0}'.format(error_msg), file=sys.stderr)
                        sleep(retries_sleep)
                        retries -= 1
                        continue
                    else:
                        if response.status_code == 503:
                            log.info('maximum retries exceeded, upload failed.')
                        break
                response.raise_for_status()
                log.info('uploaded {f} to {u}'.format(f=key, u=url))
                if delete and response.status_code == 200:
                    log.info(
                        '{f} successfully uploaded to '
                        'https://archive.org/download/{i}/{f} and verified, deleting '
                        'local copy'.format(i=self.identifier,
                                            f=key))
                    os.remove(body.name)
                return response
            except HTTPError as exc:
                msg = get_s3_xml_text(exc.response.content)
                error_msg = (' error uploading {0} to {1}, '
                             '{2}'.format(key, self.identifier, msg))
                log.error(error_msg)
                if verbose:
                    print(' error uploading {0}: {1}'.format(key, msg), file=sys.stderr)
                # Raise HTTPError with error message.
                raise type(exc)(error_msg, response=exc.response, request=exc.request)

Example 31

Project: dash-hack
Source File: rst_parser.py
View license
    def parse_section(self, module_name, section_name, file_name, lineno, lines):
        self.sections_total += 1
        # skip sections having whitespace in name
        #if section_name.find(" ") >= 0 and section_name.find("::operator") < 0:
        if (section_name.find(" ") >= 0 and not bool(re.match(r"(\w+::)*operator\s*(\w+|>>|<<|\(\)|->|\+\+|--|=|==|\+=|-=)", section_name)) ) or section_name.endswith(":"):
            if show_errors:
                print >> sys.stderr, "RST parser warning W%03d:  SKIPPED: \"%s\" File: %s:%s" % (WARNING_002_HDRWHITESPACE, section_name, file_name, lineno)
            self.sections_skipped += 1
            return

        func = {}
        func["name"] = section_name
        func["file"] = file_name
        func["line"] = lineno
        func["module"] = module_name

        # parse section name
        section_name = self.parse_namespace(func, section_name)
        class_separator_idx = section_name.find("::")
        if class_separator_idx > 0:
            func["class"] = section_name[:class_separator_idx]
            func["method"] = section_name[class_separator_idx+2:]
        else:
            func["method"] = section_name

        capturing_seealso = False
        skip_code_lines = False
        expected_brief = True
        was_code_line = False
        fdecl = DeclarationParser()
        pdecl = ParamParser()
        ll = None

        for l in lines:
            # read tail of function/method declaration if needed
            if not fdecl.isready():
                fdecl.append(ll)
                if fdecl.isready():
                    self.add_new_fdecl(func, fdecl)
                continue

            # continue capture seealso
            if capturing_seealso:
                if not l or l.startswith(" "):
                    seealso = func.get("seealso", [])
                    seealso.extend(l.split(","))
                    func["seealso"] = seealso
                    continue
                else:
                    capturing_seealso = False

            ll = l.strip()
            if ll == "..":
                expected_brief = False
                skip_code_lines = False
                continue

            # skip lines if line-skipping mode is activated
            if skip_code_lines:
                if not l:
                    continue
                if not l.startswith(" "):
                    skip_code_lines = False

            if ll.startswith(".. code-block::") or ll.startswith(".. image::"):
                skip_code_lines = True

                continue

            # todo: parse structure members; skip them for now
            if ll.startswith(".. ocv:member::"):
        #print ll
                skip_code_lines = True
                continue

            #ignore references (todo: collect them)
            if l.startswith(".. ["):
                continue

            if ll.startswith(".. "):
                expected_brief = False
            elif ll.endswith("::"):
                # turn on line-skipping mode for code fragments
                #print ll
                skip_code_lines = True
                ll = ll[:len(ll)-2]

            # continue param parsing (process params after processing .. at the beginning of the line and :: at the end)
            if pdecl.active:
                pdecl.append(l)
                if pdecl.active:
                    continue
                else:
                    self.add_new_pdecl(func, pdecl)
                    # do not continue - current line can contain next parameter definition

            # parse ".. seealso::" blocks
            if ll.startswith(".. seealso::"):
                if ll.endswith(".. seealso::"):
                    capturing_seealso = True
                else:
                    seealso = func.get("seealso", [])
                    seealso.extend(ll[ll.find("::")+2:].split(","))
                    func["seealso"] = seealso
                continue

            # skip ".. index::"
            if ll.startswith(".. index::"):
                continue

            # parse class & struct definitions
            if ll.startswith(".. ocv:class::"):
                func["class"] = ll[ll.find("::")+2:].strip()
                if "method" in func:
                    del func["method"]
                func["isclass"] = True
                expected_brief = True
                continue

            if ll.startswith(".. ocv:struct::"):
                func["class"] = ll[ll.find("::")+2:].strip()
                if "method" in func:
                    del func["method"]
                func["isstruct"] = True
                expected_brief = True
                continue

            # parse function/method definitions
            if fdecl.hasDeclaration(ll):
                fdecl = DeclarationParser(ll)
                if fdecl.isready():
                    self.add_new_fdecl(func, fdecl)
                continue

            # parse parameters
            if pdecl.hasDeclaration(l):
                pdecl = ParamParser(l)
                continue

            # record brief description
            if expected_brief:
                func["brief"] = func.get("brief", "") + "\n" + ll
                if skip_code_lines:
                    expected_brief = False # force end brief if code block begins
                continue

            # record other lines as long description
            if (skip_code_lines):
                ll = ll.replace("/*", "/ *")
                ll = ll.replace("*/", "* /")
                if (was_code_line):
                    func["long"] = func.get("long", "") + "\n" + ll + "\n"
                else:
                    was_code_line = True
                    func["long"] = func.get("long", "") + ll +"\n<code>\n\n // C++ code:\n\n"
            else:
                if (was_code_line):
                    func["long"] = func.get("long", "") + "\n" + ll + "\n</code>\n"
                    was_code_line = False
                else:
                    func["long"] = func.get("long", "") + "\n" + ll
        # endfor l in lines

        if fdecl.balance != 0:
            if show_critical_errors:
                print >> sys.stderr, "RST parser error E%03d: invalid parentheses balance in \"%s\" at %s:%s" % (ERROR_003_PARENTHESES, section_name, file_name, lineno)
            return

        # save last parameter if needed
        if pdecl.active:
            self.add_new_pdecl(func, pdecl)

        # add definition to list
        func = self.normalize(func)
        if self.validate(func):
            self.definitions[func["name"]] = func
            self.sections_parsed += 1
            if verbose:
                self.print_info(func)
        elif func:
            if func["name"] in known_text_sections_names:
                if show_errors:
                    print >> sys.stderr, "RST parser warning W%03d:  SKIPPED: \"%s\" File: %s:%s" % (WARNING_002_HDRWHITESPACE, section_name, file_name, lineno)
                self.sections_skipped += 1
            elif show_errors:
                self.print_info(func, True, sys.stderr)

Example 32

Project: gramps
Source File: argparser.py
View license
    def parse_args(self):
        """
        Fill in lists with open, exports, imports, and actions options.

        Any errors are added to self.errors
        """
        try:
            options, leftargs = getopt.getopt(self.args[1:],
                                              SHORTOPTS, LONGOPTS)
        except getopt.GetoptError as msg:
            # Extract the arguments in the list.
            # The % operator replaces the list elements
            # with repr() of the list elements
            # which is OK for latin characters,
            # but not for non latin characters in list elements
            cliargs = "[ "
            for arg in range(len(self.args) - 1):
                cliargs += self.args[arg + 1] + " "
            cliargs += "]"
            # Must first do str() of the msg object.
            msg = str(msg)
            self.errors += [(_('Error parsing the arguments'),
                             msg + '\n' +
                             _("Error parsing the arguments: %s \n"
                               "Type gramps --help for an overview of "
                               "commands, or read the manual pages."
                              ) % cliargs)]
            return

        # Some args can work on a list of databases:
        if leftargs:
            for opt_ix in range(len(options)):
                option, value = options[opt_ix]
                if option in ['-L', '-l', '-t']:
                    self.database_names = leftargs
                    leftargs = []

        if leftargs:
            # if there were an argument without option,
            # use it as a file to open and return
            self.open_gui = leftargs[0]
            print(_("Trying to open: %s ..."
                   ) % leftargs[0],
                  file=sys.stderr)
            #see if force open is on
            for opt_ix in range(len(options)):
                option, value = options[opt_ix]
                if option in ('-u', '--force-unlock'):
                    self.force_unlock = True
                    break
            return

        # Go over all given option and place them into appropriate lists
        cleandbg = []
        need_to_quit = False
        for opt_ix in range(len(options)):
            option, value = options[opt_ix]
            if option in ['-O', '--open']:
                self.open = value
            elif option in ['-C', '--create']:
                self.create = value
            elif option in ['-i', '--import']:
                family_tree_format = None
                if (opt_ix < len(options) - 1
                        and options[opt_ix + 1][0] in ('-f', '--format')):
                    family_tree_format = options[opt_ix + 1][1]
                self.imports.append((value, family_tree_format))
            elif option in ['-r', '--remove']:
                self.removes.append(value)
            elif option in ['-e', '--export']:
                family_tree_format = None
                if (opt_ix < len(options) - 1
                        and options[opt_ix + 1][0] in ('-f', '--format')):
                    family_tree_format = options[opt_ix + 1][1]
                self.exports.append((value, family_tree_format))
            elif option in ['-a', '--action']:
                action = value
                if action not in ('report', 'tool', 'book'):
                    print(_("Unknown action: %s. Ignoring."
                           ) % action,
                          file=sys.stderr)
                    continue
                options_str = ""
                if (opt_ix < len(options)-1
                        and options[opt_ix+1][0] in ('-p', '--options')):
                    options_str = options[opt_ix+1][1]
                self.actions.append((action, options_str))
            elif option in ['-d', '--debug']:
                print(_('setup debugging'), value, file=sys.stderr)
                logger = logging.getLogger(value)
                logger.setLevel(logging.DEBUG)
                cleandbg += [opt_ix]
            elif option in ['-l']:
                self.list = True
            elif option in ['-L']:
                self.list_more = True
            elif option in ['-t']:
                self.list_table = True
            elif option in ['-s', '--show']:
                print(_("Gramps config settings from %s:"
                       ) % config.filename)
                for sect in config.data:
                    for setting in config.data[sect]:
                        print("%s.%s=%s" % (sect, setting,
                                            repr(config.data[sect][setting])))
                    print()
                sys.exit(0)
            elif option in ['-b', '--databases']:
                default = config.data["database"]["backend"]
                pmgr = BasePluginManager.get_instance()
                pmgr.reg_plugins(PLUGINS_DIR, self, None)
                pmgr.reg_plugins(USER_PLUGINS, self, None, load_on_reg=True)
                for plugin in pmgr.get_reg_databases():
                    pdata = pmgr.get_plugin(plugin.id)
                    mod = pmgr.load_plugin(pdata)
                    if mod:
                        database = getattr(mod, pdata.databaseclass)
                        summary = database.get_class_summary()
                        print("Database backend ID:",
                              pdata.id,
                              "(default)" if pdata.id == default else "")
                        for key in sorted(summary.keys()):
                            print("   ", _("%s:") % key, summary[key])
                sys.exit(0)
            elif option in ['-c', '--config']:
                cfg_name = value
                set_value = False
                if cfg_name:
                    if ":" in cfg_name:
                        cfg_name, new_value = cfg_name.split(":", 1)
                        set_value = True
                    if config.has_default(cfg_name):
                        setting_value = config.get(cfg_name)
                        print(_("Current Gramps config setting: "
                                "%(name)s:%(value)s"
                               ) % {'name'  : cfg_name,
                                    'value' : repr(setting_value)},
                              file=sys.stderr)
                        if set_value:
                            # does a user want the default config value?
                            if new_value in ("DEFAULT", _("DEFAULT")):
                                new_value = config.get_default(cfg_name)
                            else:
                                converter = get_type_converter(setting_value)
                                new_value = converter(new_value)
                            config.set(cfg_name, new_value)
                            # translators: indent "New" to match "Current"
                            print(_("    New Gramps config setting: "
                                    "%(name)s:%(value)s"
                                   ) % {'name'  : cfg_name,
                                        'value' : repr(config.get(cfg_name))},
                                  file=sys.stderr)
                        else:
                            need_to_quit = True
                    else:
                        print(_("Gramps: no such config setting: '%s'"
                               ) % cfg_name,
                              file=sys.stderr)
                        need_to_quit = True
                cleandbg += [opt_ix]
            elif option in ['-h', '-?', '--help']:
                self.help = True
            elif option in ['-u', '--force-unlock']:
                self.force_unlock = True
            elif option in ['--usage']:
                self.usage = True
            elif option in ['-y', '--yes']:
                self.auto_accept = True
            elif option in ['-q', '--quiet']:
                self.quiet = True

        #clean options list
        cleandbg.reverse()
        for ind in cleandbg:
            del options[ind]

        if (len(options) > 0
                and self.open is None
                and self.imports == []
                and self.removes == []
                and not (self.list
                         or self.list_more
                         or self.list_table
                         or self.help)):
            # Extract and convert to unicode the arguments in the list.
            # The % operator replaces the list elements with repr() of
            # the list elements, which is OK for latin characters
            # but not for non-latin characters in list elements
            cliargs = "[ "
            for arg in range(len(self.args) - 1):
                cliargs += self.args[arg + 1] + ' '
            cliargs += "]"
            self.errors += [(_('Error parsing the arguments'),
                             _("Error parsing the arguments: %s \n"
                               "To use in the command-line mode, supply at "
                               "least one input file to process."
                              ) % cliargs)]
        if need_to_quit:
            sys.exit(0)

Example 33

Project: python-future
Source File: main.py
View license
def main(args=None):
    """Main program.

    Args:
        fixer_pkg: the name of a package where the fixers are located.
        args: optional; a list of command line arguments. If omitted,
              sys.argv[1:] is used.

    Returns a suggested exit status (0, 1, 2).
    """
    
    # Set up option parser
    parser = optparse.OptionParser(usage="futurize [options] file|dir ...")
    parser.add_option("-V", "--version", action="store_true",
                      help="Report the version number of futurize")
    parser.add_option("-a", "--all-imports", action="store_true",
                      help="Add all __future__ and future imports to each module")
    parser.add_option("-1", "--stage1", action="store_true",
                      help="Modernize Python 2 code only; no compatibility with Python 3 (or dependency on ``future``)")
    parser.add_option("-2", "--stage2", action="store_true",
                      help="Take modernized (stage1) code and add a dependency on ``future`` to provide Py3 compatibility.")
    parser.add_option("-0", "--both-stages", action="store_true",
                      help="Apply both stages 1 and 2")
    parser.add_option("-u", "--unicode-literals", action="store_true",
                      help="Add ``from __future__ import unicode_literals`` to implicitly convert all unadorned string literals '' into unicode strings")
    parser.add_option("-f", "--fix", action="append", default=[],
                      help="Each FIX specifies a transformation; default: all.\nEither use '-f division -f metaclass' etc. or use the fully-qualified module name: '-f lib2to3.fixes.fix_types -f libfuturize.fixes.fix_unicode_keep_u'")
    parser.add_option("-j", "--processes", action="store", default=1,
                      type="int", help="Run 2to3 concurrently")
    parser.add_option("-x", "--nofix", action="append", default=[],
                      help="Prevent a fixer from being run.")
    parser.add_option("-l", "--list-fixes", action="store_true",
                      help="List available transformations")
    parser.add_option("-p", "--print-function", action="store_true",
                      help="Modify the grammar so that print() is a function")
    parser.add_option("-v", "--verbose", action="store_true",
                      help="More verbose logging")
    parser.add_option("--no-diffs", action="store_true",
                      help="Don't show diffs of the refactoring")
    parser.add_option("-w", "--write", action="store_true",
                      help="Write back modified files")
    parser.add_option("-n", "--nobackups", action="store_true", default=False,
                      help="Don't write backups for modified files.")
    parser.add_option("-o", "--output-dir", action="store", type="str",
                      default="", help="Put output files in this directory "
                      "instead of overwriting the input files.  Requires -n. "
                      "For Python >= 2.7 only.")
    parser.add_option("-W", "--write-unchanged-files", action="store_true",
                      help="Also write files even if no changes were required"
                      " (useful with --output-dir); implies -w.")
    parser.add_option("--add-suffix", action="store", type="str", default="",
                      help="Append this string to all output filenames."
                      " Requires -n if non-empty. For Python >= 2.7 only."
                      "ex: --add-suffix='3' will generate .py3 files.")

    # Parse command line arguments
    flags = {}
    refactor_stdin = False
    options, args = parser.parse_args(args)

    if options.write_unchanged_files:
        flags["write_unchanged_files"] = True
        if not options.write:
            warn("--write-unchanged-files/-W implies -w.")
        options.write = True
    # If we allowed these, the original files would be renamed to backup names
    # but not replaced.
    if options.output_dir and not options.nobackups:
        parser.error("Can't use --output-dir/-o without -n.")
    if options.add_suffix and not options.nobackups:
        parser.error("Can't use --add-suffix without -n.")

    if not options.write and options.no_diffs:
        warn("not writing files and not printing diffs; that's not very useful")
    if not options.write and options.nobackups:
        parser.error("Can't use -n without -w")
    if "-" in args:
        refactor_stdin = True
        if options.write:
            print("Can't write to stdin.", file=sys.stderr)
            return 2
    # Is this ever necessary?
    if options.print_function:
        flags["print_function"] = True

    # Set up logging handler
    level = logging.DEBUG if options.verbose else logging.INFO
    logging.basicConfig(format='%(name)s: %(message)s', level=level)
    logger = logging.getLogger('libfuturize.main')

    if options.stage1 or options.stage2:
        assert options.both_stages is None
        options.both_stages = False
    else:
        options.both_stages = True

    avail_fixes = set()

    if options.stage1 or options.both_stages:
        avail_fixes.update(lib2to3_fix_names_stage1)
        avail_fixes.update(libfuturize_fix_names_stage1)
    if options.stage2 or options.both_stages:
        avail_fixes.update(lib2to3_fix_names_stage2)
        avail_fixes.update(libfuturize_fix_names_stage2)

    if options.unicode_literals:
        avail_fixes.add('libfuturize.fixes.fix_unicode_literals_import')

    if options.version:
        print(__version__)
        return 0
    if options.list_fixes:
        print("Available transformations for the -f/--fix option:")
        # for fixname in sorted(refactor.get_all_fix_names(fixer_pkg)):
        for fixname in sorted(avail_fixes):
            print(fixname)
        if not args:
            return 0
    if not args:
        print("At least one file or directory argument required.",
              file=sys.stderr)
        print("Use --help to show usage.", file=sys.stderr)
        return 2

    unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix)

    extra_fixes = set()
    if options.all_imports:
        if options.stage1:
            prefix = 'libfuturize.fixes.'
            extra_fixes.add(prefix +
                            'fix_add__future__imports_except_unicode_literals')
        else:
            # In case the user hasn't run stage1 for some reason:
            prefix = 'libpasteurize.fixes.'
            extra_fixes.add(prefix + 'fix_add_all__future__imports')
            extra_fixes.add(prefix + 'fix_add_future_standard_library_import')
            extra_fixes.add(prefix + 'fix_add_all_future_builtins')
    explicit = set()
    if options.fix:
        all_present = False
        for fix in options.fix:
            if fix == 'all':
                all_present = True
            else:
                if ".fix_" in fix:
                    explicit.add(fix)
                else:
                    # Infer the full module name for the fixer.
                    # First ensure that no names clash (e.g.
                    # lib2to3.fixes.fix_blah and libfuturize.fixes.fix_blah):
                    found = [f for f in avail_fixes
                             if f.endswith('fix_{0}'.format(fix))]
                    if len(found) > 1:
                        print("Ambiguous fixer name. Choose a fully qualified "
                              "module name instead from these:\n" +
                              "\n".join("  " + myf for myf in found),
                              file=sys.stderr)
                        return 2
                    elif len(found) == 0:
                        print("Unknown fixer. Use --list-fixes or -l for a list.",
                              file=sys.stderr)
                        return 2
                    explicit.add(found[0])
        if len(explicit & unwanted_fixes) > 0:
            print("Conflicting usage: the following fixers have been "
                  "simultaneously requested and disallowed:\n" +
                  "\n".join("  " + myf for myf in (explicit & unwanted_fixes)),
                  file=sys.stderr)
            return 2
        requested = avail_fixes.union(explicit) if all_present else explicit
    else:
        requested = avail_fixes.union(explicit)
    fixer_names = (requested | extra_fixes) - unwanted_fixes

    input_base_dir = os.path.commonprefix(args)
    if (input_base_dir and not input_base_dir.endswith(os.sep)
        and not os.path.isdir(input_base_dir)):
        # One or more similar names were passed, their directory is the base.
        # os.path.commonprefix() is ignorant of path elements, this corrects
        # for that weird API.
        input_base_dir = os.path.dirname(input_base_dir)
    if options.output_dir:
        input_base_dir = input_base_dir.rstrip(os.sep)
        logger.info('Output in %r will mirror the input directory %r layout.',
                    options.output_dir, input_base_dir)

    # Initialize the refactoring tool
    if future.utils.PY26:
        extra_kwargs = {}
    else:
        extra_kwargs = {
                        'append_suffix': options.add_suffix,
                        'output_dir': options.output_dir,
                        'input_base_dir': input_base_dir,
                       }

    rt = StdoutRefactoringTool(
            sorted(fixer_names), flags, sorted(explicit),
            options.nobackups, not options.no_diffs,
            **extra_kwargs)

    # Refactor all files and directories passed as arguments
    if not rt.errors:
        if refactor_stdin:
            rt.refactor_stdin()
        else:
            try:
                rt.refactor(args, options.write, None,
                            options.processes)
            except refactor.MultiprocessingUnsupported:
                assert options.processes > 1
                print("Sorry, -j isn't " \
                      "supported on this platform.", file=sys.stderr)
                return 1
        rt.summarize()

    # Return error status (0 if rt.errors is zero)
    return int(bool(rt.errors))

Example 34

Project: plasma
Source File: console.py
View license
    def __init__(self, gctx):
        self.gctx = gctx
        self.db = gctx.db
        gctx.vim = False

        # After exiting the visual mode we copy last addresses we have visited.
        # Then we can just enter 'v' and we go where we were.
        self.last_entry = None
        self.last_stack = []
        self.last_saved_stack = []

        # A hack to allow window resizing
        os.environ['LINES']="blah"
        del os.environ['LINES']
        os.environ['COLUMNS']="blah"
        del os.environ['COLUMNS']

        self.COMMANDS = {
            "analyzer": Command(
                0,
                self.__exec_analyzer,
                None,
                [
                "",
                "Analyzer status.",
                ]
            ),

            "push_analyze_symbols": Command(
                0,
                self.push_analyze_symbols,
                None,
                [
                "",
                "Force to analyze the entry point, symbols and a memory scan will be done.",
                ]
            ),

            "help": Command(
                0,
                self.__exec_help,
                None,
                [
                "",
                "Display this help."
                ]
            ),

            "history": Command(
                0,
                self.__exec_history,
                None,
                [
                "",
                "Display the command history.",
                ]
            ),

            "save": Command(
                0,
                self.__exec_save,
                None,
                [
                "",
                "Save the database.",
                ]
            ),

            "x": Command(
                1,
                self.__exec_x,
                self.__complete_x,
                [
                "[SYMBOL|0xXXXX|EP]",
                "Decompile and print on stdout. By default it will be main.",
                "The decompilation is forced, it dosn't check if addresses",
                "are defined as code."
                ]
            ),

            "v": Command(
                1,
                self.__exec_v,
                self.__complete_x,
                [
                "[SYMBOL|0xXXXX|EP]",
                "Visual mode: if no address or symbol is given, you go at the",
                "previous address before exiting the visual mode.",
                "",
                "Main shortcuts:",
                "c       create code",
                "b/w/d/Q create byte/word/dword/qword",
                "a       create ascii string",
                "p       create function",
                "o       set [d|q]word as an offset",
                "*       create an array",
                "x       show xrefs",
                "r       rename",
                "space   highlight current word (ctrl-k to clear)",
                ";       edit inline comment (enter/escape to validate/cancel)",
                "U       undefine",
                "",
                "Options:",
                "I       switch to traditional instruction string output (3 modes)",
                "M       show/hide mangling",
                "B       show/hide bytes",
                "",
                "Navigation:",
                "/       binary search: if the first char is ! you can put an",
                "        hexa string example: /!ab 13 42",
                "        the search is case sensitive.",
                "n/N     next/previous search occurence",
                "g       top",
                "G       bottom",
                "z       set current line on the middle",
                "%       goto next bracket",
                "{ }     previous/next paragraph",
                "tab     switch between dump/decompilation",
                "enter   follow address",
                "escape  go back",
                "u       re-enter",
                "q       quit",
                ]
            ),

            "hexdump": Command(
                2,
                self.__exec_hexdump,
                self.__complete_x,
                [
                "SYMBOL|0xXXXX|EP [NB_LINES]",
                "Dump memory in hexa."
                ]
            ),

            # by default it will be gctx.nb_lines
            "dump": Command(
                2,
                self.__exec_dump,
                self.__complete_x,
                [
                "SYMBOL|0xXXXX|EP [NB_LINES]",
                "Print contents at the specified address.",
                ]
            ),

            "sym": Command(
                3,
                self.__exec_sym,
                self.__complete_x,
                [
                "[SYMBOL 0xXXXX] [| FILTER]",
                "Print all symbols or set a new symbol.",
                "You can filter symbols by searching the word FILTER.",
                "If FILTER starts with -, the match is inversed."
                ]
            ),

            "rename": Command(
                2,
                self.__exec_rename,
                self.__complete_x,
                [
                "OLD_SYM NEW_SYM",
                "Rename a symbol."
                ]
            ),

            "exit": Command(
                0,
                self.__exec_exit,
                None,
                [
                "",
                "Exit"
                ]
            ),

            "sections": Command(
                0,
                self.__exec_sections,
                None,
                [
                "",
                "Print all sections.",
                ]
            ),

            "info": Command(
                0,
                self.__exec_info,
                None,
                [
                "",
                "Information about the current binary."
                ]
            ),

            "jmptable": Command(
                4,
                self.__exec_jmptable,
                None,
                [
                "INST_ADDR TABLE_ADDR NB_ENTRIES SIZE_ENTRY",
                "Create a jump table referenced at TABLE_ADDR and called",
                "from INST_ADDR."
                ]
            ),

            "py": Command(
                -1,
                self.__exec_py,
                self.__complete_file,
                [
                "[!][FILE]",
                "Run an interactive python shell or execute a script.",
                "Global variables api and args will be passed to the script.",
                "The character ! is an alias to the scripts directory."
                ]
            ),

            "mips_set_gp": Command(
                1,
                self.__exec_mips_set_gp,
                None,
                [
                "ADDR",
                "Set the register $gp to a fixed value. Note that it will",
                "erase all defined memory."
                ]
            ),

            "functions": Command(
                1,
                self.__exec_functions,
                None,
                [
                "",
                "Print the function list."
                ]
            ),

            "xrefs": Command(
                1,
                self.__exec_xrefs,
                self.__complete_x,
                [
                "SYMBOL|0xXXXX|EP",
                "Print cross references to the specified address."
                ]
            ),

            "memmap": Command(
                0,
                self.__exec_memmap,
                None,
                [
                "",
                "Open a qt window to display the memory."
                ]
            ),
        }

        if gctx.dis.is_x86:
            import plasma.lib.arch.x86.analyzer as arch_analyzer
        elif gctx.dis.is_mips:
            import plasma.lib.arch.mips.analyzer as arch_analyzer
        elif gctx.dis.is_arm:
            import plasma.lib.arch.arm.analyzer as arch_analyzer

        self.analyzer = Analyzer()
        self.analyzer.init()
        self.analyzer.start()
        self.api = Api(gctx, self.analyzer)
        gctx.api = self.api
        self.analyzer.set(gctx, arch_analyzer)

        self.gctx.dis.binary.api = self.api

        if gctx.dis.is_mips and not gctx.dis.mips_gp:
            print("please run first these commands :")
            print("mips_set_gp 0xADDRESS")
            print("push_analyze_symbols")
        else:
            # If false it means that the first analysis was already done
            if gctx.autoanalyzer and len(self.db.mem) == 0:
                self.push_analyze_symbols(None)

        print("new feature: an instruction preceded by ! means that the analyzer", file=sys.stderr)
        print("has computed an immediate value. In the visual mode, use the shortcut", file=sys.stderr)
        print("I to show original instructions.", file=sys.stderr)

        self.comp = Completer(self)
        self.comp.set_history(self.db.history)

        while 1:
            self.comp.loop()
            if SHOULD_EXIT:
                break
            if not self.check_db_modified():
                break

        self.analyzer.msg.put("exit")

Example 35

Project: spinoff
Source File: win32fix.py
View license
def fix_unicode_on_win32():
    from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int
    from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID

    original_stderr = sys.stderr

    # If any exception occurs in this code, we'll probably try to print it on stderr,
    # which makes for frustrating debugging if stderr is directed to our wrapper.
    # So be paranoid about catching errors and reporting them to original_stderr,
    # so that we can at least see them.
    def _complain(message):
        print(original_stderr, message if isinstance(message, str) else repr(message))

    # Work around <http://bugs.python.org/issue6058>.
    codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)

    # Make Unicode console output work independently of the current code page.
    # This also fixes <http://bugs.python.org/issue1602>.
    # Credit to Michael Kaplan <http://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx>
    # and TZOmegaTZIOY
    # <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>.
    try:
        # <http://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx>
        # HANDLE WINAPI GetStdHandle(DWORD nStdHandle);
        # returns INVALID_HANDLE_VALUE, NULL, or a valid handle
        #
        # <http://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx>
        # DWORD WINAPI GetFileType(DWORD hFile);
        #
        # <http://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx>
        # BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode);

        GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(("GetStdHandle", windll.kernel32))
        STD_OUTPUT_HANDLE = DWORD(-11)
        STD_ERROR_HANDLE = DWORD(-12)
        GetFileType = WINFUNCTYPE(DWORD, DWORD)(("GetFileType", windll.kernel32))
        FILE_TYPE_CHAR = 0x0002
        FILE_TYPE_REMOTE = 0x8000
        GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))(("GetConsoleMode", windll.kernel32))
        INVALID_HANDLE_VALUE = DWORD(-1).value

        def not_a_console(handle):
            if handle == INVALID_HANDLE_VALUE or handle is None:
                return True
            return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
                    or GetConsoleMode(handle, byref(DWORD())) == 0)

        old_stdout_fileno = None
        old_stderr_fileno = None
        if hasattr(sys.stdout, 'fileno'):
            old_stdout_fileno = sys.stdout.fileno()
        if hasattr(sys.stderr, 'fileno'):
            old_stderr_fileno = sys.stderr.fileno()

        STDOUT_FILENO = 1
        STDERR_FILENO = 2
        real_stdout = (old_stdout_fileno == STDOUT_FILENO)
        real_stderr = (old_stderr_fileno == STDERR_FILENO)

        if real_stdout:
            hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
            if not_a_console(hStdout):
                real_stdout = False

        if real_stderr:
            hStderr = GetStdHandle(STD_ERROR_HANDLE)
            if not_a_console(hStderr):
                real_stderr = False

        if real_stdout or real_stderr:
            # BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, LPDWORD lpCharsWritten, LPVOID lpReserved);
            WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)(("WriteConsoleW", windll.kernel32))

            class UnicodeOutput:
                def __init__(self, hConsole, stream, fileno, name):
                    self._hConsole = hConsole
                    self._stream = stream
                    self._fileno = fileno
                    self.closed = False
                    self.softspace = False
                    self.mode = 'w'
                    self.encoding = 'utf-8'
                    self.name = name
                    self.flush()

                def isatty(self):
                    return False

                def close(self):
                    # don't really close the handle, that would only cause problems
                    self.closed = True

                def fileno(self):
                    return self._fileno

                def flush(self):
                    if self._hConsole is None:
                        try:
                            self._stream.flush()
                        except Exception as e:
                            _complain("%s.flush: %r from %r" % (self.name, e, self._stream))
                            raise

                def write(self, text):
                    try:
                        if self._hConsole is None:
                            if isinstance(text, unicode):
                                text = text.encode('utf-8')
                            self._stream.write(text)
                        else:
                            if not isinstance(text, unicode):
                                text = str(text).decode('utf-8')
                            remaining = len(text)
                            while remaining > 0:
                                n = DWORD(0)
                                # There is a shorter-than-documented limitation on the
                                # length of the string passed to WriteConsoleW (see
                                # <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>.
                                retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None)

                                if retval == 0 or n.value == 0:
                                    raise IOError("WriteConsoleW returned %r, n.value = %r" % (retval, n.value))
                                remaining -= n.value
                                if remaining == 0:
                                    break
                                text = text[n.value:]
                    except Exception as e:
                        _complain("%s.write: %r" % (self.name, e))
                        raise

                def writelines(self, lines):
                    try:
                        for line in lines:
                            self.write(line)
                    except Exception as e:
                        _complain("%s.writelines: %r" % (self.name, e))
                        raise

                def __repr__(self):
                    return self.name

            sys.stdout = (UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>')
                          if real_stdout else
                          UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>'))
            sys.stderr = (UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>')
                          if real_stderr else
                          UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>'))
    except Exception as e:
        _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,))

    if UNMANGLE_CMDLINE_ARGS:
        # This works around <http://bugs.python.org/issue2128>.
        GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
        CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(("CommandLineToArgvW", windll.shell32))

        argc = c_int(0)
        argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))

        argv = [argv_unicode[i].encode('utf-8') for i in xrange(0, argc.value)]

        if not hasattr(sys, 'frozen'):
            # If this is an executable produced by py2exe or bbfreeze, then it will
            # have been invoked directly. Otherwise, unicode_argv[0] is the Python
            # interpreter, so skip that.
            argv = argv[1:]

            # Also skip option arguments to the Python interpreter.
            while len(argv) > 0:
                arg = argv[0]
                if not arg.startswith(u"-") or arg == u"-":
                    break
                argv = argv[1:]
                if arg == u'-m':
                    # sys.argv[0] should really be the absolute path of the module source,
                    # but never mind
                    break
                if arg == u'-c':
                    argv[0] = u'-c'
                    break

        # if you like:
        sys.argv = argv

Example 36

Project: inkscape-silhouette
Source File: Strategy.py
View license
  def process_pyramids_barrier(s, y_slice, max_y, left2right=True):
    """ finding the next point involves overshadowing other points.
        Our assumption is, that it is save to cut the paper at point A, 
        whenever there is a triangle sitting on the baseline (where the 
        transport rollers are) with 2x 45 degree coming from both sides, 
        meeting at 90 degrees at point A, so that the inside of the 
        triangle is free of any cuts.

        We prefer to cut away from the rollers, if possible, but that is 
        subordinate rule -- applicable, whenever the cut direction can 
        be freely chosen. If point A is already part of a cut, then we cut the
        path A-B always towards A, never starting at A.

        A horizontal barrier Y_bar exists, that limits our downwards movement temporarily.
        We assume to be called again with lowered Y_bar (increased max_y, it counts downwards).

        Another barrier Xf_bar is a forward slanted 45 degree barrier that is swept sideways. 
        Points become eligible, if they are above Y_bar and behind Xf_bar.

        We start with the sideways barrier from left to right aka increasing x.
        In this case 'behind' means to the left of Xf_bar. (Every second sweep
        will be the opposite direction, but below only left to right is
        discussed).
        The very first point that is behind Xf_bar is the starting point A. Then we iterate:

        From any previous point A, we prefer to follow a line segment to reach 
        the next point B.  Segments are eligible, if their B is rightward from A, 
        (B.x greater or equal A.x). We chose the segment with the lowest B.y coordinate
        if there is any choice and check the following conditions:

        a) B is below Y_bar. 
           Compute point C as the intersection of Y_bar with [AB]. Replace 
           the segment [AB] by segments [AC], [CB]. Let B and C swap names.
        b) B is 45 degrees or more downwards from A (B.x-A.x < B.y-A.y) 
           We make an extra check to see if B would overshadow any point in the other 
           direction. Temporarily apply a backwards slanted barrier Xb_bar in A. 
           While moving the barrier to B, stop at the first point D to the left of AB 
           (i.e. ccw(A,B,D) == True) it hits, if any.
           If so, position Xb_bar in D, compute point E as the intersection of Xb_bar 
           with A-B. Replace the segment [AB] by segments [AE], [EB]. 
           If we have a point C remembered from a), then replace segments [EB], [BC] with [EC]
           and garbage collect point B and swap back roles B and C.
           Let B and E swap names.
        c) B is more than 45 degrees upwards from A. This overshadows A. But we ignore 
           that A may have more segments to be done. We keep that B and consider 
           the issue with A unsolvable.
           Note that 'advancing' from A to B in this case is actually backwards in 
           Xf_bar's view.

        If we now have no B, then we simply move the sideways barrier to reveal our 
        next A -- very likely a jump rather than a cut. If no todo segments are left in 
        the old A, drop that old A. Iterate.

        But if we have a B and it is not case c), then we tentatively advance Xf_bar 
        from A to B and record all new points F[] in the order we pass them. We
        don't care about them, if they are all 'below' (on the right hand side
        of) segment [AB].  For the first point F, that has ccw(A,B,F) == True,
        we position Xf_bar in F, if any.  If so, we compute point G as the
        intersection of Xf_bar with [AB]. Replace the segment [AB] by segments
        [AG], [GB]. We cut segment [AG]. We make F our next A - very likely a
        jump.  If no todo segments are left in the old A, drop that old A.
        Iterate.

        Exception for all subdivide actions above: if the segment is shorter than 
        self.min_subdivide, then just keep it as is.
        If subdividing produces segments shorter than self.min_segmentlen, then we later 
        garbage collect such segments. overshoot shoud be more than min_segmentlen to 
        compensate for this.

        If iteration exhausts, we are done with this processing sweep and
        report back the lowest remaining min_y coordinate of all points we left
        behind with segments todo. The next sweep will go the other direction.

        Caller should call us again with direction toggled the other way, and
        possibly advancing max_y = min_y + monotone_back_travel. The
        variable barrier_increment is not used here, as we compute the
        increment.

        In the above context, 'cutting' a segment means, to add it to the output
        list to deactivate its seg[] entries in the endpoints. Endpoints
        without active segments do not contribute to the min_y computation
        above, they are dropped.

        When all points are dropped, we did our final sweep and return min_y =
        None.  It is caller's responsibility to check the direction of each cut
        in the s.output list with regards to sharp points and cutting-towards-the-rollers.

        Assert that we cut at least one segment per sweep or drop at least one
        point per sweep.  Also the number of added segments and points should
        be less than what we eventually output and drop.  
        If not, the above algorithm may never end.

    """
    Xf_bar = Barrier(y_slice, key=lambda a:  a[0]+a[1] if a else 0) # forward:   / moving ->
    Xb_bar = Barrier(y_slice, key=lambda a: -a[0]+a[1] if a else 0) # backwards: \ moving <-

    if not left2right:                                        # forward:   \ moving <-
      Xf_bar,Xb_bar = Xb_bar,Xf_bar                           # backwards: / moving ->

    A = Xf_bar.point()
    while True:
      if A is None:
        Ai = Xf_bar.next()
        if Ai is None: break
        A = Xf_bar.point()
        continue
      print >>sys.stderr, "process_pyramids_barrier", left2right, A, A.att()

      B = None
      a_todo = 0
      for Bi in A.seg:
        if Bi >= 0:                              # segment already done
          a_todo += 1
          pt = s.points[Bi]
          if A.y+s.min_segmentlen >= max_y and pt.y > A.y:
            continue                             # Do not look downward when close to max_y.
                                                 # This avoids a busyloop after hitting Y_bar:
                                                 # ... subdiv, advance, retry, hit, subdiv, ...
          if left2right:
            if pt.x >= A.x:
              if B is None or not ccw(A,B,pt):   # find the leftmost of all [AB]
                B = pt
          else: # not left2right 
            if pt.x <= A.x:
              if B is None or ccw(A,B,pt):       # find the rightmost of all [AB]
                B = pt

      if B is None:
        print "no more forward segments", A, a_todo
        Xb_bar.find(A, start=0)
        if a_todo == 0:
          s.points[A.id] = None                 # drop A
        while True:
          Ai = Xf_bar.next()
          A = None
          print >>sys.stderr, "xx next Ai candidate", Ai
          if Ai is None: break
          A = Xf_bar.point()
          print >>sys.stderr, "xx next A candidate", A
          if A is None: break
          if not Xb_bar.ahead(A): 
            break
          else:
            print >>sys.stderr, "process_pyramids_barrier jump: Ignored A, ahead of Xb_bar", A
        print >>sys.stderr, "process_pyramids_barrier jump to next A", A
        continue                                # just advance Xf_bar: jump
      print "segment to check a), b)", A, B

      if False:                                  # fake to trigger check a)
        b_id = B.id
        B = XY_a((1.3,20-2.1))                  
        B.id = b_id
        B.seg = [1,2,A.id,3,4]
        print "faked segment to check a), b)", A, B
      #

      subdividable_ab = bool(dist_sq(A,B) > s.min_subdivide_sq)

      C = None
      E = None
      if subdividable_ab and B.y > max_y:        # check a)
        C = XY_a((intersect_y(A,B, max_y), max_y))
        ## same, but more expensive:
        # C2 = intersect_lines(A,B,XY_a((0,max_y)),XY_a((.5,max_y)))
        print "B below barrier, C=", C
        s.subdivide_segment(A,B,C)
        Xf_bar.insert(C)
        Xb_bar.insert(C)
        B,C = C,B
        # print A.seg, B.seg, C.seg
      #

      # All of the following shortens [AB] sufficiently, so that B does not 
      # cast shadow upwards on any other point: \B/ 
      # Such a point would become our B.
      # This asserts that there is no other point, whose pyramid would bury B.

      # check b)
      if (subdividable_ab and (
          (left2right and B.x-A.x < B.y-A.y) or (not left2right and A.x-B.x < B.y-A.y))):                     
        Xb_a_idx = Xb_bar.find(A, start=0)      # could also use lookup() here. It does not matter.
        Xb_b_idx = Xb_bar.find(B)               # could also use lookup() here. It does not matter.
        print "check b), moving Xb_bar from A to B", A, B, Xb_a_idx, Xb_b_idx, Xb_bar.key(A), Xb_bar.key(B)
        D = None
        for n in range(Xb_a_idx, Xb_b_idx+1):   # sweep from A to B
          pt = Xb_bar.point(n)
          if pt.id != A.id and pt.id != B.id and ccw(A,B,pt) == True:      
            D = pt                              # found a D that is clearly left of AB.
            break
          else:
            print "backsweep ignoring pt", pt, Xb_bar.key(pt)
        #
        if D is not None:                       # compute intersection of Xb_bar with [AB]
          E = intersect_lines(D,XY_a((D.x+1,D.y+1)),A,B,limit2=True)
          if E is None: raise ValueError("finding a shadowed D failed:", A, B, D)
          E = XY_a(E)
          s.subdivide_segment(A,B,E)
          Xf_bar.insert(E)
          Xb_bar.insert(E)
          if C is not None:
            s.shortcut_segment(E,C,B)           # replace segments [EB], [BC] with [EC]
            B,C = C,B
          B,E = E,B

      # tentatively advance Xf_bar from A to B
      Xf_a_idx = Xf_bar.pos()                   # unused, we never move back to A.
      Xf_b_idx = Xf_bar.lookup(lambda b: b.id==B.id if b else False) 
      if Xf_b_idx is None:                      # Should never happen!
        print "Xf_bar.lookup(B)=None. B=",B     # Okayish fallback, but find() may return
        Xf_b_idx = Xf_bar.find(B)               # a different point with the same key().
      print "line A,B:", A, B, Xf_a_idx, Xf_b_idx, Xf_bar.point(Xf_b_idx)
      F = None
      Xf_f_idx = None
      for n in range(Xf_a_idx, Xf_b_idx+1):     # sweep from A to B (inclusive)
        pt = Xf_bar.point(n)
        if pt is None: continue
        if subdividable_ab and pt.id != A.id and pt.id != B.id and ccw(A,B,pt) == (not left2right):      
          F = pt                                # found an F that is clearly right of AB.
          Xf_f_idx = n
          break
        else:
          print "forward sweep ignoring pt", n, pt, pt.id, Xf_bar.key(pt)
      #
      if F is not None:                       # compute intersection of Xb_bar with [AB]
        _F_back = (F.x-1,F.y+1) if left2right else (F.x+1,F.y+1)
        G = intersect_lines(F,XY_a(_F_back),A,B,limit2=True)
        if G is None: raise ValueError("finding a shadowed G failed:", A, B, F, _F_back)
        G = XY_a(G)
        s.subdivide_segment(A,B,G)
        Xf_bar.insert(G)
        Xb_bar.insert(G)
        if E is not None:
          pass
          ## FIXME: should s.shortcut_segment(...E) something here.
        s.output_add(A,G,cut=True)
        s.unlink_segment(A,G)
        Xf_bar.pos(Xf_f_idx)                 # advance to F, further up on the same barrier as G
        A = Xf_bar.point()
      #
      else:
        s.output_add(A,B,cut=True)
        s.unlink_segment(A,B)
        Xf_bar.pos(Xf_b_idx)                  # advance
        A = Xf_bar.point()
      print >>sys.stderr, "advanced A to",A

    ##  barrier has moved all the way to the other end.
    print >>sys.stderr, "barrier moved all the way", Xf_bar.points, max_y, A.att() if A else None, A.seg if A else None

Example 37

View license
def main():
  parser = optparse.OptionParser()
  parser.add_option('-o', dest='output_file', help='write HTML to FILE',
                    default='trace.html', metavar='FILE')
  parser.add_option('-t', '--time', dest='trace_time', type='int',
                    help='trace for N seconds', metavar='N')
  parser.add_option('-b', '--buf-size', dest='trace_buf_size', type='int',
                    help='use a trace buffer size of N KB', metavar='N')
  parser.add_option('-d', '--disk', dest='trace_disk', default=False,
                    action='store_true', help='trace disk I/O (requires root)')
  parser.add_option('-f', '--cpu-freq', dest='trace_cpu_freq', default=False,
                    action='store_true', help='trace CPU frequency changes')
  parser.add_option('-i', '--cpu-idle', dest='trace_cpu_idle', default=False,
                    action='store_true', help='trace CPU idle events')
  parser.add_option('-l', '--cpu-load', dest='trace_cpu_load', default=False,
                    action='store_true', help='trace CPU load')
  parser.add_option('-s', '--no-cpu-sched', dest='trace_cpu_sched', default=True,
                    action='store_false', help='inhibit tracing CPU ' +
                    'scheduler (allows longer trace times by reducing data ' +
                    'rate into buffer)')
  parser.add_option('-u', '--bus-utilization', dest='trace_bus_utilization',
                    default=False, action='store_true',
                    help='trace bus utilization (requires root)')
  parser.add_option('-w', '--workqueue', dest='trace_workqueue', default=False,
                    action='store_true', help='trace the kernel workqueues ' +
                    '(requires root)')
  parser.add_option('--set-tags', dest='set_tags', action='store',
                    help='set the enabled trace tags and exit; set to a ' +
                    'comma separated list of: ' +
                    ', '.join(trace_tag_bits.iterkeys()))
  parser.add_option('--link-assets', dest='link_assets', default=False,
                    action='store_true', help='link to original CSS or JS resources '
                    'instead of embedding them')
  parser.add_option('--from-file', dest='from_file', action='store',
                    help='read the trace from a file rather than running a live trace')
  parser.add_option('--asset-dir', dest='asset_dir', default='trace-viewer',
                    type='string', help='')
  parser.add_option('-e', '--serial', dest='device_serial', type='string',
                    help='adb device serial number')
  options, args = parser.parse_args()

  if options.set_tags:
    flags = 0
    tags = options.set_tags.split(',')
    for tag in tags:
      try:
        flags |= trace_tag_bits[tag]
      except KeyError:
        parser.error('unrecognized tag: %s\nknown tags are: %s' %
                     (tag, ', '.join(trace_tag_bits.iterkeys())))
    atrace_args = ['adb', 'shell', 'setprop', 'debug.atrace.tags.enableflags', hex(flags)]
    add_adb_serial(atrace_args, options.device_serial)
    try:
      subprocess.check_call(atrace_args)
    except subprocess.CalledProcessError, e:
      print >> sys.stderr, 'unable to set tags: %s' % e
    print '\nSet enabled tags to: %s\n' % ', '.join(tags)
    print ('You will likely need to restart the Android framework for this to ' +
          'take effect:\n\n    adb shell stop\n    adb shell ' +
          'start\n')
    return

  atrace_args = ['adb', 'shell', 'atrace', '-z']
  add_adb_serial(atrace_args, options.device_serial)

  if options.trace_disk:
    atrace_args.append('-d')
  if options.trace_cpu_freq:
    atrace_args.append('-f')
  if options.trace_cpu_idle:
    atrace_args.append('-i')
  if options.trace_cpu_load:
    atrace_args.append('-l')
  if options.trace_cpu_sched:
    atrace_args.append('-s')
  if options.trace_bus_utilization:
    atrace_args.append('-u')
  if options.trace_workqueue:
    atrace_args.append('-w')
  if options.trace_time is not None:
    if options.trace_time > 0:
      atrace_args.extend(['-t', str(options.trace_time)])
    else:
      parser.error('the trace time must be a positive number')
  if options.trace_buf_size is not None:
    if options.trace_buf_size > 0:
      atrace_args.extend(['-b', str(options.trace_buf_size)])
    else:
      parser.error('the trace buffer size must be a positive number')

  if options.from_file is not None:
    atrace_args = ['cat', options.from_file]

  script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))

  if options.link_assets:
    src_dir = os.path.join(script_dir, options.asset_dir, 'src')
    build_dir = os.path.join(script_dir, options.asset_dir, 'build')

    js_files, js_flattenizer, css_files, templates = get_assets(src_dir, build_dir)

    css = '\n'.join(linked_css_tag % (os.path.join(src_dir, f)) for f in css_files)
    js = '<script language="javascript">\n%s</script>\n' % js_flattenizer
    js += '\n'.join(linked_js_tag % (os.path.join(src_dir, f)) for f in js_files)

  else:
    css_filename = os.path.join(script_dir, flattened_css_file)
    js_filename = os.path.join(script_dir, flattened_js_file)
    css = compiled_css_tag % (open(css_filename).read())
    js = compiled_js_tag % (open(js_filename).read())
    templates = ''

  html_filename = options.output_file

  trace_started = False
  leftovers = ''
  adb = subprocess.Popen(atrace_args, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  dec = zlib.decompressobj()
  while True:
    ready = select.select([adb.stdout, adb.stderr], [], [adb.stdout, adb.stderr])
    if adb.stderr in ready[0]:
      err = os.read(adb.stderr.fileno(), 4096)
      sys.stderr.write(err)
      sys.stderr.flush()
    if adb.stdout in ready[0]:
      out = leftovers + os.read(adb.stdout.fileno(), 4096)
      if options.from_file is None:
        out = out.replace('\r\n', '\n')
      if out.endswith('\r'):
        out = out[:-1]
        leftovers = '\r'
      else:
        leftovers = ''
      if not trace_started:
        lines = out.splitlines(True)
        out = ''
        for i, line in enumerate(lines):
          if line == 'TRACE:\n':
            sys.stdout.write("downloading trace...")
            sys.stdout.flush()
            out = ''.join(lines[i+1:])
            html_prefix = read_asset(script_dir, 'prefix.html')
            html_file = open(html_filename, 'w')
            html_file.write(html_prefix % (css, js, templates))
            trace_started = True
            break
          elif 'TRACE:'.startswith(line) and i == len(lines) - 1:
            leftovers = line + leftovers
          else:
            sys.stdout.write(line)
            sys.stdout.flush()
      if len(out) > 0:
        out = dec.decompress(out)
      html_out = out.replace('\n', '\\n\\\n')
      if len(html_out) > 0:
        html_file.write(html_out)
    result = adb.poll()
    if result is not None:
      break
  if result != 0:
    print >> sys.stderr, 'adb returned error code %d' % result
  elif trace_started:
    html_out = dec.flush().replace('\n', '\\n\\\n').replace('\r', '')
    if len(html_out) > 0:
      html_file.write(html_out)
    html_suffix = read_asset(script_dir, 'suffix.html')
    html_file.write(html_suffix)
    html_file.close()
    print " done\n\n    wrote file://%s\n" % (os.path.abspath(options.output_file))
  else:
    print >> sys.stderr, ('An error occured while capturing the trace.  Output ' +
      'file was not written.')

Example 38

Project: p2ptv-pi
Source File: WebUI.py
View license
    def doget(self, urlpath):
        if not urlpath.startswith(URLPATH_WEBIF_PREFIX):
            return streaminfo404()
        else:
            self.lastreqtime = time.time()
            try:
                fakeurl = 'http://127.0.0.1' + urlpath[len(URLPATH_WEBIF_PREFIX):]
                if DEBUG:
                    log('webui::doget: fakeurl', fakeurl)
                request_url = urlparse.urlparse(fakeurl)
            except:
                print_exc()
                return

            path = request_url[2]
            query_string = request_url[4]
            query_params = urlparse.parse_qs(query_string)
            if DEBUG:
                log('webui::doget: urlpath', urlpath, 'request_url', request_url, 'path', path, 'query_params', query_params)
            if len(path) == 0:
                if DEBUG:
                    log('webui::doget: show status page')
                page = self.statusPage()
                pageStream = StringIO(page)
                return {'statuscode': 200,
                 'mimetype': 'text/html',
                 'stream': pageStream,
                 'length': len(page)}
            if path == 'permid.js':
                try:
                    permid = encodestring(self.bgApp.s.get_permid()).replace('\n', '')
                    txt = "var permid = '%s';" % permid
                    dataStream = StringIO(txt)
                except:
                    print_exc()
                    return {'statuscode': 500,
                     'statusmsg': 'Bad permid'}

                return {'statuscode': 200,
                 'mimetype': 'text/javascript',
                 'stream': dataStream,
                 'length': len(txt)}
            if path == '/createstream':
                if DEBUG:
                    log('webui::doget: show create stream page')
                page = self.createStreamPage()
                pageStream = StringIO(page)
                return {'statuscode': 200,
                 'mimetype': 'text/html',
                 'stream': pageStream,
                 'length': len(page)}
            if path == '/dispatch':
                if 'url' not in query_params:
                    if DEBUG:
                        log('webui::doget:dispatch: missing url')
                    return streaminfo404()
                url = query_params['url'][0]
                redirect_url = 'http://127.0.0.1:6878/webui/' + url
                params = []
                for name, val in query_params.iteritems():
                    if name != 'url':
                        params.append(urllib.quote_plus(name) + '=' + urllib.quote_plus(val[0]))

                if len(params):
                    redirect_url += '?' + '&'.join(params)
                if DEBUG:
                    log('webui::doget:dispatch: redirect_url', redirect_url)
                page = '<!DOCTYPE html><html><head><script type="text/javascript">'
                page += 'parent.location.href = "' + redirect_url + '";'
                page += '</script></head><body></body></html>'
                pageStream = StringIO(page)
                return {'statuscode': 200,
                 'mimetype': 'text/html',
                 'stream': pageStream,
                 'length': len(page)}
            if path.startswith('/player/') and query_params.has_key('a') and query_params['a'][0] == 'check':
                player_id = path.split('/')[2]
                redirect_url = 'http://127.0.0.1:6878/webui/player/' + player_id
                params = []
                for name, val in query_params.iteritems():
                    if name != 'a':
                        params.append(urllib.quote_plus(name) + '=' + urllib.quote_plus(val[0]))

                if len(params):
                    redirect_url += '?' + '&'.join(params)
                if DEBUG:
                    log('webui::doget:dispatch: redirect_url', redirect_url)
                page = '<!DOCTYPE html><html><head><script type="text/javascript">'
                page += 'parent.location.href = "' + redirect_url + '";'
                page += '</script></head><body></body></html>'
                pageStream = StringIO(page)
                return {'statuscode': 200,
                 'mimetype': 'text/html',
                 'stream': pageStream,
                 'length': len(page)}
            if path.startswith('/player/'):
                player_id = path.split('/')[2]
                if DEBUG:
                    log('webui::doget: show player page: id', player_id)
                params = {}
                for name, val in query_params.iteritems():
                    params[name] = val[0]

                page = self.playerPage(player_id, params)
                pageStream = StringIO(page)
                return {'statuscode': 200,
                 'mimetype': 'text/html',
                 'stream': pageStream,
                 'length': len(page)}
            static_path = None
            json_query = None
            if path.startswith('/json/'):
                json_query = request_url[4]
            else:
                static_path = os.path.join(self.webUIPath, path[1:])
            if DEBUG:
                log('webui::doget: request parsed: static_path', static_path, 'json_query', json_query)
            if static_path is not None:
                if not os.path.isfile(static_path):
                    if DEBUG:
                        log('webui::doget: file not found:', static_path)
                    return streaminfo404()
                extension = os.path.splitext(static_path)[1]
                if extension in self.binaryExtensions:
                    mode = 'rb'
                else:
                    mode = 'r'
                fp = open(static_path, mode)
                data = fp.read()
                fp.close()
                dataStream = StringIO(data)
                return {'statuscode': 200,
                 'mimetype': self.getContentType(extension),
                 'stream': dataStream,
                 'length': len(data)}
            if json_query is not None:
                params = {}
                for s in json_query.split('&'):
                    name, value = s.split('=')
                    params[name] = value

                if DEBUG:
                    log('webui:doget: got json request:', json_query, 'params', params)
                if 'q' not in params:
                    return
                try:
                    req = urllib.unquote(params['q'])
                    if DEBUG:
                        log('webui::doget: parse json: req', req)
                    jreq = json.loads(req)
                    if DEBUG:
                        log('webui::doget: parse json done: jreq', jreq)
                except:
                    print_exc()
                    return

                try:
                    method = jreq['method']
                except:
                    return {'statuscode': 504,
                     'statusmsg': 'Json request in wrong format! At least a method has to be specified!'}

                try:
                    args = jreq['arguments']
                    if DEBUG:
                        print >> sys.stderr, 'webUI: Got JSON request: ', jreq, '; method: ', method, '; arguments: ', args
                except:
                    args = None
                    if DEBUG:
                        print >> sys.stderr, 'webUI: Got JSON request: ', jreq, '; method: ', method

                if args is None:
                    data = self.process_json_request(method)
                    if DEBUG:
                        print >> sys.stderr, 'WebUI: response to JSON ', method, ' request: ', data
                else:
                    data = self.process_json_request(method, args)
                    if DEBUG:
                        print >> sys.stderr, 'WebUI: response to JSON ', method, ' request: ', data, ' arguments: ', args
                if data == 'Args missing':
                    return {'statuscode': 504,
                     'statusmsg': 'Json request in wrong format! Arguments have to be specified!'}
                dataStream = StringIO(data)
                return {'statuscode': 200,
                 'mimetype': 'application/json',
                 'stream': dataStream,
                 'length': len(data)}
            if DEBUG:
                log('webui::doget: unknow request format: request_url', request_url)
            return streaminfo404()

Example 39

Project: nesoni
Source File: fill_scaffolds.py
View license
def fill_scaffolds(args):
    max_filler_length, args = grace.get_option_value(args, '--max-filler', int, 4000)
    
    if len(args) < 2:
        print USAGE
        return 1
    
    (output_dir, graph_dir), args = args[:2], args[2:]

    scaffolds = [ ]
    
    def scaffold(args):
        circular, args = grace.get_option_value(args, '--circular', grace.as_bool, False)
        
        scaffold = [ ]
        for item in args:
            scaffold.append( ('contig', int(item)) )
            scaffold.append( ('gap', None) )
        
        if not circular: scaffold = scaffold[:-1]
        
        name = 'custom_scaffold_%d' % (len(scaffolds)+1)
        scaffolds.append( (name, scaffold) )
            
    grace.execute(args, [scaffold])
    
    custom_scaffolds = (len(scaffolds) != 0)    
    
    sequences = dict( 
        (a.split()[0], b.upper()) 
          for a,b in 
            io.read_sequences(os.path.join(
              graph_dir, '454AllContigs.fna')))
    
    sequence_names = sorted(sequences)
    sequence_ids = dict(zip(sequence_names, xrange(1,len(sequence_names)+1)))
    
    contexts = { }
    context_names = { }
    context_depths = { }
    for i in xrange(1,len(sequence_names)+1):
        seq = sequences[sequence_names[i-1]]
        contexts[ i ] = seq
        context_names[ i ] = sequence_names[i-1]+'-fwd'
        contexts[ -i ] = bio.reverse_complement(seq)
        context_names[ -i ] = sequence_names[i-1]+'-rev'
    
    links = collections.defaultdict(list)
    
    for line in open(
      os.path.join(graph_dir, '454ContigGraph.txt'),
      'rU'):
        parts = line.rstrip('\n').split('\t')
        
        if parts[0].isdigit():
            seq = sequence_ids[parts[1]]
            context_depths[ seq] = float(parts[3])
            context_depths[-seq] = float(parts[3])
        
        if parts[0] == 'C':    
            name1 = 'contig%05d' % int(parts[1])
            dir1 = {"3'" : 1, "5'" : -1 }[parts[2]]
            name2 = 'contig%05d' % int(parts[3])
            dir2 = {"5'" : 1, "3'" : -1 }[parts[4]]
            depth = int(parts[5])
            #print name1, dir1, name2, dir2, depth
            
            links[ sequence_ids[name1] * dir1 ].append( (depth, sequence_ids[name2] * dir2) )
            links[ sequence_ids[name2] * -dir2 ].append( (depth, sequence_ids[name1] * -dir1) )
    
        if parts[0] == 'S' and not custom_scaffolds:  
            name = 'scaffold%05d' % int(parts[2])  
            components = parts[3].split(';')
            scaffold = [ ]
            for component in components:
                a,b = component.split(':')
                if a == 'gap':
                    scaffold.append( ('gap',int(b)) )
                else:
                    strand = { '+': +1, '-': -1 }[ b ]
                    scaffold.append( ('contig', sequence_ids['contig%05d'%int(a)] * strand) )
            scaffolds.append( (name, scaffold) )
    
    
    
    #paths = { }
    #
    #todo = [ ]
    #for i in contexts:
    #    for depth_left, neg_left in links[-i]:
    #        left = -neg_left
    #        for depth_right, right in links[i]:
    #            todo.append( ( max(-depth_left,-depth_right,-context_depths[i]), left, right, (i,)) )
    #
    #heapq.heapify(todo)
    #while todo:
    #    score, source, dest, path = heapq.heappop(todo)
    #    if (source,dest) in paths: continue
    #    
    #    paths[(source,dest)] = path
    #    
    #    if len(contexts[dest]) > max_filler_length: continue
    #    
    #    for depth, next in links[dest]:
    #        heapq.heappush(todo,
    #            ( max(score,-depth,-context_depths[dest]), source, next, path+(dest,))
    #        )
    
    
    path_source_dest = collections.defaultdict(dict) # source -> dest -> next
    path_dest_source = collections.defaultdict(dict) # dest -> source -> next
    
    
    # Use links, in order to depth of coverage, to construct paths between contigs
    # Thus: paths have maximum minimum depth
    #       subsections of paths also have this property
    
    todo = [ ]
    for i in contexts:    
        for depth_link, right in links[i]:
            todo.append( ( depth_link, i, right) )
    todo.sort(reverse=True)
    for score, left, right in todo:
        if right in path_source_dest[left]: continue
        
        sources = [(left,right)]
        if len(contexts[left]) <= max_filler_length:
            sources += path_dest_source[left].items()
        destinations = [right]
        if len(contexts[right]) <= max_filler_length:
            destinations += path_source_dest[right].keys()
        
        for source, next in sources:
            for dest in destinations:
                if dest in path_source_dest[source]: continue
                path_source_dest[source][dest] = next
                path_dest_source[dest][source] = next
    
    
    workspace = io.Workspace(output_dir)
    scaffold_f = workspace.open('scaffolds.fa','wb')
    
    #comments = [ ]
    features = [ ]
    
    used = set()
    previous_total = 0
    
    for i, (name, scaffold) in enumerate(scaffolds):
        result = '' # Inefficient. Meh.
        n_filled = 0
        n_failed = 0
        for j, item in enumerate(scaffold):
            if item[0] == 'contig':
                result += contexts[item[1]]
                used.add(abs(item[1]))
            else:
                left = scaffold[j-1]
                right = scaffold[ (j+1) % len(scaffold) ] #If gap at end, assume circular
                assert left[0] == 'contig'
                assert right[0] == 'contig'
                
                gap_start = len(result)
    
                can_fill = right[1] in path_source_dest[left[1]]
                if can_fill:
                    n = 0
                    k = path_source_dest[left[1]][right[1]]
                    while k != right[1]:
                        n += len(contexts[k])
                        result += contexts[k].lower()
                        used.add(abs(k))
                        
                        k = path_source_dest[k][right[1]]
                    
                    n_filled += 1
                        
                    if item[1] is not None and max(n,item[1]) > min(n,item[1])*4:
                        print >> sys.stderr, 'Warning: gap size changed from %d to %d in scaffold %d' % (item[1],n,i+1)
                else:
                    n_failed += 1
                    
                    #print >> sys.stderr, 'Warning: No path to fill a gap in scaffold %d' % (i+1)
                    result += 'n' * (9 if item[1] is None else item[1])
    
                gap_end = len(result)
                
                #features.append( '%s\t%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s' % (
                #    'all-scaffolds',
                #    'fill-scaffolds',
                #    'gap',
                #    previous_total + gap_start+1,
                #    previous_total + max(gap_end, gap_start+1), #Allow for zeroed out gaps. Hmm.
                #    '.', #score
                #    '+', #strand
                #    '.', #frame
                #    '' #properties
                #))
                features.append( '%s\t%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s' % (
                    name,
                    'fill-scaffolds',
                    'gap',
                    gap_start+1,
                    max(gap_end, gap_start+1), #Allow for zeroed out gaps. Hmm.
                    '.', #score
                    '+', #strand
                    '.', #frame
                    '' #properties
                ))
                    
    
        io.write_fasta(scaffold_f, name, result)
        previous_total += len(result)
        #comments.append('##sequence-region    %s %d %d' % (name, 1, len(result)))
        print >> sys.stderr, 'Scaffold%05d: %d gaps filled, %d could not be filled' % (i+1, n_filled, n_failed)
    
    scaffold_f.close()
    
    gff_f = workspace.open('scaffolds.gff', 'wb')
    #print >>gff_f, '##gff-version    3'
    #for comment in comments:
    #    print >>gff_f, comment
    for feature in features:
        print >>gff_f, feature
    gff_f.close()
    
    
    leftovers_f = workspace.open('leftovers.fa', 'wb')
    for name in sequence_names:
        if sequence_ids[name] not in used:
            io.write_fasta(leftovers_f, name, sequences[name])
    leftovers_f.close()
    
    ends = { }
    for i, (name, scaffold) in enumerate(scaffolds):
        if scaffold[-1][0] == 'gap': continue
        ends[ '%s start' % name ] = scaffold[-1][1]
        ends[ '%s end  ' % name ] = -scaffold[0][1] 
    
    for end1 in sorted(ends):
        options = [ end2 for end2 in ends if -ends[end2] in path_source_dest[ends[end1]] ]
        if len(options) == 1:
            print >> sys.stderr, 'Note: from', end1, 'only', options[0], 'is reachable'

Example 40

Project: dx-toolkit
Source File: app_unbuilder.py
View license
def dump_executable(executable, destination_directory, omit_resources=False, describe_output=[]):
    """
    Reconstitutes executable into a directory that would create a
    functionally identical executable if "dx build" were run on it.
    destination_directory will be the root source directory for the
    applet.

    :param executable: executable, i.e. app or applet,  to be dumped
    :type executable: DXExecutable (only DXApp or DXApplet now)
    :param destination_directory: an existing, empty, and writable directory
    :type destination_directory: str
    """

    old_cwd = os.getcwd()
    os.chdir(destination_directory)

    try:
        info = executable.get()

        if info["runSpec"]["interpreter"] == "bash":
            suffix = "sh"
        elif info["runSpec"]["interpreter"] == "python2.7":
            suffix = "py"
        else:
            print('Sorry, I don\'t know how to get executables with interpreter ' +
                  info["runSpec"]["interpreter"] + '\n', file=sys.stderr)
            sys.exit(1)

        # Entry point script
        script = "src/code.%s" % (suffix,)
        os.mkdir("src")
        with open(script, "w") as f:
            f.write(info["runSpec"]["code"])

        # Get all the asset bundles
        asset_depends = []
        deps_to_remove = []

        # When an applet is built bundledDepends are added in the following order:
        # 1. bundledDepends explicitly specified in the dxapp.json
        # 2. resources (contents of resources directory added as bundledDepends)
        # 3. assetDepends (translated into bundledDepends)
        #
        # Therefore while translating bundledDepends to assetDepends, we are traversing the
        # list in reverse order and exiting when we can't find the "AssetBundle" property
        # with the tarball file.
        #
        # NOTE: If last item (and contiguous earlier items) of bundledDepends (#1 above) refers to an
        # AssetBundle tarball, those items will be converted to assetDepends.
        #
        # TODO: The bundledDepends should be annotated with another field called {"asset": true}
        # to distinguish it from non assets. It will be needed to annotate the bundleDepends,
        # when the wrapper record object is no more accessible.

        for dep in reversed(info["runSpec"]["bundledDepends"]):
            file_handle = get_handler(dep["id"])
            if isinstance(file_handle, dxpy.DXFile):
                asset_record_id = file_handle.get_properties().get("AssetBundle")
                asset_record = None
                if asset_record_id:
                    asset_record = dxpy.DXRecord(asset_record_id)
                    if asset_record:
                        try:
                            asset_depends.append({"name": asset_record.describe().get("name"),
                                                  "project": asset_record.get_proj_id(),
                                                  "folder": asset_record.describe().get("folder"),
                                                  "version": asset_record.describe(fields={"properties": True}
                                                                                   )["properties"]["version"]
                                                  })
                            deps_to_remove.append(dep)
                        except DXError:
                            print("Describe failed on the assetDepends record object with ID - " +
                                  asset_record_id + "\n", file=sys.stderr)
                            pass
                else:
                    break
        # Reversing the order of the asset_depends[] so that original order is maintained
        asset_depends.reverse()
        # resources/ directory
        created_resources_directory = False
        if not omit_resources:
            for dep in info["runSpec"]["bundledDepends"]:
                if dep in deps_to_remove:
                    continue
                handler = get_handler(dep["id"])
                if isinstance(handler, dxpy.DXFile):
                    if not created_resources_directory:
                        os.mkdir("resources")
                        created_resources_directory = True
                    handler_id = handler.get_id()
                    fname = "resources/%s.tar.gz" % (handler_id)
                    download_dxfile(handler_id, fname)
                    print("Unpacking resources", file=sys.stderr)
                    tar = tarfile.open(fname)
                    tar.extractall("resources")
                    tar.close()
                    os.unlink(fname)
                    deps_to_remove.append(dep)

        # TODO: if output directory is not the same as executable name we
        # should print a warning and/or offer to rewrite the "name"
        # field in the 'dxapp.json'
        dxapp_json = collections.OrderedDict()
        all_keys = executable._get_required_keys() + executable._get_optional_keys()
        for key in all_keys:
            if key in executable._get_describe_output_keys() and key in describe_output:
                dxapp_json[key] = describe_output[key]
            if key in info:
                dxapp_json[key] = info[key]
        if info.get("hidden", False):
            dxapp_json["hidden"] = True

        # TODO: inputSpec and outputSpec elements should have their keys
        # printed in a sensible (or at least consistent) order too

        # Un-inline code
        del dxapp_json["runSpec"]["code"]
        dxapp_json["runSpec"]["file"] = script

        # Remove resources from bundledDepends
        for dep in deps_to_remove:
            dxapp_json["runSpec"]["bundledDepends"].remove(dep)

        # Add assetDepends to dxapp.json
        if len(asset_depends) > 0:
            dxapp_json["runSpec"]["assetDepends"] = asset_depends

        # Ordering input/output spec keys
        ordered_spec_keys = ("name", "label", "help", "class", "type", "patterns", "optional", "default", "choices",
                             "suggestions", "group")
        for spec_key in "inputSpec", "outputSpec":
            if spec_key not in dxapp_json.keys():
                continue
            for i, spec in enumerate(dxapp_json[spec_key]):
                ordered_spec = collections.OrderedDict()
                # Adding keys, for which the ordering is defined
                for key in ordered_spec_keys:
                    if key in spec.keys():
                        ordered_spec[key] = spec[key]
                # Adding the rest of the keys
                for key in spec.keys():
                    if key not in ordered_spec_keys:
                        ordered_spec[key] = spec[key]
                dxapp_json[spec_key][i] = ordered_spec

        # Remove dx-toolkit from execDepends
        dx_toolkit = {"name": "dx-toolkit", "package_manager": "apt"}
        if dx_toolkit in dxapp_json["runSpec"]["execDepends"]:
            dxapp_json["runSpec"]["execDepends"].remove(dx_toolkit)

        # Remove "bundledDependsByRegion" field from "runSpec". This utility
        # will reconstruct the resources directory based on the
        # "bundledDepends" field, which should be equivalent to
        # "bundledDependsByRegion".
        dxapp_json["runSpec"].pop("bundledDependsByRegion", None)

        # Cleanup of empty elements. Be careful not to let this step
        # introduce any semantic changes to the app specification. For
        # example, an empty input (output) spec is not equivalent to a
        # missing input (output) spec.
        if 'runSpec' in dxapp_json:
            _recursive_cleanup(dxapp_json['runSpec'])
        if 'access' in dxapp_json:
            _recursive_cleanup(dxapp_json['access'])
        for key in executable._get_cleanup_keys():
            if key in dxapp_json and not dxapp_json[key]:
                del dxapp_json[key]

        readme = info.get("description", "")
        devnotes = info.get("developerNotes", "")

        # Write dxapp.json, Readme.md, and Readme.developer.md
        with open("dxapp.json", "w") as f:
            f.write(flatten_json_array(json.dumps(dxapp_json, indent=2, separators=(',', ': ')), "patterns"))
            f.write('\n')
        if readme:
            with open("Readme.md", "w") as f:
                f.write(readme)
        if devnotes:
            with open("Readme.developer.md", "w") as f:
                f.write(devnotes)
    except:
        err_exit()
    finally:
        os.chdir(old_cwd)

Example 41

Project: lowrank-gru
Source File: RNN_adding.py
View license
def main(n_iter, n_batch, n_hidden, time_steps, learning_rate, savefile, scale_penalty, use_scale,
         model, n_hidden_lstm, loss_function, n_gru_lr_proj, initial_b_u):

    #import pdb; pdb.set_trace()
 
    # --- Set optimization params --------
    gradient_clipping = np.float32(50000)

    # --- Set data params ----------------
    n_input = 2
    n_output = 1
  

    # --- Manage data --------------------
    n_train = 1e5
    n_test = 1e4
    num_batches = n_train / n_batch
    
    train_x = np.asarray(np.zeros((time_steps, n_train, 2)),
                         dtype=theano.config.floatX)
    

    train_x[:,:,0] = np.asarray(np.random.uniform(low=0.,
                                                  high=1.,
                                                  size=(time_steps, n_train)),
                                dtype=theano.config.floatX)
    
#    inds = np.asarray([np.random.choice(time_steps, 2, replace=False) for i in xrange(train_x.shape[1])])    
    inds = np.asarray(np.random.randint(time_steps/2, size=(train_x.shape[1],2)))
    inds[:, 1] += time_steps/2  
    
    for i in range(train_x.shape[1]):
        train_x[inds[i, 0], i, 1] = 1.0
        train_x[inds[i, 1], i, 1] = 1.0
 
    train_y = (train_x[:,:,0] * train_x[:,:,1]).sum(axis=0)
    train_y = np.reshape(train_y, (n_train, 1))

    test_x = np.asarray(np.zeros((time_steps, n_test, 2)),
                        dtype=theano.config.floatX)
    

    test_x[:,:,0] = np.asarray(np.random.uniform(low=0.,
                                                 high=1.,
                                                 size=(time_steps, n_test)),
                                dtype=theano.config.floatX)
    
    inds = np.asarray([np.random.choice(time_steps, 2, replace=False) for i in xrange(test_x.shape[1])])    
    for i in range(test_x.shape[1]):
        test_x[inds[i, 0], i, 1] = 1.0
        test_x[inds[i, 1], i, 1] = 1.0
 
   
    test_y = (test_x[:,:,0] * test_x[:,:,1]).sum(axis=0)
    test_y = np.reshape(test_y, (n_test, 1)) 


   #######################################################################

    gradient_clipping = np.float32(1)

    if (model == 'LSTM'):   
        #inputs, parameters, costs = LSTM(n_input, n_hidden_lstm, n_output, loss_function=loss_function)
        inputs, parameters, costs = LSTM(n_input, n_hidden_lstm, n_output, loss_function=loss_function, initial_b_f=initial_b_u)
        gradients = T.grad(costs[0], parameters)
        gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]
    
    #by AnvaMiba
    elif (model == 'GRU'):   
        inputs, parameters, costs = GRU(n_input, n_hidden_lstm, n_output, loss_function=loss_function, initial_b_u=initial_b_u)
        gradients = T.grad(costs[0], parameters)
        gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]
    #by AnvaMiba
    elif (model == 'GRU_LR'):   
        inputs, parameters, costs = GRU_LR(n_input, n_hidden_lstm, n_output, n_gru_lr_proj, loss_function=loss_function, initial_b_u=initial_b_u)
        gradients = T.grad(costs[0], parameters)
        gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]

    elif (model == 'complex_RNN'):
        inputs, parameters, costs = complex_RNN(n_input, n_hidden, n_output, scale_penalty, loss_function=loss_function)
        if use_scale is False:
            parameters.pop()
        gradients = T.grad(costs[0], parameters)

    elif (model == 'complex_RNN_LSTM'):
        inputs, parameters, costs = complex_RNN_LSTM(n_input, n_hidden, n_hidden_lstm, n_output, scale_penalty, loss_function=loss_function)

    elif (model == 'IRNN'):
        inputs, parameters, costs = IRNN(n_input, n_hidden, n_output, loss_function=loss_function)
        gradients = T.grad(costs[0], parameters)
        gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]

    elif (model == 'RNN'):
        inputs, parameters, costs = tanhRNN(n_input, n_hidden, n_output, loss_function=loss_function)
        gradients = T.grad(costs[0], parameters)
        gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]

    else:
        print >> sys.stderr, "Unsuported model:", model
        return
 

   




    s_train_x = theano.shared(train_x)
    s_train_y = theano.shared(train_y)

    s_test_x = theano.shared(test_x)
    s_test_y = theano.shared(test_y)


    # --- Compile theano functions --------------------------------------------------

    index = T.iscalar('i')

    updates, rmsprop = rms_prop(learning_rate, parameters, gradients)

    givens = {inputs[0] : s_train_x[:, n_batch * index : n_batch * (index + 1), :],
              inputs[1] : s_train_y[n_batch * index : n_batch * (index + 1), :]}

    givens_test = {inputs[0] : s_test_x,
                   inputs[1] : s_test_y}
    
   
    
    train = theano.function([index], costs[0], givens=givens, updates=updates)
    test = theano.function([], costs[1], givens=givens_test)

    # --- Training Loop ---------------------------------------------------------------

    # f1 = file('/data/lisatmp3/shahamar/adding/complexRNN_400.pkl', 'rb')
    # data1 = cPickle.load(f1)
    # f1.close()
    # train_loss = data1['train_loss']
    # test_loss = data1['test_loss']
    # best_params = data1['best_params']
    # best_test_loss = data1['best_test_loss']

    # for i in xrange(len(parameters)):
    #     parameters[i].set_value(data1['parameters'][i])

    # for i in xrange(len(parameters)):
    #     rmsprop[i].set_value(data1['rmsprop'][i])
    
#    import pdb; pdb.set_trace()

    train_loss = []
    test_loss = []
    best_params = [p.get_value() for p in parameters]
    best_test_loss = 1e6
    for i in xrange(n_iter):
#        start_time = timeit.default_timer()


        if (n_iter % int(num_batches) == 0):
            #import pdb; pdb.set_trace()
            inds = np.random.permutation(int(n_train))
            data_x = s_train_x.get_value()
            s_train_x.set_value(data_x[:,inds,:])
            data_y = s_train_y.get_value()
            s_train_y.set_value(data_y[inds,:])


        mse = train(i % int(num_batches))
        train_loss.append(mse)
        print >> sys.stderr, "Iteration:", i
        print >> sys.stderr, "mse:", mse
        print >> sys.stderr, ''

        #if (i % 50==0):
        if (i % 500==0):
            mse = test()
            print >> sys.stderr, ''
            print >> sys.stderr, "TEST"
            print >> sys.stderr, "mse:", mse
            print >> sys.stderr, '' 
            test_loss.append(mse)

            if mse < best_test_loss:
                best_params = [p.get_value() for p in parameters]
                best_test_loss = mse

            
            save_vals = {'parameters': [p.get_value() for p in parameters],
                         'rmsprop': [r.get_value() for r in rmsprop],
                         'train_loss': train_loss,
                         'test_loss': test_loss,
                         'best_params': best_params,
                         'best_test_loss': best_test_loss,
                         'model': model,
                         'time_steps': time_steps}

            cPickle.dump(save_vals,
                         file(savefile, 'wb'),
                         cPickle.HIGHEST_PROTOCOL)

Example 42

Project: biocode
Source File: batch_attributor.py
View license
def main():
    parser = argparse.ArgumentParser( description='Generates a shell script of Attributor runs given a range of pipeline IDs')

    ## output file to be written
    parser.add_argument('-o', '--output_file', type=str, required=True, help='Path to an output shell file to be created' )
    parser.add_argument('-od', '--output_directory', type=str, required=True, help='Directory where output files will be written' )
    args = parser.parse_args()

    # Either set this variable to a path, or use the pipeline_min/pipeline_max below
    #  If you dont' want to use this, set it to None
    srs_id_list_file = '/path/to/some.id.list'
    srs_ids_to_keep = list()

    # inclusive.  These are NOT read if the srs_id_list_file is defined above
    #pipeline_min = 10927248802
    #pipeline_max = 11214274766
    pipeline_min = 11305294429
    pipeline_max = 11373273817

    # CONFIG
    project_area = '/local/hmp/dacc/t3/'
    owners = ['hhuot', 'cmccracken']
    config_template = '/home/hhuot/git/Attributor/assign_functional_annotation.example_hhc.config'
    # assumes all child names are like: SRS147134.rapsearch2.m8.gz
    rapsearch2_base = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmgi/rapsearch'
    attributor_path = '/home/hhuot/git/Attributor/assign_functional_annotation.py'
    # names are like either SRS045739.metagenemark.gff3 or SRS045763.metagenemark3.gff3
    gff_base_dir = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmorf/gff3'
    # names are like either SRS018791.metagenemark.faa or SRS018794.metagenemark3.faa
    fasta_base_dir = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmorf/faa'

    # key = pipeline ID, value = SRS ID
    pipelines = dict()

    # We're either getting our SRS ID list to keep from a file or a range of pipeline IDs
    if srs_id_list_file is not None:
        for line in open(srs_id_list_file):
            line = line.rstrip()
            srs_ids_to_keep.append(line)

    ## make sure the config template exists
    if not os.path.exists(config_template):
        raise Exception("ERROR: config template file not found: {0}".format(config_template))

    for pipeline_id in os.listdir("{0}/workflow/runtime/pipeline".format(project_area)):
        m = re.match("^\d+$", pipeline_id)
        if m:
            pipeline_id = int(pipeline_id)
            if srs_id_list_file is None:
                if pipeline_id > pipeline_max or pipeline_id < pipeline_min:
                    continue
        else:
            continue

        pipeline_path = "{0}/workflow/runtime/pipeline/{1}/pipeline.xml".format(project_area, pipeline_id)

        owner = find_owner(pipeline_path)
        if owner not in owners:
            continue

        pipeline_comment_path = "{0}/workflow/runtime/pipeline/{1}/pipeline.xml.comment".format(project_area, pipeline_id)
        if os.path.exists(pipeline_comment_path):
            comment = open(pipeline_comment_path).read()

            m = re.search("(SRS\d+)", comment)
            if m:
                srs_id = m.group(1)
                pipelines[pipeline_id] = srs_id
            else:
                m = re.search("(\d+\..{2,4})", comment)
                if m:
                    srs_id = m.group(1)
                    pipelines[pipeline_id] = srs_id
                else:
                    print("WARNING: Pipeline skipped without an SRS comment: {0}".format(pipeline_comment_path), file=sys.stderr)

    print("INFO: found {0} pipelines".format(len(pipelines)))

    batch_fh = open(args.output_file, 'wt')

    for pipeline_id in pipelines:
        # do we have a limit on the SRS IDs?
        if srs_id_list_file is not None:
            if pipelines[pipeline_id] not in srs_ids_to_keep:
                continue
        
        config_path = "{0}/{1}.config".format(args.output_directory, pipelines[pipeline_id])
        tmp_config_path = "/tmp/{0}.config".format(pipeline_id)

        ## get the source FASTA path
        if os.path.exists("{0}/{1}.metagenemark.faa".format(fasta_base_dir, pipelines[pipeline_id])):
            fasta_path = "{0}/{1}.metagenemark.faa".format(fasta_base_dir, pipelines[pipeline_id])
        elif os.path.exists("{0}/{1}.metagenemark3.faa".format(fasta_base_dir, pipelines[pipeline_id])):
            fasta_path = "{0}/{1}.metagenemark3.faa".format(fasta_base_dir, pipelines[pipeline_id])
        else:
            print("WARNING: failed to find FASTA file for {0}".format(pipelines[pipeline_id]), file=sys.stderr)
            continue
            #raise Exception("ERROR: failed to find FASTA file for {0}".format(pipelines[pipeline_id]))

        ## get the source GFF path
        if os.path.exists("{0}/{1}.metagenemark.gff3".format(gff_base_dir, pipelines[pipeline_id])):
            gff3_path = "{0}/{1}.metagenemark.gff3".format(gff_base_dir, pipelines[pipeline_id])
        elif os.path.exists("{0}/{1}.metagenemark3.gff3".format(gff_base_dir, pipelines[pipeline_id])):
            gff3_path = "{0}/{1}.metagenemark3.gff3".format(gff_base_dir, pipelines[pipeline_id])
        else:
            print("WARNING: failed to find GFF3 file for {0}".format(pipelines[pipeline_id]), file=sys.stderr)
            continue
            #raise Exception("ERROR: failed to find GFF3 file for {0}".format(pipelines[pipeline_id]))

        ## copy the config file
        shutil.copy(config_template, config_path)

        ## modify it with these paths
        ofh = open(tmp_config_path, 'wt')
        last_label = None

        for line in open(config_path):
            line = line.rstrip()
            m = re.match("   - label: (.+)", line)
            if m:
                last_label = m.group(1)

            m = re.match("     path: (.+)", line)
            if m:
                if last_label == 'coding_hmm_lib__equivalog':
                    ofh.write("     path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'rapsearch2__uniref100__all_full':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'rapsearch2__uniref100__all_partial':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'rapsearch2__uniref100__trusted_full':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'rapsearch2__uniref100__trusted_partial':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'coding_hmm_lib__equivalog_domain':
                    ofh.write("     path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'coding_hmm_lib__all_trusted':
                    ofh.write("     path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'tmhmm':
                    ofh.write("     path: {0}/output_repository/tmhmm/{1}_default/tmhmm.raw.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'lipoprotein_motif':
                    ofh.write("     path: {0}/output_repository/lipoprotein_motif/{1}_default/lipoprotein_motif.bsml.list\n".format(project_area, pipeline_id))
                    continue

            m = re.match("   polypeptide_fasta: (\S+)", line)
            if m:
                ofh.write("   polypeptide_fasta: {0}\n".format(fasta_path))
                continue

            m = re.match("   gff3: (\S+)", line)
            if m:
                ofh.write("   gff3: {0}\n".format(gff3_path))
                continue

            ofh.write("{0}\n".format(line))

        ofh.close()
        shutil.move(tmp_config_path, config_path)
        batch_fh.write("{0} -f gff3 -c {1} -o {2}/{3}\n".format(attributor_path, config_path, args.output_directory,
                                                        pipelines[pipeline_id]))

Example 43

Project: camr
Source File: amr_parsing.py
View license
def main():
    '''
    usage = "Usage:%prog [options] amr_file"
    opt = OptionParser(usage=usage)
    opt.add_option("-v",action="store",dest="verbose",type='int',
                   default=0,help="set up verbose level")
    opt.add_option("-a",action="store_true",dest="align",
                   default=False,help="do alignment between sentence and amr")
    opt.add_option("-b",action="store",dest="begin",type='int',
                   default=0,help="for debugging"
                   "When do alignment, where the alignment begins"
                   "When test oracle, where to begin")
    opt.add_option("-s",action="store",dest="start_step",type='int',
                   default=0,help="where the step begins,for testing oracle")
    opt.add_option("-o",action="store",dest="sentfilep",
                   help="output sentences to file and parse the sentence into dependency graph")
    opt.add_option("-i",action="store",dest="parsedfilep",
                   help="read parsed dependency graph from file")
    opt.add_option("-g",action="store",dest="userActfile",
                   help="read user input action sequences as guide")
    opt.add_option("-d",action="store",dest="oracle",type='int',default=0,\
                   help="test the output actions of deterministic oracle: "
                         "1: tree oracle 2: list-based oracle")
    '''
    arg_parser = argparse.ArgumentParser(description="Brandeis transition-based AMR parser 1.0")
    
    arg_parser.add_argument('-v','--verbose',type=int,default=0,help='set up verbose level for debug')
    arg_parser.add_argument('-b','--begin',type=int,default=0,help='specify which sentence to begin the alignment or oracle testing for debug')
    arg_parser.add_argument('-s','--start_step',type=int,default=0,help='specify which step to begin oracle testing;for debug')
    #arg_parser.add_argument('-i','--input_file',help='the input: preprocessed data instances file for aligner or training')
    arg_parser.add_argument('-d','--dev',help='development file')
    arg_parser.add_argument('-a','--add',help='additional training file')
    arg_parser.add_argument('-as','--actionset',choices=['basic'],default='basic',help='choose different action set')
    arg_parser.add_argument('-m','--mode',choices=['preprocess','test_gold_graph','align','userGuide','oracleGuide','train','parse','eval'],help="preprocess:generate pos tag, dependency tree, ner\n" "align:do alignment between AMR graph and sentence string")
    arg_parser.add_argument('-dp','--depparser',choices=['stanford','stanfordConvert','stdconv+charniak','clear','mate','turbo'],default='stdconv+charniak',help='choose the dependency parser')
    arg_parser.add_argument('--coref',action='store_true',help='flag to enable coreference information')
    arg_parser.add_argument('--prop',action='store_true',help='flag to enable semantic role labeling information')
    arg_parser.add_argument('--rne',action='store_true',help='flag to enable rich name entity')
    arg_parser.add_argument('--verblist',action='store_true',help='flag to enable verbalization list')
    #arg_parser.add_argument('--onto',action='store_true',help='flag to enable charniak parse result trained on ontonotes')
    arg_parser.add_argument('--onto',choices=['onto','onto+bolt','wsj'],default='wsj',help='choose which charniak parse result trained on ontonotes')
    arg_parser.add_argument('--model',help='specify the model file')
    arg_parser.add_argument('--feat',help='feature template file')
    arg_parser.add_argument('-iter','--iterations',default=1,type=int,help='training iterations')
    arg_parser.add_argument('amr_file',nargs='?',help='amr annotation file/input sentence file for parsing')
    arg_parser.add_argument('--amrfmt',choices=['sent','amr','amreval'],default='sent',help='specifying the input file format')
    arg_parser.add_argument('--smatcheval',action='store_true',help='give evaluation score using smatch')
    arg_parser.add_argument('-e','--eval',nargs=2,help='Error Analysis: give parsed AMR file and gold AMR file')
    arg_parser.add_argument('--section',choices=['proxy','all'],default='all',help='choose section of the corpus. Only works for LDC2014T12 dataset.')

    args = arg_parser.parse_args()

    amr_file = args.amr_file
    instances = None
    train_instance = None
    constants.FLAG_COREF=args.coref
    constants.FLAG_PROP=args.prop
    constants.FLAG_RNE=args.rne
    constants.FLAG_VERB=args.verblist
    constants.FLAG_ONTO=args.onto
    constants.FLAG_DEPPARSER=args.depparser

    # using corenlp to preprocess the sentences 
    if args.mode == 'preprocess':
        instances = preprocess(amr_file,START_SNLP=True,INPUT_AMR=args.amrfmt)
        print "Done preprocessing!"
    # preprocess the JAMR aligned amr
    elif args.mode == 'test_gold_graph':     
        instances = preprocess(amr_file,False)
        #instances = pickle.load(open('data/gold_edge_graph.pkl','rb'))
        gold_amr = []
        for inst in instances:
            GraphState.sent = inst.tokens
            gold_amr.append(GraphState.get_parsed_amr(inst.gold_graph))
        #pseudo_gold_amr = [GraphState.get_parsed_amr(inst.gold_graph) for inst in instances]
        write_parsed_amr(gold_amr,instances,amr_file,'abt.gold')
        #instances = preprocess_aligned(amr_file)
        print "Done output AMR!"
    # do alignment
    elif args.mode == 'align':

        if args.input_file:
            instances = pickle.load(open(args.input_file,'rb'))
        else:
            raise ValueError("Missing data file! specify it using --input or using preprocessing!")
        gold_instances_file = args.input_file.split('.')[0]+'_gold.p'

        print >> log, "Doing alignment..."

        if LOGGED:
            saveerr = sys.stderr
            sys.stderr = open('./log/alignment.log','w')

        amr_aligner = Aligner(verbose=args.verbose)
        ref_graphs = []
        begin = args.begin 
        counter = 1
        #for snt, amr in zip(snts[begin:],amrs[begin:]):
        for i in range(len(instances)):
            snt = instances[i].text
            amr = instances[i].amr
            if args.verbose > 1:
                print >> log, counter
                print >> log, "Sentence:"
                print >> log, snt+'\n'
                
                print >> log, "AMR:"                
                print >> log, amr.to_amr_string()

            alresult = amr_aligner.apply_align(snt,amr)
            ref_amr_graph = SpanGraph.init_ref_graph(amr,alresult)
            #ref_graphs.append(ref_amr_graph)
            instances[i].addGoldGraph(ref_amr_graph)
            if args.verbose > 1:
                #print >> log, "Reference tuples:"
                #print >> log, ref_depGraph.print_tuples()
                print >> log, amr_aligner.print_align_result(alresult,amr)
                #raw_input('ENTER to continue')
            counter += 1

        pickle.dump(instances,open(gold_instances_file,'wb'),pickle.HIGHEST_PROTOCOL)
        #pickle.dump(ref_graphs,open('./data/ref_graph.p','wb'),pickle.HIGHEST_PROTOCOL)
        if LOGGED:
            sys.stderr.close() 
            sys.stderr = saveerr
        print >> log, "Done alignment and gold graph generation."
        sys.exit()
        
    # test user guide actions
    elif args.mode == 'userGuide':
        print 'Read in training instances...'
        train_instances = preprocess(amr_file,False)

        sentID = int(raw_input("Input the sent ID:"))
        amr_parser = Parser()
        amr_parser.testUserGuide(train_instances[sentID])

        sys.exit()

    # test deterministic oracle 
    elif args.mode == 'oracleGuide':
        
        train_instances = preprocess(amr_file,START_SNLP=False)
        try:
            hand_alignments = load_hand_alignments(amr_file+str('.hand_aligned'))
        except IOError:
            hand_alignments = []


        start_step = args.start_step
        begin = args.begin
        amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT,verbose=args.verbose)
        #ref_graphs = pickle.load(open('./data/ref_graph.p','rb'))
        n_correct_total = .0
        n_parsed_total = .0
        n_gold_total = .0
        pseudo_gold_amr = []
        n_correct_tag_total = .0
        n_parsed_tag_total = 0.
        n_gold_tag_total = .0

        
        gold_amr = []
        aligned_instances = []
        #print "shuffling training instances"
        #random.shuffle(train_instances)
        for instance in train_instances[begin:]:
            
            if hand_alignments and instance.comment['id'] not in hand_alignments: continue
            state = amr_parser.testOracleGuide(instance,start_step)
            n_correct_arc,n1,n_parsed_arc, n_gold_arc,n_correct_tag,n_parsed_tag,n_gold_tag = state.evaluate()
            #assert n_correct_arc == n1
            if n_correct_arc != n1:
                import pdb
                pdb.set_trace()
            n_correct_total += n_correct_arc
            n_parsed_total += n_parsed_arc
            n_gold_total += n_gold_arc
            p = n_correct_arc/n_parsed_arc if n_parsed_arc else .0
            r = n_correct_arc/n_gold_arc if n_gold_arc else .0
            indicator = 'PROBLEM!' if p < 0.5 else ''
            if args.verbose > 2: print >> sys.stderr, "Precision: %s Recall: %s  %s\n" % (p,r,indicator)
            n_correct_tag_total +=  n_correct_tag
            n_parsed_tag_total +=  n_parsed_tag
            n_gold_tag_total += n_gold_tag
            p1 = n_correct_tag/n_parsed_tag if n_parsed_tag else .0
            r1 = n_correct_tag/n_gold_tag if n_gold_tag else .0
            if args.verbose > 2: print >> sys.stderr,"Tagging Precision:%s Recall:%s" % (p1,r1)

            instance.comment['alignments'] += ''.join(' %s-%s|%s'%(idx-1,idx,instance.amr.get_pid(state.A.abt_node_table[idx])) for idx in state.A.abt_node_table if isinstance(idx,int))

            aligned_instances.append(instance)
            pseudo_gold_amr.append(GraphState.get_parsed_amr(state.A))
            #gold_amr.append(instance.amr)
            #assert set(state.A.tuples()) == set(instance.gold_graph.tuples())
        pt = n_correct_total/n_parsed_total if n_parsed_total != .0 else .0
        rt = n_correct_total/n_gold_total if n_gold_total !=.0 else .0
        ft = 2*pt*rt/(pt+rt) if pt+rt != .0 else .0
        write_parsed_amr(pseudo_gold_amr,aligned_instances,amr_file,'pseudo-gold',hand_alignments)
        print "Total Accuracy: %s, Recall: %s, F-1: %s" % (pt,rt,ft)

        tp = n_correct_tag_total/n_parsed_tag_total if n_parsed_tag_total != .0 else .0
        tr = n_correct_tag_total/n_gold_tag_total if n_gold_tag_total != .0 else .0
        print "Tagging Precision:%s Recall:%s" % (tp,tr)

        #amr_parser.record_actions('data/action_set.txt')
    elif args.mode == 'train': # training
        print "Parser Config:"
        print "Incorporate Coref Information: %s"%(constants.FLAG_COREF)
        print "Incorporate SRL Information: %s"%(constants.FLAG_PROP)
        print "Substitue the normal name entity tag with rich name entity tag: %s"%(constants.FLAG_RNE)
        print "Using verbalization list: %s"%(constants.FLAG_VERB)
        print "Using charniak parser trained on ontonotes: %s"%(constants.FLAG_ONTO)
        print "Dependency parser used: %s"%(constants.FLAG_DEPPARSER)
        train_instances = preprocess(amr_file,START_SNLP=False)
        if args.add: train_instances = train_instances + preprocess(args.add,START_SNLP=False)
        if args.dev: dev_instances = preprocess(args.dev,START_SNLP=False)


        if args.section != 'all':
            print "Choosing corpus section: %s"%(args.section)
            tcr = constants.get_corpus_range(args.section,'train')
            train_instances = train_instances[tcr[0]:tcr[1]]
            if args.dev:
                dcr = constants.get_corpus_range(args.section,'dev')
                dev_instances = dev_instances[dcr[0]:dcr[1]]

        
        feat_template = args.feat if args.feat else None
        model = Model(elog=experiment_log)
        #model.output_feature_generator()
        parser = Parser(model=model,oracle_type=DET_T2G_ORACLE_ABT,action_type=args.actionset,verbose=args.verbose,elog=experiment_log)
        model.setup(action_type=args.actionset,instances=train_instances,parser=parser,feature_templates_file=feat_template)
        
        print >> experiment_log, "BEGIN TRAINING!"
        best_fscore = 0.0
        best_pscore = 0.0
        best_rscore = 0.0
        best_model = None
        best_iter = 1
        for iter in xrange(1,args.iterations+1):
            print >> experiment_log, "shuffling training instances"
            random.shuffle(train_instances)
            
            print >> experiment_log, "Iteration:",iter
            begin_updates = parser.perceptron.get_num_updates()
            parser.parse_corpus_train(train_instances)
            parser.perceptron.average_weight()
            
            if args.dev:
                print >> experiment_log ,"Result on develop set:"                
                _,parsed_amr = parser.parse_corpus_test(dev_instances)
                parsed_suffix = args.section+'.'+args.model.split('.')[-1]+'.'+str(iter)+'.parsed'
                write_parsed_amr(parsed_amr,dev_instances,args.dev,parsed_suffix)
                if args.smatcheval:
                    smatch_path = "./smatch_2.0.2/smatch.py"
                    python_path = 'python'
                    options = '--pr -f'
                    parsed_filename = args.dev+'.'+parsed_suffix
                    command = '%s %s %s %s %s' % (python_path, smatch_path, options, parsed_filename, args.dev)
                    
                    print 'Evaluation using command: ' + (command)
                    #print subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
                    eval_output = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
                    print eval_output
                    pscore = float(eval_output.split('\n')[0].split(':')[1].rstrip())
                    rscore = float(eval_output.split('\n')[1].split(':')[1].rstrip())
                    fscore = float(eval_output.split('\n')[2].split(':')[1].rstrip())
                    if fscore > best_fscore:
                        best_model = model
                        best_iter = iter
                        best_fscore = fscore
                        best_pscore = pscore
                        best_rscore = rscore

        if best_model is not None:
            print >> experiment_log, "Best result on iteration %d:\n Precision: %f\n Recall: %f\n F-score: %f" % (best_iter, best_pscore, best_rscore, best_fscore)
            best_model.save_model(args.model+'.m')
        print >> experiment_log ,"DONE TRAINING!"
        
    elif args.mode == 'parse': # actual parsing
        test_instances = preprocess(amr_file,START_SNLP=False,INPUT_AMR=args.amrfmt)
        if args.section != 'all':
            print "Choosing corpus section: %s"%(args.section)
            tcr = constants.get_corpus_range(args.section,'test')
            test_instances = test_instances[tcr[0]:tcr[1]]
            
        #random.shuffle(test_instances)
        print >> experiment_log, "Loading model: ", args.model 
        model = Model.load_model(args.model)
        parser = Parser(model=model,oracle_type=DET_T2G_ORACLE_ABT,action_type=args.actionset,verbose=args.verbose,elog=experiment_log)
        print >> experiment_log ,"BEGIN PARSING"
        span_graph_pairs,results = parser.parse_corpus_test(test_instances)
        parsed_suffix = '%s.%s.parsed'%(args.section,args.model.split('.')[-2])
        write_parsed_amr(results,test_instances,amr_file,suffix=parsed_suffix)
        #write_span_graph(span_graph_pairs,test_instances,amr_file,suffix='spg.50')
        ################
        # for eval     #
        ################
        #pickle.dump(span_graph_pairs,open('data/eval/%s_spg_pair.pkl'%(amr_file),'wb'),pickle.HIGHEST_PROTOCOL)
        #pickle.dump(test_instances,open('data/eval/%s_instances.pkl'%(amr_file),'wb'),pickle.HIGHEST_PROTOCOL)
        print >> experiment_log ,"DONE PARSING"
        if args.smatcheval:
            smatch_path = "./smatch_2.0.2/smatch.py"
            python_path = 'python'
            options = '--pr -f'
            parsed_filename = amr_file+'.'+parsed_suffix
            command = '%s %s %s %s %s' % (python_path,smatch_path,options,parsed_filename, amr_file)
                    
            print 'Evaluation using command: ' + (command)
            print subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)

            
        #plt.hist(results)
        #plt.savefig('result.png')

    elif args.mode == 'eval':
        '''break down error analysis'''
        # TODO: here use pickled file, replace it with parsed AMR and gold AMR
        span_graph_pairs = pickle.load(open(args.eval[0],'rb'))
        instances = pickle.load(open(args.eval[1],'rb'))
        
        amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT,verbose=args.verbose)
        error_stat = defaultdict(lambda:defaultdict(lambda:defaultdict(list)))
        for spg_pair,instance in zip(span_graph_pairs,instances):
            amr_parser.errorAnalyze(spg_pair[0],spg_pair[1],instance,error_stat)

    else:
        arg_parser.print_help()

Example 44

Project: camr
Source File: oracle.py
View license
    def give_ref_action_aux(self,currentIdx,currentChildIdx,currentGraph,ref_graph):
        
        def isCorrectReplace(childIdx,node,rgraph):
            for p in node.parents:
                if p in rgraph.nodes and childIdx in rgraph.nodes[p].children:
                    return True
            return False

        def getABTParent(gn):
            for p in gn.parents:
                if not isinstance(p,int):
                    return p
            return None
        def need_infer(cidx,gn,cn,gnset,cg,rg):
            for p in cn.parents:
                #if p not in gnset and rg.isContained(p):
                #    return False
                if p in gn.parents or gn.contains(cg.nodes[p]) or p in gn.children: # correct/merge/swap arc
                    return False
            return True
        def is_aligned(abtIdx,rg,cg):
            return abtIdx in rg.abt_node_table and rg.abt_node_table[abtIdx] in cg.nodes
            
        def need_swap(idx,idx_p,cg,gc,rg):
            cur_head_p = cg.find_true_head(idx_p)
            try:
                cur_head = cg.abt_node_table[cur_head_p] if not isinstance(cur_head_p,int) else cur_head_p
            except KeyError:
                print >> sys.stderr,"inferred abt node cur_head_p not in ref graph!"
                return None
                
            
            if cur_head in gc.children:
                return cur_head
            cur_grand_head = getABTParent(rg.nodes[cur_head])
            if (not isinstance(cur_head,int) and cur_grand_head in gc.children):
                return cur_grand_head
            return None            
            
        if currentIdx == START_ID:
            return {'type':NEXT2},None            

        currentNode = currentGraph.nodes[currentIdx] #state.get_current_node()
        currentChild = currentGraph.nodes[currentChildIdx] if currentChildIdx in currentGraph.nodes else None #state.get_current_child()
        goldNodeSet = [ref_graph.abt_node_table[k] if k in ref_graph.abt_node_table else k for k in ref_graph.nodes] 
        result_act_type = None
        result_act_label = None


        
        if currentIdx in goldNodeSet:
            currentIdx_p = currentIdx
            if currentIdx not in ref_graph.nodes:
                currentIdx = currentGraph.abt_node_table[currentIdx]            
            goldNode = ref_graph.nodes[currentIdx]                
            
            #for child in currentNode.children: 
            if currentChildIdx:
                if currentChildIdx == START_ID:
                    abtParentIdx = getABTParent(goldNode)
                    #if state.action_history and state.action_history[-1]['type'] in [REPLACEHEAD,NEXT2,DELETENODE] 
                    if need_infer(currentIdx,goldNode,currentNode,goldNodeSet,currentGraph,ref_graph) \
                            and abtParentIdx and not is_aligned(abtParentIdx,ref_graph,currentGraph) \
                            and (len(ref_graph.nodes[abtParentIdx].children) == 1 or not isinstance(currentIdx_p,int) or ((goldNode.words[0] in NOMLIST or currentGraph.sent[currentIdx_p]['lemma'].lower() in NOMLIST) and len(goldNode.words) == 1)):
                        gold_tag = ref_graph.nodes[abtParentIdx].tag
                        abt_node_index = ABT_PREFIX+str(currentGraph.abt_node_num)
                        ref_graph.add_abt_mapping(abtParentIdx,abt_node_index)
                        currentGraph.add_abt_mapping(abt_node_index,abtParentIdx)
                        return {'type':INFER},gold_tag
                    else: 
                        return {'type':NEXT1},START_EDGE

                if currentChildIdx in goldNodeSet:
                    currentChildIdx_p = currentChildIdx
                    if currentChildIdx not in ref_graph.nodes:
                        currentChildIdx = currentGraph.abt_node_table[currentChildIdx]
                    try:
                        goldChild = ref_graph.nodes[currentChildIdx] 
                    except KeyError:
                        import pdb
                        pdb.set_trace()

                    if goldChild.contains(currentNode) or goldNode.contains(currentChild):
                        return {'type':MERGE}, None # merge
                        #result_act_type = {'type':MERGE}
                    #if currentIdx in goldChild.children and \
                    #   currentChildIdx in goldNode.children:
                    '''
                    if currentChildIdx in goldNode.children and ref_graph.is_cycle(currentIdx):
                        print >> sys.stderr, "Circle detected in gold graph!"
                        gold_edge = ref_graph.get_edge_label(currentIdx,currentChildIdx)
                        return {'type':NEXT1}, gold_edge # next
                    '''
                        #result_act_type = {'type':NEXT1}
                        #result_act_label = gold_edge
                    if currentChildIdx in goldNode.children: # correct
                        #parents_to_add = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_reentrance_constrained(currentIdx_p,currentChildIdx_p)]
                        parents_to_add = [p for p in goldChild.parents if p not in [currentGraph.abt_node_table[cp] if cp in currentGraph.abt_node_table else cp for cp in currentChild.parents]]
                        
                        if parents_to_add:
                            pta = parents_to_add[0]
                            if pta in ref_graph.abt_node_table: pta = ref_graph.abt_node_table[pta]
                            if pta in currentGraph.get_possible_parent_unconstrained(currentIdx_p,currentChildIdx_p):
                                gold_edge = ref_graph.get_edge_label(parents_to_add[0],currentChildIdx)
                                return {'type':REENTRANCE,'parent_to_add':pta},gold_edge
                            else:
                                gold_edge = ref_graph.get_edge_label(currentIdx,currentChildIdx)
                                return {'type':NEXT1}, gold_edge # next
                        else:
                            gold_edge = ref_graph.get_edge_label(currentIdx,currentChildIdx)
                            return {'type':NEXT1}, gold_edge # next

                    thead = need_swap(currentIdx,currentIdx_p,currentGraph,goldChild,ref_graph)
                    if thead is not None:#in goldChild.children:
                        gold_edge = ref_graph.get_edge_label(currentChildIdx,thead)
                        return {'type':SWAP}, gold_edge # swap
                        #result_act_type = {'type':SWAP}                        

                    parents_to_attach = [p for p in goldChild.parents if p not in [currentGraph.abt_node_table[cp] if cp in currentGraph.abt_node_table else cp for cp in currentChild.parents]]
                    if parents_to_attach:
                        pta = parents_to_attach[0]
                        if pta in ref_graph.abt_node_table: pta = ref_graph.abt_node_table[pta]
                        if pta in currentGraph.get_possible_parent_unconstrained(currentIdx_p,currentChildIdx_p):
                            gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
                            return {'type':REATTACH,'parent_to_attach':pta},gold_edge
                        #elif not isinstance(pta,int): # abstract (or unaligned) nodes
                        #    abt_node_index = ABT_PREFIX+str(currentGraph.abt_node_num)
                        #    if pta == parents_to_attach[0]:
                        #        ref_graph.add_abt_mapping(pta,abt_node_index)
                        #        currentGraph.add_abt_mapping(abt_node_index,pta)
                        #    else:
                        #        currentGraph.add_abt_mapping(abt_node_index,parents_to_attach[0])
                        #    return {'type':INFER},None
                        else:
                            return {'type':NEXT1},None # violates the attachment constraints

                    else:
                        if self.verbose > 1: print >> sys.stderr, "Current child node %s doesn't have parents in gold span graph!"%(currentChildIdx)
                        return {'type':NEXT1},None
                        #return {'type':REATTACH,'parent_to_attach':None},None
                else:
                    if goldNode.contains(currentChild):
                        return {'type':MERGE},None
                        #result_act_type = {'type':MERGE}
                    else:
                        #return {'type':DELETEEDGE} # delete edge
                        #result_act_type = {'type':DELETEEDGE}
                        #k = ref_graph.isContained(currentChildIdx)
                        #if k:
                        #    return {'type':REATTACH,'parent_to_attach':k}
                        #else:    

                        return {'type':NEXT1},None
                        #return {'type':REATTACH,'parent_to_attach':None},None
                        
            else:
                #if len(currentNode.children) <= len(goldNode.children) and set(currentNode.children).issubset(set(goldNode.children)):
                #children_to_add = [c for c in goldNode.children if c not in currentNode.children and c in currentGraph.get_possible_children_constrained(currentIdx)]

                #if children_to_add:
                #    child_to_add = children_to_add[0]
                #    gold_edge = ref_graph.get_edge_label(currentIdx,child_to_add)
                #    return {'type':ADDCHILD,'child_to_add':child_to_add,'edge_label':gold_edge}
                #else:
                gold_tag = goldNode.tag
                return {'type':NEXT2, 'tag':gold_tag},None # next: done with the current node move to next one
                #else:
                #    if self.verbose > 2:
                #        print >> sys.stderr, "ERROR: Missing actions, current node's and gold node's children:%s  %s"%(str(currentNode.children), str(goldNode.children))
                #    pass            
                
                                    
        elif ref_graph.isContained(currentIdx):
            if currentChildIdx:
                if currentChildIdx in goldNodeSet:
                    #goldChild = ref_graph.nodes[currentChildIdx] if currentChildIdx in ref_graph.nodes else ref_graph.nodes[currentGraph.abt_node_table[currentChildIdx]]
                    currentChildIdx_p = currentChildIdx
                    if currentChildIdx not in ref_graph.nodes:
                        currentChildIdx = currentGraph.abt_node_table[currentChildIdx]
                    goldChild = ref_graph.nodes[currentChildIdx] 
                    if goldChild.contains(currentNode):
                        return {'type':MERGE},None
                    else:
                        parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx_p)]
                        #parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx)]
                        if parents_to_attach:
                            if ref_graph.nodes[parents_to_attach[0]].contains(currentNode):                                
                                return {'type':NEXT1},None # delay action for future merge
                            else:
                                gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
                                #if gold_edge == 'x': # not actually gold edge, skip this
                                #    return {'type':NEXT1},None                                                                                                     
                                #else:
                                return {'type':REATTACH,'parent_to_attach':parents_to_attach[0]},gold_edge
                        else:
                            return {'type':NEXT1},None
                            #return {'type':REATTACH,'parent_to_attach':None},None
                else:
                    return {'type':NEXT1},None
                    #return {'type':REATTACH,'parent_to_attach':None},None                    
            else:                
                return {'type':NEXT2},None

        else:
            '''
            if currentChildIdx:
                #assert len(currentNode.children) == 1
                if currentChildIdx in goldNodeSet:
                    goldChild = ref_graph.nodes[currentChildIdx]
                    if (isCorrectReplace(currentChildIdx,currentNode,ref_graph,state.beta) or len(currentNode.children) == 1):
                        return {'type':REPLACEHEAD},None # replace head
                        #result_act_type = {'type':REPLACEHEAD}
                    else:
                        parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_constrained(currentIdx,currentChildIdx)]
                        #parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx)]
                        if parents_to_attach:
                            if isCorrectReplace(parents_to_attach[0],currentNode,ref_graph,state.beta):
                                return {'type':NEXT1},None # delay action for future replace head
                            else:
                                gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
                                #if gold_edge == 'x': # not actually gold edge, skip this
                                #    return {'type':NEXT1},None
                                #else:
                                return {'type':REATTACH,'parent_to_attach':parents_to_attach[0]},gold_edge
                        else:
                            return {'type':NEXT1},None
                            #return {'type':REATTACH,'parent_to_attach':None},None
                else:
                    #return {'type':DELETEEDGE}
                    #result_act_type = {'type':DELETEEDGE}

                    return {'type':NEXT1},None
                    #return {'type':REATTACH,'parent_to_attach':None},None
                    '''
            if currentChildIdx:
                if currentChildIdx == START_ID:
                    return {'type':NEXT1},None
                if currentChildIdx in goldNodeSet:
                    #goldChild = ref_graph.nodes[currentChildIdx] if currentChildIdx in ref_graph.nodes else ref_graph.nodes[currentGraph.abt_node_table[currentChildIdx]]
                    currentChildIdx_p = currentChildIdx
                    if currentChildIdx not in ref_graph.nodes:
                        currentChildIdx = currentGraph.abt_node_table[currentChildIdx]
                    goldChild = ref_graph.nodes[currentChildIdx] 
                    if isCorrectReplace(currentChildIdx,currentNode,ref_graph) or len(currentNode.children) == 1: #current node's parents are already aligned
                        return {'type':REPLACEHEAD},None # replace head
                    else:
                        parents_to_attach = [p for p in goldChild.parents if p not in [currentGraph.abt_node_table[cp] if cp in currentGraph.abt_node_table else cp for cp in currentChild.parents]]
                        if parents_to_attach:
                            pta = parents_to_attach[0]
                            if pta in ref_graph.abt_node_table: pta = ref_graph.abt_node_table[pta]
                            if pta in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx_p):
                                gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
                                return {'type':REATTACH,'parent_to_attach':pta},gold_edge
                            #elif not isinstance(pta,int) and pta not in currentGraph.nodes: # abstract (or unaligned) nodes
                                
                                '''
                                #heuristic 1: just assign the abstract node index according to its first gold child  
                                #assert pta != currentIdx
                                if pta == parents_to_attach[0] and state.is_possible_align(currentIdx,parents_to_attach[0],ref_graph):
                                    ref_graph.add_abt_mapping(pta,currentIdx)
                                    currentGraph.add_abt_mapping(currentIdx,pta)
                                    gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
                                    return {'type':NEXT1},gold_edge           
                                else:
                                    return {'type':REPLACEHEAD},None
                                '''
                            else:
                                return {'type':NEXT1},None # violates the attachment constraints
                        else:
                            if self.verbose > 1: print >> sys.stderr, "Current child node %s doesn't have parents in gold span graph!"%(currentChildIdx)
                            return {'type':NEXT1},None
                else:
                    if self.verbose > 1: print >> sys.stderr, "Current child node %s should have been deleted!"%(currentChildIdx)
                    return {'type':NEXT1},None 
            else:
                # here currentNode.children must be empty
                if currentNode.children and self.verbose > 1: print >> sys.stderr, "Unaligned node %s has children"%(currentNode.start)
                # avoid delete name entity; will delete that in further reentrance action
                #if currentGraph.sent[currentIdx]['ne'] == 'O' or currentGraph.sent[currentIdx]['pos'] in FUNCTION_TAG:
                return {'type':DELETENODE},None
                #else:
                #    return {'type':NEXT2},None



        skip = NEXT1 if currentChildIdx else NEXT2
        return {'type':skip},None

Example 45

Project: calibre
Source File: cli.py
View license
def main():
    from calibre.utils.terminal import geometry
    cols = geometry()[0]

    parser = OptionParser(usage="usage: %prog [options] command args\n\ncommand "+
            "is one of: info, books, df, ls, cp, mkdir, touch, cat, rm, eject, test_file\n\n"+
    "For help on a particular command: %prog command", version=__appname__+" version: " + __version__)
    parser.add_option("--log-packets", help="print out packet stream to stdout. "+
                    "The numbers in the left column are byte offsets that allow the packet size to be read off easily.",
    dest="log_packets", action="store_true", default=False)
    parser.remove_option("-h")
    parser.disable_interspersed_args()  # Allow unrecognized options
    options, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        return 1

    command = args[0]
    args = args[1:]
    dev = None
    scanner = DeviceScanner()
    scanner.scan()
    connected_devices = []

    for d in device_plugins():
        try:
            d.startup()
        except:
            print ('Startup failed for device plugin: %s'%d)
        if d.MANAGES_DEVICE_PRESENCE:
            cd = d.detect_managed_devices(scanner.devices)
            if cd is not None:
                connected_devices.append((cd, d))
                dev = d
                break
            continue
        ok, det = scanner.is_device_connected(d)
        if ok:
            dev = d
            dev.reset(log_packets=options.log_packets, detected_device=det)
            connected_devices.append((det, dev))

    if dev is None:
        print >>sys.stderr, 'Unable to find a connected ebook reader.'
        shutdown_plugins()
        return 1

    for det, d in connected_devices:
        try:
            d.open(det, None)
        except:
            continue
        else:
            dev = d
            d.specialize_global_preferences(device_prefs)
            break

    try:
        if command == "df":
            total = dev.total_space(end_session=False)
            free = dev.free_space()
            where = ("Memory", "Card A", "Card B")
            print "Filesystem\tSize \tUsed \tAvail \tUse%"
            for i in range(3):
                print "%-10s\t%s\t%s\t%s\t%s"%(where[i], human_readable(total[i]), human_readable(total[i]-free[i]), human_readable(free[i]),
                                                                            str(0 if total[i]==0 else int(100*(total[i]-free[i])/(total[i]*1.)))+"%")
        elif command == 'eject':
            dev.eject()
        elif command == "books":
            print "Books in main memory:"
            for book in dev.books():
                print book
            print "\nBooks on storage carda:"
            for book in dev.books(oncard='carda'):
                print book
            print "\nBooks on storage cardb:"
            for book in dev.books(oncard='cardb'):
                print book
        elif command == "mkdir":
            parser = OptionParser(usage="usage: %prog mkdir [options] path\nCreate a directory on the device\n\npath must begin with / or card:/")
            if len(args) != 1:
                parser.print_help()
                sys.exit(1)
            dev.mkdir(args[0])
        elif command == "ls":
            parser = OptionParser(usage="usage: %prog ls [options] path\nList files on the device\n\npath must begin with / or card:/")
            parser.add_option(
                "-l", help="In addition to the name of each file, print the file type, permissions, and  timestamp  (the  modification time, in the local timezone). Times are local.",  # noqa
                              dest="ll", action="store_true", default=False)
            parser.add_option("-R", help="Recursively list subdirectories encountered. /dev and /proc are omitted",
                              dest="recurse", action="store_true", default=False)
            parser.remove_option("-h")
            parser.add_option("-h", "--human-readable", help="show sizes in human readable format", dest="hrs", action="store_true", default=False)
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            print ls(dev, args[0], recurse=options.recurse, ll=options.ll, human_readable_size=options.hrs, cols=cols),
        elif command == "info":
            info(dev)
        elif command == "cp":
            usage="usage: %prog cp [options] source destination\nCopy files to/from the device\n\n"+\
            "One of source or destination must be a path on the device. \n\nDevice paths have the form\n"+\
            "dev:mountpoint/my/path\n"+\
            "where mountpoint is one of / or carda: or cardb:/\n\n"+\
            "source must point to a file for which you have read permissions\n"+\
            "destination must point to a file or directory for which you have write permissions"
            parser = OptionParser(usage=usage)
            parser.add_option('-f', '--force', dest='force', action='store_true', default=False,
                              help='Overwrite the destination file if it exists already.')
            options, args = parser.parse_args(args)
            if len(args) != 2:
                parser.print_help()
                return 1
            if args[0].startswith("dev:"):
                outfile = args[1]
                path = args[0][4:]
                if path.endswith("/"):
                    path = path[:-1]
                if os.path.isdir(outfile):
                    outfile = os.path.join(outfile, path[path.rfind("/")+1:])
                try:
                    outfile = lopen(outfile, "wb")
                except IOError as e:
                    print >> sys.stderr, e
                    parser.print_help()
                    return 1
                dev.get_file(path, outfile)
                fsync(outfile)
                outfile.close()
            elif args[1].startswith("dev:"):
                try:
                    infile = lopen(args[0], "rb")
                except IOError as e:
                    print >> sys.stderr, e
                    parser.print_help()
                    return 1
                dev.put_file(infile, args[1][4:], replace_file=options.force)
                infile.close()
            else:
                parser.print_help()
                return 1
        elif command == "cat":
            outfile = sys.stdout
            parser = OptionParser(
                usage="usage: %prog cat path\nShow file on the device\n\npath should point to a file on the device and must begin with /,a:/ or b:/")
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            if args[0].endswith("/"):
                path = args[0][:-1]
            else:
                path = args[0]
            outfile = sys.stdout
            dev.get_file(path, outfile)
        elif command == "rm":
            parser = OptionParser(usage="usage: %prog rm path\nDelete files from the device\n\npath should point to a file or empty directory on the device "+
                                  "and must begin with / or card:/\n\n"+
                                  "rm will DELETE the file. Be very CAREFUL")
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            dev.rm(args[0])
        elif command == "touch":
            parser = OptionParser(usage="usage: %prog touch path\nCreate an empty file on the device\n\npath should point to a file on the device and must begin with /,a:/ or b:/\n\n"+  # noqa
            "Unfortunately, I cant figure out how to update file times on the device, so if path already exists, touch does nothing")
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            dev.touch(args[0])
        elif command == 'test_file':
            parser = OptionParser(usage=("usage: %prog test_file path\n"
                'Open device, copy file specified by path to device and '
                'then eject device.'))
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            path = args[0]
            from calibre.ebooks.metadata.meta import get_metadata
            mi = get_metadata(lopen(path, 'rb'), path.rpartition('.')[-1].lower())
            print dev.upload_books([args[0]], [os.path.basename(args[0])],
                    end_session=False, metadata=[mi])
            dev.eject()
        else:
            parser.print_help()
            if getattr(dev, 'handle', False):
                dev.close()
            return 1
    except DeviceLocked:
        print >> sys.stderr, "The device is locked. Use the --unlock option"
    except (ArgumentError, DeviceError) as e:
        print >>sys.stderr, e
        return 1
    finally:
        shutdown_plugins()

    return 0

Example 46

Project: calibre
Source File: cli.py
View license
def main():
    from calibre.utils.terminal import geometry
    cols = geometry()[0]

    parser = OptionParser(usage="usage: %prog [options] command args\n\ncommand "+
            "is one of: info, books, df, ls, cp, mkdir, touch, cat, rm, eject, test_file\n\n"+
    "For help on a particular command: %prog command", version=__appname__+" version: " + __version__)
    parser.add_option("--log-packets", help="print out packet stream to stdout. "+
                    "The numbers in the left column are byte offsets that allow the packet size to be read off easily.",
    dest="log_packets", action="store_true", default=False)
    parser.remove_option("-h")
    parser.disable_interspersed_args()  # Allow unrecognized options
    options, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        return 1

    command = args[0]
    args = args[1:]
    dev = None
    scanner = DeviceScanner()
    scanner.scan()
    connected_devices = []

    for d in device_plugins():
        try:
            d.startup()
        except:
            print ('Startup failed for device plugin: %s'%d)
        if d.MANAGES_DEVICE_PRESENCE:
            cd = d.detect_managed_devices(scanner.devices)
            if cd is not None:
                connected_devices.append((cd, d))
                dev = d
                break
            continue
        ok, det = scanner.is_device_connected(d)
        if ok:
            dev = d
            dev.reset(log_packets=options.log_packets, detected_device=det)
            connected_devices.append((det, dev))

    if dev is None:
        print >>sys.stderr, 'Unable to find a connected ebook reader.'
        shutdown_plugins()
        return 1

    for det, d in connected_devices:
        try:
            d.open(det, None)
        except:
            continue
        else:
            dev = d
            d.specialize_global_preferences(device_prefs)
            break

    try:
        if command == "df":
            total = dev.total_space(end_session=False)
            free = dev.free_space()
            where = ("Memory", "Card A", "Card B")
            print "Filesystem\tSize \tUsed \tAvail \tUse%"
            for i in range(3):
                print "%-10s\t%s\t%s\t%s\t%s"%(where[i], human_readable(total[i]), human_readable(total[i]-free[i]), human_readable(free[i]),
                                                                            str(0 if total[i]==0 else int(100*(total[i]-free[i])/(total[i]*1.)))+"%")
        elif command == 'eject':
            dev.eject()
        elif command == "books":
            print "Books in main memory:"
            for book in dev.books():
                print book
            print "\nBooks on storage carda:"
            for book in dev.books(oncard='carda'):
                print book
            print "\nBooks on storage cardb:"
            for book in dev.books(oncard='cardb'):
                print book
        elif command == "mkdir":
            parser = OptionParser(usage="usage: %prog mkdir [options] path\nCreate a directory on the device\n\npath must begin with / or card:/")
            if len(args) != 1:
                parser.print_help()
                sys.exit(1)
            dev.mkdir(args[0])
        elif command == "ls":
            parser = OptionParser(usage="usage: %prog ls [options] path\nList files on the device\n\npath must begin with / or card:/")
            parser.add_option(
                "-l", help="In addition to the name of each file, print the file type, permissions, and  timestamp  (the  modification time, in the local timezone). Times are local.",  # noqa
                              dest="ll", action="store_true", default=False)
            parser.add_option("-R", help="Recursively list subdirectories encountered. /dev and /proc are omitted",
                              dest="recurse", action="store_true", default=False)
            parser.remove_option("-h")
            parser.add_option("-h", "--human-readable", help="show sizes in human readable format", dest="hrs", action="store_true", default=False)
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            print ls(dev, args[0], recurse=options.recurse, ll=options.ll, human_readable_size=options.hrs, cols=cols),
        elif command == "info":
            info(dev)
        elif command == "cp":
            usage="usage: %prog cp [options] source destination\nCopy files to/from the device\n\n"+\
            "One of source or destination must be a path on the device. \n\nDevice paths have the form\n"+\
            "dev:mountpoint/my/path\n"+\
            "where mountpoint is one of / or carda: or cardb:/\n\n"+\
            "source must point to a file for which you have read permissions\n"+\
            "destination must point to a file or directory for which you have write permissions"
            parser = OptionParser(usage=usage)
            parser.add_option('-f', '--force', dest='force', action='store_true', default=False,
                              help='Overwrite the destination file if it exists already.')
            options, args = parser.parse_args(args)
            if len(args) != 2:
                parser.print_help()
                return 1
            if args[0].startswith("dev:"):
                outfile = args[1]
                path = args[0][4:]
                if path.endswith("/"):
                    path = path[:-1]
                if os.path.isdir(outfile):
                    outfile = os.path.join(outfile, path[path.rfind("/")+1:])
                try:
                    outfile = lopen(outfile, "wb")
                except IOError as e:
                    print >> sys.stderr, e
                    parser.print_help()
                    return 1
                dev.get_file(path, outfile)
                fsync(outfile)
                outfile.close()
            elif args[1].startswith("dev:"):
                try:
                    infile = lopen(args[0], "rb")
                except IOError as e:
                    print >> sys.stderr, e
                    parser.print_help()
                    return 1
                dev.put_file(infile, args[1][4:], replace_file=options.force)
                infile.close()
            else:
                parser.print_help()
                return 1
        elif command == "cat":
            outfile = sys.stdout
            parser = OptionParser(
                usage="usage: %prog cat path\nShow file on the device\n\npath should point to a file on the device and must begin with /,a:/ or b:/")
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            if args[0].endswith("/"):
                path = args[0][:-1]
            else:
                path = args[0]
            outfile = sys.stdout
            dev.get_file(path, outfile)
        elif command == "rm":
            parser = OptionParser(usage="usage: %prog rm path\nDelete files from the device\n\npath should point to a file or empty directory on the device "+
                                  "and must begin with / or card:/\n\n"+
                                  "rm will DELETE the file. Be very CAREFUL")
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            dev.rm(args[0])
        elif command == "touch":
            parser = OptionParser(usage="usage: %prog touch path\nCreate an empty file on the device\n\npath should point to a file on the device and must begin with /,a:/ or b:/\n\n"+  # noqa
            "Unfortunately, I cant figure out how to update file times on the device, so if path already exists, touch does nothing")
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            dev.touch(args[0])
        elif command == 'test_file':
            parser = OptionParser(usage=("usage: %prog test_file path\n"
                'Open device, copy file specified by path to device and '
                'then eject device.'))
            options, args = parser.parse_args(args)
            if len(args) != 1:
                parser.print_help()
                return 1
            path = args[0]
            from calibre.ebooks.metadata.meta import get_metadata
            mi = get_metadata(lopen(path, 'rb'), path.rpartition('.')[-1].lower())
            print dev.upload_books([args[0]], [os.path.basename(args[0])],
                    end_session=False, metadata=[mi])
            dev.eject()
        else:
            parser.print_help()
            if getattr(dev, 'handle', False):
                dev.close()
            return 1
    except DeviceLocked:
        print >> sys.stderr, "The device is locked. Use the --unlock option"
    except (ArgumentError, DeviceError) as e:
        print >>sys.stderr, e
        return 1
    finally:
        shutdown_plugins()

    return 0

Example 47

Project: UMI-tools
Source File: Utilities.py
View license
def Start(parser=None,
          argv=sys.argv,
          quiet=False,
          add_pipe_options=True,
          return_parser=False):
    """set up an experiment.

    The :py:func:`Start` method will set up a file logger and add some
    default and some optional options to the command line parser.  It
    will then parse the command line and set up input/output
    redirection and start a timer for benchmarking purposes.

    The default options added by this method are:

    ``-v/--verbose``
        the :term:`loglevel`

    ``timeit``
        turn on benchmarking information and save to file

    ``timeit-name``
         name to use for timing information,

    ``timeit-header``
         output header for timing information.

    ``seed``
         the random seed. If given, the python random
         number generator will be initialized with this
         seed.

    Optional options added are:

    Arguments
    ---------

    param parser : :py:class:`U.OptionParser`
       instance with command line options.

    argv : list
        command line options to parse. Defaults to
        :py:data:`sys.argv`

    quiet : bool
        set :term:`loglevel` to 0 - no logging

    return_parser : bool
        return the parser object, no parsing. Useful for inspecting
        the command line options of a script without running it.

    add_pipe_options : bool
        add common options for redirecting input/output

    Returns
    -------
    tuple
       (:py:class:`U.OptionParser` object, list of positional
       arguments)

    """

    if not parser:
        parser = OptionParser(
            version="%prog version: $Id$")

    global global_options, global_args, global_starting_time

    # save default values given by user
    user_defaults = copy.copy(parser.defaults)

    global_starting_time = time.time()

    group = OptionGroup(parser, "Script timing options")

    group.add_option("--timeit", dest='timeit_file', type="string",
                     help="store timeing information in file [%default].")
    group.add_option("--timeit-name", dest='timeit_name', type="string",
                     help="name in timing file for this class of jobs "
                     "[%default].")
    group.add_option("--timeit-header", dest='timeit_header',
                     action="store_true",
                     help="add header for timing information [%default].")
    parser.add_option_group(group)

    group = OptionGroup(parser, "Common options")

    group.add_option("--random-seed", dest='random_seed', type="int",
                     help="random seed to initialize number generator "
                     "with [%default].")

    group.add_option("-v", "--verbose", dest="loglevel", type="int",
                     help="loglevel [%default]. The higher, the more output.")

    group.add_option("-?", dest="short_help", action="callback",
                     callback=callbackShortHelp,
                     help="output short help (command line options only.")
    parser.add_option_group(group)

    if quiet:
        parser.set_defaults(loglevel=0)
    else:
        parser.set_defaults(loglevel=1)

    parser.set_defaults(
        timeit_file=None,
        timeit_name='all',
        timeit_header=None,
        random_seed=None,
    )

    if add_pipe_options:
        group = OptionGroup(parser, "Input/output options")
        group.add_option("-I", "--stdin", dest="stdin", type="string",
                         help="file to read stdin from [default = stdin].",
                         metavar="FILE")
        group.add_option("-L", "--log", dest="stdlog", type="string",
                         help="file with logging information "
                         "[default = stdout].",
                         metavar="FILE")
        group.add_option("-E", "--error", dest="stderr", type="string",
                         help="file with error information "
                         "[default = stderr].",
                         metavar="FILE")
        group.add_option("-S", "--stdout", dest="stdout", type="string",
                         help="file where output is to go "
                         "[default = stdout].",
                         metavar="FILE")
        group.add_option("--log2stderr", dest="log2stderr",
                         action="store_true", help="send logging information"
                         " to stderr [default = False].")

        parser.set_defaults(stderr=sys.stderr)
        parser.set_defaults(stdout=sys.stdout)
        parser.set_defaults(stdlog=sys.stdout)
        parser.set_defaults(stdin=sys.stdin)
        parser.set_defaults(log2stderr=False)

    parser.add_option_group(group)

    # restore user defaults
    parser.defaults.update(user_defaults)

    if return_parser:
        return parser

    global_options, global_args = parser.parse_args(argv[1:])

    if global_options.random_seed is not None:
        random.seed(global_options.random_seed)

    if add_pipe_options:
        if global_options.stdout != sys.stdout:
            global_options.stdout = openFile(global_options.stdout, "w")
        if global_options.stderr != sys.stderr:
            if global_options.stderr == "stderr":
                global_options.stderr = global_options.stderr
            else:
                global_options.stderr = openFile(global_options.stderr, "w")
        if global_options.stdlog != sys.stdout:
            global_options.stdlog = openFile(global_options.stdlog, "a")
        elif global_options.log2stderr:
            global_options.stdlog = global_options.stderr
        if global_options.stdin != sys.stdin:
            global_options.stdin = openFile(global_options.stdin, "r")
    else:
        global_options.stderr = sys.stderr
        global_options.stdout = sys.stdout
        global_options.stdin = sys.stdin
        if global_options.log2stderr:
            global_options.stdlog = sys.stderr
        else:
            global_options.stdlog = sys.stdout

    if global_options.loglevel >= 1:
        global_options.stdlog.write(getHeader() + "\n")
        global_options.stdlog.write(getParams(global_options) + "\n")
        global_options.stdlog.flush()

    # configure logging
    # map from 0-10 to logging scale
    # 0: quiet
    # 1: little verbositiy
    # >1: increased verbosity
    if global_options.loglevel == 0:
        lvl = logging.ERROR
    elif global_options.loglevel == 1:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG

    if global_options.stdout == global_options.stdlog:
        format = '# %(asctime)s %(levelname)s %(message)s'
    else:
        format = '%(asctime)s %(levelname)s %(message)s'

    logging.basicConfig(
        level=lvl,
        format=format,
        stream=global_options.stdlog)

    # set up multi-line logging
    # Note that .handlers is not part of the API, might change
    # Solution is to configure handlers explicitely.
    for handler in logging.getLogger().handlers:
        handler.setFormatter(MultiLineFormatter(format))

    return global_options, global_args

Example 48

Project: OWASP-ZSC
Source File: rlmain.py
View license
    def read_inputrc(self, #in 2.4 we cannot call expanduser with unicode string
                     inputrcpath=os.path.expanduser(ensure_str("~/pyreadlineconfig.ini"))):
        modes = dict([(x.mode, x) for x in self.editingmodes])
        mode = self.editingmodes[0].mode

        def setmode(name):
            self.mode = modes[name]

        def bind_key(key, name):
            import types
            if callable(name):
                modes[mode]._bind_key(key, types.MethodType(name, modes[mode]))
            elif hasattr(modes[mode], name):
                modes[mode]._bind_key(key, getattr(modes[mode], name))
            else:
                print("Trying to bind unknown command '%s' to key '%s'" %
                      (name, key))

        def un_bind_key(key):
            keyinfo = make_KeyPress_from_keydescr(key).tuple()
            if keyinfo in modes[mode].key_dispatch:
                del modes[mode].key_dispatch[keyinfo]

        def bind_exit_key(key):
            modes[mode]._bind_exit_key(key)

        def un_bind_exit_key(key):
            keyinfo = make_KeyPress_from_keydescr(key).tuple()
            if keyinfo in modes[mode].exit_dispatch:
                del modes[mode].exit_dispatch[keyinfo]

        def setkill_ring_to_clipboard(killring):
            import pyreadline.lineeditor.lineobj
            pyreadline.lineeditor.lineobj.kill_ring_to_clipboard = killring

        def sethistoryfilename(filename):
            self.mode._history.history_filename = os.path.expanduser(
                ensure_str(filename))

        def setbellstyle(mode):
            self.bell_style = mode

        def disable_readline(mode):
            self.disable_readline = mode

        def sethistorylength(length):
            self.mode._history.history_length = int(length)

        def allow_ctrl_c(mode):
            log("allow_ctrl_c:%s:%s" % (self.allow_ctrl_c, mode))
            self.allow_ctrl_c = mode

        def setbellstyle(mode):
            self.bell_style = mode

        def show_all_if_ambiguous(mode):
            self.mode.show_all_if_ambiguous = mode

        def ctrl_c_tap_time_interval(mode):
            self.ctrl_c_tap_time_interval = mode

        def mark_directories(mode):
            self.mode.mark_directories = mode

        def completer_delims(delims):
            self.mode.completer_delims = delims

        def complete_filesystem(delims):
            self.mode.complete_filesystem = delims.lower()

        def enable_ipython_paste_for_paths(boolean):
            self.mode.enable_ipython_paste_for_paths = boolean

        def debug_output(
                on, filename="pyreadline_debug_log.txt"):  #Not implemented yet
            if on in ["on", "on_nologfile"]:
                self.debug = True

            if on == "on":
                logger.start_file_log(filename)
                logger.start_socket_log()
                logger.log("STARTING LOG")
            elif on == "on_nologfile":
                logger.start_socket_log()
                logger.log("STARTING LOG")
            else:
                logger.log("STOPING LOG")
                logger.stop_file_log()
                logger.stop_socket_log()

        _color_trtable = {"black": 0,
                          "darkred": 4,
                          "darkgreen": 2,
                          "darkyellow": 6,
                          "darkblue": 1,
                          "darkmagenta": 5,
                          "darkcyan": 3,
                          "gray": 7,
                          "red": 4 + 8,
                          "green": 2 + 8,
                          "yellow": 6 + 8,
                          "blue": 1 + 8,
                          "magenta": 5 + 8,
                          "cyan": 3 + 8,
                          "white": 7 + 8}

        def set_prompt_color(color):
            self.prompt_color = self._color_trtable.get(color.lower(), 7)

        def set_input_color(color):
            self.command_color = self._color_trtable.get(color.lower(), 7)

        loc = {"branch": release.branch,
               "version": release.version,
               "mode": mode,
               "modes": modes,
               "set_mode": setmode,
               "bind_key": bind_key,
               "disable_readline": disable_readline,
               "bind_exit_key": bind_exit_key,
               "un_bind_key": un_bind_key,
               "un_bind_exit_key": un_bind_exit_key,
               "bell_style": setbellstyle,
               "mark_directories": mark_directories,
               "show_all_if_ambiguous": show_all_if_ambiguous,
               "completer_delims": completer_delims,
               "complete_filesystem": complete_filesystem,
               "debug_output": debug_output,
               "history_filename": sethistoryfilename,
               "history_length": sethistorylength,
               "set_prompt_color": set_prompt_color,
               "set_input_color": set_input_color,
               "allow_ctrl_c": allow_ctrl_c,
               "ctrl_c_tap_time_interval": ctrl_c_tap_time_interval,
               "kill_ring_to_clipboard": setkill_ring_to_clipboard,
               "enable_ipython_paste_for_paths":
               enable_ipython_paste_for_paths, }
        if os.path.isfile(inputrcpath):
            try:
                execfile(inputrcpath, loc, loc)
            except Exception as x:
                raise
                import traceback
                print("Error reading .pyinputrc", file=sys.stderr)
                filepath, lineno = traceback.extract_tb(sys.exc_traceback)[
                    1][:2]
                print("Line: %s in file %s" % (lineno, filepath),
                      file=sys.stderr)
                print(x, file=sys.stderr)
                raise ReadlineError("Error reading .pyinputrc")

Example 49

Project: mutagen
Source File: mid3v2.py
View license
def write_files(edits, filenames, escape):
    # unescape escape sequences and decode values
    encoded_edits = []
    for frame, value in edits:
        if not value:
            continue

        try:
            frame = frame_from_fsnative(frame)
        except ValueError as err:
            print_(text_type(err), file=sys.stderr)

        assert isinstance(frame, str)

        # strip "--"
        frame = frame[2:]

        try:
            value = value_from_fsnative(value, escape)
        except ValueError as err:
            error(u"%s: %s" % (frame, text_type(err)))

        assert isinstance(value, text_type)

        encoded_edits.append((frame, value))
    edits = encoded_edits

    # preprocess:
    #   for all [frame,value] pairs in the edits list
    #      gather values for identical frames into a list
    tmp = {}
    for frame, value in edits:
        if frame in tmp:
            tmp[frame].append(value)
        else:
            tmp[frame] = [value]
    # edits is now a dictionary of frame -> [list of values]
    edits = tmp

    # escape also enables escaping of the split separator
    if escape:
        string_split = split_escape
    else:
        string_split = lambda s, *args, **kwargs: s.split(*args, **kwargs)

    for filename in filenames:
        with _sig.block():
            if verbose:
                print_(u"Writing", filename, file=sys.stderr)
            try:
                id3 = mutagen.id3.ID3(filename)
            except mutagen.id3.ID3NoHeaderError:
                if verbose:
                    print_(u"No ID3 header found; creating a new tag",
                          file=sys.stderr)
                id3 = mutagen.id3.ID3()
            except Exception as err:
                print_(str(err), file=sys.stderr)
                continue
            for (frame, vlist) in edits.items():
                if frame == "POPM":
                    for value in vlist:
                        values = string_split(value, ":")
                        if len(values) == 1:
                            email, rating, count = values[0], 0, 0
                        elif len(values) == 2:
                            email, rating, count = values[0], values[1], 0
                        else:
                            email, rating, count = values

                        frame = mutagen.id3.POPM(
                            email=email, rating=int(rating), count=int(count))
                        id3.add(frame)
                elif frame == "APIC":
                    for value in vlist:
                        values = string_split(value, ":")
                        # FIXME: doesn't support filenames with an invalid
                        # encoding since we have already decoded at that point
                        fn = values[0]

                        if len(values) >= 2:
                            desc = values[1]
                        else:
                            desc = u"cover"

                        if len(values) >= 3:
                            try:
                                picture_type = int(values[2])
                            except ValueError:
                                error(u"Invalid picture type: %r" % values[1])
                        else:
                            picture_type = PictureType.COVER_FRONT

                        if len(values) >= 4:
                            mime = values[3]
                        else:
                            mime = mimetypes.guess_type(fn)[0] or "image/jpeg"

                        if len(values) >= 5:
                            error("APIC: Invalid format")

                        encoding = get_frame_encoding(frame, desc)

                        try:
                            with open(fn, "rb") as h:
                                data = h.read()
                        except IOError as e:
                            error(text_type(e))

                        frame = mutagen.id3.APIC(encoding=encoding, mime=mime,
                            desc=desc, type=picture_type, data=data)

                        id3.add(frame)
                elif frame == "COMM":
                    for value in vlist:
                        values = string_split(value, ":")
                        if len(values) == 1:
                            value, desc, lang = values[0], "", "eng"
                        elif len(values) == 2:
                            desc, value, lang = values[0], values[1], "eng"
                        else:
                            value = ":".join(values[1:-1])
                            desc, lang = values[0], values[-1]
                        frame = mutagen.id3.COMM(
                            encoding=3, text=value, lang=lang, desc=desc)
                        id3.add(frame)
                elif frame == "UFID":
                    for value in vlist:
                        values = string_split(value, ":")
                        if len(values) != 2:
                            error(u"Invalid value: %r" % values)
                        owner = values[0]
                        data = values[1].encode("utf-8")
                        frame = mutagen.id3.UFID(owner=owner, data=data)
                        id3.add(frame)
                elif frame == "TXXX":
                    for value in vlist:
                        values = string_split(value, ":", 1)
                        if len(values) == 1:
                            desc, value = "", values[0]
                        else:
                            desc, value = values[0], values[1]
                        frame = mutagen.id3.TXXX(
                            encoding=3, text=value, desc=desc)
                        id3.add(frame)
                elif issubclass(mutagen.id3.Frames[frame],
                                mutagen.id3.UrlFrame):
                    frame = mutagen.id3.Frames[frame](encoding=3, url=vlist)
                    id3.add(frame)
                else:
                    frame = mutagen.id3.Frames[frame](encoding=3, text=vlist)
                    id3.add(frame)
            id3.save(filename)

Example 50

Project: Live-Blog
Source File: cmdline.py
View license
def main(argv):
    if not color_terminal():
        # Windows' poor cmd box doesn't understand ANSI sequences
        nocolor()

    try:
        opts, args = getopt.getopt(argv[1:], 'ab:t:d:c:CD:A:ng:NEqQWw:P')
        allopts = set(opt[0] for opt in opts)
        srcdir = confdir = path.abspath(args[0])
        if not path.isdir(srcdir):
            print('Error: Cannot find source directory.', file=sys.stderr)
            return 1
        if not path.isfile(path.join(srcdir, 'conf.py')) and \
               '-c' not in allopts and '-C' not in allopts:
            print(('Error: Source directory doesn\'t '
                                 'contain conf.py file.'), file=sys.stderr)
            return 1
        outdir = path.abspath(args[1])
        if not path.isdir(outdir):
            print('Making output directory...', file=sys.stderr)
            os.makedirs(outdir)
    except (IndexError, getopt.error):
        usage(argv)
        return 1

    filenames = args[2:]
    err = 0
    for filename in filenames:
        if not path.isfile(filename):
            print('Cannot find file %r.' % filename, file=sys.stderr)
            err = 1
    if err:
        return 1

    # likely encoding used for command-line arguments
    try:
        locale = __import__('locale')  # due to submodule of the same name
        likely_encoding = locale.getpreferredencoding()
    except Exception:
        likely_encoding = None

    buildername = None
    force_all = freshenv = warningiserror = use_pdb = False
    status = sys.stdout
    warning = sys.stderr
    error = sys.stderr
    warnfile = None
    confoverrides = {}
    tags = []
    doctreedir = path.join(outdir, '.doctrees')
    for opt, val in opts:
        if opt == '-b':
            buildername = val
        elif opt == '-a':
            if filenames:
                usage(argv, 'Cannot combine -a option and filenames.')
                return 1
            force_all = True
        elif opt == '-t':
            tags.append(val)
        elif opt == '-d':
            doctreedir = path.abspath(val)
        elif opt == '-c':
            confdir = path.abspath(val)
            if not path.isfile(path.join(confdir, 'conf.py')):
                print(('Error: Configuration directory '
                                     'doesn\'t contain conf.py file.'), file=sys.stderr)
                return 1
        elif opt == '-C':
            confdir = None
        elif opt == '-D':
            try:
                key, val = val.split('=')
            except ValueError:
                print(('Error: -D option argument must be '
                                     'in the form name=value.'), file=sys.stderr)
                return 1
            try:
                val = int(val)
            except ValueError:
                if likely_encoding and isinstance(val, bytes):
                    try:
                        val = val.decode(likely_encoding)
                    except UnicodeError:
                        pass
            confoverrides[key] = val
        elif opt == '-A':
            try:
                key, val = val.split('=')
            except ValueError:
                print(('Error: -A option argument must be '
                                     'in the form name=value.'), file=sys.stderr)
                return 1
            try:
                val = int(val)
            except ValueError:
                if likely_encoding and isinstance(val, bytes):
                    try:
                        val = val.decode(likely_encoding)
                    except UnicodeError:
                        pass
            confoverrides['html_context.%s' % key] = val
        elif opt == '-n':
            confoverrides['nitpicky'] = True
        elif opt == '-N':
            nocolor()
        elif opt == '-E':
            freshenv = True
        elif opt == '-q':
            status = None
        elif opt == '-Q':
            status = None
            warning = None
        elif opt == '-W':
            warningiserror = True
        elif opt == '-w':
            warnfile = val
        elif opt == '-P':
            use_pdb = True

    if warning and warnfile:
        warnfp = open(warnfile, 'w')
        warning = Tee(warning, warnfp)
        error = warning

    try:
        app = Sphinx(srcdir, confdir, outdir, doctreedir, buildername,
                     confoverrides, status, warning, freshenv,
                     warningiserror, tags)
        app.build(force_all, filenames)
        return app.statuscode
    except KeyboardInterrupt:
        if use_pdb:
            import pdb
            print(red('Interrupted while building, starting debugger:'), file=error)
            traceback.print_exc()
            pdb.post_mortem(sys.exc_info()[2])
        return 1
    except Exception as err:
        if use_pdb:
            import pdb
            print(red('Exception occurred while building, '
                               'starting debugger:'), file=error)
            traceback.print_exc()
            pdb.post_mortem(sys.exc_info()[2])
        else:
            print(file=error)
            if isinstance(err, SystemMessage):
                print(red('reST markup error:'), file=error)
                print(terminal_safe(err.args[0]), file=error)
            elif isinstance(err, SphinxError):
                print(red('%s:' % err.category), file=error)
                print(terminal_safe(str(err)), file=error)
            else:
                print(red('Exception occurred:'), file=error)
                print(format_exception_cut_frames().rstrip(), file=error)
                tbpath = save_traceback()
                print(red('The full traceback has been saved '
                                   'in %s, if you want to report the '
                                   'issue to the developers.' % tbpath), file=error)
                print(('Please also report this if it was a user '
                                'error, so that a better error message '
                                'can be provided next time.'), file=error)
                print((
                    'Either send bugs to the mailing list at '
                    '<http://groups.google.com/group/sphinx-dev/>,\n'
                    'or report them in the tracker at '
                    '<http://bitbucket.org/birkenfeld/sphinx/issues/>. Thanks!'), file=error)
            return 1