os.path.split

Here are the examples of the python api os.path.split taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

Example 1

Project: helloworld Source File: media.py
def save_locally(parent_url, full_path, data, skip_write=False,
    disallow_zip=False, overwrite=False):
  # check dupe
  if not skip_write and not overwrite:
    full_path = get_unique_name(full_path)

  media_type = detect_media_type(full_path)
  original_size_url = ''
  url = os.path.join(parent_url, os.path.split(full_path)[1])
  thumb_url = ''

  if media_type == 'image':
    split_path = os.path.split(full_path)
    splitext_path = os.path.splitext(full_path)
    thumb_dir = os.path.join(split_path[0], 'thumbs')
    original_size_dir = os.path.join(split_path[0], 'original')
    leaf_name = os.path.split(full_path)[1]

    if not os.path.isdir(thumb_dir):
      os.makedirs(thumb_dir)
    if not os.path.isdir(original_size_dir):
      os.makedirs(original_size_dir)

    thumb_filename = os.path.join(thumb_dir, leaf_name)
    if not overwrite:
      thumb_filename = get_unique_name(thumb_filename)
    original_size_filename = os.path.join(original_size_dir, leaf_name)
    if not overwrite:
      original_size_filename = get_unique_name(original_size_filename)
    thumb_url = os.path.join(os.path.join(parent_url, 'thumbs'),
        os.path.split(thumb_filename)[1])
    original_size_url = os.path.join(os.path.join(parent_url, 'original'),
        os.path.split(original_size_filename)[1])

    if skip_write:
      os.rename(full_path, original_size_filename)
    else:
      f = open(original_size_filename, 'w+')
      f.write(data)
      f.close()

    is_animated = False
    if splitext_path[1] == '.apng':
      is_animated = True
    elif splitext_path[1] == '.gif':
      gif = Image.open(original_size_filename)

      try:
        gif.seek(1)
      except EOFError:
        is_animated = False
      else:
        is_animated = True

    if not is_animated:
      original_img = Image.open(original_size_filename)
      try:
        info = original_img._getexif()
      except:
        info = None
      exif = {}
      if info:
        for tag, value in info.items():
          decoded = TAGS.get(tag, tag)    
          exif[decoded] = value

      if 'Orientation' in exif:
        orientation = exif['Orientation']
        changed = True
        if orientation == 1:
          changed = False
        elif orientation == 2:
          # Vertical Mirror
          original_img = original_img.transpose(Image.FLIP_LEFT_RIGHT)
        elif orientation == 3:
          # Rotation 180
          original_img = original_img.transpose(Image.ROTATE_180)
        elif orientation == 4:
          # Horizontal Mirror
          original_img = original_img.transpose(Image.FLIP_TOP_BOTTOM)
        elif orientation == 5:
          # Horizontal Mirror + Rotation 270
          original_img = original_img.transpose(
              Image.FLIP_TOP_BOTTOM).transpose(Image.ROTATE_270)
        elif orientation == 6:
          # Rotation 270
          original_img = original_img.transpose(Image.ROTATE_270)
        elif orientation == 7:
          # Vertical Mirror + Rotation 270
          original_img = original_img.transpose(
              Image.FLIP_LEFT_RIGHT).transpose(Image.ROTATE_270)
        elif orientation == 8:
          # Rotation 90
          original_img = original_img.transpose(Image.ROTATE_90)

        if changed:
          original_img.save(original_size_filename, quality=95)

    thumb = Image.open(original_size_filename)
    thumb.thumbnail((content_logic.THUMB_WIDTH, content_logic.THUMB_HEIGHT),
        Image.ANTIALIAS)
    thumb.save(thumb_filename, quality=95)

    if not is_animated:
      normal = Image.open(original_size_filename)
      normal.thumbnail((content_logic.PHOTO_WIDTH, content_logic.PHOTO_HEIGHT),
          Image.ANTIALIAS)
      normal.save(full_path, quality=95)
    else:
      shutil.copyfile(original_size_filename, full_path)
  else:
    if not skip_write:
      f = open(full_path, 'w')
      f.write(data)
      f.close()

    splitext_path = os.path.splitext(full_path)
    if splitext_path[1] == '.zip' and not disallow_zip:
      z = zipfile.ZipFile(full_path)
      dir_path = os.path.join(os.path.dirname(full_path), splitext_path[0])
      for f in z.namelist():
        filename = url_factory.clean_filename(f)
        if f.endswith('/'):
          os.makedirs(os.path.join(dir_path, filename))
        else:
          if detect_media_type(filename) not in ('video', 'image', 'audio',
              'web', 'zip'):
            raise tornado.web.HTTPError(400, "i call shenanigans")
          z.extract(filename, dir_path)

  return original_size_url, url, thumb_url

Example 2

Project: OpenUpgrade Source File: translate.py
def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, module_name=None, context=None):
    """Populates the ir_translation table."""
    if verbose:
        _logger.info('loading translation file for language %s', lang)
    if context is None:
        context = {}
    db_name = cr.dbname
    registry = openerp.registry(db_name)
    lang_obj = registry.get('res.lang')
    trans_obj = registry.get('ir.translation')
    iso_lang = misc.get_iso_codes(lang)
    try:
        ids = lang_obj.search(cr, SUPERUSER_ID, [('code','=', lang)])

        if not ids:
            # lets create the language with locale information
            lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name)

        # Parse also the POT: it will possibly provide additional targets.
        # (Because the POT comments are correct on Launchpad but not the
        # PO comments due to a Launchpad limitation. See LP bug 933496.)
        pot_reader = []

        # now, the serious things: we read the language file
        fileobj.seek(0)
        if fileformat == 'csv':
            reader = csv.reader(fileobj, quotechar='"', delimiter=',')
            # read the first line of the file (it contains columns titles)
            for row in reader:
                fields = row
                break
        elif fileformat == 'po':
            reader = TinyPoFile(fileobj)
            fields = ['type', 'name', 'res_id', 'src', 'value', 'comments']

            # Make a reader for the POT file and be somewhat defensive for the
            # stable branch.
            if fileobj.name.endswith('.po'):
                try:
                    # Normally the path looks like /path/to/xxx/i18n/lang.po
                    # and we try to find the corresponding
                    # /path/to/xxx/i18n/xxx.pot file.
                    # (Sometimes we have 'i18n_extra' instead of just 'i18n')
                    addons_module_i18n, _ignored = os.path.split(fileobj.name)
                    addons_module, i18n_dir = os.path.split(addons_module_i18n)
                    addons, module = os.path.split(addons_module)
                    pot_handle = misc.file_open(os.path.join(
                        addons, module, i18n_dir, module + '.pot'))
                    pot_reader = TinyPoFile(pot_handle)
                except:
                    pass

        else:
            _logger.info('Bad file format: %s', fileformat)
            raise Exception(_('Bad file format: %s') % fileformat)

        # Read the POT references, and keep them indexed by source string.
        class Target(object):
            def __init__(self):
                self.value = None
                self.targets = set()            # set of (type, name, res_id)
                self.comments = None

        pot_targets = defaultdict(Target)
        for type, name, res_id, src, _ignored, comments in pot_reader:
            if type is not None:
                target = pot_targets[src]
                target.targets.add((type, name, res_id))
                target.comments = comments

        # read the rest of the file
        irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context)

        def process_row(row):
            """Process a single PO (or POT) entry."""
            # dictionary which holds values for this line of the csv file
            # {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
            #  'src': ..., 'value': ..., 'module':...}
            dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
                                 'comments', 'imd_model', 'imd_name', 'module'))
            dic['lang'] = lang
            dic.update(zip(fields, row))

            # discard the target from the POT targets.
            src = dic['src']
            if src in pot_targets:
                target = pot_targets[src]
                target.value = dic['value']
                target.targets.discard((dic['type'], dic['name'], dic['res_id']))

            # This would skip terms that fail to specify a res_id
            res_id = dic['res_id']
            if not res_id:
                return

            if isinstance(res_id, (int, long)) or \
                    (isinstance(res_id, basestring) and res_id.isdigit()):
                dic['res_id'] = int(res_id)
                if module_name:
                    dic['module'] = module_name
            else:
                # res_id is an xml id
                dic['res_id'] = None
                dic['imd_model'] = dic['name'].split(',')[0]
                if '.' in res_id:
                    dic['module'], dic['imd_name'] = res_id.split('.', 1)
                else:
                    dic['module'], dic['imd_name'] = module_name, res_id

            irt_cursor.push(dic)

        # First process the entries from the PO file (doing so also fills/removes
        # the entries from the POT file).
        for row in reader:
            process_row(row)

        # Then process the entries implied by the POT file (which is more
        # correct w.r.t. the targets) if some of them remain.
        pot_rows = []
        for src, target in pot_targets.iteritems():
            if target.value:
                for type, name, res_id in target.targets:
                    pot_rows.append((type, name, res_id, src, target.value, target.comments))
        pot_targets.clear()
        for row in pot_rows:
            process_row(row)

        irt_cursor.finish()
        trans_obj.clear_caches()
        if verbose:
            _logger.info("translation file loaded succesfully")

    except IOError:
        filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
        _logger.exception("couldn't read translation file %s", filename)

Example 3

Project: numscons Source File: tex.py
Function: internallatexauxaction
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
    """A builder for LaTeX files that checks the output in the aux file
    and decides how many times to use LaTeXAction, and BibTeXAction."""

    global must_rerun_latex

    # This routine is called with two actions. In this file for DVI builds
    # with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
    # set this up now for the case where the user requests a different extension
    # for the target filename
    if (XXXLaTeXAction == LaTeXAction):
       callerSuffix = ".dvi"
    else:
       callerSuffix = env['PDFSUFFIX']

    basename = SCons.Util.splitext(str(source[0]))[0]
    basedir = os.path.split(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    abspath = os.path.abspath(basedir)

    targetext = os.path.splitext(str(target[0]))[1]
    targetdir = os.path.split(str(target[0]))[0]

    saved_env = {}
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        saved_env[var] = modify_env_var(env, var, abspath)

    # Create base file names with the target directory since the auxiliary files
    # will be made there.   That's because the *COM variables have the cd
    # command in the prolog. We check
    # for the existence of files before opening them--even ones like the
    # aux file that TeX always creates--to make it possible to write tests
    # with stubs that don't necessarily generate all of the same files.

    targetbase = os.path.join(targetdir, basefile)

    # if there is a \makeindex there will be a .idx and thus
    # we have to run makeindex at least once to keep the build
    # happy even if there is no index.
    # Same for glossaries and nomenclature
    src_content = source[0].get_text_contents()
    run_makeindex = makeindex_re.search(src_content) and not os.path.exists(targetbase + '.idx')
    run_nomenclature = makenomenclature_re.search(src_content) and not os.path.exists(targetbase + '.nlo')
    run_glossary = makeglossary_re.search(src_content) and not os.path.exists(targetbase + '.glo')
    run_glossaries = makeglossaries_re.search(src_content) and not os.path.exists(targetbase + '.glo')
    run_acronyms = makeacronyms_re.search(src_content) and not os.path.exists(targetbase + '.acn')

    saved_hashes = {}
    suffix_nodes = {}

    for suffix in all_suffixes:
        theNode = env.fs.File(targetbase + suffix)
        suffix_nodes[suffix] = theNode
        saved_hashes[suffix] = theNode.get_csig()

    if Verbose:
        print "hashes: ",saved_hashes

    must_rerun_latex = True

    #
    # routine to update MD5 hash and compare
    #
    # TODO(1.5):  nested scopes
    def check_MD5(filenode, suffix, saved_hashes=saved_hashes, targetbase=targetbase):
        global must_rerun_latex
        # two calls to clear old csig
        filenode.clear_memoized_values()
        filenode.ninfo = filenode.new_ninfo()
        new_md5 = filenode.get_csig()

        if saved_hashes[suffix] == new_md5:
            if Verbose:
                print "file %s not changed" % (targetbase+suffix)
            return False        # unchanged
        saved_hashes[suffix] = new_md5
        must_rerun_latex = True
        if Verbose:
            print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
        return True     # changed

    # generate the file name that latex will generate
    resultfilename = targetbase + callerSuffix

    count = 0

    while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
        result = XXXLaTeXAction(target, source, env)
        if result != 0:
            return result

        count = count + 1

        must_rerun_latex = False
        # Decide if various things need to be run, or run again.

        # Read the log file to find warnings/errors
        logfilename = targetbase + '.log'
        logContent = ''
        if os.path.exists(logfilename):
            logContent = open(logfilename, "rb").read()


        # Read the fls file to find all .aux files
        flsfilename = targetbase + '.fls'
        flsContent = ''
        auxfiles = []
        if os.path.exists(flsfilename):
            flsContent = open(flsfilename, "rb").read()
            auxfiles = openout_aux_re.findall(flsContent)
        if Verbose:
            print "auxfiles ",auxfiles

        # Now decide if bibtex will need to be run.
        # The information that bibtex reads from the .aux file is
        # pass-independent. If we find (below) that the .bbl file is unchanged,
        # then the last latex saw a correct bibliography.
        # Therefore only do this on the first pass
        if count == 1:
            for auxfilename in auxfiles:
                target_aux = os.path.join(targetdir, auxfilename)
                if os.path.exists(target_aux):
                    content = open(target_aux, "rb").read()
                    if string.find(content, "bibdata") != -1:
                        if Verbose:
                            print "Need to run bibtex"
                        bibfile = env.fs.File(targetbase)
                        result = BibTeXAction(bibfile, bibfile, env)
                        if result != 0:
                            print env['BIBTEX']," returned an error, check the blg file"
                            return result
                        must_rerun_latex = check_MD5(suffix_nodes['.bbl'],'.bbl')
                        break

        # Now decide if latex will need to be run again due to index.
        if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex"
            idxfile = suffix_nodes['.idx']
            result = MakeIndexAction(idxfile, idxfile, env)
            if result != 0:
                print env['MAKEINDEX']," returned an error, check the ilg file"
                return result

        # TO-DO: need to add a way for the user to extend this list for whatever
        # auxiliary files they create in other (or their own) packages
        # Harder is case is where an action needs to be called -- that should be rare (I hope?)

        for index in check_suffixes:
            check_MD5(suffix_nodes[index],index)

        # Now decide if latex will need to be run again due to nomenclature.
        if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for nomenclature"
            nclfile = suffix_nodes['.nlo']
            result = MakeNclAction(nclfile, nclfile, env)
            if result != 0:
                print env['MAKENCL']," (nomenclature) returned an error, check the nlg file"
                return result

        # Now decide if latex will need to be run again due to glossary.
        if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for glossary"
            glofile = suffix_nodes['.glo']
            result = MakeGlossaryAction(glofile, glofile, env)
            if result != 0:
                print env['MAKEGLOSSARY']," (glossary) returned an error, check the glg file"
                return result

        # Now decide if latex will need to be run again due to acronyms.
        if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for acronyms"
            acrfile = suffix_nodes['.acn']
            result = MakeAcronymsAction(acrfile, acrfile, env)
            if result != 0:
                print env['MAKEACRONYMS']," (acronymns) returned an error, check the alg file"
                return result

        # Now decide if latex needs to be run yet again to resolve warnings.
        if warning_rerun_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to latex or package rerun warning"

        if rerun_citations_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to 'Rerun to get citations correct' warning"

        if undefined_references_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to undefined references or citations"

        if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
            print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop

    # rename Latex's output to what the target name is
    if not (str(target[0]) == resultfilename  and  os.path.exists(resultfilename)):
        if os.path.exists(resultfilename):
            print "move %s to %s" % (resultfilename, str(target[0]), )
            shutil.move(resultfilename,str(target[0]))

    # Original comment (when TEXPICTS was not restored):
    # The TEXPICTS enviroment variable is needed by a dvi -> pdf step
    # later on Mac OSX so leave it
    #
    # It is also used when searching for pictures (implicit dependencies).
    # Why not set the variable again in the respective builder instead
    # of leaving local modifications in the environment? What if multiple
    # latex builds in different directories need different TEXPICTS?
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        if var == 'TEXPICTS':
            continue
        if saved_env[var] is _null:
            try:
                del env['ENV'][var]
            except KeyError:
                pass # was never set
        else:
            env['ENV'][var] = saved_env[var]

    return result

Example 4

Project: Nuitka Source File: tex.py
Function: internallatexauxaction
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
    """A builder for LaTeX files that checks the output in the aux file
    and decides how many times to use LaTeXAction, and BibTeXAction."""

    global must_rerun_latex

    # This routine is called with two actions. In this file for DVI builds
    # with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
    # set this up now for the case where the user requests a different extension
    # for the target filename
    if (XXXLaTeXAction == LaTeXAction):
       callerSuffix = ".dvi"
    else:
       callerSuffix = env['PDFSUFFIX']

    basename = SCons.Util.splitext(str(source[0]))[0]
    basedir = os.path.split(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    abspath = os.path.abspath(basedir)

    targetext = os.path.splitext(str(target[0]))[1]
    targetdir = os.path.split(str(target[0]))[0]

    saved_env = {}
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        saved_env[var] = modify_env_var(env, var, abspath)

    # Create base file names with the target directory since the auxiliary files
    # will be made there.   That's because the *COM variables have the cd
    # command in the prolog. We check
    # for the existence of files before opening them--even ones like the
    # aux file that TeX always creates--to make it possible to write tests
    # with stubs that don't necessarily generate all of the same files.

    targetbase = os.path.join(targetdir, basefile)

    # if there is a \makeindex there will be a .idx and thus
    # we have to run makeindex at least once to keep the build
    # happy even if there is no index.
    # Same for glossaries, nomenclature, and acronyms
    src_content = source[0].get_text_contents()
    run_makeindex = makeindex_re.search(src_content) and not os.path.isfile(targetbase + '.idx')
    run_nomenclature = makenomenclature_re.search(src_content) and not os.path.isfile(targetbase + '.nlo')
    run_glossary = makeglossary_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
    run_glossaries = makeglossaries_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
    run_acronyms = makeacronyms_re.search(src_content) and not os.path.isfile(targetbase + '.acn')

    saved_hashes = {}
    suffix_nodes = {}


    for suffix in all_suffixes+sum(newglossary_suffix, []):
        theNode = env.fs.File(targetbase + suffix)
        suffix_nodes[suffix] = theNode
        saved_hashes[suffix] = theNode.get_csig()

    if Verbose:
        print "hashes: ",saved_hashes

    must_rerun_latex = True

    # .aux files already processed by BibTex
    already_bibtexed = []

    #
    # routine to update MD5 hash and compare
    #
    def check_MD5(filenode, suffix):
        global must_rerun_latex
        # two calls to clear old csig
        filenode.clear_memoized_values()
        filenode.ninfo = filenode.new_ninfo()
        new_md5 = filenode.get_csig()

        if saved_hashes[suffix] == new_md5:
            if Verbose:
                print "file %s not changed" % (targetbase+suffix)
            return False        # unchanged
        saved_hashes[suffix] = new_md5
        must_rerun_latex = True
        if Verbose:
            print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
        return True     # changed

    # generate the file name that latex will generate
    resultfilename = targetbase + callerSuffix

    count = 0

    while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
        result = XXXLaTeXAction(target, source, env)
        if result != 0:
            return result

        count = count + 1

        must_rerun_latex = False
        # Decide if various things need to be run, or run again.

        # Read the log file to find warnings/errors
        logfilename = targetbase + '.log'
        logContent = ''
        if os.path.isfile(logfilename):
            logContent = open(logfilename, "rb").read()


        # Read the fls file to find all .aux files
        flsfilename = targetbase + '.fls'
        flsContent = ''
        auxfiles = []
        if os.path.isfile(flsfilename):
            flsContent = open(flsfilename, "rb").read()
            auxfiles = openout_aux_re.findall(flsContent)
            # remove duplicates
            dups = {}
            for x in auxfiles:
                dups[x] = 1
            auxfiles = list(dups.keys())

        bcffiles = []
        if os.path.isfile(flsfilename):
            flsContent = open(flsfilename, "rb").read()
            bcffiles = openout_bcf_re.findall(flsContent)
            # remove duplicates
            dups = {}
            for x in bcffiles:
                dups[x] = 1
            bcffiles = list(dups.keys())

        if Verbose:
            print "auxfiles ",auxfiles
            print "bcffiles ",bcffiles

        # Now decide if bibtex will need to be run.
        # The information that bibtex reads from the .aux file is
        # pass-independent. If we find (below) that the .bbl file is unchanged,
        # then the last latex saw a correct bibliography.
        # Therefore only do this once
        # Go through all .aux files and remember the files already done.
        for auxfilename in auxfiles:
            if auxfilename not in already_bibtexed:
                already_bibtexed.append(auxfilename)
                target_aux = os.path.join(targetdir, auxfilename)
                if os.path.isfile(target_aux):
                    content = open(target_aux, "rb").read()
                    if content.find("bibdata") != -1:
                        if Verbose:
                            print "Need to run bibtex on ",auxfilename
                        bibfile = env.fs.File(SCons.Util.splitext(target_aux)[0])
                        result = BibTeXAction(bibfile, bibfile, env)
                        if result != 0:
                            check_file_error_message(env['BIBTEX'], 'blg')
                        must_rerun_latex = True

        # Now decide if biber will need to be run.
        # When the backend for biblatex is biber (by choice or default) the
        # citation information is put in the .bcf file.
        # The information that biber reads from the .bcf file is
        # pass-independent. If we find (below) that the .bbl file is unchanged,
        # then the last latex saw a correct bibliography.
        # Therefore only do this once
        # Go through all .bcf files and remember the files already done.
        for bcffilename in bcffiles:
            if bcffilename not in already_bibtexed:
                already_bibtexed.append(bcffilename)
                target_bcf = os.path.join(targetdir, bcffilename)
                if os.path.isfile(target_bcf):
                    content = open(target_bcf, "rb").read()
                    if content.find("bibdata") != -1:
                        if Verbose:
                            print "Need to run biber on ",bcffilename
                        bibfile = env.fs.File(SCons.Util.splitext(target_bcf)[0])
                        result = BiberAction(bibfile, bibfile, env)
                        if result != 0:
                            check_file_error_message(env['BIBER'], 'blg')
                        must_rerun_latex = True

        # Now decide if latex will need to be run again due to index.
        if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex"
            idxfile = suffix_nodes['.idx']
            result = MakeIndexAction(idxfile, idxfile, env)
            if result != 0:
                check_file_error_message(env['MAKEINDEX'], 'ilg')
                return result

        # TO-DO: need to add a way for the user to extend this list for whatever
        # auxiliary files they create in other (or their own) packages
        # Harder is case is where an action needs to be called -- that should be rare (I hope?)

        for index in check_suffixes:
            check_MD5(suffix_nodes[index],index)

        # Now decide if latex will need to be run again due to nomenclature.
        if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for nomenclature"
            nclfile = suffix_nodes['.nlo']
            result = MakeNclAction(nclfile, nclfile, env)
            if result != 0:
                check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
                                         'nlg')
                #return result

        # Now decide if latex will need to be run again due to glossary.
        if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for glossary"
            glofile = suffix_nodes['.glo']
            result = MakeGlossaryAction(glofile, glofile, env)
            if result != 0:
                check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
                                         'glg')
                #return result

        # Now decide if latex will need to be run again due to acronyms.
        if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for acronyms"
            acrfile = suffix_nodes['.acn']
            result = MakeAcronymsAction(acrfile, acrfile, env)
            if result != 0:
                check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
                                         'alg')
                return result

        # Now decide if latex will need to be run again due to newglossary command.
        for ig in range(len(newglossary_suffix)):
            if check_MD5(suffix_nodes[newglossary_suffix[ig][2]],newglossary_suffix[ig][2]) or (count == 1):
                # We must run makeindex
                if Verbose:
                    print "Need to run makeindex for newglossary"
                newglfile = suffix_nodes[newglossary_suffix[ig][2]]
                MakeNewGlossaryAction = SCons.Action.Action("$MAKENEWGLOSSARY ${SOURCE.filebase}%s -s ${SOURCE.filebase}.ist -t ${SOURCE.filebase}%s -o ${SOURCE.filebase}%s" % (newglossary_suffix[ig][2],newglossary_suffix[ig][0],newglossary_suffix[ig][1]), "$MAKENEWGLOSSARYCOMSTR")

                result = MakeNewGlossaryAction(newglfile, newglfile, env)
                if result != 0:
                    check_file_error_message('%s (newglossary)' % env['MAKENEWGLOSSARY'],
                                             newglossary_suffix[ig][0])
                    return result

        # Now decide if latex needs to be run yet again to resolve warnings.
        if warning_rerun_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to latex or package rerun warning"

        if rerun_citations_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to 'Rerun to get citations correct' warning"

        if undefined_references_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to undefined references or citations"

        if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
            print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop

    # rename Latex's output to what the target name is
    if not (str(target[0]) == resultfilename  and  os.path.isfile(resultfilename)):
        if os.path.isfile(resultfilename):
            print "move %s to %s" % (resultfilename, str(target[0]), )
            shutil.move(resultfilename,str(target[0]))

    # Original comment (when TEXPICTS was not restored):
    # The TEXPICTS enviroment variable is needed by a dvi -> pdf step
    # later on Mac OSX so leave it
    #
    # It is also used when searching for pictures (implicit dependencies).
    # Why not set the variable again in the respective builder instead
    # of leaving local modifications in the environment? What if multiple
    # latex builds in different directories need different TEXPICTS?
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        if var == 'TEXPICTS':
            continue
        if saved_env[var] is _null:
            try:
                del env['ENV'][var]
            except KeyError:
                pass # was never set
        else:
            env['ENV'][var] = saved_env[var]

    return result

Example 5

Project: mdcs-py Source File: MDCS.py
Function: main
def main(argc, argv):

    if (argc < 2):
    #command-line argument codes.
    #-i:config file.
    #-c:command codes
    #-m:mosaic dataset name
    #-s:Source data paths. (as inputs to command (AR/AR)
    #-l:Full path to log file (including file name)

        user_args = \
        [
        "-m: Mosaic dataset path including GDB and MD name [e.g. c:\WorldElevation.gdb\Portland]",
        "-s: Source data paths. (As inputs to command (AR)",
        "-l: Log file output path [path+file name]",
        "-artdem: Update DEM path in ART file"
        ]

        print ("\nMDCS.py v5.8a [20150611]\nUsage: MDCS.py -c:<Optional:command> -i:<config_file>" \
        "\n\nFlags to override configuration values,") \

        for arg in user_args:
            print (arg)

        print (\
        "\nNote: Commands can be combined with '+' to do multiple operations." \
        "\nAvailable commands:")

        user_cmds = solutionsLib.Solutions().getAvailableCommands()
        for key in user_cmds:
            print ("\t" + key + ' = ' + user_cmds[key]['desc'])
        sys.exit(1)

    base = Base.Base()
    if (not g_cli_callback is None):
        base.m_cli_callback_ptr = g_cli_callback
    if (not g_cli_msg_callback is None):
        base.m_cli_msg_callback_ptr = g_cli_msg_callback
    global log
    log = logger.Logger(base);
    base.setLog(log)

    argIndx = 0
    md_path_ = artdem = config = com = log_folder = code_base =  ''

    while(argIndx < argc):
        (values) = argv[argIndx].split(':')
        if (len(values[0]) < 2 or
            values[0][:1] != '-' and
            values[0][:1] != '#'):
            argIndx += 1
            continue

        exSubCode = values[0][1:len(values[0])].lower()
        subCode = values.pop(0)[1].lower()

        value = ':'.join(values).strip()

        if (subCode == 'c'):
            com = value.replace(' ', '')        #remove spaces in between.
        elif(subCode == 'i'):
            config = value
        elif(subCode == 'm'):
            md_path_ = value
        elif(subCode == 's'):
            base.m_sources = value
        elif(subCode == 'l'):
            log_folder =  value
        elif(subCode == 'b'):
            code_base =  value
        elif(exSubCode == 'artdem'):
            artdem =  value
        elif(exSubCode == 'gprun'):
            log.isGPRun = True                  # direct log messages also to (arcpy.AddMessage)
        elif(subCode == 'p'):
            pMax = value.rfind('$')
            if (pMax == -1):
                argIndx += 1
                continue

            dynamic_var = value[pMax + 1:].upper()
            v =  value[0: pMax]
            if (dynamic_var.strip() != ''):
                if ((dynamic_var in base.m_dynamic_params.keys()) == False):
                    base.m_dynamic_params[dynamic_var] = v

        argIndx += 1


    if (code_base != ''):
        base.setCodeBase(code_base)


    if (md_path_ != ''):
        (p, f) = os.path.split(md_path_)
        f = f.strip()
        const_gdb_ext_len_ = len(base.const_geodatabase_ext)
        ext = p[-const_gdb_ext_len_:].lower()
        if ((ext == base.const_geodatabase_ext.lower() or
            ext == base.const_geodatabase_SDE_ext.lower()) and
            f != ''):
            p = p.replace('\\', '/')
            w = p.split('/')
            workspace_ = ''
            for i in range(0, len(w) - 1):
                workspace_ += w[i] + '/'

            gdb_ = w[len(w) -1]
            base.m_workspace = workspace_
            base.m_geodatabase = w[len(w) - 1]
            base.m_mdName = f


    configName, ext = os.path.splitext(config)
    configName = os.path.basename(configName)

    # setup log
    log.Project ('MDCS')
    log.LogNamePrefix(configName)
    log.StartLog()

    log_output_folder  = os.path.join(os.path.dirname(solutionLib_path), 'logs')

    if (log_folder != ''):
        (path, fileName) = os.path.split(log_folder)
        if (path != ''):
            log_output_folder = path
        if (fileName != ''):
            log.LogFileName(fileName)

    log.SetLogFolder(log_output_folder)
    # ends

    if (os.path.isfile(config) == False):
        log.Message('Input config file is not specified/not found! ({})'.format(config), logger.Logger.const_critical_text)
        log.Message(base.CCMD_STATUS_FAILED, logger.Logger.const_status_text)    # set (failed) status
        log.WriteLog('#all')
        return False

    if (artdem != ''):
        (base.m_art_ws, base.m_art_ds) = os.path.split(artdem)
        base.m_art_apply_changes = True

    comInfo = {
    'AR' : { 'cb' : postAddData }       #assign a callback function to run custom user code when adding rasters.
    }

    if (com == ''):
        com = base.const_cmd_default_text

    solutions = solutionsLib.Solutions(base)
    success = solutions.run(config, com, comInfo)

    log.Message ("Done...", log.const_general_text)
    log.WriteLog('#all')   #persist information/errors collected.

Example 6

Project: cgstudiomap Source File: translate.py
def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, module_name=None, context=None):
    """Populates the ir_translation table."""
    if verbose:
        _logger.info('loading translation file for language %s', lang)
    if context is None:
        context = {}
    db_name = cr.dbname
    registry = openerp.registry(db_name)
    lang_obj = registry.get('res.lang')
    trans_obj = registry.get('ir.translation')
    iso_lang = misc.get_iso_codes(lang)
    try:
        ids = lang_obj.search(cr, SUPERUSER_ID, [('code','=', lang)])

        if not ids:
            # lets create the language with locale information
            lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name)

        # Parse also the POT: it will possibly provide additional targets.
        # (Because the POT comments are correct on Launchpad but not the
        # PO comments due to a Launchpad limitation. See LP bug 933496.)
        pot_reader = []

        # now, the serious things: we read the language file
        fileobj.seek(0)
        if fileformat == 'csv':
            reader = csv.reader(fileobj, quotechar='"', delimiter=',')
            # read the first line of the file (it contains columns titles)
            for row in reader:
                fields = row
                break
        elif fileformat == 'po':
            reader = TinyPoFile(fileobj)
            fields = ['type', 'name', 'res_id', 'src', 'value', 'comments']

            # Make a reader for the POT file and be somewhat defensive for the
            # stable branch.
            if fileobj.name.endswith('.po'):
                try:
                    # Normally the path looks like /path/to/xxx/i18n/lang.po
                    # and we try to find the corresponding
                    # /path/to/xxx/i18n/xxx.pot file.
                    # (Sometimes we have 'i18n_extra' instead of just 'i18n')
                    addons_module_i18n, _ = os.path.split(fileobj.name)
                    addons_module, i18n_dir = os.path.split(addons_module_i18n)
                    addons, module = os.path.split(addons_module)
                    pot_handle = misc.file_open(os.path.join(
                        addons, module, i18n_dir, module + '.pot'))
                    pot_reader = TinyPoFile(pot_handle)
                except:
                    pass

        else:
            _logger.error('Bad file format: %s', fileformat)
            raise Exception(_('Bad file format'))

        # Read the POT references, and keep them indexed by source string.
        class Target(object):
            def __init__(self):
                self.value = None
                self.targets = set()            # set of (type, name, res_id)
                self.comments = None

        pot_targets = defaultdict(Target)
        for type, name, res_id, src, _, comments in pot_reader:
            if type is not None:
                target = pot_targets[src]
                target.targets.add((type, name, res_id))
                target.comments = comments

        # read the rest of the file
        irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context)

        def process_row(row):
            """Process a single PO (or POT) entry."""
            # dictionary which holds values for this line of the csv file
            # {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
            #  'src': ..., 'value': ..., 'module':...}
            dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
                                 'comments', 'imd_model', 'imd_name', 'module'))
            dic['lang'] = lang
            dic.update(zip(fields, row))

            # discard the target from the POT targets.
            src = dic['src']
            if src in pot_targets:
                target = pot_targets[src]
                target.value = dic['value']
                target.targets.discard((dic['type'], dic['name'], dic['res_id']))

            # This would skip terms that fail to specify a res_id
            res_id = dic['res_id']
            if not res_id:
                return

            if isinstance(res_id, (int, long)) or \
                    (isinstance(res_id, basestring) and res_id.isdigit()):
                dic['res_id'] = int(res_id)
                if module_name:
                    dic['module'] = module_name
            else:
                # res_id is an xml id
                dic['res_id'] = None
                dic['imd_model'] = dic['name'].split(',')[0]
                if '.' in res_id:
                    dic['module'], dic['imd_name'] = res_id.split('.', 1)
                else:
                    dic['module'], dic['imd_name'] = module_name, res_id

            irt_cursor.push(dic)

        # First process the entries from the PO file (doing so also fills/removes
        # the entries from the POT file).
        for row in reader:
            process_row(row)

        # Then process the entries implied by the POT file (which is more
        # correct w.r.t. the targets) if some of them remain.
        pot_rows = []
        for src, target in pot_targets.iteritems():
            if target.value:
                for type, name, res_id in target.targets:
                    pot_rows.append((type, name, res_id, src, target.value, target.comments))
        pot_targets.clear()
        for row in pot_rows:
            process_row(row)

        irt_cursor.finish()
        trans_obj.clear_caches()
        if verbose:
            _logger.info("translation file loaded succesfully")

    except IOError:
        filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
        _logger.exception("couldn't read translation file %s", filename)

Example 7

Project: nansat Source File: mapper_landsat.py
Function: init
    def __init__(self, fileName, gdalDataset, gdalMetadata,
                       resolution='low', **kwargs):
        ''' Create LANDSAT VRT from multiple tif files or single tar.gz file'''
        mtlFileName = ''
        bandFileNames = []
        bandSizes = []
        bandDatasets = []
        fname = os.path.split(fileName)[1]

        if   (fileName.endswith('.tar') or
              fileName.endswith('.tar.gz') or
              fileName.endswith('.tgz')):
            # try to open .tar or .tar.gz or .tgz file with tar
            try:
                tarFile = tarfile.open(fileName)
            except:
                raise WrongMapperError

            # collect names of bands and corresponding sizes
            # into bandsInfo dict and bandSizes list
            tarNames = sorted(tarFile.getnames())
            for tarName in tarNames:
                # check if TIF files inside TAR qualify
                if   (tarName[0] in ['L', 'M'] and
                      os.path.splitext(tarName)[1] in ['.TIF', '.tif']):
                    # open TIF file from TAR using VSI
                    sourceFilename = '/vsitar/%s/%s' % (fileName, tarName)
                    gdalDatasetTmp = gdal.Open(sourceFilename)
                    # keep name, GDALDataset and size
                    bandFileNames.append(sourceFilename)
                    bandSizes.append(gdalDatasetTmp.RasterXSize)
                    bandDatasets.append(gdalDatasetTmp)
                elif (tarName.endswith('MTL.txt') or
                      tarName.endswith('MTL.TXT')):
                    # get mtl file
                    mtlFileName = tarName

        elif ((fname.startswith('L') or fname.startswith('M')) and
              (fname.endswith('.tif') or
               fname.endswith('.TIF') or
               fname.endswith('._MTL.txt'))):

            # try to find TIF/tif files with the same name as input file
            path, coreName = os.path.split(fileName)
            coreName = os.path.splitext(coreName)[0].split('_')[0]
            coreNameMask = coreName+'*[tT][iI][fF]'
            tifNames = sorted(glob.glob(os.path.join(path, coreNameMask)))
            for tifName in tifNames:
                sourceFilename = tifName
                gdalDatasetTmp = gdal.Open(sourceFilename)
                # keep name, GDALDataset and size
                bandFileNames.append(sourceFilename)
                bandSizes.append(gdalDatasetTmp.RasterXSize)
                bandDatasets.append(gdalDatasetTmp)

            # get mtl file
            mtlFiles = glob.glob(coreName+'*[mM][tT][lL].[tT][xX][tT]')
            if len(mtlFiles) > 0:
                mtlFileName = mtlFiles[0]
        else:
            raise WrongMapperError

        # if not TIF files found - not appropriate mapper
        if not bandFileNames:
            raise WrongMapperError

        # get appropriate band size based on number of unique size and
        # required resoltuion
        if resolution == 'low':
            bandXSise = min(bandSizes)
        elif resolution in ['high', 'hi']:
            bandXSise = max(bandSizes)
        else:
            raise OptionError('Wrong resolution %s for file %s' % (resolution, fileName))

        # find bands with appropriate size and put to metaDict
        metaDict = []
        for bandFileName, bandSize, bandDataset in zip(bandFileNames,
                                                       bandSizes,
                                                       bandDatasets):
            if bandSize == bandXSise:
                # let last part of file name be suffix
                bandSuffix = os.path.splitext(bandFileName)[0].split('_')[-1]

                metaDict.append({
                    'src': {'SourceFilename': bandFileName,
                            'SourceBand':  1,
                            'ScaleRatio': 0.1},
                    'dst': {'wkv': 'toa_outgoing_spectral_radiance',
                            'suffix': bandSuffix}})
                gdalDataset4Use = bandDataset

        # create empty VRT dataset with geolocation only
        VRT.__init__(self, gdalDataset4Use)

        # add bands with metadata and corresponding values to the empty VRT
        self._create_bands(metaDict)

        if len(mtlFileName) > 0:
            mtlFileName = os.path.join(os.path.split(bandFileNames[0])[0],
                                        mtlFileName)
            mtlFileLines = [line.strip() for line in
                            self.read_xml(mtlFileName).split('\n')]
            dateString = [line.split('=')[1].strip()
                          for line in mtlFileLines
                            if ('DATE_ACQUIRED' in line or
                              'ACQUISITION_DATE' in line)][0]
            timeStr = [line.split('=')[1].strip()
                        for line in mtlFileLines
                            if ('SCENE_CENTER_TIME' in line or
                                'SCENE_CENTER_SCAN_TIME' in line)][0]
            time_start = parse_time(dateString + 'T' + timeStr).isoformat()
            time_end = (parse_time(dateString + 'T' + timeStr) +
                        datetime.timedelta(microseconds=60000000)).isoformat()

        self.dataset.SetMetadataItem('time_coverage_start', time_start)
        self.dataset.SetMetadataItem('time_coverage_end', time_end)

        # set platform
        platform = 'LANDSAT'
        if fname[2].isdigit():
            platform += '-'+fname[2]
        ee = pti.get_gcmd_platform(platform)
        self.dataset.SetMetadataItem('platform', json.dumps(ee))

        # set instrument
        instrument = {
        'LANDSAT' : 'MSS',
        'LANDSAT-1' : 'MSS',
        'LANDSAT-2' : 'MSS',
        'LANDSAT-3' : 'MSS',
        'LANDSAT-4' : 'TM',
        'LANDSAT-5' : 'TM',
        'LANDSAT-7' : 'ETM+',
        'LANDSAT-8' : 'OLI'}[platform]
        ee = pti.get_gcmd_instrument(instrument)
        self.dataset.SetMetadataItem('instrument', json.dumps(ee))

Example 8

Project: backupchecker Source File: generatelistforgzip.py
Function: init
    def __init__(self, __genparams):
        '''The constructor for the GenerateListForGzip class'''
        __arcpath = __genparams['arcpath']
        __delimiter = __genparams['delimiter']
        self._genfull = __genparams['genfull']
        self.__confoutput = __genparams['confoutput']
        self.__listoutput = __genparams['listoutput']
        self.__fulloutput  = __genparams['fulloutput']
        self.__getallhashes  = __genparams['getallhashes']
        self.__hashtype = __genparams['hashtype']
        self.__parsingexceptions = __genparams['parsingexceptions']
        self.__confname = __genparams['confname']
        __arcstat = os.stat(__arcpath)
        __listoffiles = ['[archive]\nmtime{} {}\n\n[files]\n'.format(__delimiter,__arcstat.st_mtime)]
        __fileinfo = os.lstat(__arcpath)
        __filetype = 'f'
        if not self.__hashtype:
            __filehash = get_hash(gzip.open(__arcpath, 'rb'), 'md5')
        else:
            __filehash = get_hash(gzip.open(__arcpath, 'rb'), self.__hashtype)
        with open(__arcpath, 'rb') as __gzip:
            __filesize = self.__extract_size(__gzip)
            __filename = self.__extract_initial_filename(__gzip,
                        os.path.split(__arcpath)[-1][:-2])
        if self.__getallhashes:
            if not self.__hashtype:
                __onelinewithhash = '{value}{delimiter} ={value} type{delimiter}{value} md5{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
            else:
                __onelinewithhash = '{value}{delimiter} ={value} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__hashtype, delimiter=__delimiter)
            __listoffiles.append(__onelinewithhash.format(
                                    __filename,
                                    str(__filesize),
                                    __filetype,
                                    __filehash))
        else:
            if self.__parsingexceptions :
                for __file in self.__parsingexceptions:
                    if fnmatch.fnmatch(__filename, __file):
                        __filehash = get_hash(gzip.open(__arcpath, 'rb'), self.__parsingexceptions[__file])
                        __onelinewithhash = '{value}{delimiter} ={value} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__parsingexceptions[__file], delimiter=__delimiter)
                        __listoffiles.append(__onelinewithhash.format(
                                                __filename,
                                                str(__filesize),
                                                __filetype,
                                                __filehash))
                    else:
                        __onelinewithouthash = '{value}{delimiter} ={value} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
                        __listoffiles.append(__onelinewithouthash.format(
                                                __filename,
                                                str(__filesize),
                                                __filetype))
            else:
                __onelinewithouthash = '{value}{delimiter} ={value} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
                __listoffiles.append(__onelinewithouthash.format(
                                        __filename,
                                        str(__filesize),
                                        __filetype))

        # define the flexible file list path
        __arcwithext = os.path.split(''.join([__arcpath[:-2], 'list']))[1]
        if self.__listoutput:
            if self.__confname:
                # --gen-list and --list-output-dir and --configuration-name
                __arclistpath = os.path.join(self.__listoutput, '.'.join([self.__confname, 'list']))
            else:
                # --gen-list and --list-output-dir
                __arclistpath = os.path.join(self.__listoutput, __arcwithext)
        elif self.__fulloutput:
            if self.__confname:
                # --gen-list and --output-list-and-conf-dir and --configuration-name
                __arclistpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'list']))
            else:
                # --gen-list and --output-list-and-conf-dir
                __arclistpath = os.path.join(self.__fulloutput, __arcwithext)
        else:
            # --gen-list only
            if self.__confname:
                __arc = os.path.dirname(__arcpath)
                __arclistpath =  os.path.join(__arc, '.'.join([self.__confname, 'list']))
            else:
                __arclistpath = ''.join([__arcpath[:-2], 'list'])

        # call the method to write information in a file
        __listconfinfo = {'arclistpath': __arclistpath,
                            'listoffiles':  __listoffiles}
        self._generate_list(__listconfinfo)
        # call the method to write the configuration file if --gen-full was required
        if self._genfull:
            # generate the hash sum of the list of files
            __listhashsum = self._get_list_hash(__listconfinfo['arclistpath'])
            # define the flexible configuration file path
            __arcwithext = os.path.split(''.join([__arcpath[:-2], 'conf']))[1]
            if self.__confoutput:
                if self.__confname:
                    # --gen-full and --output-conf-dir and --configuration-name
                    __arcconfpath = os.path.join(self.__confoutput, '.'.join([self.__confname, 'conf']))
                else:
                    # --gen-full and --output-conf-dir
                    __arcconfpath = os.path.join(self.__confoutput, __arcwithext)
            elif self.__fulloutput:
                if self.__confname:
                    # --gen-full and --output-list-and-conf-dir and --configuration-name
                    __arcconfpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'conf']))
                else:
                    # --gen-full and --output-list-and-conf-dir
                    __arcconfpath = os.path.join(self.__fulloutput, __arcwithext)
            else:
                # --gen-full only
                if self.__confname:
                    __arc = os.path.dirname(__arcpath)
                    __arcconfpath =  os.path.join(__arc, '.'.join([self.__confname, 'conf']))
                else:
                    __arcconfpath = ''.join([__arcpath[:-2], 'conf'])
            # the name of the backup inside the configuration file
            if self.__confname:
                __arcname =  self.__confname
            else:
                __arcname =  os.path.basename(__arcpath[:-3])
            __confinfo = {'arcname': __arcname,
                            'arcpath': __arcpath,
                            'arcconfpath': __arcconfpath,
                            'arclistpath': __listconfinfo['arclistpath'],
                            'arctype': 'archive',
                            'sha512': __listhashsum}
            self._generate_conf(__confinfo)

Example 9

Project: chimera Source File: image.py
Function: make_file_name
    @staticmethod
    def makeFilename(path='$DATE-$TIME',
                     subs={},
                     dateFormat="%Y%m%d",
                     timeFormat="%H%M%S"):
        """
        Helper method to create filenames with increasing sequence number
        appended.
        It can do variable substitution in the given path. Standard
        variables are $DATE and $TIME (you can define the format of
        these field passint the appropriate format string in
        dateFormat and timeFormat, respectively).
        Any other variable can be defined passing an subs dictionary
        with key as variable name.

        @param path: Filename path, with directories, environmnt variables.
        @type  path: str

        @param subs: Dictionary of {VAR=NAME,...} to create aditional
                     variable substitutions.
        @type subs: dict

        @param dateFormat: Date format, as used in time.strftime, to
                           be used by $DATE variable.
        @type dateFormat: str

        @param timeFormat: Time format, as used in time.strftime, to
                           be used by $TIME variable.
        @type timeFormat: str

        @return: Filename.
        @rtype: str
        """

        localtime = dt.datetime.now()
        utctime = dt.datetime.utcnow()

        if localtime.hour < 12:
            jd_day = localtime - dt.timedelta(days=1)
        else:
            jd_day = localtime

        subs_dict = {"LAST_NOON_DATE": jd_day.strftime(dateFormat),
                     "DATE": utctime.strftime(dateFormat),
                     "TIME": utctime.strftime(timeFormat)}

        # add any user-specific keywords
        subs_dict.update(subs)

        dirname, filename = os.path.split(path)
        dirname = os.path.expanduser(dirname)
        dirname = os.path.expandvars(dirname)
        dirname = os.path.realpath(dirname)

        basename, ext = os.path.splitext(filename)
        if not ext:
            ext = "fits"
        else:
            # remove first dot
            ext = ext[1:]

        # make substitutions
        dirname = string.Template(dirname).safe_substitute(subs_dict)
        basename = string.Template(basename).safe_substitute(subs_dict)
        ext = string.Template(ext).safe_substitute(subs_dict)

        fullpath = os.path.join(dirname, basename)
        finalname = os.path.join(dirname, "%s%s%s" % (basename, os.path.extsep, ext))

        if not os.path.exists(dirname):
            os.makedirs(dirname)

        if not os.path.isdir(dirname):
            raise OSError(
                "A file with the same name as the desired directory already exists. ('%s')" % dirname)

        # If filename exists, append -NNN to the end of the file name.
        # A maximum of 1000 files can be generated with the same filename.
        if os.path.exists(finalname):
            base, ext = os.path.splitext(finalname)
            i = 1
            while os.path.exists("%s-%03i%s" % (base, i, ext)):
                i += 1
                if i == 1000:
                    raise OSError("Reached the maximum of 999 files with the same name (%s)." % finalname)

            finalname = "%s-%03i%s" % (base, i, ext)

        return finalname

Example 10

Project: deepcca Source File: dcca.py
def test_dcca(learning_rate=0.01, L1_reg=0.0001, L2_reg=0.0001, n_epochs=1000,
             dataset='mnist.pkl.gz', batch_size=20, n_hidden=500):
    """
    Demonstrate stochastic gradient descent optimization for a multilayer
    perceptron

    This is demonstrated on MNIST.

    :type learning_rate: float
    :param learning_rate: learning rate used (factor for the stochastic
    gradient

    :type L1_reg: float
    :param L1_reg: L1-norm's weight when added to the cost (see
    regularization)

    :type L2_reg: float
    :param L2_reg: L2-norm's weight when added to the cost (see
    regularization)

    :type n_epochs: int
    :param n_epochs: maximal number of epochs to run the optimizer

    :type dataset: string
    :param dataset: the path of the MNIST dataset file from
                 http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz


   """
    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print '... building the model'

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x1 = T.matrix('x1')  # the data is presented as rasterized images
    x2 = T.matrix('x2')  # the labels are presented as 1D vector of
                        # [int] labels
    h1 = T.matrix('h1')  # the data is presented as rasterized images
    h2 = T.matrix('h2')  # the labels are presented as 1D vector of

    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    net1 = DCCA(
        rng=rng,
        input=x1,
        n_in=28 * 28,
        n_hidden=300,
        n_out=8
    )
    net2 = DCCA(
        rng=rng,
        input=x2,
        n_in=10,
        n_hidden=20,
        n_out=8
    )
    if 1:
        cost1 = (
            net1.correlation(h1, h2)
            + L1_reg * net1.L1
            + L2_reg * net1.L2_sqr
        )
        cost2 = (
            net2.correlation(h1, h2)
            + L1_reg * net2.L1
            + L2_reg * net2.L2_sqr
        )
   
    """
    test_model = theano.function(
        inputs=[index],
        outputs=net1.errors(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: valid_set_x[index * batch_size:(index + 1) * batch_size],
            y: valid_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )
    """
    fprop_model1 = theano.function(
        inputs=[],
        outputs=(net1.hiddenLayer.output, net1.output),
        givens={
            x1: test_set_x
        }
    )
    fprop_model2 = theano.function(
        inputs=[],
        outputs=(net2.hiddenLayer.output, net2.output),
        givens={
            x2: test_set_y
        }
    )
    
    if 1: # grad compute for net1 in theano
        U, V, D = theano.tensor.nlinalg.svd(net1.lastLayer.Tval)
        UVT = T.dot(U, V.T)
        Delta12 = T.dot(net1.lastLayer.SigmaHat11**(-0.5), T.dot(UVT, net1.lastLayer.SigmaHat22**(-0.5)))
        UDUT = T.dot(U, T.dot(D, U.T))
        Delta11 = (-0.5) * T.dot(net1.lastLayer.SigmaHat11**(-0.5), T.dot(UDUT, net1.lastLayer.SigmaHat22**(-0.5)))
        grad_E_to_o = (1.0/8) * (2*Delta11*net1.lastLayer.H1bar+Delta12*net1.lastLayer.H2bar)
        gparam1_W = (grad_E_to_o) * (net1.lastLayer.output*(1-net1.lastLayer.output)) * (net1.hiddenLayer.output)
        gparam1_b = (grad_E_to_o) * (net1.lastLayer.output*(1-net1.lastLayer.output)) * theano.shared(numpy.array([1.0],dtype=theano.config.floatX), borrow=True)
        #gparams1 = [T.grad(cost1, param) for param in net1.params]
        gparams1 = [T.grad(cost1, param) for param in net1.hiddenLayer.params]
        gparams1.append(gparam1_W)
        #gparams1.append(gparam1_b)
    if 1: # grad compute for net2
        U, V, D = theano.tensor.nlinalg.svd(net2.lastLayer.Tval)
        UVT = T.dot(U, V.T)
        Delta12 = T.dot(net2.lastLayer.SigmaHat11**(-0.5), T.dot(UVT, net2.lastLayer.SigmaHat22**(-0.5)))
        UDUT = T.dot(U, T.dot(D, U.T))
        Delta11 = (-0.5) * T.dot(net2.lastLayer.SigmaHat11**(-0.5), T.dot(UVT, net2.lastLayer.SigmaHat22**(-0.5)))
        grad_E_to_o = (1.0/8) * (2*Delta11*net2.lastLayer.H1bar+Delta12*net2.lastLayer.H2bar)
        gparam2_W = (grad_E_to_o) * (net2.lastLayer.output*(1-net2.lastLayer.output)) * (net2.hiddenLayer.output)
        gparam2_b = (grad_E_to_o) * (net2.lastLayer.output*(1-net2.lastLayer.output)) * 1
        #gparams1 = [T.grad(cost1, param) for param in net1.params]
        gparams2 = [T.grad(cost2, param) for param in net2.hiddenLayer.params]
        gparams2.append(gparam2_W)
        gparams2.append(gparam2_b)

        #gparams2 = [T.grad(cost2, param) for param in net2.params]

    updates1 = [
        (param, param - learning_rate * gparam)
        for param, gparam in zip(net1.params, gparams1)
    ]
    updates2 = [
        (param, param - learning_rate * gparam)
        for param, gparam in zip(net2.params, gparams2)
    ]
    
    ###############
    # TRAIN MODEL #
    ###############
    print '... training'

    # early-stopping parameters
    patience = 10000  # look as this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is
                           # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatche before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = time.clock()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        print 'epoch', epoch
        #net1.fprop(test_set_x)
        #net2.fprop(test_set_y)
        h1hidden, h1tmpval = fprop_model1()
        h2hidden, h2tmpval = fprop_model2()
        h1hidden = h1hidden.T
        h2hidden = h2hidden.T
        h1tmpval = h1tmpval.T
        h2tmpval = h2tmpval.T
        if 1: # compute cost(H1, H2)
            
            H1 = h1tmpval
            H2 = h2tmpval
            m = H1.shape[1]

            H1bar = H1 - (1.0/m)*numpy.dot(H1, numpy.ones((m,m), dtype=numpy.float32))
            H2bar = H2 - (1.0/m)*numpy.dot(H2, numpy.ones((m,m), dtype=numpy.float32))
            SigmaHat12 = (1.0/(m-1))*numpy.dot(H1bar, H2bar.T)
            SigmaHat11 = (1.0/(m-1))*numpy.dot(H1bar, H1bar.T)
            SigmaHat11 = SigmaHat11 + 0.0001*numpy.identity(SigmaHat11.shape[0], dtype=numpy.float32)
            SigmaHat22 = (1.0/(m-1))*numpy.dot(H2bar, H2bar.T)
            SigmaHat22 = SigmaHat22 + 0.0001*numpy.identity(SigmaHat22.shape[0], dtype=numpy.float32)

            Tval = numpy.dot(mat_pow(SigmaHat11), numpy.dot(SigmaHat12, mat_pow(SigmaHat22)))
            
            corr =  numpy.trace(numpy.dot(Tval.T, Tval))**(0.5)
        if 1: # compute gradient dcost(H1,H2)/dH1
            
            U, D, V, = numpy.linalg.svd(Tval)
            UVT = numpy.dot(U, V.T)
            Delta12 = numpy.dot(mat_pow(SigmaHat11), numpy.dot(UVT, mat_pow(SigmaHat22)))
            UDUT = numpy.dot(U, numpy.dot(D, U.T))
            Delta11 = (-0.5) * numpy.dot(mat_pow(SigmaHat11), numpy.dot(UDUT, mat_pow(SigmaHat22)))
            grad_E_to_o = (1.0/m) * (2*numpy.dot(Delta11,H1bar)+numpy.dot(Delta12,H2bar))
            ##gparam1_W = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * (h1hidden)
            gparam1_W = numpy.dot((h1hidden), ((grad_E_to_o) * (h1tmpval*(1-h1tmpval))).T)
            ##gparam1_b = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * theano.shared(numpy.array([1.0],dtype=theano.config.floatX), borrow=True)
            gparam1_b = numpy.dot(numpy.ones((1,10000),dtype=theano.config.floatX), ((grad_E_to_o) * (h1tmpval*(1-h1tmpval))).T)
            gparam1_W = theano.shared(gparam1_W, borrow=True)
            gparam1_b = theano.shared(gparam1_b[0,:], borrow=True)

            #gparams1 = [T.grad(cost1, param) for param in net1.params]
            gparams1 = [T.grad(cost1, param) for param in net1.hiddenLayer.params]
            gparams1.append(gparam1_W)
            updates1 = [
                (param, param - learning_rate * gparam)
                for param, gparam in zip(net1.params, gparams1)
            ]
            #gparams1.append(gparam1_b)
        if 1: # compute gradient dcost(H1,H2)/dH2
            Tval2 = numpy.dot(mat_pow(SigmaHat22), numpy.dot(SigmaHat12.T, mat_pow(SigmaHat11)))
            U, D, V, = numpy.linalg.svd(Tval2)
            UVT = numpy.dot(U, V.T)
            Delta12 = numpy.dot(mat_pow(SigmaHat22), numpy.dot(UVT, mat_pow(SigmaHat11)))
            UDUT = numpy.dot(U, numpy.dot(D, U.T))
            Delta11 = (-0.5) * numpy.dot(mat_pow(SigmaHat22), numpy.dot(UDUT, mat_pow(SigmaHat11)))
            grad_E_to_o = (1.0/m) * (2*numpy.dot(Delta11,H2bar)+numpy.dot(Delta12,H1bar))
            ##gparam1_W = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * (h1hidden)
            gparam2_W = numpy.dot((h2hidden), ((grad_E_to_o) * (h2tmpval*(1-h2tmpval))).T)
            ##gparam1_b = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * theano.shared(numpy.array([1.0],dtype=theano.config.floatX), borrow=True)
            gparam2_b = numpy.dot(numpy.ones((1,10000),dtype=theano.config.floatX), ((grad_E_to_o) * (h2tmpval*(1-h2tmpval))).T)
            gparam2_W = theano.shared(gparam2_W, borrow=True)
            gparam2_b = theano.shared(gparam2_b[0,:], borrow=True)

            #gparams1 = [T.grad(cost1, param) for param in net1.params]
            gparams2 = [T.grad(cost2, param) for param in net2.hiddenLayer.params]
            gparams2.append(gparam2_W)
            updates2 = [
                (param, param - learning_rate * gparam)
                for param, gparam in zip(net2.params, gparams2)
            ]
            #gparams1.append(gparam1_b)
        
        #X_theano = theano.shared(value=X, name='inputs')
        #h1tmp = theano.shared( value=h1tmpval, name='hidden1_rep', dtype=theano.config.floatX , borrow=True)
        h1tmp = theano.shared(numpy.asarray(H1bar,dtype=theano.config.floatX),
                                 borrow=True)
        #h2tmp = theano.shared( value=h2tmpval, name='hidden2_rep', dtype=theano.config.floatX , borrow=True)
        h2tmp = theano.shared(numpy.asarray(H2bar,dtype=theano.config.floatX),
                                 borrow=True)
        #h1tmp = T.shared( value=net1.output.eval(), name='hidden1_rep' )
        #h2tmp = T.shared( net2.output.eval() )

        train_model1 = theano.function(
            inputs=[],
            #outputs=cost1,
            updates=updates1,
            givens={
                #x1: test_set_x,
                h1: h1tmp,
                h2: h2tmp
            }
        )
        train_model2 = theano.function(
            inputs=[],
            #outputs=cost2,
            updates=updates2,
            givens={
                #x2: test_set_y,
                h1: h1tmp,
                h2: h2tmp

            }
        )
        
        minibatch_avg_cost1 = train_model1()
        minibatch_avg_cost2 = train_model2()
        #print 'corr1', minibatch_avg_cost1
        #print 'corr2', minibatch_avg_cost2
        print 'corr', corr
        if epoch > 10:
            break

    end_time = time.clock()
    print(('Optimization complete. Best validation score of %f %% '
           'obtained at iteration %i, with test performance %f %%') %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print >> sys.stderr, ('The code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))


    ''' Loads the dataset

    :type dataset: string
    :param dataset: the path to the dataset (here MNIST)
    '''

    #############
    # LOAD DATA #
    #############

    # Download the MNIST dataset if it is not present
    data_dir, data_file = os.path.split(dataset)
    if data_dir == "" and not os.path.isfile(dataset):
        # Check if dataset is in the data directory.
        new_path = os.path.join(
            os.path.split(__file__)[0],
            "..",
            "data",
            dataset
        )
        if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz':
            dataset = new_path

    if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz':
        import urllib
        origin = (
            'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz'
        )
        print 'Downloading data from %s' % origin
        urllib.urlretrieve(origin, dataset)

    print '... loading data'

    # Load the dataset
    f = gzip.open(dataset, 'rb')
    train_set, valid_set, test_set = cPickle.load(f)
    f.close()
    #train_set, valid_set, test_set format: tuple(input, target)
    #input is an numpy.ndarray of 2 dimensions (a matrix)
    #witch row's correspond to an example. target is a
    #numpy.ndarray of 1 dimensions (vector)) that have the same length as
    #the number of rows in the input. It should give the target
    #target to the example with the same index in the input.

    def shared_dataset(data_xy, borrow=True):
        """ Function that loads the dataset into shared variables

        The reason we store our dataset in shared variables is to allow
        Theano to copy it into the GPU memory (when code is run on GPU).
        Since copying data into the GPU is slow, copying a minibatch everytime
        is needed (the default behaviour if the data is not in a shared
        variable) would lead to a large decrease in performance.
        """
        #import copy
        data_x, data_y = data_xy
        #daya_y = copy.deepcopy(data_x)
        data_y_new = numpy.zeros((data_y.shape[0], data_y.max()+1))
        for i in range(data_y.shape[0]):
            data_y_new[i, data_y[i]] = 1
        data_y = data_y_new
        shared_x = theano.shared(numpy.asarray(data_x,
                                               dtype=theano.config.floatX),
                                 borrow=borrow)
        shared_y = theano.shared(numpy.asarray(data_y,
                                               dtype=theano.config.floatX),
                                 borrow=borrow)
        # When storing data on the GPU it has to be stored as floats
        # therefore we will store the labels as ``floatX`` as well
        # (``shared_y`` does exactly that). But during our computations
        # we need them as ints (we use labels as index, and if they are
        # floats it doesn't make sense) therefore instead of returning
        # ``shared_y`` we will have to cast it to int. This little hack
        # lets ous get around this issue
        return shared_x, shared_y

    test_set_x, test_set_y = shared_dataset(test_set)
    valid_set_x, valid_set_y = shared_dataset(valid_set)
    train_set_x, train_set_y = shared_dataset(train_set)

    rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),
            (test_set_x, test_set_y)]
    return rval

Example 11

Project: inasafe Source File: gdal_ogr_tools.py
def polygonize_thresholds(
        raster_file_name,
        threshold_min=0.0,
        threshold_max=float('inf')):
    """
    Function to polygonize raster. Areas (pixels) with threshold_min <
    pixel_values < threshold_max will be converted to polygons.

    :param raster_file_name:  Raster file name
    :type raster_file_name: string

    :param threshold_min: Value that splits raster to
                    flooded or not flooded.
    :type threshold_min: float

    :param threshold_max: Value that splits raster to
                    flooded or not flooded.
    :type threshold_max: float

    :returns:   Polygon shape file name
    :rtype:     string

    """

    # all values that are in the threshold are set to 1, others are set to 0
    base_name = unique_filename()
    outfile = base_name + '.tif'

    indataset = gdal.Open(raster_file_name, gdal.GA_ReadOnly)
    out_driver = gdal.GetDriverByName('GTiff')
    outdataset = out_driver.Create(
        outfile,
        indataset.RasterXSize,
        indataset.RasterYSize,
        indataset.RasterCount,
        gdal.GDT_Byte)

    gt = indataset.GetGeoTransform()
    if gt is not None and gt != (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
        outdataset.SetGeoTransform(gt)
    prj = indataset.GetProjectionRef()
    if prj is not None and len(prj) > 0:
        outdataset.SetProjection(prj)

    outNoData = 1
    for iBand in range(1, indataset.RasterCount + 1):
        inband = indataset.GetRasterBand(iBand)
        outband = outdataset.GetRasterBand(iBand)

        for i in range(inband.YSize - 1, -1, -1):
            scanline = inband.ReadAsArray(
                0, i, inband.XSize, 1, inband.XSize, 1)

            if threshold_min >= 0:
                scanline = \
                    numpy.choose(numpy.less(scanline, float(threshold_min)),
                                 (scanline, 0))

            if threshold_max > 0 and threshold_max > threshold_min:
                scanline = \
                    numpy.choose(numpy.greater(scanline, float(threshold_max)),
                                 (scanline, 0))

            scanline = numpy.choose(numpy.not_equal(scanline, 0),
                                    (scanline, outNoData))
            outband.WriteArray(scanline, 0, i)

    # polygonize
    spat_ref = osr.SpatialReference()
    proj = indataset.GetProjectionRef()
    spat_ref.ImportFromWkt(proj)
    drv = ogr.GetDriverByName("ESRI Shapefile")
    base_name = unique_filename()
    out_shape_file = base_name + ".shp"

    dst_ds = drv.CreateDataSource(out_shape_file)
    # ogr_layer_name = 'polygonized'
    ogr_layer_name = os.path.splitext(os.path.split(out_shape_file)[1])[0]
    dst_layer = dst_ds.CreateLayer(ogr_layer_name, spat_ref)
    # fd = ogr.FieldDefn("DN", ogr.OFTInteger )
    fd = ogr.FieldDefn("DN", ogr.OFTReal)
    dst_layer.CreateField(fd)
    dst_field = 0

    # gdal.Polygonize(
    #     outband, outband, dst_layer, dst_field, [], callback=None)
    gdal.Polygonize(outband, None, dst_layer, dst_field, [], callback=None)

    # produce in and out polygon layers
    base_name = unique_filename()
    inside_shape_file = base_name + "_inside.shp"
    inside_layer_name = \
        os.path.splitext(os.path.split(inside_shape_file)[1])[0]

    outside_shape_file = base_name + "_outside.shp"
    outside_layer_name = \
        os.path.splitext(os.path.split(outside_shape_file)[1])[0]

    inside_ds = drv.CreateDataSource(inside_shape_file)
    inside_layer = inside_ds.CreateLayer(inside_layer_name, spat_ref)

    outside_ds = drv.CreateDataSource(outside_shape_file)
    outside_layer = inside_ds.CreateLayer(outside_layer_name, spat_ref)

    for feature in dst_layer:
        value = feature.GetField("DN")
        geom = feature.GetGeometryRef()
        if value == 1:
            new_feature = ogr.Feature(inside_layer.GetLayerDefn())
            new_feature.SetGeometry(geom)
            inside_layer.CreateFeature(new_feature)
        else:
            new_feature = ogr.Feature(outside_layer.GetLayerDefn())
            new_feature.SetGeometry(geom)
            outside_layer.CreateFeature(new_feature)

    inside_ds.Destroy()
    outside_ds.Destroy()
    dst_ds.Destroy()
    return (
        inside_shape_file,
        inside_layer_name,
        outside_shape_file,
        outside_layer_name)

Example 12

Project: WikiDAT Source File: etl.py
    def run(self):
        """
        Execute workflow to import logging records of actions on pages and
        users from dump file

        The data loading workflow is composed of a number of processor
        elements, which can be:

            - Producer (P): raw input data --> input element queue
            - ConsumerProducer (CP): input element queue --> insert db queue
            - Consumer (C): insert db queue --> database (MySQL/MariaDB)

        In this case, the usual combination is 1:N:1 (P, CP, C)
        """
        start = time.time()
        print("Starting LoggingETL workflow at %s" % (
              time.strftime("%Y-%m-%d %H:%M:%S %Z",
                            time.localtime())))

        # DATA EXTRACTION
        xml_reader_name = '-'.join([self.name, 'xml_reader'])
        logitem_proc_name = '-'.join([self.name, 'process_logitem'])
        logitem_insert_name = '-'.join([self.name, 'insert_logitem'])
        # Start subprocess to extract elements from logging dump file
        file_path = self.path[0]
        dump_file = DumpFile(file_path)
        xml_reader = Producer(name=xml_reader_name,
                              target=process_xml,
                              kwargs=dict(
                                  dump_file=dump_file),
                              consumers=self.log_fan,
                              push_logs_port=self.base_port,
                              control_port=self.control_port)
        xml_reader.start()
        print(xml_reader_name, "started")
        print(self.name, "Extracting data from XML revision history file:")
        print(str(self.path[0]))

        # List to keep tracking of logitem workers
        workers = []
        # Create and start page processes
        for worker in range(self.log_fan):
            worker_name = '-'.join([logitem_proc_name, str(worker)])
            process_logitems = Processor(name=worker_name,
                                         target=logitem_to_file,
                                         producers=1, consumers=1,
                                         pull_port=self.base_port,
                                         push_port=self.base_port+2,
                                         control_port=self.control_port)
            process_logitems.start()
            workers.append(process_logitems)
            print(worker_name, "started")

        # Create directory for logging files if it does not exist
        log_dir = os.path.join(os.path.split(file_path)[0], 'logs')
        tmp_dir = os.path.join(os.getcwd(), os.path.split(file_path)[0], 'tmp')
        file_name = os.path.split(file_path)[1]

        if not os.path.exists(log_dir):
            os.makedirs(log_dir)
        if not os.path.exists(tmp_dir):
            os.makedirs(tmp_dir)
        log_file = os.path.join(log_dir, file_name + '.log')

        db_log = MySQLDB(host='localhost', port=3306, user=self.db_user,
                         passwd=self.db_passw, db=self.db_name)
        db_log.connect()
        logitem_insert_db = Consumer(name=logitem_insert_name,
                                     target=logitem_file_to_db,
                                     kwargs=dict(con=db_log,
                                                 log_file=log_file,
                                                 tmp_dir=tmp_dir,
                                                 file_rows=self.log_cache_size,
                                                 etl_prefix=self.name),
                                     producers=self.log_fan,
                                     pull_port=self.base_port+2)

        print(logitem_insert_name, "started")
        logitem_insert_db.start()

        print("Waiting for all processes to finish...")
        print()
        xml_reader.join()
        for w in workers:
            w.join()
        logitem_insert_db.join()

        # All operations finished
        end = time.time()
        print("All tasks done in %.4f sec." % ((end-start)/1.))
        print()
        db_log.close()

Example 13

Project: plugin.audio.tuneinradio Source File: tunein.py
    def tune(self, id):
        '''Returns a list of streams associated with the station.
        '''
        self.log_debug('tune')
        self.log_debug('id: %s' % id)
        if (not self.is_station_id(id) and not self.is_topic_id(id)):
            raise TuneIn.TuneInError(-1, 'Id is not of the correct type.')
        params = [{'param': 'id', 'value': id}]
        req = urllib2.Request(
            self.__add_params_to_url('Tune.ashx', params, addrender=False))
        f = urllib2.urlopen(req)

        self.log_debug('First pass of streams.')

        streams = []
        for stream in f:
            stream = stream.rsplit()[0]
            self.log_debug('stream: %s' % stream)
            (filepath, filename) = os.path.split(stream)
            (shortname, extension) = os.path.splitext(filename)
            filepath = filepath.lower()
            filename = filename.lower()
            shortname = shortname.lower()
            extension = extension.lower()
            self.log_debug('filepath: %s' % filepath)
            self.log_debug('filename: %s' % filename)
            self.log_debug('shortname: %s' % shortname)
            self.log_debug('extension: %s' % extension)
            if (extension == '.pls'):
                self.log_debug('PLS Playlist')
                for stream in self.__parse_pls(stream):
                    streams.append(stream)
            elif (extension == '.asx'):
                self.log_debug('ASX Playlist')
                for stream in self.__parse_asx(stream):
                    streams.append(stream)
            elif (extension == '.m3u'):
                self.log_debug('M3U Playlist')
                for stream in self.__parse_m3u(stream):
                    streams.append(stream)
            elif (re.search('streamtheworld.com', filepath)):
                ''' StreamTheWorld Support
                '''
                self.log_debug('StreamTheWorld stream')
                pattern = re.compile('(.*)callsign\=(.*)$')
                result = pattern.match(filename)
                if (result):
                    stw = streamtheworld.StreamTheWorld(result.group(2))
                    stw_url = stw.get_stream_url(result.group(2))
                    streams.append(stw_url)
            elif (stream.find('player.amri.ca') != -1):
                ''' Astral Radio Support
                '''
                self.log_debug('Astral Radio stream')
                astral = astralradio.AstralRadio(stream)
                for stream in astral.get_streams():
                    streams.append(stream)
            else:
                self.log_debug('Unknown stream')
                try:
                    request = urllib2.Request(stream)
                    opener = urllib2.build_opener()
                    f = opener.open(request)
                    if f.url != stream:
                        stream = f.url
                        streams.append(stream)
                    elif f.info().gettype() == 'video/x-ms-asf':
                        try:
                            for stream in self.__parse_asf(stream):
                                streams.append(stream)
                        except:
                            try:
                                for stream in self.__parse_asx(stream):
                                    streams.append(stream)
                            except:
                                pass
                    else:
                        streams.append(stream)
                except urllib2.URLError as e:
                    self.log_debug('Ignoring URLError: %s' % e)
                    streams.append(stream)

        self.log_debug('Second pass of streams.')

        tmpstreams = streams
        streams = []
        for stream in tmpstreams:
            stream = stream.rsplit()[0]
            self.log_debug('stream: %s' % stream)
            (filepath, filename) = os.path.split(stream)
            (shortname, extension) = os.path.splitext(filename)
            filepath = filepath.lower()
            filename = filename.lower()
            shortname = shortname.lower()
            extension = extension.lower()
            self.log_debug('filepath: %s' % filepath)
            self.log_debug('filename: %s' % filename)
            self.log_debug('shortname: %s' % shortname)
            self.log_debug('extension: %s' % extension)
            if (extension == '.pls'):
                self.log_debug('PLS Playlist')
                for stream in self.__parse_pls(stream):
                    streams.append(stream)
            elif (extension == '.asx'):
                self.log_debug('ASX Playlist')
                for stream in self.__parse_asx(stream):
                    streams.append(stream)
            elif (extension == '.m3u'):
                self.log_debug('M3U Playlist')
                for stream in self.__parse_m3u(stream):
                    streams.append(stream)
            else:
                self.log_debug('Unknown stream')
                streams.append(stream)

        return streams

Example 14

Project: connectomeviewer Source File: plot_directive.py
def plot_directive(name, arguments, options, content, lineno,
                   content_offset, block_text, state, state_machine):
    """
    Handle the plot directive.
    """
    formats = setup.config.plot_formats
    if type(formats) == str:
        formats = eval(formats)

    # The user may provide a filename *or* Python code content, but not both
    if len(arguments) == 1:
        reference = directives.uri(arguments[0])
        basedir, fname = os.path.split(reference)
        basename, ext = os.path.splitext(fname)
        basedir = relpath(basedir, setup.app.builder.srcdir)
        if len(content):
            raise ValueError("plot directive may not specify both a filename and inline content")
        content = None
    else:
        basedir = "inline"
        content = '\n'.join(content)
        # Since we don't have a filename, use a hash based on the content
        reference = basename = md5(content).hexdigest()[-10:]
        fname = None

    # Get the directory of the rst file, and determine the relative
    # path from the resulting html file to the plot_directive links
    # (linkdir).  This relative path is used for html links *only*,
    # and not the embedded image.  That is given an absolute path to
    # the temporary directory, and then sphinx moves the file to
    # build/html/_images for us later.
    rstdir, rstfile = os.path.split(state_machine.docuement.attributes['source'])
    reldir = rstdir[len(setup.confdir)+1:]
    relparts = [p for p in os.path.split(reldir) if p.strip()]
    nparts = len(relparts)
    outdir = os.path.join('plot_directive', basedir)
    linkdir = ('../' * nparts) + outdir

    # tmpdir is where we build all the output files.  This way the
    # plots won't have to be redone when generating latex after html.

    # Prior to Sphinx 0.6, absolute image paths were treated as
    # relative to the root of the filesystem.  0.6 and after, they are
    # treated as relative to the root of the docuementation tree.  We need
    # to support both methods here.
    tmpdir = os.path.join('build', outdir)
    if sphinx_version < (0, 6):
        tmpdir = os.path.abspath(tmpdir)
        prefix = ''
    else:
        prefix = '/'
    if not os.path.exists(tmpdir):
        cbook.mkdirs(tmpdir)

    # destdir is the directory within the output to store files
    # that we'll be linking to -- not the embedded images.
    destdir = os.path.abspath(os.path.join(setup.app.builder.outdir, outdir))
    if not os.path.exists(destdir):
        cbook.mkdirs(destdir)

    # Generate the figures, and return the number of them
    num_figs = makefig(reference, content, tmpdir)

    if options.has_key('include-source'):
        if content is None:
            content = open(reference, 'r').read()
        lines = ['::', ''] + ['    %s'%row.rstrip() for row in content.split('\n')]
        del options['include-source']
    else:
        lines = []

    if num_figs > 0:
        options = ['      :%s: %s' % (key, val) for key, val in
                   options.items()]
        options = "\n".join(options)
        if fname is not None:
            shutil.copyfile(reference, os.path.join(destdir, fname))

        for i in range(num_figs):
            if num_figs == 1:
                outname = basename
            else:
                outname = "%s_%02d" % (basename, i)

            # Copy the linked-to files to the destination within the build tree,
            # and add a link for them
            links = []
            if fname is not None:
                links.append('`source code <%(linkdir)s/%(basename)s.py>`__')
            for format in formats[1:]:
                shutil.copyfile(os.path.join(tmpdir, outname + "." + format),
                                os.path.join(destdir, outname + "." + format))
                links.append('`%s <%s/%s.%s>`__' % (format, linkdir, outname, format))
            links = ', '.join(links) % locals()

            # Output the resulting reST
            lines.extend((template % locals()).split('\n'))
    else:
        lines.extend((exception_template % locals()).split('\n'))

    if len(lines):
        state_machine.insert_input(
            lines, state_machine.input_lines.source(0))

    return []

Example 15

Project: attention-lvcsr Source File: nvcc_compiler.py
    @staticmethod
    def compile_str(
            module_name, src_code,
            location=None, include_dirs=[], lib_dirs=[], libs=[], preargs=[],
            rpaths=rpath_defaults, py_module=True, hide_symbols=True):
        """

        Parameters
        ----------
        module_name: str
             This has been embedded in the src_code.
        src_code
            A complete c or c++ source listing for the module.
        location
            A pre-existing filesystem directory where the
            cpp file and .so will be written.
        include_dirs
            A list of include directory names (each gets prefixed with -I).
        lib_dirs
            A list of library search path directory names (each gets
            prefixed with -L).
        libs
            A list of libraries to link with (each gets prefixed with -l).
        preargs
            A list of extra compiler arguments.
        rpaths
            List of rpaths to use with Xlinker. Defaults to `rpath_defaults`.
        py_module
            If False, compile to a shared library, but
            do not import as a Python module.
        hide_symbols
            If True (the default), hide all symbols from the library symbol
            table unless explicitely exported.

        Returns
        -------
        module
            Dynamically-imported python module of the compiled code.
            (unless py_module is False, in that case returns None.)

        Notes
        -----
        On Windows 7 with nvcc 3.1 we need to compile in the real directory
        Otherwise nvcc never finish.

        """
        # Remove empty string directory
        include_dirs = [d for d in include_dirs if d]
        lib_dirs = [d for d in lib_dirs if d]

        rpaths = list(rpaths)

        if sys.platform == "win32":
            # Remove some compilation args that cl.exe does not understand.
            # cl.exe is the compiler used by nvcc on Windows.
            for a in ["-Wno-write-strings", "-Wno-unused-label",
                      "-Wno-unused-variable", "-fno-math-errno"]:
                if a in preargs:
                    preargs.remove(a)
        if preargs is None:
            preargs = []
        else:
            preargs = list(preargs)
        if sys.platform != 'win32':
            preargs.append('-fPIC')
        if config.cmodule.remove_gxx_opt:
            preargs = [p for p in preargs if not p.startswith('-O')]

        cuda_root = config.cuda.root

        # The include dirs gived by the user should have precedence over
        # the standards ones.
        include_dirs = include_dirs + std_include_dirs()
        if os.path.abspath(os.path.split(__file__)[0]) not in include_dirs:
            include_dirs.append(os.path.abspath(os.path.split(__file__)[0]))

        libs = libs + std_libs()
        if 'cudart' not in libs:
            libs.append('cudart')

        lib_dirs = lib_dirs + std_lib_dirs()

        # config.dnn.include_path add this by default for cudnn in the
        # new back-end. This should not be used in this back-end. So
        # just remove them.
        lib_dirs = [ld for ld in lib_dirs if
                    not(ld == os.path.join(cuda_root, 'lib') or
                        ld == os.path.join(cuda_root, 'lib64'))]

        if sys.platform != 'darwin':
            # sometimes, the linker cannot find -lpython so we need to tell it
            # explicitly where it is located
            # this returns somepath/lib/python2.x
            python_lib = distutils.sysconfig.get_python_lib(plat_specific=1,
                                                            standard_lib=1)
            python_lib = os.path.dirname(python_lib)
            if python_lib not in lib_dirs:
                lib_dirs.append(python_lib)

        cppfilename = os.path.join(location, 'mod.cu')
        with open(cppfilename, 'w') as cppfile:

            _logger.debug('Writing module C++ code to %s', cppfilename)
            cppfile.write(src_code)

        lib_filename = os.path.join(location, '%s.%s' %
                (module_name, get_lib_extension()))

        _logger.debug('Generating shared lib %s', lib_filename)
        # TODO: Why do these args cause failure on gtx285 that has 1.3
        # compute capability? '--gpu-architecture=compute_13',
        # '--gpu-code=compute_13',
        # nvcc argument
        preargs1 = []
        preargs2 = []
        for pa in preargs:
            if pa.startswith('-Wl,'):
                preargs1.append('-Xlinker')
                preargs1.append(pa[4:])
                continue
            for pattern in ['-O', '-arch=', '-ccbin=', '-G', '-g', '-I',
                            '-L', '--fmad', '--ftz', '--maxrregcount',
                            '--prec-div', '--prec-sqrt',  '--use_fast_math',
                            '-fmad', '-ftz', '-maxrregcount',
                            '-prec-div', '-prec-sqrt', '-use_fast_math',
                            '--use-local-env', '--cl-version=']:

                if pa.startswith(pattern):
                    preargs1.append(pa)
                    break
            else:
                preargs2.append(pa)

        # Don't put -G by default, as it slow things down.
        # We aren't sure if -g slow things down, so we don't put it by default.
        cmd = [nvcc_path, '-shared'] + preargs1
        if config.nvcc.compiler_bindir:
            cmd.extend(['--compiler-bindir', config.nvcc.compiler_bindir])

        if sys.platform == 'win32':
            # add flags for Microsoft compiler to create .pdb files
            preargs2.extend(['/Zi', '/MD'])
            cmd.extend(['-Xlinker', '/DEBUG'])
            # remove the complaints for the duplication of `double round(double)`
            # in both math_functions.h and pymath.h,
            # by not including the one in pymath.h
            cmd.extend(['-D HAVE_ROUND'])
        else:
            if hide_symbols:
                preargs2.append('-fvisibility=hidden')

        if local_bitwidth() == 64:
            cmd.append('-m64')
        else:
            cmd.append('-m32')

        if len(preargs2) > 0:
            cmd.extend(['-Xcompiler', ','.join(preargs2)])

        # We should not use rpath if possible. If the user provided
        # provided an cuda.root flag, we need to add one, but
        # otherwise, we don't add it. See gh-1540 and
        # https://wiki.debian.org/RpathIssue for details.

        if (not type(config.cuda).root.is_default and
            os.path.exists(os.path.join(config.cuda.root, 'lib'))):

            rpaths.append(os.path.join(config.cuda.root, 'lib'))
            if sys.platform != 'darwin':
                # the CUDA libs are universal (contain both 32-bit and 64-bit)
                rpaths.append(os.path.join(config.cuda.root, 'lib64'))
        if sys.platform != 'win32':
            # the -rpath option is not understood by the Microsoft linker
            for rpath in rpaths:
                cmd.extend(['-Xlinker', ','.join(['-rpath', rpath])])
        cmd.extend('-I%s' % idir for idir in include_dirs)
        cmd.extend(['-o', lib_filename])
        cmd.append(os.path.split(cppfilename)[-1])
        cmd.extend(['-L%s' % ldir for ldir in lib_dirs])
        cmd.extend(['-l%s' % l for l in libs])
        if sys.platform == 'darwin':
            # This tells the compiler to use the already-loaded python
            # symbols (which should always be the right ones).
            cmd.extend(['-Xcompiler', '-undefined,dynamic_lookup'])

        # Remove "-u Symbol" arguments, since they are usually not
        # relevant for the new compilation, even if they were used for
        # compiling python.  If they are necessary, the nvcc syntax is
        # "-U Symbol" with a capital U.
        done = False
        while not done:
            try:
                indexof = cmd.index('-u')
                cmd.pop(indexof)  # Remove -u
                cmd.pop(indexof)  # Remove argument to -u
            except ValueError as e:
                done = True

        # CUDA Toolkit v4.1 Known Issues:
        # Host linker on Mac OS 10.7 (and 10.6 for me) passes -no_pie option
        # to nvcc this option is not recognized and generates an error
        # http://stackoverflow.com/questions/9327265/nvcc-unknown-option-no-pie
        # Passing -Xlinker -pie stops -no_pie from getting passed
        if sys.platform == 'darwin' and nvcc_version >= '4.1':
            cmd.extend(['-Xlinker', '-pie'])

        # cmd.append("--ptxas-options=-v") #uncomment this to see
        # register and shared-mem requirements
        _logger.debug('Running cmd %s', ' '.join(cmd))
        orig_dir = os.getcwd()
        try:
            os.chdir(location)
            p = subprocess.Popen(
                    cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            nvcc_stdout, nvcc_stderr = decode_iter(p.communicate()[:2])
        finally:
            os.chdir(orig_dir)

        for eline in nvcc_stderr.split('\n'):
            if not eline:
                continue
            if 'skipping incompatible' in eline:
                # ld is skipping an incompatible library
                continue
            if 'declared but never referenced' in eline:
                continue
            if 'statement is unreachable' in eline:
                continue
            _logger.info("NVCC: %s", eline)

        if p.returncode:
            for i, l in enumerate(src_code.split('\n')):
                print(i + 1, l, file=sys.stderr)
            print('===============================', file=sys.stderr)
            # filter the output from the compiler
            for l in nvcc_stderr.split('\n'):
                if not l:
                    continue
                # filter out the annoying declaration warnings

                try:
                    if l[l.index(':'):].startswith(': warning: variable'):
                        continue
                    if l[l.index(':'):].startswith(': warning: label'):
                        continue
                except Exception:
                    pass
                print(l, file=sys.stderr)
            print(nvcc_stdout)
            print(cmd)
            raise Exception('nvcc return status', p.returncode,
                            'for cmd', ' '.join(cmd))
        elif config.cmodule.compilation_warning and nvcc_stdout:
            print(nvcc_stdout)

        if nvcc_stdout:
            # this doesn't happen to my knowledge
            print("DEBUG: nvcc STDOUT", nvcc_stdout, file=sys.stderr)

        if py_module:
            # touch the __init__ file
            open(os.path.join(location, "__init__.py"), 'w').close()
            return dlimport(lib_filename)

Example 16

Project: mysql-utilities Source File: binlog_admin.py
def move_binlogs_from_server(server_cnx_val, destination, options,
                             bin_basename=None, bin_index=None,
                             relay_basename=None):
    """Relocate binary logs from the given server to a new location.

    This function relocate the binary logs from a MySQL server to the specified
    destination directory, attending to the specified options.

    server_cnx_val[in]  Dictionary with the connection values for the server.
    destination[in]     Path of the destination directory for the binary log
                        files.
    options[in]         Dictionary of options (log_type, skip_flush_binlogs,
                        modified_before, sequence, verbosity).
    bin_basename[in]    Base name for the binlog files, i.e., same as the
                        value for the server option --log-bin. It replaces
                        the server variable 'log_bin_basename' for versions
                        < 5.6.2, otherwise it is ignored.
    bin_index[in]       Path of the binlog index file. It replaces the server
                        variable 'log_bin_index' for versions < 5.6.4,
                        otherwise it is ignored.
    relay_basename[in]  Base name for the relay log files, i.e., filename
                        without the extension (sequence number). Same as the
                        value for the server option --relay-log. It replaces
                        the server variable 'relay_log_basename' for versions
                        < 5.6.2, otherwise it is ignored.
    """

    log_type = options.get('log_type', LOG_TYPE_BIN)
    skip_flush = options['skip_flush_binlogs']
    verbosity = options['verbosity']
    # Connect to server
    server_options = {
        'conn_info': server_cnx_val,
    }
    srv = Server(server_options)
    srv.connect()

    # Check if the server is running locally (not remote server).
    if not srv.is_alias('localhost'):
        raise UtilError("You are using a remote server. This utility must be "
                        "run on the local server. It does not support remote "
                        "access to the binary log files.")

    # Check privileges.
    _check_privileges_to_move_binlogs(srv, options)

    # Process binlog files.
    if log_type in (LOG_TYPE_BIN, LOG_TYPE_ALL):
        # Get log_bin_basename (available since MySQL 5.6.2).
        if srv.check_version_compat(5, 6, 2):
            if bin_basename:
                print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
                    value='bin basename', min_version='5.6.2',
                    var_name='log_bin_basename'))
            binlog_basename = srv.select_variable('log_bin_basename')
            if verbosity > 0:
                print("#")
                print("# log_bin_basename: {0}".format(binlog_basename))
            binlog_source, binlog_file = os.path.split(binlog_basename)
            # Get log_bin_index (available since MySQL 5.6.4).
            if srv.check_version_compat(5, 6, 4):
                if bin_index:
                    print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
                        value='bin index', min_version='5.6.4',
                        var_name='log_bin_index'))
                binlog_index = srv.select_variable('log_bin_index')
            else:
                binlog_index = None
                action = _ACTION_SEARCH_INDEX.format(file_type='bin-log')
                print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
                    var_name='log_bin_basename', host=srv.host, port=srv.port,
                    min_version='5.6.4', action=action))
            if verbosity > 0:
                print("# log_bin_index: {0}".format(binlog_index))
        else:
            if bin_basename:
                binlog_source, binlog_file = os.path.split(bin_basename)
            else:
                action = _ACTION_DATADIR_USED.format(file_type='bin-log')
                print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
                    var_name='log_bin_basename', host=srv.host, port=srv.port,
                    min_version='5.6.2', action=action))
                # Get datadir value.
                binlog_source = srv.select_variable('datadir')
                binlog_file = None
                if verbosity > 0:
                    print("#")
                    print("# datadir: {0}".format(binlog_source))
            binlog_index = bin_index

        # Move binlog files.
        num_files = _move_binlogs(
            binlog_source, destination, LOG_TYPE_BIN, options,
            basename=binlog_file, index_file=binlog_index, skip_latest=True)
        print("#")

        # Flush binary logs to reload server's cache after move.
        if not skip_flush and num_files > 0:
            # Note: log_type for FLUSH available since MySQL 5.5.3.
            if srv.check_version_compat(5, 5, 3):
                print(_INFO_MSG_FLUSH_LOGS.format(log_type='binary'))
                srv.flush_logs(log_type='BINARY')
            else:
                print(_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='binary',
                                                      host=srv.host,
                                                      port=srv.port))
            print("#")

    if log_type in (LOG_TYPE_RELAY, LOG_TYPE_ALL):
        # Get relay_log_basename (available since MySQL 5.6.2).
        if srv.check_version_compat(5, 6, 2):
            if relay_basename:
                print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
                    value='relay basename', min_version='5.6.2',
                    var_name='relay_log_basename'))
            relay_log_basename = srv.select_variable('relay_log_basename')
            if verbosity > 0:
                print("#")
                print("# relay_log_basename: {0}".format(relay_log_basename))
            relay_source, relay_file = os.path.split(relay_log_basename)
        else:
            if relay_basename:
                relay_source, relay_file = os.path.split(relay_basename)
            else:
                action = _ACTION_DATADIR_USED.format(file_type='relay-log')
                print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
                    var_name='relay_log_basename', host=srv.host,
                    port=srv.port, min_version='5.6.2', action=action))
                # Get datadir value.
                relay_source = srv.select_variable('datadir')
                relay_file = None
                if verbosity > 0:
                    print("#")
                    print("# datadir: {0}".format(relay_source))
        # Get relay_log_index (available for all supported versions).
        relay_log_index = srv.select_variable('relay_log_index')
        if verbosity > 0:
            print("# relay_log_index: {0}".format(relay_log_index))

        # Move relay log files.
        num_files = _move_binlogs(
            relay_source, destination, LOG_TYPE_RELAY, options,
            basename=relay_file, index_file=relay_log_index, skip_latest=True)
        print("#")

        # Flush relay logs to reload server's cache after move.
        if not skip_flush and num_files > 0:
            # Note: log_type for FLUSH available since MySQL 5.5.3.
            if srv.check_version_compat(5, 5, 3):
                print(_INFO_MSG_FLUSH_LOGS.format(log_type='relay'))
                srv.flush_logs(log_type='RELAY')
            else:
                print(_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='relay',
                                                      host=srv.host,
                                                      port=srv.port))
            print("#")

    print("#...done.\n#")

Example 17

Project: bigcouch Source File: tex.py
Function: internallatexauxaction
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
    """A builder for LaTeX files that checks the output in the aux file
    and decides how many times to use LaTeXAction, and BibTeXAction."""

    global must_rerun_latex

    # This routine is called with two actions. In this file for DVI builds
    # with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
    # set this up now for the case where the user requests a different extension
    # for the target filename
    if (XXXLaTeXAction == LaTeXAction):
       callerSuffix = ".dvi"
    else:
       callerSuffix = env['PDFSUFFIX']

    basename = SCons.Util.splitext(str(source[0]))[0]
    basedir = os.path.split(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    abspath = os.path.abspath(basedir)

    targetext = os.path.splitext(str(target[0]))[1]
    targetdir = os.path.split(str(target[0]))[0]

    saved_env = {}
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        saved_env[var] = modify_env_var(env, var, abspath)

    # Create base file names with the target directory since the auxiliary files
    # will be made there.   That's because the *COM variables have the cd
    # command in the prolog. We check
    # for the existence of files before opening them--even ones like the
    # aux file that TeX always creates--to make it possible to write tests
    # with stubs that don't necessarily generate all of the same files.

    targetbase = os.path.join(targetdir, basefile)

    # if there is a \makeindex there will be a .idx and thus
    # we have to run makeindex at least once to keep the build
    # happy even if there is no index.
    # Same for glossaries and nomenclature
    src_content = source[0].get_text_contents()
    run_makeindex = makeindex_re.search(src_content) and not os.path.exists(targetbase + '.idx')
    run_nomenclature = makenomenclature_re.search(src_content) and not os.path.exists(targetbase + '.nlo')
    run_glossary = makeglossary_re.search(src_content) and not os.path.exists(targetbase + '.glo')
    run_glossaries = makeglossaries_re.search(src_content) and not os.path.exists(targetbase + '.glo')
    run_acronyms = makeacronyms_re.search(src_content) and not os.path.exists(targetbase + '.acn')

    saved_hashes = {}
    suffix_nodes = {}

    for suffix in all_suffixes:
        theNode = env.fs.File(targetbase + suffix)
        suffix_nodes[suffix] = theNode
        saved_hashes[suffix] = theNode.get_csig()

    if Verbose:
        print "hashes: ",saved_hashes

    must_rerun_latex = True

    #
    # routine to update MD5 hash and compare
    #
    def check_MD5(filenode, suffix):
        global must_rerun_latex
        # two calls to clear old csig
        filenode.clear_memoized_values()
        filenode.ninfo = filenode.new_ninfo()
        new_md5 = filenode.get_csig()

        if saved_hashes[suffix] == new_md5:
            if Verbose:
                print "file %s not changed" % (targetbase+suffix)
            return False        # unchanged
        saved_hashes[suffix] = new_md5
        must_rerun_latex = True
        if Verbose:
            print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
        return True     # changed

    # generate the file name that latex will generate
    resultfilename = targetbase + callerSuffix

    count = 0

    while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
        result = XXXLaTeXAction(target, source, env)
        if result != 0:
            return result

        count = count + 1

        must_rerun_latex = False
        # Decide if various things need to be run, or run again.

        # Read the log file to find warnings/errors
        logfilename = targetbase + '.log'
        logContent = ''
        if os.path.exists(logfilename):
            logContent = open(logfilename, "rb").read()


        # Read the fls file to find all .aux files
        flsfilename = targetbase + '.fls'
        flsContent = ''
        auxfiles = []
        if os.path.exists(flsfilename):
            flsContent = open(flsfilename, "rb").read()
            auxfiles = openout_aux_re.findall(flsContent)
        if Verbose:
            print "auxfiles ",auxfiles

        # Now decide if bibtex will need to be run.
        # The information that bibtex reads from the .aux file is
        # pass-independent. If we find (below) that the .bbl file is unchanged,
        # then the last latex saw a correct bibliography.
        # Therefore only do this on the first pass
        if count == 1:
            for auxfilename in auxfiles:
                target_aux = os.path.join(targetdir, auxfilename)
                if os.path.exists(target_aux):
                    content = open(target_aux, "rb").read()
                    if content.find("bibdata") != -1:
                        if Verbose:
                            print "Need to run bibtex"
                        bibfile = env.fs.File(targetbase)
                        result = BibTeXAction(bibfile, bibfile, env)
                        if result != 0:
                            check_file_error_message(env['BIBTEX'], 'blg')
                            return result
                        must_rerun_latex = check_MD5(suffix_nodes['.bbl'],'.bbl')
                        break

        # Now decide if latex will need to be run again due to index.
        if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex"
            idxfile = suffix_nodes['.idx']
            result = MakeIndexAction(idxfile, idxfile, env)
            if result != 0:
                check_file_error_message(env['MAKEINDEX'], 'ilg')
                return result

        # TO-DO: need to add a way for the user to extend this list for whatever
        # auxiliary files they create in other (or their own) packages
        # Harder is case is where an action needs to be called -- that should be rare (I hope?)

        for index in check_suffixes:
            check_MD5(suffix_nodes[index],index)

        # Now decide if latex will need to be run again due to nomenclature.
        if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for nomenclature"
            nclfile = suffix_nodes['.nlo']
            result = MakeNclAction(nclfile, nclfile, env)
            if result != 0:
                check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
                                         'nlg')
                #return result

        # Now decide if latex will need to be run again due to glossary.
        if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for glossary"
            glofile = suffix_nodes['.glo']
            result = MakeGlossaryAction(glofile, glofile, env)
            if result != 0:
                check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
                                         'glg')
                #return result

        # Now decide if latex will need to be run again due to acronyms.
        if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for acronyms"
            acrfile = suffix_nodes['.acn']
            result = MakeAcronymsAction(acrfile, acrfile, env)
            if result != 0:
                check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
                                         'alg')
                return result

        # Now decide if latex needs to be run yet again to resolve warnings.
        if warning_rerun_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to latex or package rerun warning"

        if rerun_citations_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to 'Rerun to get citations correct' warning"

        if undefined_references_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to undefined references or citations"

        if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
            print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop

    # rename Latex's output to what the target name is
    if not (str(target[0]) == resultfilename  and  os.path.exists(resultfilename)):
        if os.path.exists(resultfilename):
            print "move %s to %s" % (resultfilename, str(target[0]), )
            shutil.move(resultfilename,str(target[0]))

    # Original comment (when TEXPICTS was not restored):
    # The TEXPICTS enviroment variable is needed by a dvi -> pdf step
    # later on Mac OSX so leave it
    #
    # It is also used when searching for pictures (implicit dependencies).
    # Why not set the variable again in the respective builder instead
    # of leaving local modifications in the environment? What if multiple
    # latex builds in different directories need different TEXPICTS?
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        if var == 'TEXPICTS':
            continue
        if saved_env[var] is _null:
            try:
                del env['ENV'][var]
            except KeyError:
                pass # was never set
        else:
            env['ENV'][var] = saved_env[var]

    return result

Example 18

Project: nupic.vision Source File: BoxFixer.py
Function: process
  def process(self, image):
    """
    """

    assert image.mode == 'LA'
    smotion, mask = image.split()

    # Try to get box from info, otherwise from alpha channel
    origBox = image.info.get('tracking')
    if not origBox:
      origBox = mask.getbbox()

    expandedBox = (max(0, origBox[0] - self._zonePreExpansionX),
                   max(0, origBox[1] - self._zonePreExpansionY),
                   min(image.size[0], origBox[2] + self._zonePreExpansionX),
                   min(image.size[1], origBox[3] + self._zonePreExpansionY))

    imgbox = smotion.crop(expandedBox)
    #imgbox = smotion.crop(origBox)
    w, h = imgbox.size
    imgdata = numpy.array(imgbox.getdata())
    imgdata.shape = (h, w)

    # Create binary image indicating whether non-zero
    # S-Motion exists
    salpha = (imgdata > 0).astype(int)
    histX = salpha.mean(axis=0)
    smoothX = self._smooth(histX, window_len=self._windowX,
                                  window=self._windowType)

    #-----------------------------------------------------------------------
    # Apply tightening/splitting in horizontal direction

    # Pre-compute the minimum length of a strong
    # zone that we'll accept.
    # This is the max of an absolute length and a
    # minimum fraction of the original box.
    minZoneLen = max(self._minAbsZoneLenX,
                     int(round(self._minRelZoneLenX \
                           * float(len(smoothX)))))
    # Minimum length for a weak gap
    minWeakLen = max(self._minAbsWeakLenX,
                     int(round(self._minRelWeakLenX \
                           * float(len(smoothX)))))

    maxX = smoothX.max()
    # For now, simple threshold
    threshX = self._heightThresh * maxX
    strongX = (smoothX >= threshX).astype(int)

    # Pre-calculate the minimum peak strength for
    # each lobe to avoid being culled
    minStrength = maxX * self._secondaryHeightThresh

    # Find changes:
    # If deltas[k] == 1, then strongX[k+1] is the
    #       beginning of a new strong block;
    # If deltas[k] == -1, then strongX[k+1] is the
    #       beginning of a new weak block
    deltas = strongX[1:] - strongX[:-1]
    changes = numpy.where(deltas)[0]
    # Form our block lists
    strongZonesX = []
    if strongX[0]:
      curZoneStart = 0
    else:
      curZoneStart = None
    for changeIdx in changes:
      strongIdx = changeIdx + 1
      changeDir = deltas[changeIdx]
      # Start of new strong zone
      if changeDir == 1:
        assert curZoneStart is None
        curZoneStart = strongIdx
      # End of existing strong zone
      else:
        assert changeDir == -1
        assert curZoneStart is not None
        strongZone = (curZoneStart, strongIdx)
        self._acceptOrCull(smoothX, strongZonesX, strongZone, minZoneLen, minStrength)
        curZoneStart = None
    # Last one
    if curZoneStart is not None:
      strongZone = (curZoneStart, len(strongX))
      self._acceptOrCull(smoothX, strongZonesX, strongZone, minZoneLen, minStrength)

    # Remove tiny/thin weak gaps
    if len(strongZonesX) > 1:
      tempZones = []
      lastZone = strongZonesX[0]
      for startIdx, endIdx in strongZonesX[1:]:
        if startIdx - lastZone[1] >= minWeakLen:
          tempZones += [lastZone]
          lastZone = (startIdx, endIdx)
        else:
          lastZone = (lastZone[0], endIdx)
      tempZones += [lastZone]
      strongZonesX = tempZones

    #-----------------------------------------------------------------------
    # Apply tightening/splitting in vertical direction (to each strong zone)

    strongZonesAll = []
    for strongZoneX in strongZonesX:

      #histY = salpha.mean(axis=1)
      histY = salpha[:,strongZoneX[0]:strongZoneX[1]].mean(axis=1)
      smoothY = self._smooth(histY, window_len=self._windowY,
                                    window=self._windowType)

      # Pre-compute the minimum length of a strong
      # zone that we'll accept.
      # This is the max of an absolute length and a
      # minimum fraction of the original box.
      minZoneLen = max(self._minAbsZoneLenY,
                       int(round(self._minRelZoneLenY \
                             * float(len(smoothY)))))
      # Minimum length for a weak gap
      minWeakLen = max(self._minAbsWeakLenY,
                       int(round(self._minRelWeakLenY \
                             * float(len(smoothY)))))

      maxY = smoothY.max()
      # For now, simple threshold
      threshY = self._widthThresh * maxY
      strongY = (smoothY >= threshY).astype(int)

      # Pre-calculate the minimum peak strength for
      # each lobe to avoid being culled
      minStrength = maxY * self._secondaryWidthThresh

      # Find changes:
      deltas = strongY[1:] - strongY[:-1]
      changes = numpy.where(deltas)[0]
      # Form our block lists
      strongZonesY = []
      if strongY[0]:
        curZoneStart = 0
      else:
        curZoneStart = None
      for changeIdx in changes:
        strongIdx = changeIdx + 1
        changeDir = deltas[changeIdx]
        # Start of new strong zone
        if changeDir == 1:
          assert curZoneStart is None
          curZoneStart = strongIdx
        # End of existing strong zone
        else:
          assert changeDir == -1
          assert curZoneStart is not None
          strongZoneY = (curZoneStart, strongIdx)
          self._acceptOrCull(smoothY, strongZonesY, strongZoneY, minZoneLen, minStrength)
          curZoneStart = None
      # Last one
      if curZoneStart is not None:
        strongZoneY = (curZoneStart, len(strongY))
        self._acceptOrCull(smoothY, strongZonesY, strongZoneY, minZoneLen, minStrength)

      # Remove tiny/thin weak gaps
      if len(strongZonesY) > 1:
        tempZones = []
        lastZone = strongZonesY[0]
        for startIdx, endIdx in strongZonesY[1:]:
          if startIdx - lastZone[1] >= minWeakLen:
            tempZones += [lastZone]
            lastZone = (startIdx, endIdx)
          else:
            lastZone = (lastZone[0], endIdx)
        tempZones += [lastZone]
        strongZonesY = tempZones

      left, right = strongZoneX
      #strongZonesAll.extend([(left, top, right, bottom) for (top, bottom) in strongZonesY])
      for (top, bottom) in strongZonesY:
        expandedZone = (max(0, left - self._zonePostExpansionX),
                        max(0, top - self._zonePostExpansionY),
                        min(image.size[0], right + self._zonePostExpansionX),
                        min(image.size[1], bottom + self._zonePostExpansionY))
        if expandedZone[2] < expandedZone[0]:
          expandedZone[2] = expandedZone[0]
        if expandedZone[3] < expandedZone[1]:
          expandedZone[3] = expandedZone[1]
        strongZonesAll += [expandedZone]

    if False:
        # Obtain the videoID and sequenceID
        imgDir, imgName = os.path.split(imagePath)
        imgPrefix = os.path.split(imgDir)[1]
        # Example:
        # overlap.0550_sequence10067
        match = re.match(r"^(?P<mnemonic>[a-z]+)\.(?P<videoID>[0-9]{4})_sequence(?P<seqID>\-?[0-9]{1,5})$", imgPrefix)
        if not match:
          match = re.match(r"^vid(?P<videoID>[0-9]{4})_seq(?P<seqID>\-?[0-9]{3,4})$", imgPrefix)
        assert match
        d = match.groupdict()
        videoID = int(d['videoID'])
        seqID = int(d['seqID'])
        key = (videoID, seqID)

    numZones = len(strongZonesAll)

    # Debugging (and inefficient)
    if self._debugMode:
      # Mark up the img box
      blank = Image.new('L', image.size, 0)
      base = smotion.convert('RGB')
      alphaNew = blank.copy()
      alphaOrig = blank.copy()
      alphaOrig.paste(255, origBox)
      #for zoneStart, zoneEnd in strongZones:
      for (left, top, right, bottom) in strongZonesAll:
        zoneBox = (expandedBox[0] + left, expandedBox[1] + top,
                   expandedBox[0] + right, expandedBox[1] + bottom)
        #zoneBox = (origBox[0] + left, origBox[1] + top,
        #           origBox[0] + right, origBox[1] + bottom)
        alphaNew.paste(255, zoneBox)
      blender = Image.merge('RGB', (alphaOrig, alphaNew, blank))
      #blender = Image.merge('RGB', (alphaOrig, blank, alphaNew))
      blendFraction = 0.5
      resultImg = Image.blend(base, blender, blendFraction)

      # Dump marked-up images to disk
      #imgDir, imgName = os.path.split(imagePath)
      #imgPrefix = os.path.split(imgDir)[1]
      dstPath = "%06d.png" % self._imgCounter
      dstName, dstExt = os.path.splitext(dstPath)
      dstName += "__Z%d" % numZones
      dstPath = dstName + dstExt
      resultImg.save(dstPath)

    # Print stats
    #(left, top, right, bottom) = strongZonesAll[0]
    #print "ORIG (%3d, %3d, %3d, %3d) [%3dx%3d] ==> (%3d, %3d, %3d, %3d) [%3dx%3d]" % \
    #       (origBox[0], origBox[1], origBox[2], origBox[3],
    #        origBox[2]-origBox[0], origBox[3]-origBox[1],
    #        left, top, right, bottom, right-left, bottom-top)

    # If there is more than one box, use the biggest
    # (this is just a hack heuristic)
    tightenedZone = None
    if len(strongZonesAll) > 1:
      # Take biggest
      if self._splitPolicy == 'biggest':
        #print "WARNING: multiple (%d) split boxes...choosing biggest one..." % len(strongZonesAll)
        bigBoxIdx = None
        bigBoxArea = -1
        for boxIdx, subBox in enumerate(strongZonesAll):
          area = (subBox[2] - subBox[0]) * (subBox[3] - subBox[1])
          if area > bigBoxArea:
            bigBoxArea = area
            bigBoxIdx = boxIdx
        tightenedZone = strongZonesAll[bigBoxIdx]

      # Take biggest
      elif self._splitPolicy == 'union':
        #print "WARNING: multiple (%d) split boxes...taking union..." % len(strongZonesAll)
        left, top, right, bottom = strongZonesAll[0]
        for boxIdx, subBox in enumerate(strongZonesAll[1:]):
          left   = min(left,   subBox[0])
          top    = min(top,    subBox[1])
          right  = max(right,  subBox[2])
          bottom = max(bottom, subBox[3])
        tightenedZone = (left, top, right, bottom)

    elif not strongZonesAll:
      #print "WARNING: dissipated box...reverting to original..."
      box = origBox

    else:
      assert len(strongZonesAll) == 1
      tightenedZone = strongZonesAll[0]

    if strongZonesAll:
      subBox = tightenedZone
      # 'inference' may be None if box was culled
      box = (expandedBox[0] + subBox[0],
             expandedBox[1] + subBox[1],
             expandedBox[0] + subBox[2],
             expandedBox[1] + subBox[3])

    # Print stats
    (left, top, right, bottom) = box
    #print "%06d: (%3d, %3d, %3d, %3d) [%3dx%3d] ==> (%3d, %3d, %3d, %3d) [%3dx%3d]" % \
    #       (self._imgCounter,
    #        origBox[0], origBox[1], origBox[2], origBox[3],
    #        origBox[2]-origBox[0], origBox[3]-origBox[1],
    #        left, top, right, bottom, right-left, bottom-top)

    alphaNew = Image.new('L', smotion.size, 0)
    alphaNew.paste(255, box)
    dstImage = Image.merge('LA', (smotion, alphaNew))

    dstImage.info['tracking'] = box

    # TEMP TEMP TEMP - dump box dims (pre and post fix)
    if self._debugMode:
      if self._imgCounter == 0:
        mode = 'w'
        self._logBox = open("LOGBOX.txt", mode)
      print >>self._logBox, "%d    %d %d %d %d    %d %d %d %d" % \
                      (self._imgCounter,
                       origBox[0], origBox[1], origBox[2], origBox[3],
                       box[0], box[1], box[2], box[3])

    self._imgCounter += 1

    return dstImage

Example 19

Project: twistranet Source File: bootstrap.py
def bootstrap():
    """
    Load initial data if it's not present.
    This method is SAFE, ie. it won't destroy any existing data, only add missing stuff`.
    
    This should be called every time twistranet is started!
    """
    try:
        # Let's log in.
        __account__ = SystemAccount.objects.__booster__.get()
    except SystemAccount.DoesNotExist:
        log.info("No SystemAccount available. That means this instance has never been bootstraped, so let's do it now.")
        raise RuntimeError("Please sync your databases with 'manage.py syncdb' before bootstraping.")
    except:
        # Default fixture probably not installed yet. Don't do anything yet.
        log.info("DatabaseError while bootstraping. Your tables are probably not created yet.")
        traceback.print_exc()
        return

    # Now create the bootstrap / default / help fixture objects.
    # Import your fixture there, if you don't do so they may not be importable.
    from twistranet.fixtures.bootstrap import FIXTURES as BOOTSTRAP_FIXTURES
    from twistranet.fixtures.help_en import FIXTURES as HELP_EN_FIXTURES
    # XXX TODO: Make a fixture registry? Or fix fixture import someway?
    try:
        from twistrans.fixtures.help_fr import FIXTURES as HELP_FR_FIXTURES
        from twistrans.fixtures.bootstrap_fr import FIXTURES as BOOTSTRAP_FR_FIXTURES
    except ImportError:
        HELP_FR_FIXTURES = []
        BOOTSTRAP_FR_FIXTURES = []
        log.info("twistrans not installed, translations are not installed.")
    
    # Load fixtures
    for obj in BOOTSTRAP_FIXTURES:          obj.apply()

    # Special treatment for bootstrap: Set the GlobalCommunity owner = AdminCommunity
    glob = GlobalCommunity.objects.get()
    admin_cty = AdminCommunity.objects.get()
    glob.owner = admin_cty
    glob.publisher = glob
    glob.save()
    admin_cty.publisher = glob
    admin_cty.save()

    # Create default resources by associating them to the SystemAccount and publishing them on GlobalCommunity.
    default_resources_dir = os.path.abspath(
        os.path.join(
            os.path.split(twistranet.__file__)[0],
            'fixtures',
            'resources',
        )
    )
    log.debug("Default res. dir: %s" % default_resources_dir)
    for root, dirs, files in os.walk(default_resources_dir):
        for fname in files:
            slug = os.path.splitext(os.path.split(fname)[1])[0]
            objects = Resource.objects.filter(slug = slug)
            if objects:
                if len(objects) > 1:
                    raise IntegrityError("More than one resource with '%s' slug" % slug)
                r = objects[0]
            else:
                r = Resource()
    
            # Copy file to its actual location with the storage API
            source_fn = os.path.join(root, fname)
            r.publisher = glob
            r.resource_file = File(open(source_fn, "rb"), fname)
            r.slug = slugify(slug)
            r.save()
        break   # XXX We don't handle subdirs yet.
    
    # Set SystemAccount picture (which is a way to check if things are working properly).
    __account__.picture = Resource.objects.get(slug = "default_tn_picture")
    __account__.save()

    # Install HELP fixture.
    for obj in HELP_EN_FIXTURES:            obj.apply()
        
    # Have we got an admin account? If not, we generate one now.
    django_admins = UserAccount.objects.filter(user__is_superuser = True)
    admin_password = None
    if not django_admins.exists():
        admin_password = ""
        for i in range(6):
            admin_password = "%s%s" % (admin_password, random.choice(string.lowercase + string.digits))
        admin = User.objects.create(
            username = settings.TWISTRANET_DEFAULT_ADMIN_USERNAME,
            first_name = settings.TWISTRANET_DEFAULT_ADMIN_FIRSTNAME,
            last_name = settings.TWISTRANET_DEFAULT_ADMIN_LASTNAME,
            email = settings.TWISTRANET_ADMIN_EMAIL,
            is_superuser = True,
        )
        admin.set_password(admin_password)
        admin.save()
        
    # Sample data only imported if asked to in settings.py
    if settings.TWISTRANET_IMPORT_SAMPLE_DATA:
        from twistranet.fixtures import sample
        sample.create_users()
        for obj in sample.get_fixtures():
            obj.apply()
        
        # Add relations bwn sample users
        # A <=> admin
        # B  => admin
        # A = UserAccount.objects.get(slug = "a")
        # B = UserAccount.objects.get(slug = "b")
        # admin = UserAccount.objects.get(slug = "admin")
        # A.follow(admin)
        # admin.follow(A)
        # B.follow(admin)
        
    # Import COGIP sample (if requested)
    if settings.TWISTRANET_IMPORT_COGIP:
        fixtures_module = "twistranet.project_templates.cogip.fixtures"
        fixtures_module = import_module(fixtures_module)
        # We disable email sending
        backup_EMAIL_BACKEND = settings.EMAIL_BACKEND
        settings.EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'
        fixtures_module.load()
        settings.EMAIL_BACKEND = backup_EMAIL_BACKEND

    # Repair permissions
    repair()
    
    # Display admin password
    if admin_password is not None:
        print "\n\n" \
            "  You can now run your server with 'manage.py runserver'.\n" \
            "  Your initial administrator login/password are '%s / %s'" % (settings.TWISTRANET_DEFAULT_ADMIN_USERNAME, admin_password)

Example 20

Project: OpenCobolIDE Source File: compilers.py
Function: check_compiler
    @classmethod
    @memoized
    def check_compiler(cls, compiler):
        def get_output_path(input_path, possible_extensions):
            dirname, filename = os.path.split(input_path)
            basename = os.path.splitext(filename)[0]
            for ext in possible_extensions:
                candidate = os.path.join(dirname, basename + ext)
                if os.path.exists(candidate):
                    return candidate
            return 'none'

        def rm_dest(dest):
            if os.path.exists(dest):
                try:
                    os.remove(dest)
                except OSError:
                    # log something
                    _logger().exception('failed to remove check compiler destination')
                    return False
            return True

        from open_cobol_ide.view.dialogs.preferences import DEFAULT_TEMPLATE
        working_dir = tempfile.gettempdir()
        cbl_path = os.path.join(working_dir, 'test.cbl')
        try:
            with open(cbl_path, 'w') as f:
                f.write(DEFAULT_TEMPLATE)
        except OSError as e:
            return 'Failed to create %s, error=%r' % (cbl_path, e), -1
        dest = os.path.join(tempfile.gettempdir(),
                            'test' + ('.exe' if system.windows else ''))

        _logger().debug('removing executable test if still exists...')
        if not rm_dest(dest):
            return 'Failed to remove %r before checking if compilation of executable works.\n' \
                'Please remove this file before attempting a new compilation check!' % dest, -1
        _logger().debug('check compiler')
        success1 = False
        status, output1 = run_command(compiler, ['-v', '-x', cbl_path], working_dir=working_dir)
        dest = get_output_path(cbl_path, possible_extensions=['.exe', '.bat', ''])
        if dest:
            if os.path.exists(dest):
                success1 = True
                if status != 0:
                    _logger().warn('test executable compilation returned a non-zero return code')
                # detect default executable extension
                GnuCobolCompiler.extensions[0] = os.path.splitext(dest)[1]
                try:
                    os.remove(dest)
                except OSError:
                    _logger().exception('failed to remove destination file: %r' % dest)

        dest = os.path.join(tempfile.gettempdir(),
                            'test' + ('.dll' if system.windows else '.so' if system.linux else '.dylib'))
        _logger().debug('removing executable test if still exists...')
        if not rm_dest(dest):
            return 'Failed to remove %r before checking if compilation of module works.\n' \
                'Please remove this file before attempting a new compilation check!' % dest, -1
        success2 = False
        status, output2 = run_command(compiler, ['-v', cbl_path], working_dir=working_dir)
        dest = get_output_path(cbl_path, possible_extensions=['.so', '.dll', '.dylib'])
        if dest:
            if os.path.exists(dest):
                success2 = True
                if status != 0:
                    _logger().warn('test executable compilation returned a non-zero return code')
                # detect default executable extension
                GnuCobolCompiler.extensions[1] = os.path.splitext(dest)[1]
                try:
                    os.remove(dest)
                except OSError:
                    _logger().exception('failed to remove destination file: %r' % dest)

        _logger().info('GnuCOBOL compiler check: %s (%d/%d)',
                       'success' if success1 and success2 else 'fail',
                       success1, success2)
        if success1:
            _logger().info('Executable extension: %s' % GnuCobolCompiler.extensions[0])

        if success2:
            _logger().info('Module extension: %s' % GnuCobolCompiler.extensions[1])

        try:
            os.remove(cbl_path)
        except OSError:
            _logger().exception('failed to remove test file: %r' % dest)

        if not success1 or not success2:
            status = -1

        return '%s\n%s' % (output1, output2), status

Example 21

Project: odoo Source File: translate.py
def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, module_name=None, context=None):
    """Populates the ir_translation table."""
    if verbose:
        _logger.info('loading translation file for language %s', lang)
    if context is None:
        context = {}
    db_name = cr.dbname
    registry = openerp.registry(db_name)
    lang_obj = registry.get('res.lang')
    trans_obj = registry.get('ir.translation')
    iso_lang = misc.get_iso_codes(lang)
    try:
        ids = lang_obj.search(cr, SUPERUSER_ID, [('code','=', lang)])

        if not ids:
            # lets create the language with locale information
            lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name)

        # Parse also the POT: it will possibly provide additional targets.
        # (Because the POT comments are correct on Launchpad but not the
        # PO comments due to a Launchpad limitation. See LP bug 933496.)
        pot_reader = []

        # now, the serious things: we read the language file
        fileobj.seek(0)
        if fileformat == 'csv':
            reader = csv.reader(fileobj, quotechar='"', delimiter=',')
            # read the first line of the file (it contains columns titles)
            for row in reader:
                fields = row
                break
        elif fileformat == 'po':
            reader = TinyPoFile(fileobj)
            fields = ['type', 'name', 'res_id', 'src', 'value', 'comments']

            # Make a reader for the POT file and be somewhat defensive for the
            # stable branch.
            if fileobj.name.endswith('.po'):
                try:
                    # Normally the path looks like /path/to/xxx/i18n/lang.po
                    # and we try to find the corresponding
                    # /path/to/xxx/i18n/xxx.pot file.
                    # (Sometimes we have 'i18n_extra' instead of just 'i18n')
                    addons_module_i18n, _ = os.path.split(fileobj.name)
                    addons_module, i18n_dir = os.path.split(addons_module_i18n)
                    addons, module = os.path.split(addons_module)
                    pot_handle = misc.file_open(os.path.join(
                        addons, module, i18n_dir, module + '.pot'))
                    pot_reader = TinyPoFile(pot_handle)
                except:
                    pass

        else:
            _logger.error('Bad file format: %s', fileformat)
            raise Exception(_('Bad file format'))

        # Read the POT references, and keep them indexed by source string.
        class Target(object):
            def __init__(self):
                self.value = None
                self.targets = set()            # set of (type, name, res_id)
                self.comments = None

        pot_targets = defaultdict(Target)
        for type, name, res_id, src, _, comments in pot_reader:
            if type is not None:
                target = pot_targets[src]
                target.targets.add((type, name, res_id))
                target.comments = comments

        # read the rest of the file
        irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context)

        def process_row(row):
            """Process a single PO (or POT) entry."""
            # dictionary which holds values for this line of the csv file
            # {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
            #  'src': ..., 'value': ..., 'module':...}
            dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
                                 'comments', 'imd_model', 'imd_name', 'module'))
            dic['lang'] = lang
            dic.update(zip(fields, row))

            # discard the target from the POT targets.
            src = dic['src']
            if src in pot_targets:
                target = pot_targets[src]
                target.value = dic['value']
                target.targets.discard((dic['type'], dic['name'], dic['res_id']))

            # This would skip terms that fail to specify a res_id
            res_id = dic['res_id']
            if not res_id:
                return

            if isinstance(res_id, (int, long)) or \
                    (isinstance(res_id, basestring) and res_id.isdigit()):
                dic['res_id'] = int(res_id)
                dic['module'] = module_name
            else:
                # res_id is an xml id
                dic['res_id'] = None
                dic['imd_model'] = dic['name'].split(',')[0]
                if '.' in res_id:
                    dic['module'], dic['imd_name'] = res_id.split('.', 1)
                else:
                    dic['module'], dic['imd_name'] = False, res_id

            irt_cursor.push(dic)

        # First process the entries from the PO file (doing so also fills/removes
        # the entries from the POT file).
        for row in reader:
            process_row(row)

        # Then process the entries implied by the POT file (which is more
        # correct w.r.t. the targets) if some of them remain.
        pot_rows = []
        for src, target in pot_targets.iteritems():
            if target.value:
                for type, name, res_id in target.targets:
                    pot_rows.append((type, name, res_id, src, target.value, target.comments))
        pot_targets.clear()
        for row in pot_rows:
            process_row(row)

        irt_cursor.finish()
        trans_obj.clear_caches()
        if verbose:
            _logger.info("translation file loaded succesfully")

    except IOError:
        filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
        _logger.exception("couldn't read translation file %s", filename)

Example 22

Project: gimp-plugins Source File: saver.py
def save_both(img, drawable, filename, copyname, width, height):
    '''Save the image, and also save the copy if appropriate,
       doing any duplicating or scaling that might be necessary.
       Returns None on success, else a nonempty string error message.
    '''
    msg = "Saving " + filename

    if copyname:
        if width and height:
            msg += " and a scaled copy, " + copyname
        else:
            msg += " and a copy, " + copyname
    print msg
    # Don't use gimp_message -- it can pop up a dialog.
    # pdb.gimp_message_set_handler(MESSAGE_BOX)
    # pdb.gimp_message(msg)
    # Alternately, could use gimp_progress, though that's not ideal.
    # If you use gimp_progress, don't forget to call pdb.gimp_progress_end()
    #pdb.gimp_progress_set_text(msg)

    # Is there any point to pushing and popping the context?
    #gimp.context_pop()

    # Set up the parasite with information about the copied image,
    # so it will be saved with the main XCF image.
    if copyname:
        if (width and height):
            percent = width * 100.0 / img.width
            # XXX note that this does not guard against changed aspect ratios.
            parastring = '%s\n%d\n%d\n%d' % (copyname, percent, width, height)
        else:
            parastring = '%s\n100.0\nNone\nNone' % (copyname)

        print "Saving parasite", parastring
        para = img.attach_new_parasite('export-copy', 1, parastring)
        # The second argument is is_persistent

        # Also make sure that copyname is a full path (if filename is).
        copypath = os.path.split(copyname)
        if not copypath[0] or len(copypath) == 1:
            filepath = os.path.split(filename)
            if len(filepath) > 1 and filepath[0]:
                print "Turning", copyname, "into a full path:",
                copyname = os.path.join(filepath[0], copyname)
                print copyname

    # Now the parasite is safely attached, and we can save.
    # Alas, we can't attach the JPEG settings parasite until after
    # we've saved the copy, so that won't get saved with the main image
    # though it will be attached to the image and remembered in
    # this session, and the next time we save it should be remembered.

    def is_xcf(thefilename):
        base, ext = os.path.splitext(thefilename)
        ext = ext.lower()
        if ext == '.gz' or ext == '.bz2':
            base, ext = os.path.splitext(base)
            ext = ext.lower()
        return (ext == '.xcf')

    # First, save the original image.
    if is_xcf(filename) or len(img.layers) < 2:
        pdb.gimp_file_save(img, drawable, filename, filename)
    else:
        # It's not XCF and it has multiple layers.
        # We need to make a new image and flatten it.
        copyimg = pdb.gimp_image_duplicate(img)
        # Don't actually flatten since that will prevent saving transparent png.
        #copyimg.flatten()
        pdb.gimp_image_merge_visible_layers(copyimg, CLIP_TO_IMAGE)
        pdb.gimp_file_save(copyimg, copyimg.active_layer, filename, filename)
        gimp.delete(copyimg)

    # We've done the important part, so mark the image clean.
    img.filename = filename
    pdb.gimp_image_clean_all(img)

    # If we don't have to save a copy, return.
    if not copyname:
        return None

    # We'll need a new image if the copy is non-xcf and we have more
    # than one layer, or if we're scaling.
    if (width and height):
        # We're scaling!
        copyimg = pdb.gimp_image_duplicate(img)
        copyimg.scale(width, height)
        print "Scaling to", width, 'x', height
        print "Set parastring to", parastring, "(end of parastring)"

    elif len(img.layers) > 1 and not is_xcf(copyname):
        # We're not scaling, but we still need to flatten.
        copyimg = pdb.gimp_image_duplicate(img)
        # copyimg.flatten()
        pdb.gimp_image_merge_visible_layers(copyimg, CLIP_TO_IMAGE)
        print "Flattening but not scaling"
    else:
        copyimg = img
        print "Not scaling or flattening"

    # gimp-file-save insists on being passed a valid layer,
    # even if saving to a multilayer format such as XCF. Go figure.

    # I don't get it. What's the rule for whether gimp_file_save
    # will prompt for jpg parameters? Saving back to an existing
    # file doesn't seem to help.
    # run_mode=RUN_WITH_LAST_VALS is supposed to prevent prompting,
    # but actually seems to do nothing. Copying the -save-options
    # parasites is more effective.
    try:
        pdb.gimp_file_save(copyimg, copyimg.active_layer, copyname, copyname,
                           run_mode=RUN_WITH_LAST_VALS)
    except RuntimeError, e:
        gimp.delete(copyimg)
        print "Runtime error -- didn't save"
        return "Runtime error -- didn't save"

    # Find any image type settings parasites (e.g. jpeg settings)
    # that got set during save, so we'll be able to use them
    # next time.
    def copy_settings_parasites(fromimg, toimg):
        for pname in fromimg.parasite_list():
            if pname[-9:] == '-settings' or pname[-13:] == '-save-options':
                para = fromimg.parasite_find(pname)
                if para:
                    toimg.attach_new_parasite(pname, para.flags, para.data)

    # Copy any settings parasites we may have saved from previous runs:
    copy_settings_parasites(copyimg, img)

    gimp.delete(copyimg)
    #gimp.Display(copyimg)
    return None

Example 23

Project: deepcca Source File: dcca.py
def test_dcca_old(learning_rate=0.01, L1_reg=0.0001, L2_reg=0.0001, n_epochs=1000,
             dataset='mnist.pkl.gz', batch_size=20, n_hidden=500):
    """
    Demonstrate stochastic gradient descent optimization for a multilayer
    perceptron

    This is demonstrated on MNIST.

    :type learning_rate: float
    :param learning_rate: learning rate used (factor for the stochastic
    gradient

    :type L1_reg: float
    :param L1_reg: L1-norm's weight when added to the cost (see
    regularization)

    :type L2_reg: float
    :param L2_reg: L2-norm's weight when added to the cost (see
    regularization)

    :type n_epochs: int
    :param n_epochs: maximal number of epochs to run the optimizer

    :type dataset: string
    :param dataset: the path of the MNIST dataset file from
                 http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz


   """
    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print '... building the model'

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    y = T.matrix('y')  # the labels are presented as 1D vector of
                        # [int] labels

    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    if 0:
        net1 = MLP(
            rng=rng,
            input=x,
            n_in=28 * 28,
            n_hidden=300,
            n_out=50
        )
        net2 = MLP(
            rng=rng,
            input=y,
            n_in=10,
            n_hidden=20,
            n_out=5
        )
    
    net = DCCA(
        rng=rng,
        x1=x,
        x2=y,
        n_in1=28 * 28,
        n_hidden1=300,
        n_out1=50,
        n_in2=10,
        n_hidden2=20,
        n_out2=5
    )
  

    # start-snippet-4
    # the cost we minimize during training is the negative log likelihood of
    # the model plus the regularization terms (L1 and L2); cost is expressed
    # here symbolically
    cost1 = (
        net.correlation(y)
        + L1_reg * net.L11
        + L2_reg * net.L2_sqr1
    )
    cost2 = (
        net.correlation(y)
        + L1_reg * net.L12
        + L2_reg * net.L2_sqr2
    )
    # end-snippet-4

    # compiling a Theano function that computes the mistakes that are made
    # by the model on a minibatch
    """
    test_model = theano.function(
        inputs=[index],
        outputs=net1.errors(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: valid_set_x[index * batch_size:(index + 1) * batch_size],
            y: valid_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )
    """

    # start-snippet-5
    # compute the gradient of cost with respect to theta (sotred in params)
    # the resulting gradients will be stored in a list gparams
    gparams1 = [T.grad(cost1, param) for param in net.params1]
    gparams2 = [T.grad(cost2, param) for param in net.params2]

    # specify how to update the parameters of the model as a list of
    # (variable, update expression) pairs

    # given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of
    # same length, zip generates a list C of same size, where each element
    # is a pair formed from the two lists :
    #    C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)]
    updates1 = [
        (param, param - learning_rate * gparam)
        for param, gparam in zip(net.params1, gparams1)
    ]
    updates2 = [
        (param, param - learning_rate * gparam)
        for param, gparam in zip(net.params2, gparams2)
    ]

    # compiling a Theano function `train_model` that returns the cost, but
    # in the same time updates the parameter of the model based on the rules
    # defined in `updates`
    train_model1 = theano.function(
        inputs=[index],
        outputs=cost1,
        updates=updates1,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size],
            y: train_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )
    train_model2 = theano.function(
        inputs=[index],
        outputs=cost2,
        updates=updates2,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size],
            y: train_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )
    # end-snippet-5

    ###############
    # TRAIN MODEL #
    ###############
    print '... training'

    # early-stopping parameters
    patience = 10000  # look as this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is
                           # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatche before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = time.clock()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_model(minibatch_index)
            # iteration number
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                # compute zero-one loss on validation set
                validation_losses = [validate_model(i) for i
                                     in xrange(n_valid_batches)]
                this_validation_loss = numpy.mean(validation_losses)

                print(
                    'epoch %i, minibatch %i/%i, validation error %f %%' %
                    (
                        epoch,
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.
                    )
                )

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:
                    #improve patience if loss improvement is good enough
                    if (
                        this_validation_loss < best_validation_loss *
                        improvement_threshold
                    ):
                        patience = max(patience, iter * patience_increase)

                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = [test_model(i) for i
                                   in xrange(n_test_batches)]
                    test_score = numpy.mean(test_losses)

                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            if patience <= iter:
                done_looping = True
                break

    end_time = time.clock()
    print(('Optimization complete. Best validation score of %f %% '
           'obtained at iteration %i, with test performance %f %%') %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print >> sys.stderr, ('The code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))


    ''' Loads the dataset

    :type dataset: string
    :param dataset: the path to the dataset (here MNIST)
    '''

    #############
    # LOAD DATA #
    #############

    # Download the MNIST dataset if it is not present
    data_dir, data_file = os.path.split(dataset)
    if data_dir == "" and not os.path.isfile(dataset):
        # Check if dataset is in the data directory.
        new_path = os.path.join(
            os.path.split(__file__)[0],
            "..",
            "data",
            dataset
        )
        if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz':
            dataset = new_path

    if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz':
        import urllib
        origin = (
            'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz'
        )
        print 'Downloading data from %s' % origin
        urllib.urlretrieve(origin, dataset)

    print '... loading data'

    # Load the dataset
    f = gzip.open(dataset, 'rb')
    train_set, valid_set, test_set = cPickle.load(f)
    f.close()
    #train_set, valid_set, test_set format: tuple(input, target)
    #input is an numpy.ndarray of 2 dimensions (a matrix)
    #witch row's correspond to an example. target is a
    #numpy.ndarray of 1 dimensions (vector)) that have the same length as
    #the number of rows in the input. It should give the target
    #target to the example with the same index in the input.

    def shared_dataset(data_xy, borrow=True):
        """ Function that loads the dataset into shared variables

        The reason we store our dataset in shared variables is to allow
        Theano to copy it into the GPU memory (when code is run on GPU).
        Since copying data into the GPU is slow, copying a minibatch everytime
        is needed (the default behaviour if the data is not in a shared
        variable) would lead to a large decrease in performance.
        """
        #import copy
        data_x, data_y = data_xy
        #daya_y = copy.deepcopy(data_x)
        data_y_new = numpy.zeros((data_y.shape[0], data_y.max()+1))
        for i in range(data_y.shape[0]):
            data_y_new[i, data_y[i]] = 1
        data_y = data_y_new
        shared_x = theano.shared(numpy.asarray(data_x,
                                               dtype=theano.config.floatX),
                                 borrow=borrow)
        shared_y = theano.shared(numpy.asarray(data_y,
                                               dtype=theano.config.floatX),
                                 borrow=borrow)
        # When storing data on the GPU it has to be stored as floats
        # therefore we will store the labels as ``floatX`` as well
        # (``shared_y`` does exactly that). But during our computations
        # we need them as ints (we use labels as index, and if they are
        # floats it doesn't make sense) therefore instead of returning
        # ``shared_y`` we will have to cast it to int. This little hack
        # lets ous get around this issue
        return shared_x, shared_y

    test_set_x, test_set_y = shared_dataset(test_set)
    valid_set_x, valid_set_y = shared_dataset(valid_set)
    train_set_x, train_set_y = shared_dataset(train_set)

    rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),
            (test_set_x, test_set_y)]
    return rval

Example 24

Project: WikiDAT Source File: etl.py
    def run(self):
        """
        Execute workflow to import revision history data from dump files

        The data loading workflow is composed of a number of processor
        elements, which can be:

            - Producer (P): raw input data --> input element queue
            - ConsumerProducer (CP): input element queue --> insert db queue
            - Consumer (C): insert db queue --> database (MySQL/MariaDB)

        In this case, the logical combination is usually N:N:1 (P, CP, C)
        """
        start = time.time()
        print(self.name, "Starting PageRevisionETL workflow at %s" % (
                         time.strftime("%Y-%m-%d %H:%M:%S %Z",
                                       time.localtime())))

        db_ns = MySQLDB(host='localhost', port=3306, user=self.db_user,
                        passwd=self.db_passw, db=self.db_name)
        db_ns.connect()

        db_pages = MySQLDB(host='localhost', port=3306,
                           user=self.db_user, passwd=self.db_passw,
                           db=self.db_name)
        db_pages.connect()

        db_revs = MySQLDB(host='localhost', port=3306, user=self.db_user,
                          passwd=self.db_passw, db=self.db_name)
        db_revs.connect()

        # DATA EXTRACTION
        # Use consistent naming for all child processes
        xml_reader_name = '-'.join([self.name, 'xml_reader'])
        page_proc_name = '-'.join([self.name, 'process_page'])
        rev_proc_name = '-'.join([self.name, 'process_revision'])
        page_insert_name = '-'.join([self.name, 'insert_page'])
        rev_insert_name = '-'.join([self.name, 'insert_revision'])

        for path in iter(self.paths_queue.get, 'STOP'):
            # Start subprocess to extract elements from revision dump file
            dump_file = DumpFile(path)
            xml_reader = Producer(name=xml_reader_name,
                                  target=process_xml,
                                  kwargs=dict(
                                      dump_file=dump_file),
                                  consumers=self.page_fan + self.rev_fan,
                                  push_pages_port=self.base_port,
                                  push_revs_port=self.base_port+1,
                                  control_port=self.control_port)
            xml_reader.start()
            print(xml_reader_name, "started")
            print(self.name, "Extracting data from XML revision history file:")
            print(path)

            # List to keep tracking of page and revision workers
            workers = []
            db_workers_revs = []
            # Create and start page processes
            for worker in range(self.page_fan):
                page_worker_name = '-'.join([page_proc_name, str(worker)])
                process_page = Processor(name=page_worker_name,
                                         target=pages_to_file,
                                         producers=1, consumers=1,
                                         pull_port=self.base_port,
                                         push_port=self.base_port+2,
                                         control_port=self.control_port)
                process_page.start()
                workers.append(process_page)
                print(page_worker_name, "started")

            # Create and start revision processes
            for worker in range(self.rev_fan):
                rev_worker_name = '-'.join([rev_proc_name, str(worker)])

                db_wrev = MySQLDB(host='localhost', port=3306,
                                  user=self.db_user,
                                  passwd=self.db_passw, db=self.db_name)
                db_wrev.connect()

                process_revision = Processor(name=rev_worker_name,
                                             target=revs_to_file,
                                             kwargs=dict(
                                                 lang=self.lang),
                                             producers=1, consumers=1,
                                             pull_port=self.base_port+1,
                                             push_port=self.base_port+3,
                                             control_port=self.control_port)
                process_revision.start()
                workers.append(process_revision)
                db_workers_revs.append(db_wrev)
                print(rev_worker_name, "started")

            # Create directory for logging files if it does not exist
            log_dir = os.path.join(os.path.split(path)[0], 'logs')
            tmp_dir = os.path.join(os.getcwd(), os.path.split(path)[0], 'tmp')
            file_name = os.path.split(path)[1]

            if not os.path.exists(log_dir):
                os.makedirs(log_dir)
            if not os.path.exists(tmp_dir):
                os.makedirs(tmp_dir)
            log_file = os.path.join(log_dir, file_name + '.log')

            page_insert_db = Consumer(name=page_insert_name,
                                      target=pages_file_to_db,
                                      kwargs=dict(con=db_pages,
                                                  log_file=log_file,
                                                  tmp_dir=tmp_dir,
                                                  file_rows=self.page_cache_size,
                                                  etl_prefix=self.name),
                                      producers=self.page_fan,
                                      pull_port=self.base_port+2)

            rev_insert_db = Consumer(name=rev_insert_name,
                                     target=revs_file_to_db,
                                     kwargs=dict(con=db_revs,
                                                 log_file=log_file,
                                                 tmp_dir=tmp_dir,
                                                 file_rows=self.rev_cache_size,
                                                 etl_prefix=self.name),
                                     producers=self.rev_fan,
                                     pull_port=self.base_port+3)

            page_insert_db.start()
            print(page_insert_name, "started")
            rev_insert_db.start()
            print(rev_insert_name, "started")

            print(self.name, "Waiting for all processes to finish...")
            print()
            xml_reader.join()
            for w in workers:
                w.join()
            page_insert_db.join()
            rev_insert_db.join()

            # Mark this path as done
            self.paths_queue.task_done()

        # Mark STOP message as processed and finish
        self.paths_queue.task_done()

        end = time.time()
        print(self.name, ": All tasks done in %.4f sec." % ((end-start)/1.))
        print()
        db_ns.close()
        db_pages.close()
        db_revs.close()
        for dbcon in db_workers_revs:
            dbcon.close()

Example 25

Project: deepcca Source File: SdA_mapping.py
def test_SdA_regress(finetune_lr=0.05, pretraining_epochs=10,
             pretrain_lr=0.1, training_epochs=10000,
             dataset='mnist.pkl.gz', batch_size=20):
    datasets = load_data_half(dataset)

    train_set_x, train_set_y = datasets[0]##
    valid_set_x, valid_set_y = datasets[1]##
    test_set_x, test_set_y = datasets[2]##
    train_set_x=train_set_x.eval()
    train_set_y=train_set_y.eval()
    import theano
    train_set_x_lab=train_set_x[:,:]
    train_set_x_unlab=train_set_x[:,:]
    train_set_y_lab=train_set_y[:,:]
    train_set_y_unlab=train_set_y[:,:]
    train_set_x_lab=theano.shared(numpy.asarray(train_set_x_lab,
                                                dtype=theano.config.floatX),
                                  borrow=True)
    train_set_y_lab=theano.shared(numpy.asarray(train_set_y_lab,
                                                dtype=theano.config.floatX),
                                  borrow=True)
    train_set_x_unlab=theano.shared(numpy.asarray(train_set_x_unlab,
                                                  dtype=theano.config.floatX),
                                    borrow=True)
    train_set_y_unlab=theano.shared(numpy.asarray(train_set_y_unlab,
                                                  dtype=theano.config.floatX),
                                    borrow=True)

    # compute number of minibatches for training, validation and testing
    n_train_batches_l = train_set_y_lab.eval().shape[0]
    n_train_batches_l /= batch_size
    n_train_batches_u = train_set_y_unlab.eval().shape[0]
    n_train_batches_u /= batch_size
    # compute number of minibatches for training, validation and testing
    #n_train_batches = train_set_x.get_value(borrow=True).shape[0]
    #n_train_batches /= batch_size

    # numpy random generator
    # start-snippet-3
    numpy_rng = numpy.random.RandomState(89677)
    print '... building the model'
    # construct the stacked denoising autoencoder class
    #from SdA_orig import SdA as SdA_old
    hidden_layer_size = 100
    SdA_inp = SdA(numpy_rng,
                  n_ins=392,
                  hidden_layers_sizes=[hidden_layer_size]
    )
    SdA_out = SdA(numpy_rng,
                  n_ins=392,
                  hidden_layers_sizes=[hidden_layer_size]
    )
        
    # PRETRAINING THE MODEL #
    if 0 : # pretrain inp ae
        print '... getting the pretraining functions for INPUT AE'
        pretraining_fns = SdA_inp.pretraining_functions(train_set_x=train_set_x_unlab,
                                                    batch_size=batch_size)
    
        print '... pre-training the model'
        start_time = time.clock()
        ## Pre-train layer-wise
        corruption_levels = [.1, .2, .3]
        for i in xrange(SdA_inp.n_layers):
            # go through pretraining epochs
            for epoch in xrange(pretraining_epochs):
                # go through the training set
                c = []
                for batch_index in xrange(n_train_batches_u):
                    c.append(pretraining_fns[i](index=batch_index,
                             corruption=corruption_levels[i],
                             lr=pretrain_lr))
                print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
                print numpy.mean(c)
    
        end_time = time.clock()
    
        print >> sys.stderr, ('The pretraining code for file ' +
                              os.path.split(__file__)[1] +
                              ' ran for %.2fm' % ((end_time - start_time) / 60.))
    if 0 : # pretrain out ae
        print '... getting the pretraining functions for OUTPUT AE'
        pretraining_fns = SdA_out.pretraining_functions(train_set_x=train_set_y_unlab,
                                                    batch_size=batch_size)
    
        print '... pre-training the model'
        start_time = time.clock()
        ## Pre-train layer-wise
        corruption_levels = [.5, .2, .3]
        for i in xrange(SdA_out.n_layers):
            # go through pretraining epochs
            for epoch in xrange(pretraining_epochs):
                # go through the training set
                c = []
                for batch_index in xrange(n_train_batches_u):
                    c.append(pretraining_fns[i](index=batch_index,
                             corruption=corruption_levels[i],
                             lr=pretrain_lr))
                print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
                print numpy.mean(c)
    
        end_time = time.clock()
    
        print >> sys.stderr, ('The pretraining code for file ' +
                              os.path.split(__file__)[1] +
                              ' ran for %.2fm' % ((end_time - start_time) / 60.))
    
        
    if 0: # save aes
        f=open('aes_shallow_sig_nobias.pkl', 'w+')
        import pickle
        pickle.dump(SdA_inp, f)
        pickle.dump(SdA_out, f)
        f.flush()
        f.close() 
    if 0: # load aes
        f=open('aes_shallow_sig_nobias.pkl', 'r')
        import pickle
        SdA_inp=pickle.load(f)
        SdA_out=pickle.load(f)
        f.close()    
   
    if 1: # cca
        from dcca_numpy import netCCA_nobias, netCCA, dCCA
        from mlp_numpy import expit, logistic_prime, linear, linear_prime, relu, relu_prime, tanh, tanh_prime
        train_y1 = train_set_x_lab.eval()
        train_y2 = train_set_y_lab.eval()
        test_y1 = test_set_x.eval()
        test_y2 = test_set_y.eval()

        ##param1=((train_y1.shape[1],0,0),(2038, relu, relu_prime),(50, relu, relu_prime))
        ##param2=((train_y2.shape[1],0,0),(1608, relu, relu_prime),(50, relu, relu_prime))
        param1=((train_y1.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
        param2=((train_y2.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
        W1s = []
        b1s = []
        for i in range(len(SdA_inp.dA_layers)):
            W1s.append( SdA_inp.dA_layers[i].W.T.eval() )
            ##b1s.append( SdA_inp.dA_layers[i].b.eval() )
            ##b1s[-1] = b1s[-1].reshape((b1s[-1].shape[0], 1))
        W2s = []
        b2s = []
        for i in range(len(SdA_out.dA_layers)):
            W2s.append( SdA_out.dA_layers[i].W.T.eval() )
            ##b2s.append( SdA_out.dA_layers[i].b.eval() )
            ##b2s[-1] = b2s[-1].reshape((b2s[-1].shape[0], 1))

        numpy.random.seed(0)
        N1=netCCA_nobias(train_y1,param1, W1s)
        N2=netCCA_nobias(train_y2,param2, W2s)
        N = dCCA(train_y1, train_y2, N1, N2)
        N1.reconstruct(test_set_x.eval()[0,:])
        cnt = 0
        from dcca_numpy import cca_cost, cca, order_cost, cor_cost
        while True:
            X=N1.predict(test_set_x.eval())
            Y=N2.predict(test_set_y.eval())
            _H1 = numpy.dot(X, N.A1)
            _H2 = numpy.dot(Y, N.A2)
            print 'cuem', cnt, cor_cost(_H1, _H2)
            X1_rec = numpy.tanh(X.dot(N1.weights[0]))
            X2_rec = numpy.tanh(Y.dot(N2.weights[0]))
            param=((hidden_layer_size,0,0),(hidden_layer_size, relu, relu_prime))
            from mlp_numpy import NeuralNetwork as NN

            lr=NN(X,Y,param)
            lr.train(X[:,:],Y[:,:],10, 0.005)
            Yh=lr.predict(X[:,:])
            X2_reg = N2.fs[-1](numpy.dot(Yh,N2.weights[0]))

            #X2_reg = N2.fs[-1](numpy.dot(_H1.dot(numpy.linalg.inv(N.A1)),N2.weights[0]))

            print '****', 'mse1:', numpy.mean((X1_rec-test_set_x.eval())**2.0)
            print '****', 'mse2:', numpy.mean((X2_rec-test_set_y.eval())**2.0)
            print '****', 'mse_map:', numpy.mean((X2_reg-test_set_y.eval())**2.0)

            if cnt % 2:
                N.train(5, True, 10000.0)
            else:
                N.train(5, False, 10000.0)

            cnt += 1
            f=open('netcca.pkl', 'w+')
            import pickle
            pickle.dump(N, f)
            pickle.dump(N, f)
            f.flush()
            f.close() 
            if cnt == 200:
                break
        for i in range(len(SdA_inp.dA_layers)):
            SdA_inp.dA_layers[i].W = theano.shared( N1.weights[i].T )
            SdA_inp.dA_layers[i].b = theano.shared( N1.biases[i][:,0] )
        
        for i in range(len(SdA_out.dA_layers)):
            SdA_out.dA_layers[i].W = theano.shared( N2.weights[i].T )
            SdA_out.dA_layers[i].b = theano.shared( N2.weights[i][:,0] )

        
    if 1 : # pretrain middle layer
        print '... pre-training MIDDLE layer'

        h1 = T.matrix('x')  # the data is presented as rasterized images
        h2 = T.matrix('y')  # the labels are presented as 1D vector of
        log_reg = HiddenLayer(numpy_rng, h1, hidden_layer_size, hidden_layer_size)

        if 1: # for middle layer
            learning_rate = 0.01
            fprop_inp = theano.function(
                [],
                SdA_inp.sigmoid_layers[-1].output,
                givens={
                    SdA_inp.sigmoid_layers[0].input: train_set_x_lab
                },
                name='fprop_inp'
            )
            fprop_out = theano.function(
                [],
                SdA_out.sigmoid_layers[-1].output,
                givens={
                    SdA_out.sigmoid_layers[0].input: train_set_y_lab
                },
                name='fprop_out'
            )
            #H11=fprop_inp() 
            #H21=fprop_out()
            ##H1=N1.predict(train_set_x.eval())
            ##H2=N2.predict(train_set_y.eval())
            H1=fprop_inp()
            H2=fprop_out()
            H1=theano.shared(H1)
            H2=theano.shared(H2)
            # compute the gradients with respect to the model parameters
            logreg_cost = log_reg.mse(h2)

            gparams = T.grad(logreg_cost, log_reg.params)
    
            # compute list of fine-tuning updates
            updates = [
                (param, param - gparam * learning_rate)
                for param, gparam in zip(log_reg.params, gparams)
            ]

            train_fn_middle = theano.function(
                inputs=[],
                outputs=logreg_cost,
                updates=updates,
                givens={
                    h1: H1,
                    h2: H2
                },
                name='train_middle'
            )
        epoch = 0
        while epoch < 10:
            print epoch, train_fn_middle()
            epoch += 1
            
    sda = SdA_regress(
        SdA_inp,
        SdA_out,
        log_reg,
        numpy_rng=numpy_rng,
        n_inp=28*28//2,
        hidden_layers_sizes_inp=[hidden_layer_size],
        hidden_layers_sizes_out=[hidden_layer_size],
        n_out=28*28//2
    )
    # end-snippet-3 start-snippet-4
    # end-snippet-4
    
    # FINETUNING THE MODEL #

    # get the training, validation and testing function for the model
    print '... getting the finetuning functions'
    train_fn, validate_model, test_model = sda.build_finetune_functions(
        datasets=datasets,
        batch_size=batch_size,
        learning_rate=finetune_lr
    )
    
        
    print '... finetunning the model'
    # early-stopping parameters
    patience = 10 * n_train_batches_l  # look as this many examples regardless
    patience_increase = 2.  # wait this much longer when a new best is
                            # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches_l, patience / 2)
                                  # go through this many
                                  # minibatche before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = time.clock()

    done_looping = False
    epoch = 0
    fprop = theano.function(
        [],
        sda.sigmoid_layers[-1].output,
        givens={
            sda.x: test_set_x
        },
        name='fprop'
    )
    while True:
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches_l):
            minibatch_avg_cost = train_fn(minibatch_index)
            iter = (epoch - 1) * n_train_batches_l + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                validation_losses = validate_model()
                this_validation_loss = numpy.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches_l,
                       this_validation_loss ))

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if (
                        this_validation_loss < best_validation_loss *
                        improvement_threshold
                    ):
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = test_model()
                    test_score = numpy.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches_l,
                           test_score ))

            if patience <= iter:
                done_looping = True
                #break
            if 0: # vis weights
                fprop = theano.function(
                    [],
                    sda.sigmoid_layers[-1].output,
                    givens={
                        sda.x: test_set_x
                    },
                    name='fprop'
                )
                yh=fprop()
                yh=yh
    end_time = time.clock()
    print(
        (
            'Optimization complete with best validation score of %f %%, '
            'on iteration %i, '
            'with test performance %f %%'
        )
        % (best_validation_loss , best_iter + 1, test_score)
    )
    print >> sys.stderr, ('The training code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))

Example 26

Project: openwrt-mt7620 Source File: tex.py
Function: tex_emitter_core
def tex_emitter_core(target, source, env, graphics_extensions):
    """An emitter for TeX and LaTeX sources.
    For LaTeX sources we try and find the common created files that
    are needed on subsequent runs of latex to finish tables of contents,
    bibliographies, indices, lists of figures, and hyperlink references.
    """
    basename = SCons.Util.splitext(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    targetdir = os.path.split(str(target[0]))[0]
    targetbase = os.path.join(targetdir, basefile)

    basedir = os.path.split(str(source[0]))[0]
    abspath = os.path.abspath(basedir)
    target[0].attributes.path = abspath

    #
    # file names we will make use of in searching the sources and log file
    #
    emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
    auxfilename = targetbase + '.aux'
    logfilename = targetbase + '.log'
    flsfilename = targetbase + '.fls'

    env.SideEffect(auxfilename,target[0])
    env.SideEffect(logfilename,target[0])
    env.SideEffect(flsfilename,target[0])
    if Verbose:
        print "side effect :",auxfilename,logfilename,flsfilename
    env.Clean(target[0],auxfilename)
    env.Clean(target[0],logfilename)
    env.Clean(target[0],flsfilename)

    content = source[0].get_text_contents()

    # These variables are no longer used.
    #idx_exists = os.path.isfile(targetbase + '.idx')
    #nlo_exists = os.path.isfile(targetbase + '.nlo')
    #glo_exists = os.path.isfile(targetbase + '.glo')
    #acr_exists = os.path.isfile(targetbase + '.acn')

    # set up list with the regular expressions
    # we use to find features used
    file_tests_search = [auxfile_re,
                         makeindex_re,
                         bibliography_re,
                         bibunit_re,
                         tableofcontents_re,
                         listoffigures_re,
                         listoftables_re,
                         hyperref_re,
                         makenomenclature_re,
                         makeglossary_re,
                         makeglossaries_re,
                         makeacronyms_re,
                         beamer_re ]
    # set up list with the file suffixes that need emitting
    # when a feature is found
    file_tests_suff = [['.aux','aux_file'],
                  ['.idx', '.ind', '.ilg','makeindex'],
                  ['.bbl', '.blg','bibliography'],
                  ['.bbl', '.blg','bibunit'],
                  ['.toc','contents'],
                  ['.lof','figures'],
                  ['.lot','tables'],
                  ['.out','hyperref'],
                  ['.nlo', '.nls', '.nlg','nomenclature'],
                  ['.glo', '.gls', '.glg','glossary'],
                  ['.glo', '.gls', '.glg','glossaries'],
                  ['.acn', '.acr', '.alg','acronyms'],
                  ['.nav', '.snm', '.out', '.toc','beamer'] ]
    # build the list of lists
    file_tests = []
    for i in range(len(file_tests_search)):
        file_tests.append( [None, file_tests_suff[i]] )

    # TO-DO: need to add a way for the user to extend this list for whatever
    # auxiliary files they create in other (or their own) packages

    # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
    savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
    paths = env['ENV']['TEXINPUTS']
    if SCons.Util.is_List(paths):
        pass
    else:
        # Split at os.pathsep to convert into absolute path
        paths = paths.split(os.pathsep)

    # now that we have the path list restore the env
    if savedpath is _null:
        try:
            del env['ENV']['TEXINPUTS']
        except KeyError:
            pass # was never set
    else:
        env['ENV']['TEXINPUTS'] = savedpath
    if Verbose:
        print "search path ",paths

    aux_files = []
    file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)

    for (theSearch,suffix_list) in file_tests:
        # add side effects if feature is present.If file is to be generated,add all side effects
        if (theSearch != None) or (not source[0].exists() ):
            file_list = [targetbase,]
            # for bibunit we need a list of files
            if suffix_list[-1] == 'bibunit':
                file_basename = os.path.join(targetdir, 'bu*.aux')
                file_list = glob.glob(file_basename)
                # remove the suffix '.aux'
                for i in range(len(file_list)):
                    file_list[i] = SCons.Util.splitext(file_list[i])[0]
            # now define the side effects
            for file_name in file_list:
                for suffix in suffix_list[:-1]:
                    env.SideEffect(file_name + suffix,target[0])
                    if Verbose:
                        print "side effect :",file_name + suffix
                    env.Clean(target[0],file_name + suffix)

    for aFile in aux_files:
        aFile_base = SCons.Util.splitext(aFile)[0]
        env.SideEffect(aFile_base + '.aux',target[0])
        if Verbose:
            print "side effect :",aFile_base + '.aux'
        env.Clean(target[0],aFile_base + '.aux')
    # read fls file to get all other files that latex creates and will read on the next pass
    # remove files from list that we explicitly dealt with above
    if os.path.isfile(flsfilename):
        content = open(flsfilename, "rb").read()
        out_files = openout_re.findall(content)
        myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
        for filename in out_files[:]:
            if filename in myfiles:
                out_files.remove(filename)
        env.SideEffect(out_files,target[0])
        if Verbose:
            print "side effect :",out_files
        env.Clean(target[0],out_files)

    return (target, source)

Example 27

Project: gprMax Source File: gprMax.py
def run_model(args, modelrun, numbermodelruns, inputfile, usernamespace):
    """Runs a model - processes the input file; builds the Yee cells; calculates update coefficients; runs main FDTD loop.

    Args:
        args (dict): Namespace with command line arguments
        modelrun (int): Current model run number.
        numbermodelruns (int): Total number of model runs.
        inputfile (str): Name of the input file to open.
        usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.

    Returns:
        tsolve (int): Length of time (seconds) of main FDTD calculations
    """

    # Monitor memory usage
    p = psutil.Process()

    # Declare variable to hold FDTDGrid class
    global G

    # Normal model reading/building process; bypassed if geometry information to be reused
    if 'G' not in globals():
        inputfilestr = '\n--- Model {} of {}, input file: {}'.format(modelrun, numbermodelruns, inputfile)
        print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)

        # Add the current model run to namespace that can be accessed by user in any Python code blocks in input file
        usernamespace['current_model_run'] = modelrun

        # Read input file and process any Python or include commands
        processedlines = process_python_include_code(inputfile, usernamespace)

        # Print constants/variables in user-accessable namespace
        uservars = ''
        for key, value in sorted(usernamespace.items()):
            if key != '__builtins__':
                uservars += '{}: {}, '.format(key, value)
        print('Constants/variables used/available for Python scripting: {{{}}}\n'.format(uservars[:-2]))

        # Write a file containing the input commands after Python or include commands have been processed
        if args.write_processed:
            write_processed_file(inputfile, modelrun, numbermodelruns, processedlines)

        # Check validity of command names and that essential commands are present
        singlecmds, multicmds, geometry = check_cmd_names(processedlines)

        # Initialise an instance of the FDTDGrid class
        G = FDTDGrid()
        G.inputfilename = os.path.split(inputfile)[1]
        G.inputdirectory = os.path.dirname(os.path.abspath(inputfile))

        # Create built-in materials
        m = Material(0, 'pec')
        m.se = float('inf')
        m.type = 'builtin'
        m.averagable = False
        G.materials.append(m)
        m = Material(1, 'free_space')
        m.type = 'builtin'
        G.materials.append(m)

        # Process parameters for commands that can only occur once in the model
        process_singlecmds(singlecmds, G)

        # Process parameters for commands that can occur multiple times in the model
        print()
        process_multicmds(multicmds, G)

        # Initialise an array for volumetric material IDs (solid), boolean arrays for specifying materials not to be averaged (rigid),
        # an array for cell edge IDs (ID)
        G.initialise_geometry_arrays()

        # Initialise arrays for the field components
        G.initialise_field_arrays()

        # Process geometry commands in the order they were given
        process_geometrycmds(geometry, G)

        # Build the PMLs and calculate initial coefficients
        print()
        if all(value == 0 for value in G.pmlthickness.values()):
            if G.messages:
                print('PML boundaries: switched off')
            pass # If all the PMLs are switched off don't need to build anything
        else:
            if G.messages:
                if all(value == G.pmlthickness['xminus'] for value in G.pmlthickness.values()):
                    pmlinfo = G.pmlthickness['xminus']
                else:
                    pmlinfo = ''
                    for key, value in G.pmlthickness.items():
                        pmlinfo += '{}: {}, '.format(key, value)
                    pmlinfo = pmlinfo[:-2]
                print('PML boundaries: {} cells'.format(pmlinfo))
            pbar = tqdm(total=sum(1 for value in G.pmlthickness.values() if value > 0), desc='Building PML boundaries', ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
            build_pmls(G, pbar)
            pbar.close()

        # Build the model, i.e. set the material properties (ID) for every edge of every Yee cell
        print()
        pbar = tqdm(total=2, desc='Building main grid', ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
        build_electric_components(G.solid, G.rigidE, G.ID, G)
        pbar.update()
        build_magnetic_components(G.solid, G.rigidH, G.ID, G)
        pbar.update()
        pbar.close()

        # Process any voltage sources (that have resistance) to create a new material at the source location
        for voltagesource in G.voltagesources:
            voltagesource.create_material(G)

        # Initialise arrays of update coefficients to pass to update functions
        G.initialise_std_update_coeff_arrays()

        # Initialise arrays of update coefficients and temporary values if there are any dispersive materials
        if Material.maxpoles != 0:
            G.initialise_dispersive_arrays()

        # Process complete list of materials - calculate update coefficients, store in arrays, and build text list of materials/properties
        materialsdata = process_materials(G)
        if G.messages:
            materialstable = AsciiTable(materialsdata)
            materialstable.outer_border = False
            materialstable.justify_columns[0] = 'right'
            print(materialstable.table)

        # Check to see if numerical dispersion might be a problem
        results = dispersion_analysis(G)
        if results['deltavp'] and np.abs(results['deltavp']) > G.maxnumericaldisp:
            print(Fore.RED + "\nWARNING: Potentially significant numerical dispersion. Largest physical phase-velocity error is {:.2f}% in material '{}' with wavelength sampled by {} cells (maximum significant frequency {:g}Hz)".format(results['deltavp'], results['material'].ID, round_value(results['N']), results['maxfreq']) + Style.RESET_ALL)
        elif results['deltavp']:
            print("\nNumerical dispersion analysis: largest physical phase-velocity error is {:.2f}% in material '{}' with wavelength sampled by {} cells (maximum significant frequency {:g}Hz)".format(results['deltavp'], results['material'].ID, round_value(results['N']), results['maxfreq']))

    # If geometry information to be reused between model runs
    else:
        inputfilestr = '\n--- Model {} of {}, input file (not re-processed, i.e. geometry fixed): {}'.format(modelrun, numbermodelruns, inputfile)
        print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)

        # Clear arrays for field components
        G.initialise_field_arrays()

        # Clear arrays for fields in PML
        for pml in G.pmls:
            pml.initialise_field_arrays()

    # Adjust position of simple sources and receivers if required
    if G.srcsteps[0] > 0 or G.srcsteps[1] > 0 or G.srcsteps[2] > 0:
        for source in itertools.chain(G.hertziandipoles, G.magneticdipoles):
            if modelrun == 1:
                if source.xcoord + G.srcsteps[0] * (numbermodelruns - 1) > G.nx or source.ycoord + G.srcsteps[1] * (numbermodelruns - 1) > G.ny or source.zcoord + G.srcsteps[2] * (numbermodelruns - 1) > G.nz:
                    raise GeneralError('Source(s) will be stepped to a position outside the domain.')
            source.xcoord = source.xcoordorigin + (modelrun - 1) * G.srcsteps[0]
            source.ycoord = source.ycoordorigin + (modelrun - 1) * G.srcsteps[1]
            source.zcoord = source.zcoordorigin + (modelrun - 1) * G.srcsteps[2]
    if G.rxsteps[0] > 0 or G.rxsteps[1] > 0 or G.rxsteps[2] > 0:
        for receiver in G.rxs:
            if modelrun == 1:
                if receiver.xcoord + G.rxsteps[0] * (numbermodelruns - 1) > G.nx or receiver.ycoord + G.rxsteps[1] * (numbermodelruns - 1) > G.ny or receiver.zcoord + G.rxsteps[2] * (numbermodelruns - 1) > G.nz:
                    raise GeneralError('Receiver(s) will be stepped to a position outside the domain.')
            receiver.xcoord = receiver.xcoordorigin + (modelrun - 1) * G.rxsteps[0]
            receiver.ycoord = receiver.ycoordorigin + (modelrun - 1) * G.rxsteps[1]
            receiver.zcoord = receiver.zcoordorigin + (modelrun - 1) * G.rxsteps[2]

    # Write files for any geometry views and geometry object outputs
    if not (G.geometryviews or G.geometryobjectswrite) and args.geometry_only:
        print(Fore.RED + '\nWARNING: No geometry views or geometry objects to output found.' + Style.RESET_ALL)
    if G.geometryviews:
        print()
        for i, geometryview in enumerate(G.geometryviews):
            geometryview.set_filename(modelrun, numbermodelruns, G)
            pbar = tqdm(total=geometryview.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry view file {} of {}, {}'.format(i + 1, len(G.geometryviews), os.path.split(geometryview.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
            geometryview.write_vtk(modelrun, numbermodelruns, G, pbar)
            pbar.close()
    if G.geometryobjectswrite:
        
        for i, geometryobject in enumerate(G.geometryobjectswrite):
            pbar = tqdm(total=geometryobject.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry object file {} of {}, {}'.format(i + 1, len(G.geometryobjectswrite), os.path.split(geometryobject.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
            geometryobject.write_hdf5(G, pbar)
            pbar.close()

    # Run simulation (if not doing geometry only)
    if not args.geometry_only:

        # Prepare any snapshot files
        for snapshot in G.snapshots:
            snapshot.prepare_vtk_imagedata(modelrun, numbermodelruns, G)

        # Output filename
        inputfileparts = os.path.splitext(inputfile)
        if numbermodelruns == 1:
            outputfile = inputfileparts[0] + '.out'
        else:
            outputfile = inputfileparts[0] + str(modelrun) + '.out'
        print('\nOutput file: {}\n'.format(outputfile))

        ####################################
        #  Start - Main FDTD calculations  #
        ####################################
        tsolvestart = perf_counter()

        # Absolute time
        abstime = 0

        for timestep in tqdm(range(G.iterations), desc='Running simulation, model ' + str(modelrun) + ' of ' + str(numbermodelruns), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable):
            # Store field component values for every receiver and transmission line
            store_outputs(timestep, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G)

            # Write any snapshots to file
            for i, snap in enumerate(G.snapshots):
                if snap.time == timestep + 1:
                    snapiters = 36 * (((snap.xf - snap.xs) / snap.dx) * ((snap.yf - snap.ys) / snap.dy) * ((snap.zf - snap.zs) / snap.dz))
                    pbar = tqdm(total=snapiters, leave=False, unit='byte', unit_scale=True, desc='  Writing snapshot file {} of {}, {}'.format(i + 1, len(G.snapshots), os.path.split(snap.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
                    snap.write_vtk_imagedata(G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G, pbar)
                    pbar.close()

            # Update electric field components
            if Material.maxpoles == 0:  # All materials are non-dispersive so do standard update
                update_electric(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsE, G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
            elif Material.maxpoles == 1:  # If there are any dispersive materials do 1st part of dispersive update (it is split into two parts as it requires present and updated electric field values).
                update_electric_dispersive_1pole_A(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsE, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
            elif Material.maxpoles > 1:
                update_electric_dispersive_multipole_A(G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles, G.updatecoeffsE, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)

            # Update electric field components with the PML correction
            for pml in G.pmls:
                pml.update_electric(G)

            # Update electric field components from sources (update any Hertzian dipole sources last)
            for source in G.voltagesources + G.transmissionlines + G.hertziandipoles:
                source.update_electric(abstime, G.updatecoeffsE, G.ID, G.Ex, G.Ey, G.Ez, G)

            # If there are any dispersive materials do 2nd part of dispersive update (it is split into two parts as it requires present and updated electric field values). Therefore it can only be completely updated after the electric field has been updated by the PML and source updates.
            if Material.maxpoles == 1:
                update_electric_dispersive_1pole_B(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez)
            elif Material.maxpoles > 1:
                update_electric_dispersive_multipole_B(G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez)

            # Increment absolute time value
            abstime += 0.5 * G.dt

            # Update magnetic field components
            update_magnetic(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsH, G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)

            # Update magnetic field components with the PML correction
            for pml in G.pmls:
                pml.update_magnetic(G)

            # Update magnetic field components from sources
            for source in G.transmissionlines + G.magneticdipoles:
                source.update_magnetic(abstime, G.updatecoeffsH, G.ID, G.Hx, G.Hy, G.Hz, G)

            # Increment absolute time value
            abstime += 0.5 * G.dt

        tsolve = int(perf_counter() - tsolvestart)

        # Write an output file in HDF5 format
        write_hdf5_outputfile(outputfile, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G)

        ##################################
        #  End - Main FDTD calculations  #
        ##################################

    if G.messages:
        print('Memory (RAM) used: ~{}'.format(human_size(p.memory_info().rss)))

    # If geometry information to be reused between model runs then FDTDGrid class instance must be global so that it persists
    if not args.geometry_fixed:
        del G

    # Return time to complete solving if in benchmarking mode
    if args.benchmark:
        return tsolve

Example 28

Project: VisTrails Source File: SelectPredictorsLayers.py
    def __init__(self, inputMDS, outputMDS, displayJPEG, rPath, parent=None):
        #print inputMDS, outputMDS, rPath, modelsPath
        self.rPath = rPath
        
        self.selection_name = os.path.split(outputMDS)[1]
        self.selection_name = os.path.splitext(self.selection_name)[0]
        self.selection_name = self.selection_name.split('_')[-1]
        
        self.displayJPEG = displayJPEG
        
        QtGui.QDialog.__init__(self, parent)
        
        self.inputMDS = inputMDS
        self.outputMDS = outputMDS
        self.outputDir = os.path.split(outputMDS)[0]
        
        layout = QtGui.QVBoxLayout()
        self.setWindowFlags(QtCore.Qt.Window)
        ##begin  autogenerated code from QtDesigner
        #
        
        self.horizontalLayout_4 = QtGui.QHBoxLayout()
        self.horizontalLayout_4.setSpacing(5)
        self.horizontalLayout_4.setMargin(5)
        self.horizontalLayout_4.setObjectName(_fromUtf8("horizontalLayout_4"))
        self.verticalLayout_3 = QtGui.QVBoxLayout()
        self.verticalLayout_3.setObjectName(_fromUtf8("verticalLayout_3"))
        self.splitter = QtGui.QSplitter()
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.splitter.sizePolicy().hasHeightForWidth())
        self.splitter.setSizePolicy(sizePolicy)
        self.splitter.setMinimumSize(QtCore.QSize(0, 0))
        self.splitter.setFrameShape(QtGui.QFrame.Box)
        self.splitter.setFrameShadow(QtGui.QFrame.Plain)
        self.splitter.setLineWidth(1)
        self.splitter.setOrientation(QtCore.Qt.Horizontal)
        self.splitter.setHandleWidth(10)
        self.splitter.setObjectName(_fromUtf8("splitter"))
        self.widget = QtGui.QWidget(self.splitter)
        self.widget.setObjectName(_fromUtf8("widget"))
        self.verticalLayout = QtGui.QVBoxLayout(self.widget)
        self.verticalLayout.setSpacing(4)
        self.verticalLayout.setSizeConstraint(QtGui.QLayout.SetDefaultConstraint)
        self.verticalLayout.setContentsMargins(6, 6, 3, 3)
        self.verticalLayout.setObjectName(_fromUtf8("verticalLayout"))
        self.label_2 = QtGui.QLabel(self.widget)
        self.label_2.setObjectName(_fromUtf8("label_2"))
        self.verticalLayout.addWidget(self.label_2)
        self.treeview = QtGui.QTreeWidget(self.widget)
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Ignored, QtGui.QSizePolicy.Expanding)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.treeview.sizePolicy().hasHeightForWidth())
        self.treeview.setSizePolicy(sizePolicy)
        self.treeview.setMinimumSize(QtCore.QSize(75, 0))
        self.treeview.setMaximumSize(QtCore.QSize(16777215, 16777215))
        self.treeview.setSizeIncrement(QtCore.QSize(100, 0))
        self.treeview.setBaseSize(QtCore.QSize(0, 0))
        self.treeview.setObjectName(_fromUtf8("treeview"))
        self.verticalLayout.addWidget(self.treeview)
        self.horizontalLayout_3 = QtGui.QHBoxLayout()
        self.horizontalLayout_3.setSizeConstraint(QtGui.QLayout.SetDefaultConstraint)
        self.horizontalLayout_3.setObjectName(_fromUtf8("horizontalLayout_3"))
        self.btnRunR = QtGui.QPushButton(self.widget)
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.btnRunR.sizePolicy().hasHeightForWidth())
        self.btnRunR.setSizePolicy(sizePolicy)
        self.btnRunR.setObjectName(_fromUtf8("btnRunR"))
        self.horizontalLayout_3.addWidget(self.btnRunR)
        spacerItem = QtGui.QSpacerItem(10, 0, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
        self.horizontalLayout_3.addItem(spacerItem)
        self.label = QtGui.QLabel(self.widget)
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.label.sizePolicy().hasHeightForWidth())
        self.label.setSizePolicy(sizePolicy)
        self.label.setObjectName(_fromUtf8("label"))
        self.horizontalLayout_3.addWidget(self.label)
        self.lineEdit = QtGui.QLineEdit(self.widget)
        self.lineEdit.setText(_fromUtf8("0.7"))
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.lineEdit.sizePolicy().hasHeightForWidth())
        self.lineEdit.setSizePolicy(sizePolicy)
        self.lineEdit.setMaximumSize(QtCore.QSize(75, 16777215))
        self.lineEdit.setObjectName(_fromUtf8("lineEdit"))
        self.horizontalLayout_3.addWidget(self.lineEdit)
        
        self.chkPresence = QtGui.QCheckBox(self.widget)
        self.chkPresence.setChecked(True)
        self.chkPresence.setText(_fromUtf8("Include presence/count points"))
        self.chkPresence.setObjectName(_fromUtf8("chkPresence"))
        self.verticalLayout.addWidget(self.chkPresence)
        self.chkAbsence = QtGui.QCheckBox(self.widget)
        self.chkAbsence.setChecked(True)
        self.chkAbsence.setText(_fromUtf8("Include absence points"))
        self.chkAbsence.setObjectName(_fromUtf8("chkAbsence"))
        self.verticalLayout.addWidget(self.chkAbsence)
        self.chkBackground = QtGui.QCheckBox(self.widget)
        self.chkBackground.setChecked(True)
        self.chkBackground.setText(_fromUtf8("Include background points"))
        self.chkBackground.setObjectName(_fromUtf8("chkBackground"))
        self.verticalLayout.addWidget(self.chkBackground)
        
        self.verticalLayout.addLayout(self.horizontalLayout_3)
        
        self.scene = QtGui.QGraphicsScene() 
        self.view = QtGui.QGraphicsView(self.scene)
        #self.view = customGraphicsView(self.scene)
        self.view.setDragMode(QtGui.QGraphicsView.ScrollHandDrag)
        
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
        sizePolicy.setHorizontalStretch(4)
        sizePolicy.setVerticalStretch(0)
        #sizePolicy.setHeightForWidth(self.view.sizePolicy().hasHeightForWidth())
        self.view.setSizePolicy(sizePolicy)
        self.view.setResizeAnchor(QtGui.QGraphicsView.AnchorUnderMouse)
        self.view.setObjectName(_fromUtf8("view"))
        
        self.splitter.addWidget(self.view)
        
        self.verticalLayout_3.addWidget(self.splitter)
        self.horizontalLayout = QtGui.QHBoxLayout()
        self.horizontalLayout.setContentsMargins(-1, 3, -1, 3)
        self.horizontalLayout.setObjectName(_fromUtf8("horizontalLayout"))
        self.btnOK = QtGui.QPushButton()
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.btnOK.sizePolicy().hasHeightForWidth())
        self.btnOK.setSizePolicy(sizePolicy)
        self.btnOK.setBaseSize(QtCore.QSize(100, 0))
        self.btnOK.setToolTip(_fromUtf8(""))
        self.btnOK.setObjectName(_fromUtf8("btnOK"))
        self.horizontalLayout.addWidget(self.btnOK)
        self.btnCancel = QtGui.QPushButton()
        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.btnCancel.sizePolicy().hasHeightForWidth())
        self.btnCancel.setSizePolicy(sizePolicy)
        self.btnCancel.setBaseSize(QtCore.QSize(100, 0))
        self.btnCancel.setObjectName(_fromUtf8("btnCancel"))
        self.horizontalLayout.addWidget(self.btnCancel)
        self.verticalLayout_3.addLayout(self.horizontalLayout)
        self.horizontalLayout_4.addLayout(self.verticalLayout_3)

        self.setWindowTitle(_fromUtf8("Covariate Coorelation viewer"))
        self.label_2.setText(_fromUtf8("Covariates"))
        self.btnRunR.setText(_fromUtf8("Update"))
        self.label.setText(_fromUtf8("Threshold"))
        self.btnOK.setText(_fromUtf8("OK"))
        self.btnCancel.setText(_fromUtf8("Cancel"))
        self.resize(1100, 800)
        self.view.resize(800, 750)
        
        ##End  autogenerated code from QtDesigner
        
        layout.addLayout(self.horizontalLayout_4)
        
        self.btnCancel.setShortcut('Esc')
        self.connect(self.btnOK, QtCore.SIGNAL('clicked(bool)'),
                     self.okTriggered)
        self.connect(self.btnCancel, QtCore.SIGNAL('clicked(bool)'),
                     self.cancel)
        self.connect(self.btnRunR, QtCore.SIGNAL('clicked(bool)'),
                     self.updateROutput)
#        self.connect(self.view, QtCore.SIGNAL('resize()'),
#                     self.resize)
        
        self.scene.wheelEvent = self.wheel_event
#        self.resizeEvent = self.resize_event
        #code to populate the treeview with the contents of our MDS
        self.PopulateTreeview()
        
        self.setLayout(layout)
        self.repaint()
        #utils.breakpoint()
        #code to add in pictureviewer stuff
        outputPic = self.runR(self.inputMDS)
#        self.setup_view()
        self.load_picture(outputPic)

Example 29

Project: cassback Source File: restore_subcommand.py
Function: call
    def __call__(self):

        self.log.info("Starting sub command %s", self.command_name)

        endpoint = self._endpoint(self.args)
        manifest = self._load_manifest_by_name(endpoint, 
            self.args.backup_name)

        # We put the files that need to be restored in there.
        work_queue = Queue.Queue()
        # We put the results in here
        # So we can say what was copied to where.
        result_queue = Queue.Queue()

        # Fill the queue with components we want to restore.
        components = []
        for component in manifest.iter_components():
            msg = (
                json.dumps(manifest.serialise()),
                json.dumps(component.serialise())
            )
            work_queue.put(msg)
            components.append(component)
        self.log.info("Queued components for restore: %s", 
            ", ".join(str(c) for c in components))

        # Make worker threads to do the work. 
        workers = [
            self._create_worker_thread(i, work_queue, result_queue)
            for i in range(self.args.threads)
        ]
        for worker in workers:
            worker.start()
            
        if self.args.report_interval_secs > 0:
            reporter = SlurpReporterThread(work_queue, 
                self.args.report_interval_secs)
            reporter.start()
        else:
            self.log.info("Progress reporting is disabled.")

        # Wait for the work queue to empty. 
        self.log.info("Waiting on workers.")
        work_queue.join()
        self.log.info("Finished sub command %s", self.command_name)

        # Make a pretty message 
        op_results = []
        # result_queue has BackupFile's
        while not result_queue.empty():
            op_results.append(RestoreResult.deserialise(json.loads(
                result_queue.get_nowait())))
        
        str_bld = []
        failed_ops = [
            op_result
            for op_result in op_results
            if op_result.failed_reason
        ]
        
        skipped_ops = [
            op_result
            for op_result in op_results
            if not op_result.should_restore
        ]
        success_ops = [
            op_result
            for op_result in op_results
            if not op_result.failed_reason and op_result.should_restore
        ]
        corrupt_files = [
            op_result
            for op_result in op_results
            if op_result.corrupt_path
        ]
        
        def gen_groups(lst, key):
            lst.sort(key=key)
            for k, i in itertools.groupby(lst, key):
                yield k, i
                
        if corrupt_files:
            str_bld.append("Moved corrupt existing files:")
            for op_result in corrupt_files:
                str_bld.append("%s -> %s" % ( 
                    op_result.restore_path, op_result.corrupt_path))
            str_bld.append("")
            
        if failed_ops:
            for k, ops in gen_groups(failed_ops, lambda x: x.failed_reason):
                
                str_bld.append("Failed transfers - %s:" % (k,))
                for op_result in ops:
                    _, name = os.path.split(op_result.source_path)
                    str_bld.append("%s -> %s \tbecause %s" % (
                        name, op_result.restore_path, op_result.restore_reason))
                str_bld.append("")

        if skipped_ops:
            str_bld.append("Skipped files:")
            for op_result in skipped_ops:
                _, name = os.path.split(op_result.source_path)
                str_bld.append("%s -> %s \tbecause %s" %( 
                    name, op_result.restore_path, op_result.restore_reason))
            str_bld.append("")
                    
        if success_ops:
            str_bld.append("Restored files:")
            for op_result in success_ops:
                _, name = os.path.split(op_result.source_path)
                str_bld.append("%s -> %s \tbecause %s" %( 
                    name, op_result.restore_path, op_result.restore_reason))
            str_bld.append("")
        
        return (0, "\n".join(str_bld))

Example 30

Project: scons Source File: tex.py
Function: tex_emitter_core
def tex_emitter_core(target, source, env, graphics_extensions):
    """An emitter for TeX and LaTeX sources.
    For LaTeX sources we try and find the common created files that
    are needed on subsequent runs of latex to finish tables of contents,
    bibliographies, indices, lists of figures, and hyperlink references.
    """
    basename = SCons.Util.splitext(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    targetdir = os.path.split(str(target[0]))[0]
    targetbase = os.path.join(targetdir, basefile)

    basedir = os.path.split(str(source[0]))[0]
    abspath = os.path.abspath(basedir)
    target[0].attributes.path = abspath

    #
    # file names we will make use of in searching the sources and log file
    #
    emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
    auxfilename = targetbase + '.aux'
    logfilename = targetbase + '.log'
    flsfilename = targetbase + '.fls'

    env.SideEffect(auxfilename,target[0])
    env.SideEffect(logfilename,target[0])
    env.SideEffect(flsfilename,target[0])
    if Verbose:
        print "side effect :",auxfilename,logfilename,flsfilename
    env.Clean(target[0],auxfilename)
    env.Clean(target[0],logfilename)
    env.Clean(target[0],flsfilename)

    content = source[0].get_text_contents()

    idx_exists = os.path.exists(targetbase + '.idx')
    nlo_exists = os.path.exists(targetbase + '.nlo')
    glo_exists = os.path.exists(targetbase + '.glo')
    acr_exists = os.path.exists(targetbase + '.acn')

    # set up list with the regular expressions
    # we use to find features used
    file_tests_search = [auxfile_re,
                         makeindex_re,
                         bibliography_re,
                         bibunit_re,
                         tableofcontents_re,
                         listoffigures_re,
                         listoftables_re,
                         hyperref_re,
                         makenomenclature_re,
                         makeglossary_re,
                         makeglossaries_re,
                         makeacronyms_re,
                         beamer_re ]
    # set up list with the file suffixes that need emitting
    # when a feature is found
    file_tests_suff = [['.aux','aux_file'],
                  ['.idx', '.ind', '.ilg','makeindex'],
                  ['.bbl', '.blg','bibliography'],
                  ['.bbl', '.blg','bibunit'],
                  ['.toc','contents'],
                  ['.lof','figures'],
                  ['.lot','tables'],
                  ['.out','hyperref'],
                  ['.nlo', '.nls', '.nlg','nomenclature'],
                  ['.glo', '.gls', '.glg','glossary'],
                  ['.glo', '.gls', '.glg','glossaries'],
                  ['.acn', '.acr', '.alg','acronyms'],
                  ['.nav', '.snm', '.out', '.toc','beamer'] ]
    # build the list of lists
    file_tests = []
    for i in range(len(file_tests_search)):
        file_tests.append( [None, file_tests_suff[i]] )

    # TO-DO: need to add a way for the user to extend this list for whatever
    # auxiliary files they create in other (or their own) packages

    # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
    savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
    paths = env['ENV']['TEXINPUTS']
    if SCons.Util.is_List(paths):
        pass
    else:
        # Split at os.pathsep to convert into absolute path
        paths = paths.split(os.pathsep)

    # now that we have the path list restore the env
    if savedpath is _null:
        try:
            del env['ENV']['TEXINPUTS']
        except KeyError:
            pass # was never set
    else:
        env['ENV']['TEXINPUTS'] = savedpath
    if Verbose:
        print "search path ",paths

    aux_files = []
    file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)

    for (theSearch,suffix_list) in file_tests:
        # add side effects if feature is present.If file is to be generated,add all side effects
        if (theSearch != None) or (not source[0].exists() ):
            file_list = [targetbase,]
            # for bibunit we need a list of files
            if suffix_list[-1] == 'bibunit':
                file_basename = os.path.join(targetdir, 'bu*.aux')
                file_list = glob.glob(file_basename)
                # remove the suffix '.aux'
                for i in range(len(file_list)):
                    file_list[i] = SCons.Util.splitext(file_list[i])[0]
            # now define the side effects
            for file_name in file_list:
                for suffix in suffix_list[:-1]:
                    env.SideEffect(file_name + suffix,target[0])
                    if Verbose:
                        print "side effect :",file_name + suffix
                    env.Clean(target[0],file_name + suffix)

    for aFile in aux_files:
        aFile_base = SCons.Util.splitext(aFile)[0]
        env.SideEffect(aFile_base + '.aux',target[0])
        if Verbose:
            print "side effect :",aFile_base + '.aux'
        env.Clean(target[0],aFile_base + '.aux')
    # read fls file to get all other files that latex creates and will read on the next pass
    # remove files from list that we explicitly dealt with above
    if os.path.exists(flsfilename):
        content = open(flsfilename, "rb").read()
        out_files = openout_re.findall(content)
        myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
        for filename in out_files[:]:
            if filename in myfiles:
                out_files.remove(filename)
        env.SideEffect(out_files,target[0])
        if Verbose:
            print "side effect :",out_files
        env.Clean(target[0],out_files)

    return (target, source)

Example 31

Project: Nuitka Source File: tex.py
Function: tex_emitter_core
def tex_emitter_core(target, source, env, graphics_extensions):
    """An emitter for TeX and LaTeX sources.
    For LaTeX sources we try and find the common created files that
    are needed on subsequent runs of latex to finish tables of contents,
    bibliographies, indices, lists of figures, and hyperlink references.
    """
    basename = SCons.Util.splitext(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    targetdir = os.path.split(str(target[0]))[0]
    targetbase = os.path.join(targetdir, basefile)

    basedir = os.path.split(str(source[0]))[0]
    abspath = os.path.abspath(basedir)
    target[0].attributes.path = abspath

    #
    # file names we will make use of in searching the sources and log file
    #
    emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
    auxfilename = targetbase + '.aux'
    logfilename = targetbase + '.log'
    flsfilename = targetbase + '.fls'
    syncfilename = targetbase + '.synctex.gz'

    env.SideEffect(auxfilename,target[0])
    env.SideEffect(logfilename,target[0])
    env.SideEffect(flsfilename,target[0])
    env.SideEffect(syncfilename,target[0])
    if Verbose:
        print "side effect :",auxfilename,logfilename,flsfilename,syncfilename
    env.Clean(target[0],auxfilename)
    env.Clean(target[0],logfilename)
    env.Clean(target[0],flsfilename)
    env.Clean(target[0],syncfilename)

    content = source[0].get_text_contents()

    # These variables are no longer used.
    #idx_exists = os.path.isfile(targetbase + '.idx')
    #nlo_exists = os.path.isfile(targetbase + '.nlo')
    #glo_exists = os.path.isfile(targetbase + '.glo')
    #acr_exists = os.path.isfile(targetbase + '.acn')

    # set up list with the regular expressions
    # we use to find features used
    file_tests_search = [auxfile_re,
                         makeindex_re,
                         bibliography_re,
                         bibunit_re,
                         multibib_re,
                         addbibresource_re,
                         tableofcontents_re,
                         listoffigures_re,
                         listoftables_re,
                         hyperref_re,
                         makenomenclature_re,
                         makeglossary_re,
                         makeglossaries_re,
                         makeacronyms_re,
                         beamer_re,
                         newglossary_re,
                         biblatex_re ]
    # set up list with the file suffixes that need emitting
    # when a feature is found
    file_tests_suff = [['.aux','aux_file'],
                  ['.idx', '.ind', '.ilg','makeindex'],
                  ['.bbl', '.blg','bibliography'],
                  ['.bbl', '.blg','bibunit'],
                  ['.bbl', '.blg','multibib'],
                  ['.bbl', '.blg','.bcf','addbibresource'],
                  ['.toc','contents'],
                  ['.lof','figures'],
                  ['.lot','tables'],
                  ['.out','hyperref'],
                  ['.nlo', '.nls', '.nlg','nomenclature'],
                  ['.glo', '.gls', '.glg','glossary'],
                  ['.glo', '.gls', '.glg','glossaries'],
                  ['.acn', '.acr', '.alg','acronyms'],
                  ['.nav', '.snm', '.out', '.toc','beamer'],
                  ['newglossary',],
                  ['.bcf', '.blg','biblatex'] ]
    # for newglossary the suffixes are added as we find the command
    # build the list of lists
    file_tests = []
    for i in range(len(file_tests_search)):
        file_tests.append( [None, file_tests_suff[i]] )

    # TO-DO: need to add a way for the user to extend this list for whatever
    # auxiliary files they create in other (or their own) packages

    # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
    savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
    paths = env['ENV']['TEXINPUTS']
    if SCons.Util.is_List(paths):
        pass
    else:
        # Split at os.pathsep to convert into absolute path
        paths = paths.split(os.pathsep)

    # now that we have the path list restore the env
    if savedpath is _null:
        try:
            del env['ENV']['TEXINPUTS']
        except KeyError:
            pass # was never set
    else:
        env['ENV']['TEXINPUTS'] = savedpath
    if Verbose:
        print "search path ",paths

    # scan all sources for side effect files
    aux_files = []
    file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)

    for (theSearch,suffix_list) in file_tests:
        # add side effects if feature is present.If file is to be generated,add all side effects
        if Verbose and theSearch:
            print "check side effects for ",suffix_list[-1]
        if (theSearch != None) or (not source[0].exists() ):
            file_list = [targetbase,]
            # for bibunit we need a list of files
            if suffix_list[-1] == 'bibunit':
                file_basename = os.path.join(targetdir, 'bu*.aux')
                file_list = glob.glob(file_basename)
                # remove the suffix '.aux'
                for i in range(len(file_list)):
                    file_list.append(SCons.Util.splitext(file_list[i])[0])
            # for multibib we need a list of files
            if suffix_list[-1] == 'multibib':
                for multibibmatch in multibib_re.finditer(content):
                    if Verbose:
                        print "multibib match ",multibibmatch.group(1)
                    if multibibmatch != None:
                        baselist = multibibmatch.group(1).split(',')
                        if Verbose:
                            print "multibib list ", baselist
                        for i in range(len(baselist)):
                            file_list.append(os.path.join(targetdir, baselist[i]))
            # now define the side effects
            for file_name in file_list:
                for suffix in suffix_list[:-1]:
                    env.SideEffect(file_name + suffix,target[0])
                    if Verbose:
                        print "side effect tst :",file_name + suffix, " target is ",str(target[0])
                    env.Clean(target[0],file_name + suffix)

    for aFile in aux_files:
        aFile_base = SCons.Util.splitext(aFile)[0]
        env.SideEffect(aFile_base + '.aux',target[0])
        if Verbose:
            print "side effect aux :",aFile_base + '.aux'
        env.Clean(target[0],aFile_base + '.aux')
    # read fls file to get all other files that latex creates and will read on the next pass
    # remove files from list that we explicitly dealt with above
    if os.path.isfile(flsfilename):
        content = open(flsfilename, "rb").read()
        out_files = openout_re.findall(content)
        myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
        for filename in out_files[:]:
            if filename in myfiles:
                out_files.remove(filename)
        env.SideEffect(out_files,target[0])
        if Verbose:
            print "side effect fls :",out_files
        env.Clean(target[0],out_files)

    return (target, source)

Example 32

Project: tomato Source File: swf_image_dumper.py
    def save_image(self, object_id, v):
        # 実際に出力した画像の ID を保存
        file_base = self.output_file

        if v['tag'] == 'DefineBitsLossLess' and v['format'] == 3:
            color_table_length = ord(v['data'][0])+1
            decompress_data = zlib.decompress(v['data'][1:])
            color_table = []
            for i in range(color_table_length):
                c = (ord(decompress_data[i*3]), 
                     ord(decompress_data[i*3+1]), 
                     ord(decompress_data[i*3+2]))
                color_table.append(c)
            color_map = decompress_data[color_table_length*3:]
            width = v['width']
            height = v['height']
            width = width if width % 4 == 0 else (width/4+1)* 4
            im = Image.new("RGB", (width, height), "white")
            x = y = 0
            for i in range(len(color_map)):
                im.putpixel((x,y), color_table[ord(color_map[i])])
                x += 1
                if x == width:
                    x = 0
                    y += 1

            out_file = "%s_%s.png" % (file_base, object_id)
            im.save(out_file)
            v['file_name'] = os.path.split(out_file)[1]

        elif v['tag'] == 'DefineBitsLossless2' and v['format'] == 3:
            color_table_length = ord(v['data'][0])+1
            decompress_data = zlib.decompress(v['data'][1:])
            color_table = []
            for i in range(color_table_length):
                c = (ord(decompress_data[i*4+0]), 
                     ord(decompress_data[i*4+1]), 
                     ord(decompress_data[i*4+2]),
                     ord(decompress_data[i*4+3]))
                color_table.append(c)
            color_map = decompress_data[color_table_length*4:]
            width = v['width']
            height = v['height']
            width = width if width % 4 == 0 else (width/4+1)* 4
            im = Image.new("RGB", (width, height), "white")
            x = y = 0
            for i in range(len(color_map)):
                im.putpixel((x,y), color_table[ord(color_map[i])])
                x += 1
                if x == width:
                    x = 0
                    y += 1

            out_file = "%s_%s.png" % (file_base, object_id)
            im.save(out_file)
            v['file_name'] = os.path.split(out_file)[1]

        elif v['tag'] == 'DefineBitsLossLess' and v['format'] == 5:
            decompress_data = zlib.decompress(v['data'])
            width = v['width']
            height = v['height']
            im = Image.new("RGB", (width, height), "white")
            x = y = 0
            for i in range(0, len(decompress_data), 4):
                im.putpixel((x,y), (ord(decompress_data[i+1]),
                                    ord(decompress_data[i+2]),
                                    ord(decompress_data[i+3])))
                x += 1
                if x == width:
                    x = 0
                    y += 1

            out_file = "%s_%s.png" % (file_base, object_id)
            im.save(out_file)
            v['file_name'] = os.path.split(out_file)[1]

        elif v['tag'] == 'DefineBitsLossless2' and v['format'] == 5:
            decompress_data = zlib.decompress(v['data'])
            width = v['width']
            height = v['height']
            im = Image.new("RGBA", (width, height), "white")
            x = y = 0
            for i in range(0, len(decompress_data), 4):
                im.putpixel((x,y), (ord(decompress_data[i+1]),
                                    ord(decompress_data[i+2]),
                                    ord(decompress_data[i+3]),
                                    ord(decompress_data[i+0])))
                x += 1
                if x == width:
                    x = 0
                    y += 1

            out_file = "%s_%s.png" % (file_base, object_id)
            im.save(out_file)
            v['file_name'] = os.path.split(out_file)[1]

        elif v['tag'] == 'DefineBitsJPEG2':
            data = v['data']
            if data[:4] == '\xff\xd9\xff\xd8':
                """
                See: http://pwiki.awm.jp/~yoya/?Flash/JPEG
                    JPEG SOI marker (FF D8) -> \xff\xd8
                    JPEG EOI marker (FF D9) -> \xff\xd9
                Flash に格納されているのは基本的にはそのままの JPEG データ
                ただし、データの最初に [EOI] + [SOI] が付けられていることがあり
                その場合は除去する必要がある
                """
                data = data[4:]
            im = Image.open(StringIO(data))

            v['width'] = im.size[0]
            v['height'] = im.size[1]
            out_file = "%s_%s.jpg" % (file_base, object_id)
            im.save(out_file)
            v['file_name'] = os.path.split(out_file)[1]

Example 33

Project: romcollectionbrowser Source File: dialogeditromcollection.py
	def updateRomCollectionControls(self):
		
		Logutil.log('updateRomCollectionControls', util.LOG_LEVEL_INFO)
		
		control = self.getControlById(CONTROL_LIST_ROMCOLLECTIONS)
		selectedRomCollectionName = str(control.getSelectedItem().getLabel())
				
		Logutil.log('selected rom collection: ' +str(selectedRomCollectionName), util.LOG_LEVEL_INFO)
				
		self.selectedRomCollection = None
		
		for rcId in self.romCollections.keys():
			romCollection = self.romCollections[rcId]
			if romCollection.name == selectedRomCollectionName:
				self.selectedRomCollection = romCollection
				break
			
		if(self.selectedRomCollection == None):
			return
				
		#Import Games
		#HACK: split romPath and fileMask
		firstRomPath = ''
		fileMask = ''
		for romPath in self.selectedRomCollection.romPaths:
			
			pathParts = os.path.split(romPath)			 
			if(firstRomPath == ''):				
				firstRomPath = pathParts[0]
				fileMask = pathParts[1]
			elif(firstRomPath == pathParts[0]):
				fileMask = fileMask +',' +pathParts[1]
								
		control = self.getControlById(CONTROL_BUTTON_ROMPATH)
		util.setLabel(firstRomPath, control)
		
		control = self.getControlById(CONTROL_BUTTON_FILEMASK)
		util.setLabel(fileMask, control)		
		
		control = self.getControlById(CONTROL_BUTTON_IGNOREONSCAN)		
		control.setSelected(self.selectedRomCollection.ignoreOnScan)
		
		control = self.getControlById(CONTROL_BUTTON_ALLOWUPDATE)
		control.setSelected(self.selectedRomCollection.allowUpdate)
		
		control = self.getControlById(CONTROL_BUTTON_DISKINDICATOR)
		util.setLabel(self.selectedRomCollection.diskPrefix, control)
		
		control = self.getControlById(CONTROL_BUTTON_MAXFOLDERDEPTH)
		util.setLabel(self.selectedRomCollection.maxFolderDepth, control)
		
		control = self.getControlById(CONTROL_BUTTON_USEFOLDERASGAMENAME)
		control.setSelected(self.selectedRomCollection.useFoldernameAsGamename)
		
		#Import Game Data
		#Media Types
		mediaTypeList = []
		firstMediaPath = ''
		firstMediaFileMask = ''
		for mediaPath in self.selectedRomCollection.mediaPaths:
			mediaTypeList.append(mediaPath.fileType.name)
			if(firstMediaPath == ''):
				pathParts = os.path.split(mediaPath.path)
				firstMediaPath = pathParts[0]
				firstMediaFileMask = pathParts[1]
				
		self.addItemsToList(CONTROL_LIST_MEDIATYPES, mediaTypeList)
		
		control = self.getControlById(CONTROL_BUTTON_MEDIAPATH)
		util.setLabel(firstMediaPath, control)
		
		control = self.getControlById(CONTROL_BUTTON_MEDIAFILEMASK)
		util.setLabel(firstMediaFileMask, control)		
						
		self.selectScrapersInList(self.selectedRomCollection.scraperSites, self.availableScrapers)
		
		#Browse Games
		optionMain = self.selectedRomCollection.imagePlacingMain.name
		try:
			optionMain = config.imagePlacingDict[optionMain]
		except:
			pass
		self.selectItemInList(optionMain, CONTROL_LIST_IMAGEPLACING_MAIN)
		
		optionInfo = self.selectedRomCollection.imagePlacingInfo.name
		try:
			optionInfo = config.imagePlacingDict[optionInfo]
		except:			
			pass		
		self.selectItemInList(optionInfo, CONTROL_LIST_IMAGEPLACING_INFO)
		
		control = self.getControlById(CONTROL_BUTTON_AUTOPLAYVIDEO_MAIN)
		if(control != None):
			control.setSelected(self.selectedRomCollection.autoplayVideoMain)
		
		control = self.getControlById(CONTROL_BUTTON_AUTOPLAYVIDEO_INFO)
		if(control != None):
			control.setSelected(self.selectedRomCollection.autoplayVideoInfo)
		
		#Launch Games
		control = self.getControlById(CONTROL_BUTTON_USERETROPLAYER)
		if(control):
			control.setSelected(self.selectedRomCollection.useBuiltinEmulator)
		
		control = self.getControlById(CONTROL_BUTTON_GAMECLIENT)
		if(control):
			util.setLabel(self.selectedRomCollection.gameclient, control)		
			
		control = self.getControlById(CONTROL_BUTTON_EMUCMD)
		util.setLabel(self.selectedRomCollection.emulatorCmd, control)		
		
		control = self.getControlById(CONTROL_BUTTON_PARAMS)
		util.setLabel(self.selectedRomCollection.emulatorParams, control)		
		
		control = self.getControlById(CONTROL_BUTTON_USEEMUSOLO)
		control.setSelected(self.selectedRomCollection.useEmuSolo)
		
		control = self.getControlById(CONTROL_BUTTON_USEPOPEN)
		control.setSelected(self.selectedRomCollection.usePopen)
		
		pathParts = os.path.split(self.selectedRomCollection.saveStatePath)
		saveStatePath = pathParts[0]
		saveStateFileMask = pathParts[1]
		
		control = self.getControlById(CONTROL_BUTTON_SAVESTATEPATH)
		util.setLabel(saveStatePath, control)
		
		control = self.getControlById(CONTROL_BUTTON_SAVESTATEMASK)
		util.setLabel(saveStateFileMask, control)
		
		control = self.getControlById(CONTROL_BUTTON_SAVESTATEPARAMS)		
		util.setLabel(self.selectedRomCollection.saveStateParams, control)
		
		control = self.getControlById(CONTROL_BUTTON_DONTEXTRACTZIP)
		control.setSelected(self.selectedRomCollection.doNotExtractZipFiles)
		
		control = self.getControlById(CONTROL_BUTTON_MAKELOCALCOPY)
		control.setSelected(self.selectedRomCollection.makeLocalCopy)
		
		control = self.getControlById(CONTROL_BUTTON_PRECMD)
		util.setLabel(self.selectedRomCollection.preCmd, control)		
		
		control = self.getControlById(CONTROL_BUTTON_POSTCMD)
		util.setLabel(self.selectedRomCollection.postCmd, control)

Example 34

Project: enigma2 Source File: MovieList.py
	def buildMovieListEntry(self, serviceref, info, begin, data):
		switch = config.usage.show_icons_in_movielist.value
		ext = config.movielist.useextlist.value
		width = self.l.getItemSize().width()
		pathName = serviceref.getPath()
		res = [ None ]

		if ext != '0':
			ih = self.itemHeight / 2
		else:
			ih = self.itemHeight

		if self.screenwidth and self.screenwidth == 1920:
			listBeginX = 3
			listEndX = 3
			listMarginX = 12
			pathIconSize = 29
			dataIconSize = 25
			progressIconSize = 25
			progressBarSize = 72
			textPosY = 2
		else:
			listBeginX = 2
			listEndX = 2
			listMarginX = 8
			pathIconSize = 25
			dataIconSize = 21
			progressIconSize = 21
			progressBarSize = 48
			textPosY = 1

		textPosX = listBeginX + dataIconSize + listMarginX

		if serviceref.flags & eServiceReference.mustDescent:
			# Directory
			iconSize = pathIconSize
			iconPosX = listBeginX-1
			iconPosY = ih/2-iconSize/2
			if iconPosY < iconPosX:
				iconPosY = iconPosX
			# Name is full path name
			if info is None:
				# Special case: "parent"
				txt = ".."
			else:
				p = os.path.split(pathName)
				if not p[1]:
					# if path ends in '/', p is blank.
					p = os.path.split(p[0])
				txt = p[1]
				if txt == ".Trash":
					dateSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), _("Trashcan")).width()
					res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=self.iconTrash))
					res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = _("Deleted items")))
					res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, textPosY), size=(dateSize, self.itemHeight), font=1, flags=RT_HALIGN_RIGHT|RT_VALIGN_CENTER, text=_("Trashcan")))
					return res
			dateSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), _("Directory")).width()
			res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=self.iconFolder))
			res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = txt))
			res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, textPosY), size=(dateSize, self.itemHeight), font=1, flags=RT_HALIGN_RIGHT|RT_VALIGN_CENTER, text=_("Directory")))
			return res
		if (data == -1) or (data is None):
			data = MovieListData()
			cur_idx = self.l.getCurrentSelectionIndex()
			x = self.list[cur_idx] # x = ref,info,begin,...
			data.len = 0 #dont recalc movielist to speedup loading the list
			self.list[cur_idx] = (x[0], x[1], x[2], data) #update entry in list... so next time we don't need to recalc
			data.txt = info.getName(serviceref)
			if config.movielist.hide_extensions.value:
				fileName, fileExtension = os.path.splitext(data.txt)
				if fileExtension in KNOWN_EXTENSIONS:
					data.txt = fileName
			data.icon = None
			data.part = None
			if os.path.split(pathName)[1] in self.runningTimers:
				if switch == 'i':
					if (self.playInBackground or self.playInForeground) and serviceref == (self.playInBackground or self.playInForeground):
						data.icon = self.iconMoviePlayRec
					else:
						data.icon = self.iconMovieRec
				elif switch == 'p' or switch == 's':
					data.part = 100
					if (self.playInBackground or self.playInForeground) and serviceref == (self.playInBackground or self.playInForeground):
						data.partcol = 0xffc71d
					else:
						data.partcol = 0xff001d
			elif (self.playInBackground or self.playInForeground) and serviceref == (self.playInBackground or self.playInForeground):
				data.icon = self.iconMoviePlay
			else:
				data.part = moviePlayState(pathName + '.cuts', serviceref, data.len)
				if switch == 'i':
					if data.part is not None and data.part > 0:
						data.icon = self.iconPart[data.part // 25]
					else:
						if config.usage.movielist_unseen.value:
							data.icon = self.iconUnwatched
				elif switch == 'p' or switch == 's':
					if data.part is not None and data.part > 0:
						data.partcol = 0xffc71d
					else:
						if config.usage.movielist_unseen.value:
							data.part = 100
							data.partcol = 0x206333
		len = data.len
		if len > 0:
			len = "%d:%02d" % (len / 60, len % 60)
		else:
			len = ""

		iconSize = 0

		if switch == 'i':
			iconSize = dataIconSize
			iconPosX = listBeginX
			iconPosY = ih/2-iconSize/2
			if iconPosY < iconPosX:
				iconPosY = iconPosX
			res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=data.icon))
		elif switch == 'p':
			if data.part is not None and data.part > 0:
				iconSize = progressBarSize
				iconPosX = listBeginX
				iconPosY = ih/2-iconSize/8
				if iconPosY < iconPosX:
					iconPosY = iconPosX
				res.append(MultiContentEntryProgress(pos=(iconPosX,iconPosY), size=(iconSize,iconSize/4), percent=data.part, borderWidth=2, foreColor=data.partcol, foreColorSelected=None, backColor=None, backColorSelected=None))
			else:
				iconSize = dataIconSize
				iconPosX = listBeginX
				iconPosY = ih/2-iconSize/2
				if iconPosY < iconPosX:
					iconPosY = iconPosX
				res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=data.icon))
		elif switch == 's':
			iconSize = progressIconSize
			iconPosX = listBeginX
			iconPosY = ih/2-iconSize/2
			if iconPosY < iconPosX:
				iconPosY = iconPosX
			if data.part is not None and data.part > 0:
				res.append(MultiContentEntryProgress(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), percent=data.part, borderWidth=2, foreColor=data.partcol, foreColorSelected=None, backColor=None, backColorSelected=None))
			else:
				res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=data.icon))

		begin_string = ""
		if begin > 0:
			begin_string = ', '.join(FuzzyTime(begin, inPast = True))
		dateSize = serviceSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), begin_string).width()

		if iconSize:
			textPosX = listBeginX + iconSize + listMarginX
		else:
			textPosX = listBeginX

		if ext != '0':
			getrec = info.getName(serviceref)
			fileName, fileExtension = os.path.splitext(getrec)
			desc = None
			picon = None
			service = None
			try:
				serviceHandler = eServiceCenter.getInstance()
				info = serviceHandler.info(serviceref)
				desc = info.getInfoString(serviceref, iServiceInformation.sDescription)		# get description
				ref = info.getInfoString(serviceref, iServiceInformation.sServiceref)		# get reference
				service = ServiceReference(ref).getServiceName()							# get service name
				serviceSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), service).width()
			except Exception, e:
				print('[MovieList] load extended infos get failed: ', e)
			if ext == '2':
				try:
					picon = getPiconName(ref)
					picon = loadPNG(picon)
				except Exception, e:
					print('[MovieList] load picon get failed: ', e)

			if fileExtension in RECORD_EXTENSIONS:
				if ext == '1':
					res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-serviceSize-listMarginX-listEndX, ih), font=0, flags=RT_HALIGN_LEFT|RT_VALIGN_CENTER, text=data.txt))
					res.append(MultiContentEntryText(pos=(width-serviceSize-listEndX, textPosY), size=(serviceSize, ih), font=1, flags=RT_HALIGN_RIGHT | RT_VALIGN_CENTER, text=service))
				if ext == '2':
					piconSize = ih  * 1.667
					res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font=0, flags=RT_HALIGN_LEFT|RT_VALIGN_CENTER, text=data.txt))
					res.append(MultiContentEntryPixmapAlphaTest(pos=(width-piconSize-listEndX, listEndX), size=(piconSize, ih), png=picon, flags = BT_SCALE | BT_KEEP_ASPECT_RATIO))
				res.append(MultiContentEntryText(pos=(listBeginX, ih+textPosY), size=(width-listBeginX-dateSize-listMarginX-listEndX, ih), font=1, flags=RT_HALIGN_LEFT, text=desc))
				res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, ih+textPosY), size=(dateSize, ih), font=1, flags=RT_HALIGN_RIGHT, text=begin_string))
				return res
			else:
				res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = data.txt))
				res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, ih), size=(dateSize, ih), font=1, flags=RT_HALIGN_RIGHT, text=begin_string))
				return res
		else:
			res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = data.txt))
			res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, textPosY), size=(dateSize, ih), font=1, flags=RT_HALIGN_RIGHT|RT_VALIGN_CENTER, text=begin_string))
			return res

Example 35

Project: youtube-dl-gui Source File: downloaders.py
Function: extract_data
def extract_data(stdout):
    """Extract data from youtube-dl stdout.

    Args:
        stdout (string): String that contains the youtube-dl stdout.

    Returns:
        Python dictionary. The returned dictionary can be empty if there are
        no data to extract else it may contain one or more of the
        following keys:

        'status'         : Contains the status of the download process.
        'path'           : Destination path.
        'extension'      : The file extension.
        'filename'       : The filename without the extension.
        'percent'        : The percentage of the video being downloaded.
        'eta'            : Estimated time for the completion of the download process.
        'speed'          : Download speed.
        'filesize'       : The size of the video file being downloaded.
        'playlist_index' : The playlist index of the current video file being downloaded.
        'playlist_size'  : The number of videos in the playlist.

    """
    data_dictionary = {}

    if not stdout:
        return data_dictionary

    stdout = [string for string in stdout.split(' ') if string != '']

    stdout[0] = stdout[0].lstrip('\r')

    if stdout[0] == '[download]':
        data_dictionary['status'] = 'Downloading'

        # Get path, filename & extension
        if stdout[1] == 'Destination:':
            path, fullname = os.path.split(' '.join(stdout[2:]))
            filename, extension = os.path.splitext(fullname)

            data_dictionary['path'] = path
            data_dictionary['extension'] = extension
            # remove the format that youtube-dl adds during the merging
            # process at end of the filename
            data_dictionary['filename'] = re.sub('\.f[0-9]{1,3}$', '', filename)

        # Get progress info
        if '%' in stdout[1]:
            if stdout[1] == '100%':
                data_dictionary['speed'] = ''
                data_dictionary['eta'] = ''
                data_dictionary['percent'] = '100%'
                data_dictionary['filesize'] = stdout[3]
            else:
                data_dictionary['percent'] = stdout[1]
                data_dictionary['filesize'] = stdout[3]
                data_dictionary['speed'] = stdout[5]
                data_dictionary['eta'] = stdout[7]

        # Get playlist info
        if stdout[1] == 'Downloading' and stdout[2] == 'video':
            data_dictionary['playlist_index'] = stdout[3]
            data_dictionary['playlist_size'] = stdout[5]

        # Get file already downloaded status
        if stdout[-1] == 'downloaded':
            data_dictionary['status'] = 'Already Downloaded'

        # Get filesize abort status
        if stdout[-1] == 'Aborting.':
            data_dictionary['status'] = 'Filesize Abort'

    elif stdout[0] == '[hlsnative]':
        # native hls extractor
        # see: https://github.com/rg3/youtube-dl/blob/master/youtube_dl/downloader/hls.py#L54
        data_dictionary['status'] = 'Downloading'

        if len(stdout) == 7:
            segment_no = float(stdout[6])
            current_segment = float(stdout[4])

            # Get the percentage
            percent = '{0:.1f}%'.format(current_segment / segment_no * 100)
            data_dictionary['percent'] = percent

    elif stdout[0] == '[ffmpeg]':
        data_dictionary['status'] = 'Post Processing'

        # Get final extension after merging process
        if stdout[1] == 'Merging':
            data_dictionary['extension'] = os.path.splitext(' '.join(stdout[4:])[:-1])[1]

    else:
        data_dictionary['status'] = 'Pre Processing'

    return data_dictionary

Example 36

Project: nzbToMedia Source File: transcoder.py
def buildCommands(file, newDir, movieName, bitbucket):
    if isinstance(file, str):
        inputFile = file
        if '"concat:' in file:
            file = file.split('|')[0].replace('concat:', '')
        video_details, result = getVideoDetails(file)
        dir, name = os.path.split(file)
        name, ext = os.path.splitext(name)
        check = re.match("VTS_([0-9][0-9])_[0-9]+", name)
        if check and core.CONCAT:
            name = movieName
        elif check:
            name = ('{0}.cd{1}'.format(movieName, check.groups()[0]))
        elif core.CONCAT and re.match("(.+)[cC][dD][0-9]", name):
            name = re.sub("([\ \.\-\_\=\:]+[cC][dD][0-9])", "", name)
        if ext == core.VEXTENSION and newDir == dir:  # we need to change the name to prevent overwriting itself.
            core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION)  # adds '-transcoded.ext'
    else:
        img, data = iteritems(file).next()
        name = data['name']
        video_details, result = getVideoDetails(data['files'][0], img, bitbucket)
        inputFile = '-'
        file = '-'

    newfilePath = os.path.normpath(os.path.join(newDir, name) + core.VEXTENSION)

    map_cmd = []
    video_cmd = []
    audio_cmd = []
    audio_cmd2 = []
    sub_cmd = []
    meta_cmd = []
    other_cmd = []

    if not video_details or not video_details.get(
            "streams"):  # we couldn't read streams with ffprobe. Set defaults to try transcoding.
        videoStreams = []
        audioStreams = []
        subStreams = []

        map_cmd.extend(['-map', '0'])
        if core.VCODEC:
            video_cmd.extend(['-c:v', core.VCODEC])
            if core.VCODEC == 'libx264' and core.VPRESET:
                video_cmd.extend(['-pre', core.VPRESET])
        else:
            video_cmd.extend(['-c:v', 'copy'])
        if core.VFRAMERATE:
            video_cmd.extend(['-r', str(core.VFRAMERATE)])
        if core.VBITRATE:
            video_cmd.extend(['-b:v', str(core.VBITRATE)])
        if core.VRESOLUTION:
            video_cmd.extend(['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)])
        if core.VPRESET:
            video_cmd.extend(['-preset', core.VPRESET])
        if core.VCRF:
            video_cmd.extend(['-crf', str(core.VCRF)])
        if core.VLEVEL:
            video_cmd.extend(['-level', str(core.VLEVEL)])

        if core.ACODEC:
            audio_cmd.extend(['-c:a', core.ACODEC])
            if core.ACODEC in ['aac',
                               'dts']:  # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg
                audio_cmd.extend(['-strict', '-2'])
        else:
            audio_cmd.extend(['-c:a', 'copy'])
        if core.ACHANNELS:
            audio_cmd.extend(['-ac', str(core.ACHANNELS)])
        if core.ABITRATE:
            audio_cmd.extend(['-b:a', str(core.ABITRATE)])
        if core.OUTPUTQUALITYPERCENT:
            audio_cmd.extend(['-q:a', str(core.OUTPUTQUALITYPERCENT)])

        if core.SCODEC and core.ALLOWSUBS:
            sub_cmd.extend(['-c:s', core.SCODEC])
        elif core.ALLOWSUBS:  # Not every subtitle codec can be used for every video container format!
            sub_cmd.extend(['-c:s', 'copy'])
        else:  # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options
            sub_cmd.extend(['-sn'])  # Don't copy the subtitles over

        if core.OUTPUTFASTSTART:
            other_cmd.extend(['-movflags', '+faststart'])

    else:
        videoStreams = [item for item in video_details["streams"] if item["codec_type"] == "video"]
        audioStreams = [item for item in video_details["streams"] if item["codec_type"] == "audio"]
        subStreams = [item for item in video_details["streams"] if item["codec_type"] == "subtitle"]
        if core.VEXTENSION not in ['.mkv', '.mpegts']:
            subStreams = [item for item in video_details["streams"] if
                          item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item[
                              "codec_name"] != "pgssub"]

    for video in videoStreams:
        codec = video["codec_name"]
        fr = video.get("avg_frame_rate", 0)
        width = video.get("width", 0)
        height = video.get("height", 0)
        scale = core.VRESOLUTION
        if codec in core.VCODEC_ALLOW or not core.VCODEC:
            video_cmd.extend(['-c:v', 'copy'])
        else:
            video_cmd.extend(['-c:v', core.VCODEC])
        if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001):
            video_cmd.extend(['-r', str(core.VFRAMERATE)])
        if scale:
            w_scale = width / float(scale.split(':')[0])
            h_scale = height / float(scale.split(':')[1])
            if w_scale > h_scale:  # widescreen, Scale by width only.
                scale = "{width}:{height}".format(
                    width=scale.split(':')[0],
                    height=int((height / w_scale) / 2) * 2,
                )
                if w_scale > 1:
                    video_cmd.extend(['-vf', 'scale={width}'.format(width=scale)])
            else:  # lower or matching ratio, scale by height only.
                scale = "{width}:{height}".format(
                    width=int((width / h_scale) / 2) * 2,
                    height=scale.split(':')[1],
                )
                if h_scale > 1:
                    video_cmd.extend(['-vf', 'scale={height}'.format(height=scale)])
        if core.VBITRATE:
            video_cmd.extend(['-b:v', str(core.VBITRATE)])
        if core.VPRESET:
            video_cmd.extend(['-preset', core.VPRESET])
        if core.VCRF:
            video_cmd.extend(['-crf', str(core.VCRF)])
        if core.VLEVEL:
            video_cmd.extend(['-level', str(core.VLEVEL)])
        no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v']
        if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy):
            video_cmd[1] = core.VCODEC
        if core.VCODEC == 'copy':  # force copy. therefore ignore all other video transcoding.
            video_cmd = ['-c:v', 'copy']
        map_cmd.extend(['-map', '0:{index}'.format(index=video["index"])])
        break  # Only one video needed

    used_audio = 0
    a_mapped = []
    if audioStreams:
        try:
            audio1 = [item for item in audioStreams if item["tags"]["language"] == core.ALANGUAGE]
        except:  # no language tags. Assume only 1 language.
            audio1 = audioStreams
        try:
            audio2 = [item for item in audio1 if item["codec_name"] in core.ACODEC_ALLOW]
        except:
            audio2 = []
        try:
            audio3 = [item for item in audioStreams if item["tags"]["language"] != core.ALANGUAGE]
        except:
            audio3 = []
        try:
            audio4 = [item for item in audio3 if item["codec_name"] in core.ACODEC_ALLOW]
        except:
            audio4 = []

        if audio2:  # right (or only) language and codec...
            map_cmd.extend(['-map', '0:{index}'.format(index=audio2[0]["index"])])
            a_mapped.extend([audio2[0]["index"]])
            bitrate = int(float(audio2[0].get("bit_rate", 0))) / 1000
            channels = int(float(audio2[0].get("channels", 0)))
            audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
        elif audio1:  # right (or only) language, wrong codec.
            map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])])
            a_mapped.extend([audio1[0]["index"]])
            bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000
            channels = int(float(audio1[0].get("channels", 0)))
            audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])
        elif audio4:  # wrong language, right codec.
            map_cmd.extend(['-map', '0:{index}'.format(index=audio4[0]["index"])])
            a_mapped.extend([audio4[0]["index"]])
            bitrate = int(float(audio4[0].get("bit_rate", 0))) / 1000
            channels = int(float(audio4[0].get("channels", 0)))
            audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
        elif audio3:  # wrong language, wrong codec. just pick the default audio track
            map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])])
            a_mapped.extend([audio3[0]["index"]])
            bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000
            channels = int(float(audio3[0].get("channels", 0)))
            audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])

        if core.ACHANNELS and channels and channels > core.ACHANNELS:
            audio_cmd.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS)])
            if audio_cmd[1] == 'copy':
                audio_cmd[1] = core.ACODEC
        if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1):
            audio_cmd.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE)])
            if audio_cmd[1] == 'copy':
                audio_cmd[1] = core.ACODEC
        if core.OUTPUTQUALITYPERCENT:
            audio_cmd.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
            if audio_cmd[1] == 'copy':
                audio_cmd[1] = core.ACODEC
        if audio_cmd[1] in ['aac', 'dts']:
            audio_cmd[2:2] = ['-strict', '-2']

        if core.ACODEC2_ALLOW:
            used_audio += 1
            try:
                audio5 = [item for item in audio1 if item["codec_name"] in core.ACODEC2_ALLOW]
            except:
                audio5 = []
            try:
                audio6 = [item for item in audio3 if item["codec_name"] in core.ACODEC2_ALLOW]
            except:
                audio6 = []
            if audio5:  # right language and codec.
                map_cmd.extend(['-map', '0:{index}'.format(index=audio5[0]["index"])])
                a_mapped.extend([audio5[0]["index"]])
                bitrate = int(float(audio5[0].get("bit_rate", 0))) / 1000
                channels = int(float(audio5[0].get("channels", 0)))
                audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
            elif audio1:  # right language wrong codec.
                map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])])
                a_mapped.extend([audio1[0]["index"]])
                bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000
                channels = int(float(audio1[0].get("channels", 0)))
                if core.ACODEC2:
                    audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
                else:
                    audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
            elif audio6:  # wrong language, right codec
                map_cmd.extend(['-map', '0:{index}'.format(index=audio6[0]["index"])])
                a_mapped.extend([audio6[0]["index"]])
                bitrate = int(float(audio6[0].get("bit_rate", 0))) / 1000
                channels = int(float(audio6[0].get("channels", 0)))
                audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
            elif audio3:  # wrong language, wrong codec just pick the default audio track
                map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])])
                a_mapped.extend([audio3[0]["index"]])
                bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000
                channels = int(float(audio3[0].get("channels", 0)))
                if core.ACODEC2:
                    audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
                else:
                    audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])

            if core.ACHANNELS2 and channels and channels > core.ACHANNELS2:
                audio_cmd2.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS2)])
                if audio_cmd2[1] == 'copy':
                    audio_cmd2[1] = core.ACODEC2
            if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1):
                audio_cmd2.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE2)])
                if audio_cmd2[1] == 'copy':
                    audio_cmd2[1] = core.ACODEC2
            if core.OUTPUTQUALITYPERCENT:
                audio_cmd2.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
                if audio_cmd2[1] == 'copy':
                    audio_cmd2[1] = core.ACODEC2
            if audio_cmd2[1] in ['aac', 'dts']:
                audio_cmd2[2:2] = ['-strict', '-2']
            audio_cmd.extend(audio_cmd2)

        if core.AINCLUDE and core.ACODEC3:
            for audio in audioStreams:
                if audio["index"] in a_mapped:
                    continue
                used_audio += 1
                map_cmd.extend(['-map', '0:{index}'.format(index=audio["index"])])
                audio_cmd3 = []
                bitrate = int(float(audio.get("bit_rate", 0))) / 1000
                channels = int(float(audio.get("channels", 0)))
                if audio["codec_name"] in core.ACODEC3_ALLOW:
                    audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
                else:
                    if core.ACODEC3:
                        audio_cmd3.extend(['-c:a:{0}'.format(used_audio), core.ACODEC3])
                    else:
                        audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])

                if core.ACHANNELS3 and channels and channels > core.ACHANNELS3:
                    audio_cmd3.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS3)])
                    if audio_cmd3[1] == 'copy':
                        audio_cmd3[1] = core.ACODEC3
                if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1):
                    audio_cmd3.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE3)])
                    if audio_cmd3[1] == 'copy':
                        audio_cmd3[1] = core.ACODEC3
                if core.OUTPUTQUALITYPERCENT > 0:
                    audio_cmd3.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
                    if audio_cmd3[1] == 'copy':
                        audio_cmd3[1] = core.ACODEC3
                if audio_cmd3[1] in ['aac', 'dts']:
                    audio_cmd3[2:2] = ['-strict', '-2']
                audio_cmd.extend(audio_cmd3)

    s_mapped = []
    burnt = 0
    n = 0
    for lan in core.SLANGUAGES:
        try:
            subs1 = [item for item in subStreams if item["tags"]["language"] == lan]
        except:
            subs1 = []
        if core.BURN and not subs1 and not burnt and os.path.isfile(file):
            for subfile in get_subs(file):
                if lan in os.path.split(subfile)[1]:
                    video_cmd.extend(['-vf', 'subtitles={subs}'.format(subs=subfile)])
                    burnt = 1
        for sub in subs1:
            if core.BURN and not burnt and os.path.isfile(inputFile):
                subloc = 0
                for index in range(len(subStreams)):
                    if subStreams[index]["index"] == sub["index"]:
                        subloc = index
                        break
                video_cmd.extend(['-vf', 'subtitles={sub}:si={loc}'.format(sub=inputFile, loc=subloc)])
                burnt = 1
            if not core.ALLOWSUBS:
                break
            if sub["codec_name"] in ["dvd_subtitle", "VobSub"] and core.SCODEC == "mov_text": # We can't convert these.
                continue
            map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])])
            s_mapped.extend([sub["index"]])

    if core.SINCLUDE:
        for sub in subStreams:
            if not core.ALLOWSUBS:
                break
            if sub["index"] in s_mapped:
                continue
            if sub["codec_name"] in ["dvd_subtitle", "VobSub"] and core.SCODEC == "mov_text": # We can't convert these.
                continue
            map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])])
            s_mapped.extend([sub["index"]])

    if core.OUTPUTFASTSTART:
        other_cmd.extend(['-movflags', '+faststart'])

    command = [core.FFMPEG, '-loglevel', 'warning']

    if core.HWACCEL:
        command.extend(['-hwaccel', 'auto'])
    if core.GENERALOPTS:
        command.extend(core.GENERALOPTS)

    command.extend(['-i', inputFile])

    if core.SEMBED and os.path.isfile(file):
        for subfile in get_subs(file):
            sub_details, result = getVideoDetails(subfile)
            if not sub_details or not sub_details.get("streams"):
                continue
            command.extend(['-i', subfile])
            lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0]
            metlan = None
            try:
                if len(lan) == 3:
                    metlan = Language(lan)
                if len(lan) == 2:
                    metlan = Language.fromalpha2(lan)
            except: pass
            if metlan:
                meta_cmd.extend(['-metadata:s:s:{x}'.format(x=len(s_mapped) + n),
                                 'language={lang}'.format(lang=metlan.alpha3)])
            n += 1
            map_cmd.extend(['-map', '{x}:0'.format(x=n)])

    if not core.ALLOWSUBS or (not s_mapped and not n):
        sub_cmd.extend(['-sn'])
    else:
        if core.SCODEC:
            sub_cmd.extend(['-c:s', core.SCODEC])
        else:
            sub_cmd.extend(['-c:s', 'copy'])

    command.extend(map_cmd)
    command.extend(video_cmd)
    command.extend(audio_cmd)
    command.extend(sub_cmd)
    command.extend(meta_cmd)
    command.extend(other_cmd)
    command.append(newfilePath)
    if platform.system() != 'Windows':
        command = core.NICENESS + command
    return command

Example 37

Project: mysql-utilities Source File: diskusage.py
def show_log_usage(server, datadir, options):
    """Show binary or relay log disk space usage.

    Display binary log file information if binlog turned on if log_type =
    'binary log' (default) or show relay log file information is server is
    a slave and relay log is engaged.

    server[in]        Connected server to operate against
    datadir[in]       The datadir for the server
    options[in]       Required options for operation: format, no_headers.
                      log_type

    return True or raise exception on error
    """
    log_type = options.get("log_type", "binary log")
    have_read = options.get("have_read", False)
    is_remote = options.get("is_remote", False)
    quiet = options.get("quiet", False)

    # Check privileges to execute required queries: SUPER or REPLICATION CLIENT
    user_inst = User(server, "{0}@{1}".format(server.user, server.host))
    has_super = user_inst.has_privilege("*", "*", "SUPER")
    has_rpl_client = user_inst.has_privilege("*", "*", "REPLICATION CLIENT")

    # Verify necessary permissions (access to filesystem) and privileges
    # (execute queries) to get logs usage information.
    if log_type == 'binary log':
        # Check for binlog ON first.
        res = server.show_server_variable('log_bin')
        if res and res[0][1].upper() == 'OFF':
            print("# Binary logging is turned off on the server.")
            return True
        # Check required privileges according to the access to the datadir.
        if not is_remote and have_read:
            # Requires SUPER or REPLICATION CLIENT to execute:
            # SHOW MASTER STATUS.
            if not has_super and not has_rpl_client:
                print("# {0} information not accessible. User must have the "
                      "SUPER or REPLICATION CLIENT "
                      "privilege.".format(log_type.capitalize()))
                return True
        else:
            # Requires SUPER for server < 5.6.6 or also REPLICATION CLIENT for
            # server >= 5.6.6 to execute: SHOW BINARY LOGS.
            if (server.check_version_compat(5, 6, 6)
               and not has_super and not has_rpl_client):
                print("# {0} information not accessible. User must have the "
                      "SUPER or REPLICATION CLIENT "
                      "privilege.".format(log_type.capitalize()))
                return True
            elif not has_super:
                print("# {0} information not accessible. User must have the "
                      "SUPER "
                      "privilege.".format(log_type.capitalize()))
                return True
    else:  # relay log
        # Requires SUPER or REPLICATION CLIENT to execute SHOW SLAVE STATUS.
        if not has_super and not has_rpl_client:
            print("# {0} information not accessible. User must have the "
                  "SUPER or REPLICATION CLIENT "
                  "privilege.".format(log_type.capitalize()))
            return True
        # Can only retrieve usage information from the localhost filesystem.
        if is_remote:
            print("# {0} information not accessible from a remote host."
                  "".format(log_type.capitalize()))
            return True
        elif not have_read:
            print("# {0} information not accessible. Check your permissions "
                  "to {1}.".format(log_type.capitalize(), datadir))
            return True

    # Check server status and availability of specified log file type.
    if log_type == 'binary log':
        try:
            res = server.exec_query("SHOW MASTER STATUS")
            if res:
                current_log = res[0][0]
            else:
                print("# Cannot access files - no binary log information")
                return True
        except:
            raise UtilError("Cannot get {0} information.".format(log_type))
    else:
        try:
            res = server.exec_query("SHOW SLAVE STATUS")
            if res:
                current_log = res[0][7]
            else:
                print("# Server is not an active slave - no relay log "
                      "information.")
                return True
        except:
            raise UtilError("Cannot get {0} information.".format(log_type))

    # Enough permissions and privileges, get the usage information.
    if not quiet:
        print("# {0} information:".format(log_type.capitalize()))
        print("Current {0} file = {1}".format(log_type, current_log))

    if log_type == 'binary log' and (is_remote or not have_read):
        # Retrieve binlog usage info from SHOW BINARY LOGS.
        try:
            logs = server.exec_query("SHOW BINARY LOGS")
            if logs:
                # Calculate total size.
                total = sum([int(item[1]) for item in logs])
            else:
                print("# No binary logs data available.")
                return True
        except:
            raise UtilError("Cannot get {0} information.".format(log_type))
    else:
        # Retrieve usage info from localhost filesystem.
        # Note: as of 5.6.2, users can specify location of binlog and relaylog.
        if server.check_version_compat(5, 6, 2):
            if log_type == 'binary log':
                res = server.show_server_variable("log_bin_basename")[0]
            else:
                res = server.show_server_variable("relay_log_basename")[0]
            log_path, log_prefix = os.path.split(res[1])
            # In case log_path and log_prefix are '' (not defined) set them
            # to the default value.
            if not log_path:
                log_path = datadir
            if not log_prefix:
                log_prefix = os.path.splitext(current_log)[0]
        else:
            log_path = datadir
            log_prefix = os.path.splitext(current_log)[0]

        logs, total = _build_log_list(log_path, log_prefix)

    if not logs:
        raise UtilError("The {0}s are missing.".format(log_type))

    # Print logs usage information.
    _print_logs(logs, total, options)

    return True

Example 38

Project: backupchecker Source File: generatelistforbzip2.py
Function: init
    def __init__(self, __genparams):
        '''The constructor for the GenerateListForBzip2 class'''
        __arcpath = __genparams['arcpath']
        __delimiter = __genparams['delimiter']
        self._genfull = __genparams['genfull']
        self.__listoutput = __genparams['listoutput']
        self.__confoutput = __genparams['confoutput']
        self.__fulloutput = __genparams['fulloutput']
        self.__getallhashes  = __genparams['getallhashes']
        self.__hashtype = __genparams['hashtype']
        self.__parsingexceptions = __genparams['parsingexceptions']
        self.__confname = __genparams['confname']
        __listoffiles = ['[files]\n']
        __filetype = 'f'
        __filehash = get_hash(bz2.BZ2File(__arcpath, 'r'), 'md5')
        if self.__getallhashes:
            if not self.__hashtype:
                __onelinewithhash = '{value}{delimiter} type{delimiter}{value} md5{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
            else:
                __onelinewithhash = '{value}{delimiter} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__hashtype, delimiter=__delimiter)
            __listoffiles.append(__onelinewithhash.format(
                                    os.path.split(__arcpath)[-1][:-4],
                                    __filetype,
                                    __filehash))
        else:
            if self.__parsingexceptions :
                for __file in self.__parsingexceptions:
                    if fnmatch.fnmatch(os.path.split(__arcpath)[-1][:-4], __file):
                        __filehash = get_hash(bz2.BZ2File(__arcpath, 'r'), self.__parsingexceptions[__file])
                        __onelinewithhash = '{value}{delimiter} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__parsingexceptions[__file], delimiter=__delimiter)
                        __listoffiles.append(__onelinewithhash.format(
                                                os.path.split(__arcpath)[-1][:-4],
                                                __filetype,
                                                __filehash))
                    else:
                        __onelinewithouthash = '{value}{delimiter} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
                        __listoffiles.append(__onelinewithouthash.format(
                                                os.path.split(__arcpath)[-1][:-4],
                                                __filetype))
            else:
                __onelinewithouthash = '{value}{delimiter} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
                __listoffiles.append(__onelinewithouthash.format(
                                        os.path.split(__arcpath)[-1][:-4],
                                        __filetype))

        # define the flexible file list path
        __arcwithext = os.path.split(''.join([__arcpath[:-3], 'list']))[1]
        if self.__listoutput:
            if self.__confname:
                # --gen-list and --output-list-dir and --configuration-name
                __arclistpath = os.path.join(self.__listoutput, '.'.join([self.__confname, 'list']))
            else:
                # --gen-list and --output-list-dir 
                __arclistpath = os.path.join(self.__listoutput, __arcwithext)
        elif self.__fulloutput:
            if self.__confname:
                # --gen-list and --output-conf-and-list-dir and --configuration-name
                __arclistpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'list']))
            else:
                # --gen-list and --output-conf-and-list-dir
                __arclistpath = os.path.join(self.__fulloutput, __arcwithext)
        else:
            # --gen-list only
            if self.__confname:
                __arc = os.path.dirname(__arcpath)
                __arclistpath = os.path.join(__arc, '.'.join([self.__confname, 'list']))
            else:
                __arclistpath = ''.join([__arcpath[:-3], 'list'])
            
        # call the method to write information in a file
        __listconfinfo = {'arclistpath': __arclistpath,
                            'listoffiles':  __listoffiles}
        self.__lci = __listconfinfo
        self._generate_list(__listconfinfo)
        # call the method to write the configuration file if --gen-full was required
        if self._genfull:
            # generate the hash sum of the list of files
            __listhashsum = self._get_list_hash(__listconfinfo['arclistpath'])
            # define the flexible configuration file path
            __arcwithext = os.path.split(''.join([__arcpath[:-3], 'conf']))[1]
            if self.__confoutput:
                if self.__confname:
                    # --gen-full and --output-conf-dir and --configuration-name
                    __arcconfpath = os.path.join(self.__confoutput, '.'.join([self.__confname, 'conf']))
                else:
                    # --gen-full and --output-conf-dir
                    __arcconfpath = os.path.join(self.__confoutput, __arcwithext)
            elif self.__fulloutput:
                if self.__confname:
                    # --gen-full and --output-conf-and-list-dir and --configuration-name
                    __arcconfpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'conf']))
                else:
                    # --gen-full and --output-conf-and-list-dir
                    __arcconfpath = os.path.join(self.__fulloutput, __arcwithext)
            else:
                # --gen-full only
                if self.__confname:
                    __arc = os.path.dirname(__arcpath)
                    __arcconfpath = os.path.join(__arc, '.'.join([self.__confname, 'conf']))
                else:
                    __arcconfpath = ''.join([__arcpath[:-3], 'conf'])
            # user-define name of the archive
            if self.__confname:
                __arcname =  self.__confname
            else:
                __arcname =  os.path.basename(__arcpath[:-4])
            __confinfo = {'arcname': __arcname,
                            'arcpath': __arcpath,
                            'arcconfpath': __arcconfpath,
                            'arclistpath': __listconfinfo['arclistpath'],
                            'arctype': 'archive',
                            'sha512': __listhashsum}
            self.__ci = __confinfo
            self._generate_conf(__confinfo)

Example 39

Project: yap Source File: yap_postprocess.py
def run_postprocess(
        postprocess_cmd_arr,
        file_basecount_dict,
        workflow_prov,
        err_log,
        stat_log):
    ''' 
    Prepare postprocess command with input/output paths according to sample name, 
    pass commands to yap_tee or subprocess for execution.
    '''
    if wd.regroup_output =='yes':
    	workflow_output_path = wd.workflow_output_path + "/regroup_output"
    else:
	workflow_output_path= wd.workflow_output_path
    for zz in range(0, len(postprocess_cmd_arr)):
        postprocess_tee_arr = []
        postprocess_nontee_arr = []
        initial_pipe_commands = []
        postprocess_temp_arr = []
        cmd_type = postprocess_cmd_arr[zz][0]
        cmd_meta_data = postprocess_cmd_arr[zz][1]
        postprocess_temp_arr = postprocess_cmd_arr[zz][2]
        input_file_extension = ''
	pipe1=''
	pipe2=''
        #set default input directory for postprocess stage as aligner_output
        #user can specify "postprocess_output" through  configuration file
        input_directory = "aligner_output" 
        for kk in range(0, len(cmd_meta_data)):
            input_meta_data = cmd_meta_data[kk].split(" ")
            if input_meta_data:
                if re.search('input_file_type', input_meta_data[0]) is not None:
                    input_file_extension = input_meta_data[1]
                if re.search('input_directory', input_meta_data[0]) is not None:
                    input_directory = input_meta_data[1]
        '''iterate over filename and barcode, fetch files from the source directory,
        file extensions and python glob module'''
        for filename_key in file_basecount_dict.iterkeys():
	    #fetch original input file pair for this sample
            for tmp_arr in wd.inp_files_list:
            	if tmp_arr[2] == filename_key:
			#store it in variables pipe1 and pipe2
                	pipe1 = tmp_arr[0]
                        pipe2 = tmp_arr[1]
                	break
            postprocess_input_file_arr = []
            path, file_name = os.path.split(filename_key)
            if wd.run_preprocess_analysis == "no" and wd.run_reference_alignment == "no" and wd.run_postprocess_analysis == "yes":
            	file_name, extension = os.path.splitext(file_name)
            	file_name, extension = os.path.splitext(file_name)
                file_basecount_dict[filename_key] = {
                    'no_barcode_specified': []}
            for barcode in file_basecount_dict[filename_key]:
                barcode_value = yap_tools.rename_barcode(barcode)
                aligner_dir_path = ''
                postprocess_dir_path = ''
                aligner_dir_path = workflow_output_path + "/" + file_name + "/" + barcode + "/" + input_directory
                postprocess_input = aligner_dir_path + "/" + "*" + input_file_extension + "*"
                postprocess_input_file_arr = glob.glob(postprocess_input)
            	if wd.run_preprocess_analysis == "no" and wd.run_reference_alignment == "no" and wd.run_postprocess_analysis == "yes":
            		if input_directory == "aligner_output":
                        	aligner_dir_path = path
                                postprocess_input = filename_key
                                temp_arr = glob.glob(aligner_dir_path + "/" + "*" + input_file_extension + "*")
                                if temp_arr > 0:
                                	for k in temp_arr:
                                        	if k == postprocess_input:
                                                	postprocess_input_file_arr = [postprocess_input]
                if input_file_extension == '':
                    postprocess_input_file_arr = []
                postprocess_dir_path = workflow_output_path + "/" + file_name + "/" + barcode + "/" + "postprocess_output"
                postprocess_input_file_arr.sort()
                if (len(postprocess_input_file_arr) > 0):
                    if wd.run_preprocess_analysis == "no" and wd.run_reference_alignment == "no" and wd.run_postprocess_analysis == "yes":
                    	if input_directory == "aligner_output":
                        	if postprocess_input_file_arr[0] == filename_key:
                            		pass
                        	else:
                            		break
                    for k in range(0, len(postprocess_temp_arr)):
                        postprocess_cmd = postprocess_temp_arr[k][1]
                        postprocess_cmd_name = postprocess_temp_arr[k][0]
                        if file_name == '':
                            if barcode_value != '':
                                postprocess_outfile = postprocess_dir_path + "/" + \
                                    barcode_value + "_" + postprocess_cmd_name
                            else:
                                postprocess_outfile = postprocess_dir_path + \
                                    "/" + postprocess_cmd_name
                        else:
                            if barcode_value != '':
                                postprocess_outfile = postprocess_dir_path + "/" + \
                                    file_name + "_" + barcode_value + \
                                    "_" + postprocess_cmd_name
                            else:
                                postprocess_outfile = postprocess_dir_path + \
                                    "/" + file_name + "_" + \
                                    postprocess_cmd_name
                        #replace generic keywords with appropriate file path variables
                        postprocess_cmd = postprocess_cmd.replace(
                            'input_directory', '')
                        postprocess_cmd = postprocess_cmd.replace(
                            'input_file_type' + ' ' + input_file_extension, '')
                        postprocess_cmd = postprocess_cmd.replace(
                            "aligner_output", '')
                        postprocess_cmd = postprocess_cmd.replace(
                            "postprocess_output", '')
                        postprocess_cmd = postprocess_cmd.replace(
                            'output_file', postprocess_outfile)
                        postprocess_cmd = postprocess_cmd.replace(
                            'output_directory', postprocess_dir_path)
                        postprocess_cmd = postprocess_cmd.replace(' =', '=')
                        postprocess_cmd = postprocess_cmd.replace(
                            'sample_name', file_name)
                        if re.search("file_based_input", postprocess_cmd) is not None:
                            postprocess_cmd = postprocess_cmd.replace(
                                'file_based_input',
                                postprocess_input_file_arr[0])
                            postprocess_nontee_arr = [
                                postprocess_cmd_name, postprocess_cmd]
                        elif re.search("directory_based_input", postprocess_cmd) is not None:
                            postprocess_cmd = postprocess_cmd.replace(
                                'directory_based_input', aligner_dir_path)
                            postprocess_nontee_arr = [
                                postprocess_cmd_name, postprocess_cmd]
                        else:
                            postprocess_tee_arr.append(postprocess_cmd)
                    workflow_prov.append(
                        "INPUT: " + postprocess_input_file_arr[0])
                    for kk in postprocess_tee_arr:
                        if kk != '':
                            workflow_prov.append(kk)
                    if len(postprocess_tee_arr) != 0:
		        #pass commands to yap_tee function
                        yap_tools.yap_tee(
                            initial_pipe_commands,
                            postprocess_tee_arr,
                            postprocess_input_file_arr[0],
                            err_log,
                            stat_log)
                    if len(postprocess_nontee_arr) != 0:
		        #pass commands to non_tee function which uses subproces
                        run_postprocess_nontee(
                            postprocess_nontee_arr,
                            workflow_prov,
                            err_log,
                            stat_log)
                else:
                    if file_name == '':
                        print "Warning: No aligner output for barcode = ", barcode, " ...skipping the postprocess step for command : \n", postprocess_temp_arr, "\n"
                    else:
                        print "Warning: No aligner output for filename = ", file_name, "  barcode = ", barcode, " ...skipping the postprocess step for command: \n", postprocess_temp_arr, "\n"
    return workflow_prov

Example 40

Project: bigcouch Source File: tex.py
Function: tex_emitter_core
def tex_emitter_core(target, source, env, graphics_extensions):
    """An emitter for TeX and LaTeX sources.
    For LaTeX sources we try and find the common created files that
    are needed on subsequent runs of latex to finish tables of contents,
    bibliographies, indices, lists of figures, and hyperlink references.
    """
    basename = SCons.Util.splitext(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    targetdir = os.path.split(str(target[0]))[0]
    targetbase = os.path.join(targetdir, basefile)

    basedir = os.path.split(str(source[0]))[0]
    abspath = os.path.abspath(basedir)
    target[0].attributes.path = abspath
    
    #
    # file names we will make use of in searching the sources and log file
    #
    emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
    auxfilename = targetbase + '.aux'
    logfilename = targetbase + '.log'
    flsfilename = targetbase + '.fls'

    env.SideEffect(auxfilename,target[0])
    env.SideEffect(logfilename,target[0])
    env.SideEffect(flsfilename,target[0])
    if Verbose:
        print "side effect :",auxfilename,logfilename,flsfilename
    env.Clean(target[0],auxfilename)
    env.Clean(target[0],logfilename)
    env.Clean(target[0],flsfilename)

    content = source[0].get_text_contents()

    idx_exists = os.path.exists(targetbase + '.idx')
    nlo_exists = os.path.exists(targetbase + '.nlo')
    glo_exists = os.path.exists(targetbase + '.glo')
    acr_exists = os.path.exists(targetbase + '.acn')

    # set up list with the regular expressions
    # we use to find features used
    file_tests_search = [auxfile_re,
                         makeindex_re,
                         bibliography_re,
                         tableofcontents_re,
                         listoffigures_re,
                         listoftables_re,
                         hyperref_re,
                         makenomenclature_re,
                         makeglossary_re,
                         makeglossaries_re,
                         makeacronyms_re,
                         beamer_re ]
    # set up list with the file suffixes that need emitting
    # when a feature is found
    file_tests_suff = [['.aux'],
                  ['.idx', '.ind', '.ilg'],
                  ['.bbl', '.blg'],
                  ['.toc'],
                  ['.lof'],
                  ['.lot'],
                  ['.out'],
                  ['.nlo', '.nls', '.nlg'],
                  ['.glo', '.gls', '.glg'],
                  ['.glo', '.gls', '.glg'],
                  ['.acn', '.acr', '.alg'],
                  ['.nav', '.snm', '.out', '.toc'] ]
    # build the list of lists
    file_tests = []
    for i in range(len(file_tests_search)):
        file_tests.append( [None, file_tests_suff[i]] )

    # TO-DO: need to add a way for the user to extend this list for whatever
    # auxiliary files they create in other (or their own) packages

    # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
    savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
    paths = env['ENV']['TEXINPUTS']
    if SCons.Util.is_List(paths):
        pass
    else:
        # Split at os.pathsep to convert into absolute path
        paths = paths.split(os.pathsep)

    # now that we have the path list restore the env
    if savedpath is _null:
        try:
            del env['ENV']['TEXINPUTS']
        except KeyError:
            pass # was never set
    else:
        env['ENV']['TEXINPUTS'] = savedpath
    if Verbose:
        print "search path ",paths

    aux_files = []
    file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)

    for (theSearch,suffix_list) in file_tests:
        if theSearch:
            for suffix in suffix_list:
                env.SideEffect(targetbase + suffix,target[0])
                if Verbose:
                    print "side effect :",targetbase + suffix
                env.Clean(target[0],targetbase + suffix)

    for aFile in aux_files:
        aFile_base = SCons.Util.splitext(aFile)[0]
        env.SideEffect(aFile_base + '.aux',target[0])
        if Verbose:
            print "side effect :",aFile_base + '.aux'
        env.Clean(target[0],aFile_base + '.aux')
    # read fls file to get all other files that latex creates and will read on the next pass
    # remove files from list that we explicitly dealt with above
    if os.path.exists(flsfilename):
        content = open(flsfilename, "rb").read()
        out_files = openout_re.findall(content)
        myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
        for filename in out_files[:]:
            if filename in myfiles:
                out_files.remove(filename)
        env.SideEffect(out_files,target[0])
        if Verbose:
            print "side effect :",out_files
        env.Clean(target[0],out_files)

    return (target, source)

Example 41

Project: numscons Source File: tex.py
Function: tex_emitter_core
def tex_emitter_core(target, source, env, graphics_extensions):
    """An emitter for TeX and LaTeX sources.
    For LaTeX sources we try and find the common created files that
    are needed on subsequent runs of latex to finish tables of contents,
    bibliographies, indices, lists of figures, and hyperlink references.
    """
    basename = SCons.Util.splitext(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    targetdir = os.path.split(str(target[0]))[0]
    targetbase = os.path.join(targetdir, basefile)

    basedir = os.path.split(str(source[0]))[0]
    abspath = os.path.abspath(basedir)
    target[0].attributes.path = abspath
    
    #
    # file names we will make use of in searching the sources and log file
    #
    emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
    auxfilename = targetbase + '.aux'
    logfilename = targetbase + '.log'
    flsfilename = targetbase + '.fls'

    env.SideEffect(auxfilename,target[0])
    env.SideEffect(logfilename,target[0])
    env.SideEffect(flsfilename,target[0])
    if Verbose:
        print "side effect :",auxfilename,logfilename,flsfilename
    env.Clean(target[0],auxfilename)
    env.Clean(target[0],logfilename)
    env.Clean(target[0],flsfilename)

    content = source[0].get_text_contents()

    idx_exists = os.path.exists(targetbase + '.idx')
    nlo_exists = os.path.exists(targetbase + '.nlo')
    glo_exists = os.path.exists(targetbase + '.glo')
    acr_exists = os.path.exists(targetbase + '.acn')

    # set up list with the regular expressions
    # we use to find features used
    file_tests_search = [auxfile_re,
                         makeindex_re,
                         bibliography_re,
                         tableofcontents_re,
                         listoffigures_re,
                         listoftables_re,
                         hyperref_re,
                         makenomenclature_re,
                         makeglossary_re,
                         makeglossaries_re,
                         makeacronyms_re,
                         beamer_re ]
    # set up list with the file suffixes that need emitting
    # when a feature is found
    file_tests_suff = [['.aux'],
                  ['.idx', '.ind', '.ilg'],
                  ['.bbl', '.blg'],
                  ['.toc'],
                  ['.lof'],
                  ['.lot'],
                  ['.out'],
                  ['.nlo', '.nls', '.nlg'],
                  ['.glo', '.gls', '.glg'],
                  ['.glo', '.gls', '.glg'],
                  ['.acn', '.acr', '.alg'],
                  ['.nav', '.snm', '.out', '.toc'] ]
    # build the list of lists
    file_tests = []
    for i in range(len(file_tests_search)):
        file_tests.append( [None, file_tests_suff[i]] )

    # TO-DO: need to add a way for the user to extend this list for whatever
    # auxiliary files they create in other (or their own) packages

    # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
    savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
    paths = env['ENV']['TEXINPUTS']
    if SCons.Util.is_List(paths):
        pass
    else:
        # Split at os.pathsep to convert into absolute path
        # TODO(1.5)
        #paths = paths.split(os.pathsep)
        paths = string.split(paths, os.pathsep)

    # now that we have the path list restore the env
    if savedpath is _null:
        try:
            del env['ENV']['TEXINPUTS']
        except KeyError:
            pass # was never set
    else:
        env['ENV']['TEXINPUTS'] = savedpath
    if Verbose:
        print "search path ",paths

    file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir)

    for (theSearch,suffix_list) in file_tests:
        if theSearch:
            for suffix in suffix_list:
                env.SideEffect(targetbase + suffix,target[0])
                if Verbose:
                    print "side effect :",targetbase + suffix
                env.Clean(target[0],targetbase + suffix)

    # read fls file to get all other files that latex creates and will read on the next pass
    # remove files from list that we explicitly dealt with above
    if os.path.exists(flsfilename):
        content = open(flsfilename, "rb").read()
        out_files = openout_re.findall(content)
        myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
        for filename in out_files[:]:
            if filename in myfiles:
                out_files.remove(filename)
        env.SideEffect(out_files,target[0])
        if Verbose:
            print "side effect :",out_files
        env.Clean(target[0],out_files)

    return (target, source)

Example 42

Project: scons Source File: tex.py
Function: internallatexauxaction
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
    """A builder for LaTeX files that checks the output in the aux file
    and decides how many times to use LaTeXAction, and BibTeXAction."""

    global must_rerun_latex

    # This routine is called with two actions. In this file for DVI builds
    # with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
    # set this up now for the case where the user requests a different extension
    # for the target filename
    if (XXXLaTeXAction == LaTeXAction):
       callerSuffix = ".dvi"
    else:
       callerSuffix = env['PDFSUFFIX']

    basename = SCons.Util.splitext(str(source[0]))[0]
    basedir = os.path.split(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    abspath = os.path.abspath(basedir)

    targetext = os.path.splitext(str(target[0]))[1]
    targetdir = os.path.split(str(target[0]))[0]

    saved_env = {}
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        saved_env[var] = modify_env_var(env, var, abspath)

    # Create base file names with the target directory since the auxiliary files
    # will be made there.   That's because the *COM variables have the cd
    # command in the prolog. We check
    # for the existence of files before opening them--even ones like the
    # aux file that TeX always creates--to make it possible to write tests
    # with stubs that don't necessarily generate all of the same files.

    targetbase = os.path.join(targetdir, basefile)

    # if there is a \makeindex there will be a .idx and thus
    # we have to run makeindex at least once to keep the build
    # happy even if there is no index.
    # Same for glossaries and nomenclature
    src_content = source[0].get_text_contents()
    run_makeindex = makeindex_re.search(src_content) and not os.path.exists(targetbase + '.idx')
    run_nomenclature = makenomenclature_re.search(src_content) and not os.path.exists(targetbase + '.nlo')
    run_glossary = makeglossary_re.search(src_content) and not os.path.exists(targetbase + '.glo')
    run_glossaries = makeglossaries_re.search(src_content) and not os.path.exists(targetbase + '.glo')
    run_acronyms = makeacronyms_re.search(src_content) and not os.path.exists(targetbase + '.acn')

    saved_hashes = {}
    suffix_nodes = {}

    for suffix in all_suffixes:
        theNode = env.fs.File(targetbase + suffix)
        suffix_nodes[suffix] = theNode
        saved_hashes[suffix] = theNode.get_csig()

    if Verbose:
        print "hashes: ",saved_hashes

    must_rerun_latex = True

    #
    # routine to update MD5 hash and compare
    #
    def check_MD5(filenode, suffix):
        global must_rerun_latex
        # two calls to clear old csig
        filenode.clear_memoized_values()
        filenode.ninfo = filenode.new_ninfo()
        new_md5 = filenode.get_csig()

        if saved_hashes[suffix] == new_md5:
            if Verbose:
                print "file %s not changed" % (targetbase+suffix)
            return False        # unchanged
        saved_hashes[suffix] = new_md5
        must_rerun_latex = True
        if Verbose:
            print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
        return True     # changed

    # generate the file name that latex will generate
    resultfilename = targetbase + callerSuffix

    count = 0

    while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
        result = XXXLaTeXAction(target, source, env)
        if result != 0:
            return result

        count = count + 1

        must_rerun_latex = False
        # Decide if various things need to be run, or run again.

        # Read the log file to find warnings/errors
        logfilename = targetbase + '.log'
        logContent = ''
        if os.path.exists(logfilename):
            logContent = open(logfilename, "rb").read()


        # Read the fls file to find all .aux files
        flsfilename = targetbase + '.fls'
        flsContent = ''
        auxfiles = []
        if os.path.exists(flsfilename):
            flsContent = open(flsfilename, "rb").read()
            auxfiles = openout_aux_re.findall(flsContent)
            # remove duplicates
            dups = {}
            for x in auxfiles:
                dups[x] = 1
            auxfiles = list(dups.keys())

        if Verbose:
            print "auxfiles ",auxfiles

        # Now decide if bibtex will need to be run.
        # The information that bibtex reads from the .aux file is
        # pass-independent. If we find (below) that the .bbl file is unchanged,
        # then the last latex saw a correct bibliography.
        # Therefore only do this on the first pass
        if count == 1:
            for auxfilename in auxfiles:
                target_aux = os.path.join(targetdir, auxfilename)
                if os.path.exists(target_aux):
                    content = open(target_aux, "rb").read()
                    if content.find("bibdata") != -1:
                        if Verbose:
                            print "Need to run bibtex"
                        bibfile = env.fs.File(SCons.Util.splitext(target_aux)[0])
                        result = BibTeXAction(bibfile, bibfile, env)
                        if result != 0:
                            check_file_error_message(env['BIBTEX'], 'blg')
                        must_rerun_latex = must_rerun_latex or check_MD5(suffix_nodes['.bbl'],'.bbl')

        # Now decide if latex will need to be run again due to index.
        if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex"
            idxfile = suffix_nodes['.idx']
            result = MakeIndexAction(idxfile, idxfile, env)
            if result != 0:
                check_file_error_message(env['MAKEINDEX'], 'ilg')
                return result

        # TO-DO: need to add a way for the user to extend this list for whatever
        # auxiliary files they create in other (or their own) packages
        # Harder is case is where an action needs to be called -- that should be rare (I hope?)

        for index in check_suffixes:
            check_MD5(suffix_nodes[index],index)

        # Now decide if latex will need to be run again due to nomenclature.
        if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for nomenclature"
            nclfile = suffix_nodes['.nlo']
            result = MakeNclAction(nclfile, nclfile, env)
            if result != 0:
                check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
                                         'nlg')
                #return result

        # Now decide if latex will need to be run again due to glossary.
        if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for glossary"
            glofile = suffix_nodes['.glo']
            result = MakeGlossaryAction(glofile, glofile, env)
            if result != 0:
                check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
                                         'glg')
                #return result

        # Now decide if latex will need to be run again due to acronyms.
        if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for acronyms"
            acrfile = suffix_nodes['.acn']
            result = MakeAcronymsAction(acrfile, acrfile, env)
            if result != 0:
                check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
                                         'alg')
                return result

        # Now decide if latex needs to be run yet again to resolve warnings.
        if warning_rerun_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to latex or package rerun warning"

        if rerun_citations_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to 'Rerun to get citations correct' warning"

        if undefined_references_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to undefined references or citations"

        if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
            print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop

    # rename Latex's output to what the target name is
    if not (str(target[0]) == resultfilename  and  os.path.exists(resultfilename)):
        if os.path.exists(resultfilename):
            print "move %s to %s" % (resultfilename, str(target[0]), )
            shutil.move(resultfilename,str(target[0]))

    # Original comment (when TEXPICTS was not restored):
    # The TEXPICTS enviroment variable is needed by a dvi -> pdf step
    # later on Mac OSX so leave it
    #
    # It is also used when searching for pictures (implicit dependencies).
    # Why not set the variable again in the respective builder instead
    # of leaving local modifications in the environment? What if multiple
    # latex builds in different directories need different TEXPICTS?
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        if var == 'TEXPICTS':
            continue
        if saved_env[var] is _null:
            try:
                del env['ENV'][var]
            except KeyError:
                pass # was never set
        else:
            env['ENV'][var] = saved_env[var]

    return result

Example 43

Project: VisTrails Source File: FieldDataQuery.py
Function: process_csv
    def processCSV(self):
        
        self.validateArgs()
        if self.verbose:
            self.writetolog("Starting on Field Data Query for " + os.path.split(self.csv)[1])
            self.writetolog("  using template " + os.path.split(self.template)[1])
         
        templatename = os.path.split(self.template)[1]
        if templatename == 'hdr.adf':
            templatename = os.path.split(os.path.split(self.template)[0])[1]
         
            
        csvfile = open(self.csv, "r")
        #dialect = csv.Sniffer().sniff(csvfile.read(1024))
        reader = csv.reader(csvfile)
        usedPixels = {}
        header = reader.next()
        if header[2].lower() == 'responsecount':
            self.countdata = True
        
        #Commented this out because it is causing an error
        #to be thrown by the java, uncomment out when the 
        #java has been replaced
        header.append("frequency")
        header.append("numPresence")
        header.append("pixelColumn")
        header.append("pixelRow")
        header.append(os.path.abspath(self.template))
        header.append(os.path.abspath(self.csv))
    
        #loop through each row (observation) and 
        #if that particular pixel hasn't been encountered before
        #add it to a dictionary containing a key of the pixel X,Y
        #and values of each row encountered for that pixel
        #if pixel
        lineCount = linesInFile(self.csv)
        extraPoints = []
        pointCount = 0
        pcntDone = 0
        for row in reader:
            if self.pointInTemplate(row[0], row[1]):
                pixelColumn = int(floor((float(row[0]) - self.templateParams["ulx"]) 
                                        / self.templateParams["xScale"]))
                pixelRow = int(floor((float(row[1]) - self.templateParams["uly"]) 
                                     / self.templateParams["yScale"]))
                pixel = "".join(["X:",str(pixelColumn),":Y:",str(pixelRow)])
                #if verbose == True:
                if not pixel in usedPixels:
                    usedPixels[pixel] = [row]
                    #usedPixels[pixel] = usedPixels[pixel].append(row)
                else:
                    curVal = usedPixels[pixel]
                    curVal.append(row)
                    usedPixels[pixel] = curVal
            else:
                extraPoints.append([row[0], row[1], row[2]])
            pointCount += 1
            if self.verbose:
                if float(pointCount)/lineCount > float(pcntDone)/100:
                    pcntDone += 10
                    if self.verbose:
                        print str(pcntDone) + "...",
    
        #Open up and write to an output file
        oFile = open(self.output, 'wb')
        fOut = csv.writer(oFile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        fOut.writerow(header)
    
        #Add each used pixel to the output file
        for k in usedPixels:
            v = usedPixels[k]
            outputLine = v[0]
    
            pixelColumn = int(k.rsplit(':')[1])
            pixelRow = int(k.rsplit(':')[3])
            outPixelX = (self.templateParams["ulx"] + (self.templateParams["xScale"] * pixelColumn) + 
                                    self.templateParams["xScale"]/2)
            outPixelY = (self.templateParams["uly"] + (self.templateParams["yScale"] * pixelRow) + 
                                    self.templateParams["yScale"]/2)
            frequency = len(v)
    
            #loop though the 'points' in each pixel
            #for count data the value is the sum of the 'points'
            #    if there were any 'hits' not equal to 0
            #    otherwise if there were any absences the value is 0
            #    what's left is background points
            #for presense/absense data
            #    The value is 1 if there were any 1s in our points
            #    Or 0 if there were any zeros (absenses)
            #    Otherwise it's a background pixel.
            total = 0
            numAbsense = 0
            for i in range (frequency):
                if int(float(v[i][2])) > 0:
                    total += int(float(v[i][2]))
                if int(float(v[i][2])) == 0:
                    numAbsense += 1
            
            outputLine[0] = outPixelX
            outputLine[1] = outPixelY
            
            if self.countdata and total > 0:
                outputLine[2] = total
            elif total > 0:
                outputLine[2] = 1
            elif numAbsense > 0:
                outputLine[2] = 0
            else:
                outputLine[2] = -9999
                
            outputLine.append(frequency)
            outputLine.append(total)
            outputLine.append(pixelColumn)
            outputLine.append(pixelRow)
            
    
            fOut.writerow(outputLine)
        oFile.close
        if self.verbose:
            self.writetolog("Done\nFinished creating field data query output.\n")
            if len(extraPoints) > 0:
                self.writetolog ("  WARNING: " + str(len(extraPoints)) + " points" +
                    " out of " + str(pointCount) + " total points in the " +
                    "original CSV were outside the template extent and WERE NOT " +
                    "INCLUDED IN THE FDQ OUTPUT.")
            else:
                pass

Example 44

Project: openwrt-mt7620 Source File: tex.py
Function: internallatexauxaction
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
    """A builder for LaTeX files that checks the output in the aux file
    and decides how many times to use LaTeXAction, and BibTeXAction."""

    global must_rerun_latex

    # This routine is called with two actions. In this file for DVI builds
    # with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
    # set this up now for the case where the user requests a different extension
    # for the target filename
    if (XXXLaTeXAction == LaTeXAction):
       callerSuffix = ".dvi"
    else:
       callerSuffix = env['PDFSUFFIX']

    basename = SCons.Util.splitext(str(source[0]))[0]
    basedir = os.path.split(str(source[0]))[0]
    basefile = os.path.split(str(basename))[1]
    abspath = os.path.abspath(basedir)

    targetext = os.path.splitext(str(target[0]))[1]
    targetdir = os.path.split(str(target[0]))[0]

    saved_env = {}
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        saved_env[var] = modify_env_var(env, var, abspath)

    # Create base file names with the target directory since the auxiliary files
    # will be made there.   That's because the *COM variables have the cd
    # command in the prolog. We check
    # for the existence of files before opening them--even ones like the
    # aux file that TeX always creates--to make it possible to write tests
    # with stubs that don't necessarily generate all of the same files.

    targetbase = os.path.join(targetdir, basefile)

    # if there is a \makeindex there will be a .idx and thus
    # we have to run makeindex at least once to keep the build
    # happy even if there is no index.
    # Same for glossaries, nomenclature, and acronyms
    src_content = source[0].get_text_contents()
    run_makeindex = makeindex_re.search(src_content) and not os.path.isfile(targetbase + '.idx')
    run_nomenclature = makenomenclature_re.search(src_content) and not os.path.isfile(targetbase + '.nlo')
    run_glossary = makeglossary_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
    run_glossaries = makeglossaries_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
    run_acronyms = makeacronyms_re.search(src_content) and not os.path.isfile(targetbase + '.acn')

    saved_hashes = {}
    suffix_nodes = {}

    for suffix in all_suffixes:
        theNode = env.fs.File(targetbase + suffix)
        suffix_nodes[suffix] = theNode
        saved_hashes[suffix] = theNode.get_csig()

    if Verbose:
        print "hashes: ",saved_hashes

    must_rerun_latex = True

    #
    # routine to update MD5 hash and compare
    #
    def check_MD5(filenode, suffix):
        global must_rerun_latex
        # two calls to clear old csig
        filenode.clear_memoized_values()
        filenode.ninfo = filenode.new_ninfo()
        new_md5 = filenode.get_csig()

        if saved_hashes[suffix] == new_md5:
            if Verbose:
                print "file %s not changed" % (targetbase+suffix)
            return False        # unchanged
        saved_hashes[suffix] = new_md5
        must_rerun_latex = True
        if Verbose:
            print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
        return True     # changed

    # generate the file name that latex will generate
    resultfilename = targetbase + callerSuffix

    count = 0

    while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
        result = XXXLaTeXAction(target, source, env)
        if result != 0:
            return result

        count = count + 1

        must_rerun_latex = False
        # Decide if various things need to be run, or run again.

        # Read the log file to find warnings/errors
        logfilename = targetbase + '.log'
        logContent = ''
        if os.path.isfile(logfilename):
            logContent = open(logfilename, "rb").read()


        # Read the fls file to find all .aux files
        flsfilename = targetbase + '.fls'
        flsContent = ''
        auxfiles = []
        if os.path.isfile(flsfilename):
            flsContent = open(flsfilename, "rb").read()
            auxfiles = openout_aux_re.findall(flsContent)
            # remove duplicates
            dups = {}
            for x in auxfiles:
                dups[x] = 1
            auxfiles = list(dups.keys())

        if Verbose:
            print "auxfiles ",auxfiles

        # Now decide if bibtex will need to be run.
        # The information that bibtex reads from the .aux file is
        # pass-independent. If we find (below) that the .bbl file is unchanged,
        # then the last latex saw a correct bibliography.
        # Therefore only do this on the first pass
        if count == 1:
            for auxfilename in auxfiles:
                target_aux = os.path.join(targetdir, auxfilename)
                if os.path.isfile(target_aux):
                    content = open(target_aux, "rb").read()
                    if content.find("bibdata") != -1:
                        if Verbose:
                            print "Need to run bibtex"
                        bibfile = env.fs.File(SCons.Util.splitext(target_aux)[0])
                        result = BibTeXAction(bibfile, bibfile, env)
                        if result != 0:
                            check_file_error_message(env['BIBTEX'], 'blg')
                        must_rerun_latex = must_rerun_latex or check_MD5(suffix_nodes['.bbl'],'.bbl')

        # Now decide if latex will need to be run again due to index.
        if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex"
            idxfile = suffix_nodes['.idx']
            result = MakeIndexAction(idxfile, idxfile, env)
            if result != 0:
                check_file_error_message(env['MAKEINDEX'], 'ilg')
                return result

        # TO-DO: need to add a way for the user to extend this list for whatever
        # auxiliary files they create in other (or their own) packages
        # Harder is case is where an action needs to be called -- that should be rare (I hope?)

        for index in check_suffixes:
            check_MD5(suffix_nodes[index],index)

        # Now decide if latex will need to be run again due to nomenclature.
        if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for nomenclature"
            nclfile = suffix_nodes['.nlo']
            result = MakeNclAction(nclfile, nclfile, env)
            if result != 0:
                check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
                                         'nlg')
                #return result

        # Now decide if latex will need to be run again due to glossary.
        if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for glossary"
            glofile = suffix_nodes['.glo']
            result = MakeGlossaryAction(glofile, glofile, env)
            if result != 0:
                check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
                                         'glg')
                #return result

        # Now decide if latex will need to be run again due to acronyms.
        if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
            # We must run makeindex
            if Verbose:
                print "Need to run makeindex for acronyms"
            acrfile = suffix_nodes['.acn']
            result = MakeAcronymsAction(acrfile, acrfile, env)
            if result != 0:
                check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
                                         'alg')
                return result

        # Now decide if latex needs to be run yet again to resolve warnings.
        if warning_rerun_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to latex or package rerun warning"

        if rerun_citations_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to 'Rerun to get citations correct' warning"

        if undefined_references_re.search(logContent):
            must_rerun_latex = True
            if Verbose:
                print "rerun Latex due to undefined references or citations"

        if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
            print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop

    # rename Latex's output to what the target name is
    if not (str(target[0]) == resultfilename  and  os.path.isfile(resultfilename)):
        if os.path.isfile(resultfilename):
            print "move %s to %s" % (resultfilename, str(target[0]), )
            shutil.move(resultfilename,str(target[0]))

    # Original comment (when TEXPICTS was not restored):
    # The TEXPICTS enviroment variable is needed by a dvi -> pdf step
    # later on Mac OSX so leave it
    #
    # It is also used when searching for pictures (implicit dependencies).
    # Why not set the variable again in the respective builder instead
    # of leaving local modifications in the environment? What if multiple
    # latex builds in different directories need different TEXPICTS?
    for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
        if var == 'TEXPICTS':
            continue
        if saved_env[var] is _null:
            try:
                del env['ENV'][var]
            except KeyError:
                pass # was never set
        else:
            env['ENV'][var] = saved_env[var]

    return result

Example 45

Project: pychess Source File: which.py
Function: which_files
def which_files(file, mode=os.F_OK | os.X_OK, path=None, pathext=None):
    """ Generate full paths, where the file*is accesible under mode
        and is located in the directory passed as a part of the file name,
        or in any directory on path if a base file name is passed.

        The mode matches an existing executable file by default.

        The path defaults to the PATH environment variable,
        or to os.defpath if the PATH variable is not set.
        On Windows, a current directory is searched before directories in the PATH variable,
        but not before directories in an explicitly passed path string or iterable.

        The pathext is used to match files with any of the extensions appended to file.
        On Windows, it defaults to the ``PATHEXT`` environment variable.
        If the PATHEXT variable is not set, then the default pathext value is hardcoded
        for different Windows versions, to match the actual search performed on command execution.

        On Windows <= 4.x, ie. NT and older, it defaults to '.COM;.EXE;.BAT;.CMD'.
        On Windows 5.x, ie. 2k/XP/2003, the extensions '.VBS;.VBE;.JS;.JSE;.WSF;.WSH' are appended,
        On Windows >= 6.x, ie. Vista/2008/7, the extension '.MSC' is further appended.
        The actual search on command execution may differ under Wine,
        which may use a different default value, that is not treated specially here.

        In each directory, the file is first searched without any additional extension,
        even when a pathext string or iterable is explicitly passed.

        >>> def test(expected, *args, **argd):
        ...     result = list(which_files(*args, **argd))
        ...     assert result == expected, 'which_files: %s != %s' % (result, expected)
        ...
        ...     try:
        ...         result = [ which(*args, **argd) ]
        ...     except IOError:
        ...         result = []
        ...     assert result[:1] == expected[:1], 'which: %s != %s' % (result[:1], expected[:1])

        >>> ### Set up
        >>> import stat, tempfile
        >>> dir = tempfile.mkdtemp(prefix='test-')
        >>> ext = '.ext'
        >>> tmp = tempfile.NamedTemporaryFile(prefix='command-', suffix=ext, dir=dir)
        >>> name = tmp.name
        >>> file = os.path.basename(name)
        >>> here = os.path.join(os.curdir, file)
        >>> nonexistent = '%s-nonexistent' % name
        >>> path = os.pathsep.join([ nonexistent, name,  dir, dir ])
        ... # Test also that duplicates are removed, and non-existent objects
        ... # or non-directories in path do not trigger any exceptions.

        >>> ### Test permissions
        >>> test(_windows and [name] or [], file, path=path)
        >>> test(_windows and [name] or [], file, mode=os.X_OK, path=path)
        ... # executable flag is not needed on Windows

        >>> test([name], file, mode=os.F_OK, path=path)
        >>> test([name], file, mode=os.R_OK, path=path)
        >>> test([name], file, mode=os.W_OK, path=path)
        >>> test([name], file, mode=os.R_OK|os.W_OK, path=path)

        >>> os.chmod(name, stat.S_IRWXU)
        >>> test([name], file, mode=os.R_OK|os.W_OK|os.X_OK, path=path)

        >>> ### Test paths
        >>> _save_path = os.environ.get('PATH', '')
        >>> cwd = os.getcwd()

        >>> test([], file, path='')
        >>> test([], file, path=nonexistent)
        >>> test([], nonexistent, path=path)
        >>> test([name], file, path=path)
        >>> test([name], name, path=path)
        >>> test([name], name, path='')
        >>> test([name], name, path=nonexistent)

        >>> os.chdir(dir)
        >>> test([name], file, path=path)
        >>> test([here], file, path=os.curdir)
        >>> test([name], name, path=os.curdir)
        >>> test([], file, path='')
        >>> test([], file, path=nonexistent)

        >>> os.environ['PATH'] = path
        >>> test(_windows and [here] or [name], file)
        ... # current directory is always searched first on Windows
        >>> os.environ['PATH'] = os.curdir
        >>> test([here], file)
        >>> test([name], name)
        >>> os.environ['PATH'] = ''
        >>> test(_windows and [here] or [], file)
        >>> os.environ['PATH'] = nonexistent
        >>> test(_windows and [here] or [], file)

        >>> os.chdir(cwd)
        >>> os.environ['PATH'] = path
        >>> test([name], file)
        >>> os.environ['PATH'] = _save_path

        >>> ### Test extensions
        >>> test([], file[:-4], path=path, pathext='')
        >>> test([], file[:-4], path=path, pathext=nonexistent)
        >>> test([name], file[:-4], path=path, pathext=ext)

        >>> test([name], file, path=path, pathext=ext)
        >>> test([name], file, path=path, pathext='')
        >>> test([name], file, path=path, pathext=nonexistent)

        >>> ### Tear down
        >>> tmp.close()
        >>> os.rmdir(dir)

    """
    filepath, file = os.path.split(file)

    if filepath:
        path = (filepath, )
    elif path is None:
        path = os.environ.get('PATH', os.defpath).split(os.pathsep)
        if _windows and os.curdir not in path:
            path.insert(
                0, os.curdir
            )  # current directory is always searched first on Windows
    elif isinstance(path, basestring):
        path = path.split(os.pathsep)

    if pathext is None:
        pathext = ['']
        if _windows:
            pathext += (os.environ.get('PATHEXT', '') or
                        _getwinpathext()).lower().split(os.pathsep)
    elif isinstance(pathext, basestring):
        pathext = pathext.split(os.pathsep)

    if '' not in pathext:
        pathext.insert(
            0, ''
        )  # always check command without extension, even for an explicitly passed pathext

    seen = set()
    for dir in path:
        if dir:  # only non-empty directories are searched
            id = os.path.normcase(os.path.abspath(dir))
            if id not in seen:  # each directory is searched only once
                seen.add(id)
                woex = os.path.join(dir, file)
                for ext in pathext:
                    name = woex + ext
                    if os.path.isfile(name) and os.access(name, mode):
                        return name
#                        yield name
    return None

Example 46

Project: OwnTube Source File: btmakemetafile.py

def makeinfo(file, piece_length, encoding, flag, progress, progress_percent=1, gethash = None):
    if gethash is None:
        gethash = {}
    
    if not 'md5' in gethash:
        gethash['md5'] = False
    if not 'crc32' in gethash:
        gethash['crc32'] = False
    if not 'sha1' in gethash:
        gethash['sha1'] = False
        
    file = abspath(file)
    if isdir(file):
        subs = subfiles(file)
        subs.sort()
        pieces = []
        sh = sha()
        done = 0L
        fs = []
        totalsize = 0.0
        totalhashed = 0L
        for p, f in subs:
            totalsize += getsize(f)

        for p, f in subs:
            pos = 0L
            size = getsize(f)
            h = open(f, 'rb')

            if gethash['md5']:
                hash_md5 = md5.new()
            if gethash['sha1']:
                hash_sha1 = sha()
            if gethash['crc32']:
                hash_crc32 = zlib.crc32('')
            
            while pos < size:
                a = min(size - pos, piece_length - done)
                
                readpiece = h.read(a)

                # See if the user cancelled
                if flag.isSet():
                    return
                
                sh.update(readpiece)

                # See if the user cancelled
                if flag.isSet():
                    return

                if gethash['md5']:                
                    # Update MD5
                    hash_md5.update(readpiece)
    
                    # See if the user cancelled
                    if flag.isSet():
                        return

                if gethash['crc32']:                
                    # Update CRC32
                    hash_crc32 = zlib.crc32(readpiece, hash_crc32)
    
                    # See if the user cancelled
                    if flag.isSet():
                        return
                
                if gethash['sha1']:                
                    # Update SHA1
                    hash_sha1.update(readpiece)
    
                    # See if the user cancelled
                    if flag.isSet():
                        return
                
                done += a
                pos += a
                totalhashed += a
                
                if done == piece_length:
                    pieces.append(sh.digest())
                    done = 0
                    sh = sha()
                if progress_percent:
                    progress(totalhashed / totalsize)
                else:
                    progress(a)
                    
            newdict = {'length': size,
                       'path': uniconvertl(p, encoding) }
            if gethash['md5']:
                newdict['md5sum'] = hash_md5.hexdigest()
            if gethash['crc32']:
                newdict['crc32'] = "%08X" % hash_crc32
            if gethash['sha1']:
                newdict['sha1'] = hash_sha1.digest()
                    
            fs.append(newdict)
                    
            h.close()
        if done > 0:
            pieces.append(sh.digest())
        return {'pieces': ''.join(pieces), 
                'piece length': piece_length,
                'files': fs, 
                'name': uniconvert(split(file)[1], encoding) }
    else:
        size = getsize(file)
        pieces = []
        p = 0L
        h = open(file, 'rb')
        
        if gethash['md5']:
            hash_md5 = md5.new()
        if gethash['crc32']:
            hash_crc32 = zlib.crc32('')
        if gethash['sha1']:
            hash_sha1 = sha()
        
        while p < size:
            x = h.read(min(piece_length, size - p))

            # See if the user cancelled
            if flag.isSet():
                return
            
            if gethash['md5']:
                # Update MD5
                hash_md5.update(x)
    
                # See if the user cancelled
                if flag.isSet():
                    return
            
            if gethash['crc32']:
                # Update CRC32
                hash_crc32 = zlib.crc32(x, hash_crc32)
    
                # See if the user cancelled
                if flag.isSet():
                    return
            
            if gethash['sha1']:
                # Update SHA-1
                hash_sha1.update(x)
    
                # See if the user cancelled
                if flag.isSet():
                    return
                
            pieces.append(sha(x).digest())

            # See if the user cancelled
            if flag.isSet():
                return

            p += piece_length
            if p > size:
                p = size
            if progress_percent:
                progress(float(p) / size)
            else:
                progress(min(piece_length, size - p))
        h.close()
        newdict = {'pieces': ''.join(pieces), 
                   'piece length': piece_length,
                   'length': size, 
                   'name': uniconvert(split(file)[1], encoding) }
        if gethash['md5']:
            newdict['md5sum'] = hash_md5.hexdigest()
        if gethash['crc32']:
            newdict['crc32'] = "%08X" % hash_crc32
        if gethash['sha1']:
            newdict['sha1'] = hash_sha1.digest()
                   

Example 47

Project: RMG-Py Source File: input.py
def saveInputFile(path, rmg):
    """
    Save an RMG input file at `path` on disk from the :class:`RMG` object 
    `rmg`.
    """

    f = open(path, 'w')

    # Databases
    f.write('database(\n')
    #f.write('    "{0}",\n'.format(rmg.databaseDirectory))
    f.write('    thermoLibraries = {0!r},\n'.format(rmg.thermoLibraries))
    f.write('    reactionLibraries = {0!r},\n'.format(rmg.reactionLibraries))
    f.write('    seedMechanisms = {0!r},\n'.format(rmg.seedMechanisms))
    f.write('    kineticsDepositories = {0!r},\n'.format(rmg.kineticsDepositories))
    f.write('    kineticsFamilies = {0!r},\n'.format(rmg.kineticsFamilies))
    f.write('    kineticsEstimator = {0!r},\n'.format(rmg.kineticsEstimator))
    f.write(')\n\n')

    # Species
    for species in rmg.initialSpecies:
        f.write('species(\n')
        f.write('    label = "{0}",\n'.format(species.label))
        f.write('    reactive = {0},\n'.format(species.reactive))
        f.write('    structure = adjacencyList(\n')
        f.write('"""\n')
        f.write(species.molecule[0].toAdjacencyList())
        f.write('"""),\n')
        f.write(')\n\n')
    
    # Reaction systems
    for system in rmg.reactionSystems:
        if rmg.solvent:
            f.write('liquidReactor(\n')
            f.write('    temperature = ({0:g},"{1!s}"),\n'.format(system.T.getValue(),system.T.units))
            f.write('    initialConcentrations={\n')
            for species, conc in system.initialConcentrations.iteritems():
                f.write('        "{0!s}": ({1:g},"{2!s}"),\n'.format(species.label,conc.getValue(),conc.units))
        else:
            f.write('simpleReactor(\n')
            f.write('    temperature = ({0:g},"{1!s}"),\n'.format(system.T.getValue(),system.T.units))
            # Convert the pressure from SI pascal units to bar here
            # Do something more fancy later for converting to user's desired units for both T and P..
            f.write('    pressure = ({0:g},"{1!s}"),\n'.format(system.P.getValue(),system.P.units))
            f.write('    initialMoleFractions={\n')
            for species, molfrac in system.initialMoleFractions.iteritems():
                f.write('        "{0!s}": {1:g},\n'.format(species.label, molfrac))
        f.write('    },\n')               
        
        # Termination criteria
        conversions = ''
        for term in system.termination:
            if isinstance(term, TerminationTime):
                f.write('    terminationTime = ({0:g},"{1!s}"),\n'.format(term.time.getValue(),term.time.units))
                
            else:
                conversions += '        "{0:s}": {1:g},\n'.format(term.species.label, term.conversion)
        if conversions:        
            f.write('    terminationConversion = {\n')
            f.write(conversions)
            f.write('    },\n')
        
        # Sensitivity analysis
        if system.sensitiveSpecies:
            sensitivity = []
            for item in system.sensitiveSpecies:
                sensitivity.append(item.label)
            f.write('    sensitivity = {0},\n'.format(sensitivity))
            f.write('    sensitivityThreshold = {0},\n'.format(system.sensitivityThreshold))
        
        f.write(')\n\n')
    
    if rmg.solvent:
        f.write("solvation(\n    solvent = '{0!s}'\n)\n\n".format(rmg.solvent))
        
    # Simulator tolerances
    f.write('simulator(\n')
    f.write('    atol = {0:g},\n'.format(rmg.absoluteTolerance))
    f.write('    rtol = {0:g},\n'.format(rmg.relativeTolerance))
    f.write('    sens_atol = {0:g},\n'.format(rmg.sensitivityAbsoluteTolerance))
    f.write('    sens_rtol = {0:g},\n'.format(rmg.sensitivityRelativeTolerance))
    f.write(')\n\n')

    # Model
    f.write('model(\n')
    f.write('    toleranceMoveToCore = {0:g},\n'.format(rmg.fluxToleranceMoveToCore))
    f.write('    toleranceKeepInEdge = {0:g},\n'.format(rmg.fluxToleranceKeepInEdge))
    f.write('    toleranceInterruptSimulation = {0:g},\n'.format(rmg.fluxToleranceInterrupt))
    f.write('    maximumEdgeSpecies = {0:d},\n'.format(rmg.maximumEdgeSpecies))
    f.write('    minCoreSizeForPrune = {0:d},\n'.format(rmg.minCoreSizeForPrune))
    f.write('    minSpeciesExistIterationsForPrune = {0:d},\n'.format(rmg.minSpeciesExistIterationsForPrune))
    f.write('    filterReactions = {0:d},\n'.format(rmg.filterReactions))
    f.write(')\n\n')

    # Pressure Dependence
    if rmg.pressureDependence:
        f.write('pressureDependence(\n')
        f.write('    method = {0!r},\n'.format(rmg.pressureDependence.method))
        f.write('    maximumGrainSize = ({0:g},"{1!s}"),\n'.format(rmg.pressureDependence.grainSize.getValue(),rmg.pressureDependence.grainSize.units))
        f.write('    minimumNumberOfGrains = {0},\n'.format(rmg.pressureDependence.grainCount))
        f.write('    temperatures = ({0:g},{1:g},"{2!s}",{3:d}),\n'.format(
            rmg.pressureDependence.Tmin.getValue(),
            rmg.pressureDependence.Tmax.getValue(),
            rmg.pressureDependence.Tmax.units,
            rmg.pressureDependence.Tcount,
        ))
        f.write('    pressures = ({0:g},{1:g},"{2!s}",{3:d}),\n'.format(
            rmg.pressureDependence.Pmin.getValue(),
            rmg.pressureDependence.Pmax.getValue(),
            rmg.pressureDependence.Pmax.units,
            rmg.pressureDependence.Pcount,
        ))
        f.write('    interpolation = {0},\n'.format(rmg.pressureDependence.interpolationModel))     
        f.write('    maximumAtoms = {0}, \n'.format(rmg.pressureDependence.maximumAtoms))
        f.write(')\n\n')
    
    # Quantum Mechanics
    if rmg.quantumMechanics:
        f.write('quantumMechanics(\n')
        f.write('    software = {0!r},\n'.format(rmg.quantumMechanics.settings.software))
        f.write('    method = {0!r},\n'.format(rmg.quantumMechanics.settings.method))
        # Split paths created by QMSettings
        if rmg.quantumMechanics.settings.fileStore:
            f.write('    fileStore = {0!r},\n'.format(os.path.split(rmg.quantumMechanics.settings.fileStore)[0]))
        else:
            f.write('    fileStore = None,\n')
        if rmg.quantumMechanics.settings.scratchDirectory:
            f.write('    scratchDirectory = {0!r},\n'.format(os.path.split(rmg.quantumMechanics.settings.scratchDirectory)[0]))
        else:
            f.write('    scratchDirectory = None,\n')
        f.write('    onlyCyclics = {0},\n'.format(rmg.quantumMechanics.settings.onlyCyclics))
        f.write('    maxRadicalNumber = {0},\n'.format(rmg.quantumMechanics.settings.maxRadicalNumber))
        f.write(')\n\n')
    
    # Species Constraints
    if rmg.speciesConstraints:
        f.write('generatedSpeciesConstraints(\n')
        for constraint, value in sorted(rmg.speciesConstraints.items(), key=lambda constraint: constraint[0]):
            if value is not None: f.write('    {0} = {1},\n'.format(constraint,value))
        f.write(')\n\n')
    
    # Options
    f.write('options(\n')
    f.write('    units = "{0}",\n'.format(rmg.units))
    if rmg.saveRestartPeriod:
        f.write('    saveRestartPeriod = ({0},"{1}"),\n'.format(rmg.saveRestartPeriod.getValue(), rmg.saveRestartPeriod.units))
    else:
        f.write('    saveRestartPeriod = None,\n')
    f.write('    generateOutputHTML = {0},\n'.format(rmg.generateOutputHTML))
    f.write('    generatePlots = {0},\n'.format(rmg.generatePlots))
    f.write('    saveSimulationProfiles = {0},\n'.format(rmg.saveSimulationProfiles))
    f.write('    saveEdgeSpecies = {0},\n'.format(rmg.saveEdgeSpecies))
    f.write('    keepIrreversible = {0},\n'.format(rmg.keepIrreversible))
    f.write('    verboseComments = {0},\n'.format(rmg.verboseComments))
    f.write('    wallTime = {0},\n'.format(rmg.wallTime))
    f.write(')\n\n')
    
    f.close()

Example 48

Project: picrust Source File: predict_metagenomes.py
def main():
    option_parser, opts, args =\
       parse_command_line_parameters(**script_info)

    if opts.verbose:
        print "Loading OTU table: ",opts.input_otu_table

    otu_table = load_table(opts.input_otu_table)
    ids_to_load = otu_table.ids(axis='observation').tolist()

    if opts.verbose:
        print "Done loading OTU table containing %i samples and %i OTUs." \
          %(len(otu_table.ids()),len(otu_table.ids(axis='observation')))

    #Hardcoded loaction of the precalculated datasets for PICRUSt,
    #relative to the project directory
    precalc_data_dir=join(get_picrust_project_dir(),'picrust','data')

    # Load a table of gene counts by OTUs.
    #This can be either user-specified or precalculated
    genome_table_fp = determine_data_table_fp(precalc_data_dir,\
      opts.type_of_prediction,opts.gg_version,\
      user_specified_table=opts.input_count_table,verbose=opts.verbose)

    if opts.verbose:
        print "Loading gene count data from file: %s" %genome_table_fp

    genome_table= load_data_table(genome_table_fp,\
      load_data_table_in_biom=opts.load_precalc_file_in_biom,\
      suppress_subset_loading=opts.suppress_subset_loading,\
      ids_to_load=ids_to_load,verbose=opts.verbose,transpose=True)

    if opts.verbose:
        print "Loaded %i genes across %i OTUs from gene count table" \
          %(len(genome_table.ids(axis='observation')),len(genome_table.ids()))

    if opts.with_confidence:
        if opts.input_variance_table:
            variance_table_fp = opts.input_variance_table
        else:
            variance_table_fp = determine_data_table_fp(precalc_data_dir,\
              opts.type_of_prediction,opts.gg_version,\
              precalc_file_suffix='precalculated_variances.tab.gz',\
              user_specified_table=opts.input_count_table)

        if opts.verbose:
            print "Loading variance information from table: %s" \
            %variance_table_fp

        variance_table= load_data_table(variance_table_fp,\
          load_data_table_in_biom=opts.load_precalc_file_in_biom,\
          suppress_subset_loading=opts.suppress_subset_loading,\
          ids_to_load=ids_to_load,transpose=True)

        if opts.verbose:
            print "Loaded %i genes across %i OTUs from variance table" \
              %(len(variance_table.ids(axis='observation')),len(variance_table.ids()))
        #Raise an error if the genome table and variance table differ
        #in the genomes they contain.
        #better to find out now than have something obscure happen latter on
        if opts.verbose:
            print "Checking that genome table and variance table are consistent"
        try:
            assert set(variance_table.ids(axis='observation')) == set(genome_table.ids(axis='observation'))
        except AssertionError,e:
            for var_id in variance_table.ids(axis='observation'):
                if var_id not in genome_table.ids(axis='observation'):
                    print "Variance table ObsId %s not in genome_table ObsIds" %var_id
            raise AssertionError("Variance table and genome table contain different gene ids")
        try:
            assert set(variance_table.ids()) == set(genome_table.ids())
        except AssertionError,e:
            for var_id in variance_table.ids():
                if var_id not in genome_table.ids():
                    print "Variance table SampleId %s not in genome_table SampleIds" %var_id
            raise AssertionError("Variance table and genome table contain different OTU ids")

        #sort the ObservationIds and SampleIds to be in the same order
        variance_table=variance_table.sort_order(genome_table.ids(axis='observation'), axis='observation')
        variance_table=variance_table.sort_order(genome_table.ids(), axis='sample')

    make_output_dir_for_file(opts.output_metagenome_table)

    if opts.accuracy_metrics:
        # Calculate accuracy metrics
        weighted_nsti = calc_nsti(otu_table,genome_table,weighted=True)
        samples= weighted_nsti[0]
        nstis = list(weighted_nsti[1])
        samples_and_nstis = zip(samples,nstis)
        if opts.verbose:
            print "Writing NSTI information to file:", opts.accuracy_metrics
        accuracy_output_fh = open(opts.accuracy_metrics,'w')
        accuracy_output_fh.write("#Sample\tMetric\tValue\n")
        for sample,nsti in samples_and_nstis:
            line = "%s\tWeighted NSTI\t%s\n" %(sample,str(nsti))
            accuracy_output_fh.write(line)

    if opts.with_confidence:
        #If we are calculating variance, we get the prediction as part
        #of the process

        if opts.verbose:
            print "Predicting the metagenome, metagenome variance and confidence intervals for the metagenome..."

        predicted_metagenomes,predicted_metagenome_variances,\
        predicted_metagenomes_lower_CI_95,predicted_metagenomes_upper_CI_95=\
          predict_metagenome_variances(otu_table,genome_table,variance_table)
    else:
        #If we don't need confidence intervals, we can do a faster pure numpy prediction

        if opts.verbose:
            print "Predicting the metagenome..."
        predicted_metagenomes = predict_metagenomes(otu_table,genome_table)

    if opts.normalize_by_otu:
        #normalize (e.g. divide) the abundances by the sum of the OTUs per sample
        if opts.verbose:
            print "Normalizing functional abundances by sum of OTUs per sample"
        inverse_otu_sums = [1/x for x in otu_table.sum(axis='sample')]
        scaling_factors = dict(zip(otu_table.ids(),inverse_otu_sums))
        predicted_metagenomes = scale_metagenomes(predicted_metagenomes,scaling_factors)

    if opts.normalize_by_function:
        #normalize (e.g. divide) the abundances by the sum of the functions per sample
        #Sum of functional abundances per sample will equal 1 (e.g. relative abundance).
        if opts.verbose:
            print "Normalizing functional abundances by sum of functions per sample"
        predicted_metagenomes = predicted_metagenomes.norm(axis='sample', inplace=False)


    write_metagenome_to_file(predicted_metagenomes,opts.output_metagenome_table,\
        opts.format_tab_delimited,"metagenome prediction",verbose=opts.verbose)

    if opts.with_confidence:
        output_path,output_filename = split(opts.output_metagenome_table)
        base_output_filename,ext = splitext(output_filename)
        variance_output_fp =\
          join(output_path,"%s_variances%s" %(base_output_filename,ext))
        upper_CI_95_output_fp =\
          join(output_path,"%s_upper_CI_95%s" %(base_output_filename,ext))
        lower_CI_95_output_fp =\
          join(output_path,"%s_lower_CI_95%s" %(base_output_filename,ext))

        write_metagenome_to_file(predicted_metagenome_variances,\
          variance_output_fp,opts.format_tab_delimited,\
          "metagenome prediction variance",verbose=opts.verbose)

        write_metagenome_to_file(predicted_metagenomes_upper_CI_95,\
          upper_CI_95_output_fp,opts.format_tab_delimited,\
          "metagenome prediction upper 95% confidence interval",\
          verbose=opts.verbose)

        write_metagenome_to_file(predicted_metagenomes_lower_CI_95,\
          lower_CI_95_output_fp,opts.format_tab_delimited,\
          "metagenome prediction lower 95% confidence interval",\
          verbose=opts.verbose)

Example 49

Project: cookiecutter Source File: generate.py
def generate_files(repo_dir, context=None, output_dir='.',
                   overwrite_if_exists=False):
    """Render the templates and saves them to files.

    :param repo_dir: Project template input directory.
    :param context: Dict for populating the template's variables.
    :param output_dir: Where to output the generated project dir into.
    :param overwrite_if_exists: Overwrite the contents of the output directory
        if it exists.
    """
    template_dir = find_template(repo_dir)
    logger.debug('Generating project from {}...'.format(template_dir))
    context = context or {}

    unrendered_dir = os.path.split(template_dir)[1]
    ensure_dir_is_templated(unrendered_dir)
    env = StrictEnvironment(
        context=context,
        keep_trailing_newline=True,
    )
    try:
        project_dir = render_and_create_dir(
            unrendered_dir,
            context,
            output_dir,
            env,
            overwrite_if_exists
        )
    except UndefinedError as err:
        msg = "Unable to create project directory '{}'".format(unrendered_dir)
        raise UndefinedVariableInTemplate(msg, err, context)

    # We want the Jinja path and the OS paths to match. Consequently, we'll:
    #   + CD to the template folder
    #   + Set Jinja's path to '.'
    #
    #  In order to build our files to the correct folder(s), we'll use an
    # absolute path for the target folder (project_dir)

    project_dir = os.path.abspath(project_dir)
    logger.debug('Project directory is {}'.format(project_dir))

    _run_hook_from_repo_dir(repo_dir, 'pre_gen_project', project_dir, context)

    with work_in(template_dir):
        env.loader = FileSystemLoader('.')

        for root, dirs, files in os.walk('.'):
            # We must separate the two types of dirs into different lists.
            # The reason is that we don't want ``os.walk`` to go through the
            # unrendered directories, since they will just be copied.
            copy_dirs = []
            render_dirs = []

            for d in dirs:
                d_ = os.path.normpath(os.path.join(root, d))
                # We check the full path, because that's how it can be
                # specified in the ``_copy_without_render`` setting, but
                # we store just the dir name
                if is_copy_only_path(d_, context):
                    copy_dirs.append(d)
                else:
                    render_dirs.append(d)

            for copy_dir in copy_dirs:
                indir = os.path.normpath(os.path.join(root, copy_dir))
                outdir = os.path.normpath(os.path.join(project_dir, indir))
                logger.debug(
                    'Copying dir {} to {} without rendering'
                    ''.format(indir, outdir)
                )
                shutil.copytree(indir, outdir)

            # We mutate ``dirs``, because we only want to go through these dirs
            # recursively
            dirs[:] = render_dirs
            for d in dirs:
                unrendered_dir = os.path.join(project_dir, root, d)
                try:
                    render_and_create_dir(
                        unrendered_dir,
                        context,
                        output_dir,
                        env,
                        overwrite_if_exists
                    )
                except UndefinedError as err:
                    rmtree(project_dir)
                    _dir = os.path.relpath(unrendered_dir, output_dir)
                    msg = "Unable to create directory '{}'".format(_dir)
                    raise UndefinedVariableInTemplate(msg, err, context)

            for f in files:
                infile = os.path.normpath(os.path.join(root, f))
                if is_copy_only_path(infile, context):
                    outfile_tmpl = env.from_string(infile)
                    outfile_rendered = outfile_tmpl.render(**context)
                    outfile = os.path.join(project_dir, outfile_rendered)
                    logger.debug(
                        'Copying file {} to {} without rendering'
                        ''.format(infile, outfile)
                    )
                    shutil.copyfile(infile, outfile)
                    shutil.copymode(infile, outfile)
                    continue
                try:
                    generate_file(project_dir, infile, context, env)
                except UndefinedError as err:
                    rmtree(project_dir)
                    msg = "Unable to create file '{}'".format(infile)
                    raise UndefinedVariableInTemplate(msg, err, context)

    _run_hook_from_repo_dir(repo_dir, 'post_gen_project', project_dir, context)

    return project_dir

Example 50

Project: headphones Source File: music_encoder.py
Function: command
def command(encoder, musicSource, musicDest, albumPath, xldProfile):
    """
    Encode a given music file with a certain encoder. Returns True on success,
    or False otherwise.
    """

    startMusicTime = time.time()
    cmd = []

    if xldProfile:
        xldDestDir = os.path.split(musicDest)[0]
        cmd = [encoder]
        cmd.extend([musicSource])
        cmd.extend(['--profile'])
        cmd.extend([xldProfile])
        cmd.extend(['-o'])
        cmd.extend([xldDestDir])

    # Lame
    elif headphones.CONFIG.ENCODER == 'lame':
        cmd = [encoder]
        opts = []
        if not headphones.CONFIG.ADVANCEDENCODER:
            opts.extend(['-h'])
            if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
                opts.extend(['--resample', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-b',
                             str(headphones.CONFIG.BITRATE)])
            elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
                opts.extend(['-v', str(headphones.CONFIG.ENCODERQUALITY)])
        else:
            advanced = (headphones.CONFIG.ADVANCEDENCODER.split())
            for tok in advanced:
                opts.extend([tok.encode(headphones.SYS_ENCODING)])
        opts.extend([musicSource])
        opts.extend([musicDest])
        cmd.extend(opts)

    # FFmpeg
    elif headphones.CONFIG.ENCODER == 'ffmpeg':
        cmd = [encoder, '-i', musicSource]
        opts = []
        if not headphones.CONFIG.ADVANCEDENCODER:
            if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg':
                opts.extend(['-acodec', 'libvorbis'])
            if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a':
                opts.extend(['-strict', 'experimental'])
            if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
                opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab',
                             str(headphones.CONFIG.BITRATE) + 'k'])
            elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
                opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)])
            opts.extend(['-y', '-ac', '2', '-vn'])
        else:
            advanced = (headphones.CONFIG.ADVANCEDENCODER.split())
            for tok in advanced:
                opts.extend([tok.encode(headphones.SYS_ENCODING)])
        opts.extend([musicDest])
        cmd.extend(opts)

    # Libav
    elif headphones.CONFIG.ENCODER == "libav":
        cmd = [encoder, '-i', musicSource]
        opts = []
        if not headphones.CONFIG.ADVANCEDENCODER:
            if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg':
                opts.extend(['-acodec', 'libvorbis'])
            if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a':
                opts.extend(['-strict', 'experimental'])
            if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
                opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab',
                             str(headphones.CONFIG.BITRATE) + 'k'])
            elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
                opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)])
            opts.extend(['-y', '-ac', '2', '-vn'])
        else:
            advanced = (headphones.CONFIG.ADVANCEDENCODER.split())
            for tok in advanced:
                opts.extend([tok.encode(headphones.SYS_ENCODING)])
        opts.extend([musicDest])
        cmd.extend(opts)

    # Prevent Windows from opening a terminal window
    startupinfo = None

    if headphones.SYS_PLATFORM == "win32":
        startupinfo = subprocess.STARTUPINFO()
        try:
            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
        except AttributeError:
            startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW

    # Encode
    logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
    logger.debug(subprocess.list2cmdline(cmd))

    process = subprocess.Popen(cmd, startupinfo=startupinfo,
                               stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    stdout, stderr = process.communicate(headphones.CONFIG.ENCODER)

    # Error if return code not zero
    if process.returncode:
        logger.error(
            'Encoding failed for %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
        out = stdout if stdout else stderr
        out = out.decode(headphones.SYS_ENCODING, 'replace')
        outlast2lines = '\n'.join(out.splitlines()[-2:])
        logger.error('%s error details: %s' % (headphones.CONFIG.ENCODER, outlast2lines))
        out = out.rstrip("\n")
        logger.debug(out)
        encoded = False
    else:
        logger.info('%s encoded in %s', musicSource, getTimeEncode(startMusicTime))
        encoded = True

    return encoded
See More Examples - Go to Next Page
Page 1 Selected Page 2 Page 3 Page 4