Here are the examples of the python api os.path.split taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
155 Examples
2
Example 1
Project: helloworld Source File: media.py
def save_locally(parent_url, full_path, data, skip_write=False,
disallow_zip=False, overwrite=False):
# check dupe
if not skip_write and not overwrite:
full_path = get_unique_name(full_path)
media_type = detect_media_type(full_path)
original_size_url = ''
url = os.path.join(parent_url, os.path.split(full_path)[1])
thumb_url = ''
if media_type == 'image':
split_path = os.path.split(full_path)
splitext_path = os.path.splitext(full_path)
thumb_dir = os.path.join(split_path[0], 'thumbs')
original_size_dir = os.path.join(split_path[0], 'original')
leaf_name = os.path.split(full_path)[1]
if not os.path.isdir(thumb_dir):
os.makedirs(thumb_dir)
if not os.path.isdir(original_size_dir):
os.makedirs(original_size_dir)
thumb_filename = os.path.join(thumb_dir, leaf_name)
if not overwrite:
thumb_filename = get_unique_name(thumb_filename)
original_size_filename = os.path.join(original_size_dir, leaf_name)
if not overwrite:
original_size_filename = get_unique_name(original_size_filename)
thumb_url = os.path.join(os.path.join(parent_url, 'thumbs'),
os.path.split(thumb_filename)[1])
original_size_url = os.path.join(os.path.join(parent_url, 'original'),
os.path.split(original_size_filename)[1])
if skip_write:
os.rename(full_path, original_size_filename)
else:
f = open(original_size_filename, 'w+')
f.write(data)
f.close()
is_animated = False
if splitext_path[1] == '.apng':
is_animated = True
elif splitext_path[1] == '.gif':
gif = Image.open(original_size_filename)
try:
gif.seek(1)
except EOFError:
is_animated = False
else:
is_animated = True
if not is_animated:
original_img = Image.open(original_size_filename)
try:
info = original_img._getexif()
except:
info = None
exif = {}
if info:
for tag, value in info.items():
decoded = TAGS.get(tag, tag)
exif[decoded] = value
if 'Orientation' in exif:
orientation = exif['Orientation']
changed = True
if orientation == 1:
changed = False
elif orientation == 2:
# Vertical Mirror
original_img = original_img.transpose(Image.FLIP_LEFT_RIGHT)
elif orientation == 3:
# Rotation 180
original_img = original_img.transpose(Image.ROTATE_180)
elif orientation == 4:
# Horizontal Mirror
original_img = original_img.transpose(Image.FLIP_TOP_BOTTOM)
elif orientation == 5:
# Horizontal Mirror + Rotation 270
original_img = original_img.transpose(
Image.FLIP_TOP_BOTTOM).transpose(Image.ROTATE_270)
elif orientation == 6:
# Rotation 270
original_img = original_img.transpose(Image.ROTATE_270)
elif orientation == 7:
# Vertical Mirror + Rotation 270
original_img = original_img.transpose(
Image.FLIP_LEFT_RIGHT).transpose(Image.ROTATE_270)
elif orientation == 8:
# Rotation 90
original_img = original_img.transpose(Image.ROTATE_90)
if changed:
original_img.save(original_size_filename, quality=95)
thumb = Image.open(original_size_filename)
thumb.thumbnail((content_logic.THUMB_WIDTH, content_logic.THUMB_HEIGHT),
Image.ANTIALIAS)
thumb.save(thumb_filename, quality=95)
if not is_animated:
normal = Image.open(original_size_filename)
normal.thumbnail((content_logic.PHOTO_WIDTH, content_logic.PHOTO_HEIGHT),
Image.ANTIALIAS)
normal.save(full_path, quality=95)
else:
shutil.copyfile(original_size_filename, full_path)
else:
if not skip_write:
f = open(full_path, 'w')
f.write(data)
f.close()
splitext_path = os.path.splitext(full_path)
if splitext_path[1] == '.zip' and not disallow_zip:
z = zipfile.ZipFile(full_path)
dir_path = os.path.join(os.path.dirname(full_path), splitext_path[0])
for f in z.namelist():
filename = url_factory.clean_filename(f)
if f.endswith('/'):
os.makedirs(os.path.join(dir_path, filename))
else:
if detect_media_type(filename) not in ('video', 'image', 'audio',
'web', 'zip'):
raise tornado.web.HTTPError(400, "i call shenanigans")
z.extract(filename, dir_path)
return original_size_url, url, thumb_url
2
Example 2
Project: OpenUpgrade Source File: translate.py
def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, module_name=None, context=None):
"""Populates the ir_translation table."""
if verbose:
_logger.info('loading translation file for language %s', lang)
if context is None:
context = {}
db_name = cr.dbname
registry = openerp.registry(db_name)
lang_obj = registry.get('res.lang')
trans_obj = registry.get('ir.translation')
iso_lang = misc.get_iso_codes(lang)
try:
ids = lang_obj.search(cr, SUPERUSER_ID, [('code','=', lang)])
if not ids:
# lets create the language with locale information
lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name)
# Parse also the POT: it will possibly provide additional targets.
# (Because the POT comments are correct on Launchpad but not the
# PO comments due to a Launchpad limitation. See LP bug 933496.)
pot_reader = []
# now, the serious things: we read the language file
fileobj.seek(0)
if fileformat == 'csv':
reader = csv.reader(fileobj, quotechar='"', delimiter=',')
# read the first line of the file (it contains columns titles)
for row in reader:
fields = row
break
elif fileformat == 'po':
reader = TinyPoFile(fileobj)
fields = ['type', 'name', 'res_id', 'src', 'value', 'comments']
# Make a reader for the POT file and be somewhat defensive for the
# stable branch.
if fileobj.name.endswith('.po'):
try:
# Normally the path looks like /path/to/xxx/i18n/lang.po
# and we try to find the corresponding
# /path/to/xxx/i18n/xxx.pot file.
# (Sometimes we have 'i18n_extra' instead of just 'i18n')
addons_module_i18n, _ignored = os.path.split(fileobj.name)
addons_module, i18n_dir = os.path.split(addons_module_i18n)
addons, module = os.path.split(addons_module)
pot_handle = misc.file_open(os.path.join(
addons, module, i18n_dir, module + '.pot'))
pot_reader = TinyPoFile(pot_handle)
except:
pass
else:
_logger.info('Bad file format: %s', fileformat)
raise Exception(_('Bad file format: %s') % fileformat)
# Read the POT references, and keep them indexed by source string.
class Target(object):
def __init__(self):
self.value = None
self.targets = set() # set of (type, name, res_id)
self.comments = None
pot_targets = defaultdict(Target)
for type, name, res_id, src, _ignored, comments in pot_reader:
if type is not None:
target = pot_targets[src]
target.targets.add((type, name, res_id))
target.comments = comments
# read the rest of the file
irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context)
def process_row(row):
"""Process a single PO (or POT) entry."""
# dictionary which holds values for this line of the csv file
# {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
# 'src': ..., 'value': ..., 'module':...}
dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
'comments', 'imd_model', 'imd_name', 'module'))
dic['lang'] = lang
dic.update(zip(fields, row))
# discard the target from the POT targets.
src = dic['src']
if src in pot_targets:
target = pot_targets[src]
target.value = dic['value']
target.targets.discard((dic['type'], dic['name'], dic['res_id']))
# This would skip terms that fail to specify a res_id
res_id = dic['res_id']
if not res_id:
return
if isinstance(res_id, (int, long)) or \
(isinstance(res_id, basestring) and res_id.isdigit()):
dic['res_id'] = int(res_id)
if module_name:
dic['module'] = module_name
else:
# res_id is an xml id
dic['res_id'] = None
dic['imd_model'] = dic['name'].split(',')[0]
if '.' in res_id:
dic['module'], dic['imd_name'] = res_id.split('.', 1)
else:
dic['module'], dic['imd_name'] = module_name, res_id
irt_cursor.push(dic)
# First process the entries from the PO file (doing so also fills/removes
# the entries from the POT file).
for row in reader:
process_row(row)
# Then process the entries implied by the POT file (which is more
# correct w.r.t. the targets) if some of them remain.
pot_rows = []
for src, target in pot_targets.iteritems():
if target.value:
for type, name, res_id in target.targets:
pot_rows.append((type, name, res_id, src, target.value, target.comments))
pot_targets.clear()
for row in pot_rows:
process_row(row)
irt_cursor.finish()
trans_obj.clear_caches()
if verbose:
_logger.info("translation file loaded succesfully")
except IOError:
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
_logger.exception("couldn't read translation file %s", filename)
2
Example 3
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
"""A builder for LaTeX files that checks the output in the aux file
and decides how many times to use LaTeXAction, and BibTeXAction."""
global must_rerun_latex
# This routine is called with two actions. In this file for DVI builds
# with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
# set this up now for the case where the user requests a different extension
# for the target filename
if (XXXLaTeXAction == LaTeXAction):
callerSuffix = ".dvi"
else:
callerSuffix = env['PDFSUFFIX']
basename = SCons.Util.splitext(str(source[0]))[0]
basedir = os.path.split(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
abspath = os.path.abspath(basedir)
targetext = os.path.splitext(str(target[0]))[1]
targetdir = os.path.split(str(target[0]))[0]
saved_env = {}
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
saved_env[var] = modify_env_var(env, var, abspath)
# Create base file names with the target directory since the auxiliary files
# will be made there. That's because the *COM variables have the cd
# command in the prolog. We check
# for the existence of files before opening them--even ones like the
# aux file that TeX always creates--to make it possible to write tests
# with stubs that don't necessarily generate all of the same files.
targetbase = os.path.join(targetdir, basefile)
# if there is a \makeindex there will be a .idx and thus
# we have to run makeindex at least once to keep the build
# happy even if there is no index.
# Same for glossaries and nomenclature
src_content = source[0].get_text_contents()
run_makeindex = makeindex_re.search(src_content) and not os.path.exists(targetbase + '.idx')
run_nomenclature = makenomenclature_re.search(src_content) and not os.path.exists(targetbase + '.nlo')
run_glossary = makeglossary_re.search(src_content) and not os.path.exists(targetbase + '.glo')
run_glossaries = makeglossaries_re.search(src_content) and not os.path.exists(targetbase + '.glo')
run_acronyms = makeacronyms_re.search(src_content) and not os.path.exists(targetbase + '.acn')
saved_hashes = {}
suffix_nodes = {}
for suffix in all_suffixes:
theNode = env.fs.File(targetbase + suffix)
suffix_nodes[suffix] = theNode
saved_hashes[suffix] = theNode.get_csig()
if Verbose:
print "hashes: ",saved_hashes
must_rerun_latex = True
#
# routine to update MD5 hash and compare
#
# TODO(1.5): nested scopes
def check_MD5(filenode, suffix, saved_hashes=saved_hashes, targetbase=targetbase):
global must_rerun_latex
# two calls to clear old csig
filenode.clear_memoized_values()
filenode.ninfo = filenode.new_ninfo()
new_md5 = filenode.get_csig()
if saved_hashes[suffix] == new_md5:
if Verbose:
print "file %s not changed" % (targetbase+suffix)
return False # unchanged
saved_hashes[suffix] = new_md5
must_rerun_latex = True
if Verbose:
print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
return True # changed
# generate the file name that latex will generate
resultfilename = targetbase + callerSuffix
count = 0
while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
result = XXXLaTeXAction(target, source, env)
if result != 0:
return result
count = count + 1
must_rerun_latex = False
# Decide if various things need to be run, or run again.
# Read the log file to find warnings/errors
logfilename = targetbase + '.log'
logContent = ''
if os.path.exists(logfilename):
logContent = open(logfilename, "rb").read()
# Read the fls file to find all .aux files
flsfilename = targetbase + '.fls'
flsContent = ''
auxfiles = []
if os.path.exists(flsfilename):
flsContent = open(flsfilename, "rb").read()
auxfiles = openout_aux_re.findall(flsContent)
if Verbose:
print "auxfiles ",auxfiles
# Now decide if bibtex will need to be run.
# The information that bibtex reads from the .aux file is
# pass-independent. If we find (below) that the .bbl file is unchanged,
# then the last latex saw a correct bibliography.
# Therefore only do this on the first pass
if count == 1:
for auxfilename in auxfiles:
target_aux = os.path.join(targetdir, auxfilename)
if os.path.exists(target_aux):
content = open(target_aux, "rb").read()
if string.find(content, "bibdata") != -1:
if Verbose:
print "Need to run bibtex"
bibfile = env.fs.File(targetbase)
result = BibTeXAction(bibfile, bibfile, env)
if result != 0:
print env['BIBTEX']," returned an error, check the blg file"
return result
must_rerun_latex = check_MD5(suffix_nodes['.bbl'],'.bbl')
break
# Now decide if latex will need to be run again due to index.
if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
# We must run makeindex
if Verbose:
print "Need to run makeindex"
idxfile = suffix_nodes['.idx']
result = MakeIndexAction(idxfile, idxfile, env)
if result != 0:
print env['MAKEINDEX']," returned an error, check the ilg file"
return result
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# Harder is case is where an action needs to be called -- that should be rare (I hope?)
for index in check_suffixes:
check_MD5(suffix_nodes[index],index)
# Now decide if latex will need to be run again due to nomenclature.
if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
# We must run makeindex
if Verbose:
print "Need to run makeindex for nomenclature"
nclfile = suffix_nodes['.nlo']
result = MakeNclAction(nclfile, nclfile, env)
if result != 0:
print env['MAKENCL']," (nomenclature) returned an error, check the nlg file"
return result
# Now decide if latex will need to be run again due to glossary.
if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
# We must run makeindex
if Verbose:
print "Need to run makeindex for glossary"
glofile = suffix_nodes['.glo']
result = MakeGlossaryAction(glofile, glofile, env)
if result != 0:
print env['MAKEGLOSSARY']," (glossary) returned an error, check the glg file"
return result
# Now decide if latex will need to be run again due to acronyms.
if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
# We must run makeindex
if Verbose:
print "Need to run makeindex for acronyms"
acrfile = suffix_nodes['.acn']
result = MakeAcronymsAction(acrfile, acrfile, env)
if result != 0:
print env['MAKEACRONYMS']," (acronymns) returned an error, check the alg file"
return result
# Now decide if latex needs to be run yet again to resolve warnings.
if warning_rerun_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to latex or package rerun warning"
if rerun_citations_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to 'Rerun to get citations correct' warning"
if undefined_references_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to undefined references or citations"
if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop
# rename Latex's output to what the target name is
if not (str(target[0]) == resultfilename and os.path.exists(resultfilename)):
if os.path.exists(resultfilename):
print "move %s to %s" % (resultfilename, str(target[0]), )
shutil.move(resultfilename,str(target[0]))
# Original comment (when TEXPICTS was not restored):
# The TEXPICTS enviroment variable is needed by a dvi -> pdf step
# later on Mac OSX so leave it
#
# It is also used when searching for pictures (implicit dependencies).
# Why not set the variable again in the respective builder instead
# of leaving local modifications in the environment? What if multiple
# latex builds in different directories need different TEXPICTS?
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
if var == 'TEXPICTS':
continue
if saved_env[var] is _null:
try:
del env['ENV'][var]
except KeyError:
pass # was never set
else:
env['ENV'][var] = saved_env[var]
return result
2
Example 4
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
"""A builder for LaTeX files that checks the output in the aux file
and decides how many times to use LaTeXAction, and BibTeXAction."""
global must_rerun_latex
# This routine is called with two actions. In this file for DVI builds
# with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
# set this up now for the case where the user requests a different extension
# for the target filename
if (XXXLaTeXAction == LaTeXAction):
callerSuffix = ".dvi"
else:
callerSuffix = env['PDFSUFFIX']
basename = SCons.Util.splitext(str(source[0]))[0]
basedir = os.path.split(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
abspath = os.path.abspath(basedir)
targetext = os.path.splitext(str(target[0]))[1]
targetdir = os.path.split(str(target[0]))[0]
saved_env = {}
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
saved_env[var] = modify_env_var(env, var, abspath)
# Create base file names with the target directory since the auxiliary files
# will be made there. That's because the *COM variables have the cd
# command in the prolog. We check
# for the existence of files before opening them--even ones like the
# aux file that TeX always creates--to make it possible to write tests
# with stubs that don't necessarily generate all of the same files.
targetbase = os.path.join(targetdir, basefile)
# if there is a \makeindex there will be a .idx and thus
# we have to run makeindex at least once to keep the build
# happy even if there is no index.
# Same for glossaries, nomenclature, and acronyms
src_content = source[0].get_text_contents()
run_makeindex = makeindex_re.search(src_content) and not os.path.isfile(targetbase + '.idx')
run_nomenclature = makenomenclature_re.search(src_content) and not os.path.isfile(targetbase + '.nlo')
run_glossary = makeglossary_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
run_glossaries = makeglossaries_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
run_acronyms = makeacronyms_re.search(src_content) and not os.path.isfile(targetbase + '.acn')
saved_hashes = {}
suffix_nodes = {}
for suffix in all_suffixes+sum(newglossary_suffix, []):
theNode = env.fs.File(targetbase + suffix)
suffix_nodes[suffix] = theNode
saved_hashes[suffix] = theNode.get_csig()
if Verbose:
print "hashes: ",saved_hashes
must_rerun_latex = True
# .aux files already processed by BibTex
already_bibtexed = []
#
# routine to update MD5 hash and compare
#
def check_MD5(filenode, suffix):
global must_rerun_latex
# two calls to clear old csig
filenode.clear_memoized_values()
filenode.ninfo = filenode.new_ninfo()
new_md5 = filenode.get_csig()
if saved_hashes[suffix] == new_md5:
if Verbose:
print "file %s not changed" % (targetbase+suffix)
return False # unchanged
saved_hashes[suffix] = new_md5
must_rerun_latex = True
if Verbose:
print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
return True # changed
# generate the file name that latex will generate
resultfilename = targetbase + callerSuffix
count = 0
while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
result = XXXLaTeXAction(target, source, env)
if result != 0:
return result
count = count + 1
must_rerun_latex = False
# Decide if various things need to be run, or run again.
# Read the log file to find warnings/errors
logfilename = targetbase + '.log'
logContent = ''
if os.path.isfile(logfilename):
logContent = open(logfilename, "rb").read()
# Read the fls file to find all .aux files
flsfilename = targetbase + '.fls'
flsContent = ''
auxfiles = []
if os.path.isfile(flsfilename):
flsContent = open(flsfilename, "rb").read()
auxfiles = openout_aux_re.findall(flsContent)
# remove duplicates
dups = {}
for x in auxfiles:
dups[x] = 1
auxfiles = list(dups.keys())
bcffiles = []
if os.path.isfile(flsfilename):
flsContent = open(flsfilename, "rb").read()
bcffiles = openout_bcf_re.findall(flsContent)
# remove duplicates
dups = {}
for x in bcffiles:
dups[x] = 1
bcffiles = list(dups.keys())
if Verbose:
print "auxfiles ",auxfiles
print "bcffiles ",bcffiles
# Now decide if bibtex will need to be run.
# The information that bibtex reads from the .aux file is
# pass-independent. If we find (below) that the .bbl file is unchanged,
# then the last latex saw a correct bibliography.
# Therefore only do this once
# Go through all .aux files and remember the files already done.
for auxfilename in auxfiles:
if auxfilename not in already_bibtexed:
already_bibtexed.append(auxfilename)
target_aux = os.path.join(targetdir, auxfilename)
if os.path.isfile(target_aux):
content = open(target_aux, "rb").read()
if content.find("bibdata") != -1:
if Verbose:
print "Need to run bibtex on ",auxfilename
bibfile = env.fs.File(SCons.Util.splitext(target_aux)[0])
result = BibTeXAction(bibfile, bibfile, env)
if result != 0:
check_file_error_message(env['BIBTEX'], 'blg')
must_rerun_latex = True
# Now decide if biber will need to be run.
# When the backend for biblatex is biber (by choice or default) the
# citation information is put in the .bcf file.
# The information that biber reads from the .bcf file is
# pass-independent. If we find (below) that the .bbl file is unchanged,
# then the last latex saw a correct bibliography.
# Therefore only do this once
# Go through all .bcf files and remember the files already done.
for bcffilename in bcffiles:
if bcffilename not in already_bibtexed:
already_bibtexed.append(bcffilename)
target_bcf = os.path.join(targetdir, bcffilename)
if os.path.isfile(target_bcf):
content = open(target_bcf, "rb").read()
if content.find("bibdata") != -1:
if Verbose:
print "Need to run biber on ",bcffilename
bibfile = env.fs.File(SCons.Util.splitext(target_bcf)[0])
result = BiberAction(bibfile, bibfile, env)
if result != 0:
check_file_error_message(env['BIBER'], 'blg')
must_rerun_latex = True
# Now decide if latex will need to be run again due to index.
if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
# We must run makeindex
if Verbose:
print "Need to run makeindex"
idxfile = suffix_nodes['.idx']
result = MakeIndexAction(idxfile, idxfile, env)
if result != 0:
check_file_error_message(env['MAKEINDEX'], 'ilg')
return result
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# Harder is case is where an action needs to be called -- that should be rare (I hope?)
for index in check_suffixes:
check_MD5(suffix_nodes[index],index)
# Now decide if latex will need to be run again due to nomenclature.
if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
# We must run makeindex
if Verbose:
print "Need to run makeindex for nomenclature"
nclfile = suffix_nodes['.nlo']
result = MakeNclAction(nclfile, nclfile, env)
if result != 0:
check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
'nlg')
#return result
# Now decide if latex will need to be run again due to glossary.
if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
# We must run makeindex
if Verbose:
print "Need to run makeindex for glossary"
glofile = suffix_nodes['.glo']
result = MakeGlossaryAction(glofile, glofile, env)
if result != 0:
check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
'glg')
#return result
# Now decide if latex will need to be run again due to acronyms.
if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
# We must run makeindex
if Verbose:
print "Need to run makeindex for acronyms"
acrfile = suffix_nodes['.acn']
result = MakeAcronymsAction(acrfile, acrfile, env)
if result != 0:
check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
'alg')
return result
# Now decide if latex will need to be run again due to newglossary command.
for ig in range(len(newglossary_suffix)):
if check_MD5(suffix_nodes[newglossary_suffix[ig][2]],newglossary_suffix[ig][2]) or (count == 1):
# We must run makeindex
if Verbose:
print "Need to run makeindex for newglossary"
newglfile = suffix_nodes[newglossary_suffix[ig][2]]
MakeNewGlossaryAction = SCons.Action.Action("$MAKENEWGLOSSARY ${SOURCE.filebase}%s -s ${SOURCE.filebase}.ist -t ${SOURCE.filebase}%s -o ${SOURCE.filebase}%s" % (newglossary_suffix[ig][2],newglossary_suffix[ig][0],newglossary_suffix[ig][1]), "$MAKENEWGLOSSARYCOMSTR")
result = MakeNewGlossaryAction(newglfile, newglfile, env)
if result != 0:
check_file_error_message('%s (newglossary)' % env['MAKENEWGLOSSARY'],
newglossary_suffix[ig][0])
return result
# Now decide if latex needs to be run yet again to resolve warnings.
if warning_rerun_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to latex or package rerun warning"
if rerun_citations_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to 'Rerun to get citations correct' warning"
if undefined_references_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to undefined references or citations"
if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop
# rename Latex's output to what the target name is
if not (str(target[0]) == resultfilename and os.path.isfile(resultfilename)):
if os.path.isfile(resultfilename):
print "move %s to %s" % (resultfilename, str(target[0]), )
shutil.move(resultfilename,str(target[0]))
# Original comment (when TEXPICTS was not restored):
# The TEXPICTS enviroment variable is needed by a dvi -> pdf step
# later on Mac OSX so leave it
#
# It is also used when searching for pictures (implicit dependencies).
# Why not set the variable again in the respective builder instead
# of leaving local modifications in the environment? What if multiple
# latex builds in different directories need different TEXPICTS?
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
if var == 'TEXPICTS':
continue
if saved_env[var] is _null:
try:
del env['ENV'][var]
except KeyError:
pass # was never set
else:
env['ENV'][var] = saved_env[var]
return result
2
Example 5
def main(argc, argv):
if (argc < 2):
#command-line argument codes.
#-i:config file.
#-c:command codes
#-m:mosaic dataset name
#-s:Source data paths. (as inputs to command (AR/AR)
#-l:Full path to log file (including file name)
user_args = \
[
"-m: Mosaic dataset path including GDB and MD name [e.g. c:\WorldElevation.gdb\Portland]",
"-s: Source data paths. (As inputs to command (AR)",
"-l: Log file output path [path+file name]",
"-artdem: Update DEM path in ART file"
]
print ("\nMDCS.py v5.8a [20150611]\nUsage: MDCS.py -c:<Optional:command> -i:<config_file>" \
"\n\nFlags to override configuration values,") \
for arg in user_args:
print (arg)
print (\
"\nNote: Commands can be combined with '+' to do multiple operations." \
"\nAvailable commands:")
user_cmds = solutionsLib.Solutions().getAvailableCommands()
for key in user_cmds:
print ("\t" + key + ' = ' + user_cmds[key]['desc'])
sys.exit(1)
base = Base.Base()
if (not g_cli_callback is None):
base.m_cli_callback_ptr = g_cli_callback
if (not g_cli_msg_callback is None):
base.m_cli_msg_callback_ptr = g_cli_msg_callback
global log
log = logger.Logger(base);
base.setLog(log)
argIndx = 0
md_path_ = artdem = config = com = log_folder = code_base = ''
while(argIndx < argc):
(values) = argv[argIndx].split(':')
if (len(values[0]) < 2 or
values[0][:1] != '-' and
values[0][:1] != '#'):
argIndx += 1
continue
exSubCode = values[0][1:len(values[0])].lower()
subCode = values.pop(0)[1].lower()
value = ':'.join(values).strip()
if (subCode == 'c'):
com = value.replace(' ', '') #remove spaces in between.
elif(subCode == 'i'):
config = value
elif(subCode == 'm'):
md_path_ = value
elif(subCode == 's'):
base.m_sources = value
elif(subCode == 'l'):
log_folder = value
elif(subCode == 'b'):
code_base = value
elif(exSubCode == 'artdem'):
artdem = value
elif(exSubCode == 'gprun'):
log.isGPRun = True # direct log messages also to (arcpy.AddMessage)
elif(subCode == 'p'):
pMax = value.rfind('$')
if (pMax == -1):
argIndx += 1
continue
dynamic_var = value[pMax + 1:].upper()
v = value[0: pMax]
if (dynamic_var.strip() != ''):
if ((dynamic_var in base.m_dynamic_params.keys()) == False):
base.m_dynamic_params[dynamic_var] = v
argIndx += 1
if (code_base != ''):
base.setCodeBase(code_base)
if (md_path_ != ''):
(p, f) = os.path.split(md_path_)
f = f.strip()
const_gdb_ext_len_ = len(base.const_geodatabase_ext)
ext = p[-const_gdb_ext_len_:].lower()
if ((ext == base.const_geodatabase_ext.lower() or
ext == base.const_geodatabase_SDE_ext.lower()) and
f != ''):
p = p.replace('\\', '/')
w = p.split('/')
workspace_ = ''
for i in range(0, len(w) - 1):
workspace_ += w[i] + '/'
gdb_ = w[len(w) -1]
base.m_workspace = workspace_
base.m_geodatabase = w[len(w) - 1]
base.m_mdName = f
configName, ext = os.path.splitext(config)
configName = os.path.basename(configName)
# setup log
log.Project ('MDCS')
log.LogNamePrefix(configName)
log.StartLog()
log_output_folder = os.path.join(os.path.dirname(solutionLib_path), 'logs')
if (log_folder != ''):
(path, fileName) = os.path.split(log_folder)
if (path != ''):
log_output_folder = path
if (fileName != ''):
log.LogFileName(fileName)
log.SetLogFolder(log_output_folder)
# ends
if (os.path.isfile(config) == False):
log.Message('Input config file is not specified/not found! ({})'.format(config), logger.Logger.const_critical_text)
log.Message(base.CCMD_STATUS_FAILED, logger.Logger.const_status_text) # set (failed) status
log.WriteLog('#all')
return False
if (artdem != ''):
(base.m_art_ws, base.m_art_ds) = os.path.split(artdem)
base.m_art_apply_changes = True
comInfo = {
'AR' : { 'cb' : postAddData } #assign a callback function to run custom user code when adding rasters.
}
if (com == ''):
com = base.const_cmd_default_text
solutions = solutionsLib.Solutions(base)
success = solutions.run(config, com, comInfo)
log.Message ("Done...", log.const_general_text)
log.WriteLog('#all') #persist information/errors collected.
2
Example 6
Project: cgstudiomap Source File: translate.py
def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, module_name=None, context=None):
"""Populates the ir_translation table."""
if verbose:
_logger.info('loading translation file for language %s', lang)
if context is None:
context = {}
db_name = cr.dbname
registry = openerp.registry(db_name)
lang_obj = registry.get('res.lang')
trans_obj = registry.get('ir.translation')
iso_lang = misc.get_iso_codes(lang)
try:
ids = lang_obj.search(cr, SUPERUSER_ID, [('code','=', lang)])
if not ids:
# lets create the language with locale information
lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name)
# Parse also the POT: it will possibly provide additional targets.
# (Because the POT comments are correct on Launchpad but not the
# PO comments due to a Launchpad limitation. See LP bug 933496.)
pot_reader = []
# now, the serious things: we read the language file
fileobj.seek(0)
if fileformat == 'csv':
reader = csv.reader(fileobj, quotechar='"', delimiter=',')
# read the first line of the file (it contains columns titles)
for row in reader:
fields = row
break
elif fileformat == 'po':
reader = TinyPoFile(fileobj)
fields = ['type', 'name', 'res_id', 'src', 'value', 'comments']
# Make a reader for the POT file and be somewhat defensive for the
# stable branch.
if fileobj.name.endswith('.po'):
try:
# Normally the path looks like /path/to/xxx/i18n/lang.po
# and we try to find the corresponding
# /path/to/xxx/i18n/xxx.pot file.
# (Sometimes we have 'i18n_extra' instead of just 'i18n')
addons_module_i18n, _ = os.path.split(fileobj.name)
addons_module, i18n_dir = os.path.split(addons_module_i18n)
addons, module = os.path.split(addons_module)
pot_handle = misc.file_open(os.path.join(
addons, module, i18n_dir, module + '.pot'))
pot_reader = TinyPoFile(pot_handle)
except:
pass
else:
_logger.error('Bad file format: %s', fileformat)
raise Exception(_('Bad file format'))
# Read the POT references, and keep them indexed by source string.
class Target(object):
def __init__(self):
self.value = None
self.targets = set() # set of (type, name, res_id)
self.comments = None
pot_targets = defaultdict(Target)
for type, name, res_id, src, _, comments in pot_reader:
if type is not None:
target = pot_targets[src]
target.targets.add((type, name, res_id))
target.comments = comments
# read the rest of the file
irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context)
def process_row(row):
"""Process a single PO (or POT) entry."""
# dictionary which holds values for this line of the csv file
# {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
# 'src': ..., 'value': ..., 'module':...}
dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
'comments', 'imd_model', 'imd_name', 'module'))
dic['lang'] = lang
dic.update(zip(fields, row))
# discard the target from the POT targets.
src = dic['src']
if src in pot_targets:
target = pot_targets[src]
target.value = dic['value']
target.targets.discard((dic['type'], dic['name'], dic['res_id']))
# This would skip terms that fail to specify a res_id
res_id = dic['res_id']
if not res_id:
return
if isinstance(res_id, (int, long)) or \
(isinstance(res_id, basestring) and res_id.isdigit()):
dic['res_id'] = int(res_id)
if module_name:
dic['module'] = module_name
else:
# res_id is an xml id
dic['res_id'] = None
dic['imd_model'] = dic['name'].split(',')[0]
if '.' in res_id:
dic['module'], dic['imd_name'] = res_id.split('.', 1)
else:
dic['module'], dic['imd_name'] = module_name, res_id
irt_cursor.push(dic)
# First process the entries from the PO file (doing so also fills/removes
# the entries from the POT file).
for row in reader:
process_row(row)
# Then process the entries implied by the POT file (which is more
# correct w.r.t. the targets) if some of them remain.
pot_rows = []
for src, target in pot_targets.iteritems():
if target.value:
for type, name, res_id in target.targets:
pot_rows.append((type, name, res_id, src, target.value, target.comments))
pot_targets.clear()
for row in pot_rows:
process_row(row)
irt_cursor.finish()
trans_obj.clear_caches()
if verbose:
_logger.info("translation file loaded succesfully")
except IOError:
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
_logger.exception("couldn't read translation file %s", filename)
2
Example 7
def __init__(self, fileName, gdalDataset, gdalMetadata,
resolution='low', **kwargs):
''' Create LANDSAT VRT from multiple tif files or single tar.gz file'''
mtlFileName = ''
bandFileNames = []
bandSizes = []
bandDatasets = []
fname = os.path.split(fileName)[1]
if (fileName.endswith('.tar') or
fileName.endswith('.tar.gz') or
fileName.endswith('.tgz')):
# try to open .tar or .tar.gz or .tgz file with tar
try:
tarFile = tarfile.open(fileName)
except:
raise WrongMapperError
# collect names of bands and corresponding sizes
# into bandsInfo dict and bandSizes list
tarNames = sorted(tarFile.getnames())
for tarName in tarNames:
# check if TIF files inside TAR qualify
if (tarName[0] in ['L', 'M'] and
os.path.splitext(tarName)[1] in ['.TIF', '.tif']):
# open TIF file from TAR using VSI
sourceFilename = '/vsitar/%s/%s' % (fileName, tarName)
gdalDatasetTmp = gdal.Open(sourceFilename)
# keep name, GDALDataset and size
bandFileNames.append(sourceFilename)
bandSizes.append(gdalDatasetTmp.RasterXSize)
bandDatasets.append(gdalDatasetTmp)
elif (tarName.endswith('MTL.txt') or
tarName.endswith('MTL.TXT')):
# get mtl file
mtlFileName = tarName
elif ((fname.startswith('L') or fname.startswith('M')) and
(fname.endswith('.tif') or
fname.endswith('.TIF') or
fname.endswith('._MTL.txt'))):
# try to find TIF/tif files with the same name as input file
path, coreName = os.path.split(fileName)
coreName = os.path.splitext(coreName)[0].split('_')[0]
coreNameMask = coreName+'*[tT][iI][fF]'
tifNames = sorted(glob.glob(os.path.join(path, coreNameMask)))
for tifName in tifNames:
sourceFilename = tifName
gdalDatasetTmp = gdal.Open(sourceFilename)
# keep name, GDALDataset and size
bandFileNames.append(sourceFilename)
bandSizes.append(gdalDatasetTmp.RasterXSize)
bandDatasets.append(gdalDatasetTmp)
# get mtl file
mtlFiles = glob.glob(coreName+'*[mM][tT][lL].[tT][xX][tT]')
if len(mtlFiles) > 0:
mtlFileName = mtlFiles[0]
else:
raise WrongMapperError
# if not TIF files found - not appropriate mapper
if not bandFileNames:
raise WrongMapperError
# get appropriate band size based on number of unique size and
# required resoltuion
if resolution == 'low':
bandXSise = min(bandSizes)
elif resolution in ['high', 'hi']:
bandXSise = max(bandSizes)
else:
raise OptionError('Wrong resolution %s for file %s' % (resolution, fileName))
# find bands with appropriate size and put to metaDict
metaDict = []
for bandFileName, bandSize, bandDataset in zip(bandFileNames,
bandSizes,
bandDatasets):
if bandSize == bandXSise:
# let last part of file name be suffix
bandSuffix = os.path.splitext(bandFileName)[0].split('_')[-1]
metaDict.append({
'src': {'SourceFilename': bandFileName,
'SourceBand': 1,
'ScaleRatio': 0.1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'suffix': bandSuffix}})
gdalDataset4Use = bandDataset
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset4Use)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
if len(mtlFileName) > 0:
mtlFileName = os.path.join(os.path.split(bandFileNames[0])[0],
mtlFileName)
mtlFileLines = [line.strip() for line in
self.read_xml(mtlFileName).split('\n')]
dateString = [line.split('=')[1].strip()
for line in mtlFileLines
if ('DATE_ACQUIRED' in line or
'ACQUISITION_DATE' in line)][0]
timeStr = [line.split('=')[1].strip()
for line in mtlFileLines
if ('SCENE_CENTER_TIME' in line or
'SCENE_CENTER_SCAN_TIME' in line)][0]
time_start = parse_time(dateString + 'T' + timeStr).isoformat()
time_end = (parse_time(dateString + 'T' + timeStr) +
datetime.timedelta(microseconds=60000000)).isoformat()
self.dataset.SetMetadataItem('time_coverage_start', time_start)
self.dataset.SetMetadataItem('time_coverage_end', time_end)
# set platform
platform = 'LANDSAT'
if fname[2].isdigit():
platform += '-'+fname[2]
ee = pti.get_gcmd_platform(platform)
self.dataset.SetMetadataItem('platform', json.dumps(ee))
# set instrument
instrument = {
'LANDSAT' : 'MSS',
'LANDSAT-1' : 'MSS',
'LANDSAT-2' : 'MSS',
'LANDSAT-3' : 'MSS',
'LANDSAT-4' : 'TM',
'LANDSAT-5' : 'TM',
'LANDSAT-7' : 'ETM+',
'LANDSAT-8' : 'OLI'}[platform]
ee = pti.get_gcmd_instrument(instrument)
self.dataset.SetMetadataItem('instrument', json.dumps(ee))
2
Example 8
def __init__(self, __genparams):
'''The constructor for the GenerateListForGzip class'''
__arcpath = __genparams['arcpath']
__delimiter = __genparams['delimiter']
self._genfull = __genparams['genfull']
self.__confoutput = __genparams['confoutput']
self.__listoutput = __genparams['listoutput']
self.__fulloutput = __genparams['fulloutput']
self.__getallhashes = __genparams['getallhashes']
self.__hashtype = __genparams['hashtype']
self.__parsingexceptions = __genparams['parsingexceptions']
self.__confname = __genparams['confname']
__arcstat = os.stat(__arcpath)
__listoffiles = ['[archive]\nmtime{} {}\n\n[files]\n'.format(__delimiter,__arcstat.st_mtime)]
__fileinfo = os.lstat(__arcpath)
__filetype = 'f'
if not self.__hashtype:
__filehash = get_hash(gzip.open(__arcpath, 'rb'), 'md5')
else:
__filehash = get_hash(gzip.open(__arcpath, 'rb'), self.__hashtype)
with open(__arcpath, 'rb') as __gzip:
__filesize = self.__extract_size(__gzip)
__filename = self.__extract_initial_filename(__gzip,
os.path.split(__arcpath)[-1][:-2])
if self.__getallhashes:
if not self.__hashtype:
__onelinewithhash = '{value}{delimiter} ={value} type{delimiter}{value} md5{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
else:
__onelinewithhash = '{value}{delimiter} ={value} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__hashtype, delimiter=__delimiter)
__listoffiles.append(__onelinewithhash.format(
__filename,
str(__filesize),
__filetype,
__filehash))
else:
if self.__parsingexceptions :
for __file in self.__parsingexceptions:
if fnmatch.fnmatch(__filename, __file):
__filehash = get_hash(gzip.open(__arcpath, 'rb'), self.__parsingexceptions[__file])
__onelinewithhash = '{value}{delimiter} ={value} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__parsingexceptions[__file], delimiter=__delimiter)
__listoffiles.append(__onelinewithhash.format(
__filename,
str(__filesize),
__filetype,
__filehash))
else:
__onelinewithouthash = '{value}{delimiter} ={value} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
__listoffiles.append(__onelinewithouthash.format(
__filename,
str(__filesize),
__filetype))
else:
__onelinewithouthash = '{value}{delimiter} ={value} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
__listoffiles.append(__onelinewithouthash.format(
__filename,
str(__filesize),
__filetype))
# define the flexible file list path
__arcwithext = os.path.split(''.join([__arcpath[:-2], 'list']))[1]
if self.__listoutput:
if self.__confname:
# --gen-list and --list-output-dir and --configuration-name
__arclistpath = os.path.join(self.__listoutput, '.'.join([self.__confname, 'list']))
else:
# --gen-list and --list-output-dir
__arclistpath = os.path.join(self.__listoutput, __arcwithext)
elif self.__fulloutput:
if self.__confname:
# --gen-list and --output-list-and-conf-dir and --configuration-name
__arclistpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'list']))
else:
# --gen-list and --output-list-and-conf-dir
__arclistpath = os.path.join(self.__fulloutput, __arcwithext)
else:
# --gen-list only
if self.__confname:
__arc = os.path.dirname(__arcpath)
__arclistpath = os.path.join(__arc, '.'.join([self.__confname, 'list']))
else:
__arclistpath = ''.join([__arcpath[:-2], 'list'])
# call the method to write information in a file
__listconfinfo = {'arclistpath': __arclistpath,
'listoffiles': __listoffiles}
self._generate_list(__listconfinfo)
# call the method to write the configuration file if --gen-full was required
if self._genfull:
# generate the hash sum of the list of files
__listhashsum = self._get_list_hash(__listconfinfo['arclistpath'])
# define the flexible configuration file path
__arcwithext = os.path.split(''.join([__arcpath[:-2], 'conf']))[1]
if self.__confoutput:
if self.__confname:
# --gen-full and --output-conf-dir and --configuration-name
__arcconfpath = os.path.join(self.__confoutput, '.'.join([self.__confname, 'conf']))
else:
# --gen-full and --output-conf-dir
__arcconfpath = os.path.join(self.__confoutput, __arcwithext)
elif self.__fulloutput:
if self.__confname:
# --gen-full and --output-list-and-conf-dir and --configuration-name
__arcconfpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'conf']))
else:
# --gen-full and --output-list-and-conf-dir
__arcconfpath = os.path.join(self.__fulloutput, __arcwithext)
else:
# --gen-full only
if self.__confname:
__arc = os.path.dirname(__arcpath)
__arcconfpath = os.path.join(__arc, '.'.join([self.__confname, 'conf']))
else:
__arcconfpath = ''.join([__arcpath[:-2], 'conf'])
# the name of the backup inside the configuration file
if self.__confname:
__arcname = self.__confname
else:
__arcname = os.path.basename(__arcpath[:-3])
__confinfo = {'arcname': __arcname,
'arcpath': __arcpath,
'arcconfpath': __arcconfpath,
'arclistpath': __listconfinfo['arclistpath'],
'arctype': 'archive',
'sha512': __listhashsum}
self._generate_conf(__confinfo)
2
Example 9
@staticmethod
def makeFilename(path='$DATE-$TIME',
subs={},
dateFormat="%Y%m%d",
timeFormat="%H%M%S"):
"""
Helper method to create filenames with increasing sequence number
appended.
It can do variable substitution in the given path. Standard
variables are $DATE and $TIME (you can define the format of
these field passint the appropriate format string in
dateFormat and timeFormat, respectively).
Any other variable can be defined passing an subs dictionary
with key as variable name.
@param path: Filename path, with directories, environmnt variables.
@type path: str
@param subs: Dictionary of {VAR=NAME,...} to create aditional
variable substitutions.
@type subs: dict
@param dateFormat: Date format, as used in time.strftime, to
be used by $DATE variable.
@type dateFormat: str
@param timeFormat: Time format, as used in time.strftime, to
be used by $TIME variable.
@type timeFormat: str
@return: Filename.
@rtype: str
"""
localtime = dt.datetime.now()
utctime = dt.datetime.utcnow()
if localtime.hour < 12:
jd_day = localtime - dt.timedelta(days=1)
else:
jd_day = localtime
subs_dict = {"LAST_NOON_DATE": jd_day.strftime(dateFormat),
"DATE": utctime.strftime(dateFormat),
"TIME": utctime.strftime(timeFormat)}
# add any user-specific keywords
subs_dict.update(subs)
dirname, filename = os.path.split(path)
dirname = os.path.expanduser(dirname)
dirname = os.path.expandvars(dirname)
dirname = os.path.realpath(dirname)
basename, ext = os.path.splitext(filename)
if not ext:
ext = "fits"
else:
# remove first dot
ext = ext[1:]
# make substitutions
dirname = string.Template(dirname).safe_substitute(subs_dict)
basename = string.Template(basename).safe_substitute(subs_dict)
ext = string.Template(ext).safe_substitute(subs_dict)
fullpath = os.path.join(dirname, basename)
finalname = os.path.join(dirname, "%s%s%s" % (basename, os.path.extsep, ext))
if not os.path.exists(dirname):
os.makedirs(dirname)
if not os.path.isdir(dirname):
raise OSError(
"A file with the same name as the desired directory already exists. ('%s')" % dirname)
# If filename exists, append -NNN to the end of the file name.
# A maximum of 1000 files can be generated with the same filename.
if os.path.exists(finalname):
base, ext = os.path.splitext(finalname)
i = 1
while os.path.exists("%s-%03i%s" % (base, i, ext)):
i += 1
if i == 1000:
raise OSError("Reached the maximum of 999 files with the same name (%s)." % finalname)
finalname = "%s-%03i%s" % (base, i, ext)
return finalname
2
Example 10
Project: deepcca Source File: dcca.py
def test_dcca(learning_rate=0.01, L1_reg=0.0001, L2_reg=0.0001, n_epochs=1000,
dataset='mnist.pkl.gz', batch_size=20, n_hidden=500):
"""
Demonstrate stochastic gradient descent optimization for a multilayer
perceptron
This is demonstrated on MNIST.
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient
:type L1_reg: float
:param L1_reg: L1-norm's weight when added to the cost (see
regularization)
:type L2_reg: float
:param L2_reg: L2-norm's weight when added to the cost (see
regularization)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: the path of the MNIST dataset file from
http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x1 = T.matrix('x1') # the data is presented as rasterized images
x2 = T.matrix('x2') # the labels are presented as 1D vector of
# [int] labels
h1 = T.matrix('h1') # the data is presented as rasterized images
h2 = T.matrix('h2') # the labels are presented as 1D vector of
rng = numpy.random.RandomState(1234)
# construct the MLP class
net1 = DCCA(
rng=rng,
input=x1,
n_in=28 * 28,
n_hidden=300,
n_out=8
)
net2 = DCCA(
rng=rng,
input=x2,
n_in=10,
n_hidden=20,
n_out=8
)
if 1:
cost1 = (
net1.correlation(h1, h2)
+ L1_reg * net1.L1
+ L2_reg * net1.L2_sqr
)
cost2 = (
net2.correlation(h1, h2)
+ L1_reg * net2.L1
+ L2_reg * net2.L2_sqr
)
"""
test_model = theano.function(
inputs=[index],
outputs=net1.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
}
)
validate_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
}
)
"""
fprop_model1 = theano.function(
inputs=[],
outputs=(net1.hiddenLayer.output, net1.output),
givens={
x1: test_set_x
}
)
fprop_model2 = theano.function(
inputs=[],
outputs=(net2.hiddenLayer.output, net2.output),
givens={
x2: test_set_y
}
)
if 1: # grad compute for net1 in theano
U, V, D = theano.tensor.nlinalg.svd(net1.lastLayer.Tval)
UVT = T.dot(U, V.T)
Delta12 = T.dot(net1.lastLayer.SigmaHat11**(-0.5), T.dot(UVT, net1.lastLayer.SigmaHat22**(-0.5)))
UDUT = T.dot(U, T.dot(D, U.T))
Delta11 = (-0.5) * T.dot(net1.lastLayer.SigmaHat11**(-0.5), T.dot(UDUT, net1.lastLayer.SigmaHat22**(-0.5)))
grad_E_to_o = (1.0/8) * (2*Delta11*net1.lastLayer.H1bar+Delta12*net1.lastLayer.H2bar)
gparam1_W = (grad_E_to_o) * (net1.lastLayer.output*(1-net1.lastLayer.output)) * (net1.hiddenLayer.output)
gparam1_b = (grad_E_to_o) * (net1.lastLayer.output*(1-net1.lastLayer.output)) * theano.shared(numpy.array([1.0],dtype=theano.config.floatX), borrow=True)
#gparams1 = [T.grad(cost1, param) for param in net1.params]
gparams1 = [T.grad(cost1, param) for param in net1.hiddenLayer.params]
gparams1.append(gparam1_W)
#gparams1.append(gparam1_b)
if 1: # grad compute for net2
U, V, D = theano.tensor.nlinalg.svd(net2.lastLayer.Tval)
UVT = T.dot(U, V.T)
Delta12 = T.dot(net2.lastLayer.SigmaHat11**(-0.5), T.dot(UVT, net2.lastLayer.SigmaHat22**(-0.5)))
UDUT = T.dot(U, T.dot(D, U.T))
Delta11 = (-0.5) * T.dot(net2.lastLayer.SigmaHat11**(-0.5), T.dot(UVT, net2.lastLayer.SigmaHat22**(-0.5)))
grad_E_to_o = (1.0/8) * (2*Delta11*net2.lastLayer.H1bar+Delta12*net2.lastLayer.H2bar)
gparam2_W = (grad_E_to_o) * (net2.lastLayer.output*(1-net2.lastLayer.output)) * (net2.hiddenLayer.output)
gparam2_b = (grad_E_to_o) * (net2.lastLayer.output*(1-net2.lastLayer.output)) * 1
#gparams1 = [T.grad(cost1, param) for param in net1.params]
gparams2 = [T.grad(cost2, param) for param in net2.hiddenLayer.params]
gparams2.append(gparam2_W)
gparams2.append(gparam2_b)
#gparams2 = [T.grad(cost2, param) for param in net2.params]
updates1 = [
(param, param - learning_rate * gparam)
for param, gparam in zip(net1.params, gparams1)
]
updates2 = [
(param, param - learning_rate * gparam)
for param, gparam in zip(net2.params, gparams2)
]
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = time.clock()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
print 'epoch', epoch
#net1.fprop(test_set_x)
#net2.fprop(test_set_y)
h1hidden, h1tmpval = fprop_model1()
h2hidden, h2tmpval = fprop_model2()
h1hidden = h1hidden.T
h2hidden = h2hidden.T
h1tmpval = h1tmpval.T
h2tmpval = h2tmpval.T
if 1: # compute cost(H1, H2)
H1 = h1tmpval
H2 = h2tmpval
m = H1.shape[1]
H1bar = H1 - (1.0/m)*numpy.dot(H1, numpy.ones((m,m), dtype=numpy.float32))
H2bar = H2 - (1.0/m)*numpy.dot(H2, numpy.ones((m,m), dtype=numpy.float32))
SigmaHat12 = (1.0/(m-1))*numpy.dot(H1bar, H2bar.T)
SigmaHat11 = (1.0/(m-1))*numpy.dot(H1bar, H1bar.T)
SigmaHat11 = SigmaHat11 + 0.0001*numpy.identity(SigmaHat11.shape[0], dtype=numpy.float32)
SigmaHat22 = (1.0/(m-1))*numpy.dot(H2bar, H2bar.T)
SigmaHat22 = SigmaHat22 + 0.0001*numpy.identity(SigmaHat22.shape[0], dtype=numpy.float32)
Tval = numpy.dot(mat_pow(SigmaHat11), numpy.dot(SigmaHat12, mat_pow(SigmaHat22)))
corr = numpy.trace(numpy.dot(Tval.T, Tval))**(0.5)
if 1: # compute gradient dcost(H1,H2)/dH1
U, D, V, = numpy.linalg.svd(Tval)
UVT = numpy.dot(U, V.T)
Delta12 = numpy.dot(mat_pow(SigmaHat11), numpy.dot(UVT, mat_pow(SigmaHat22)))
UDUT = numpy.dot(U, numpy.dot(D, U.T))
Delta11 = (-0.5) * numpy.dot(mat_pow(SigmaHat11), numpy.dot(UDUT, mat_pow(SigmaHat22)))
grad_E_to_o = (1.0/m) * (2*numpy.dot(Delta11,H1bar)+numpy.dot(Delta12,H2bar))
##gparam1_W = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * (h1hidden)
gparam1_W = numpy.dot((h1hidden), ((grad_E_to_o) * (h1tmpval*(1-h1tmpval))).T)
##gparam1_b = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * theano.shared(numpy.array([1.0],dtype=theano.config.floatX), borrow=True)
gparam1_b = numpy.dot(numpy.ones((1,10000),dtype=theano.config.floatX), ((grad_E_to_o) * (h1tmpval*(1-h1tmpval))).T)
gparam1_W = theano.shared(gparam1_W, borrow=True)
gparam1_b = theano.shared(gparam1_b[0,:], borrow=True)
#gparams1 = [T.grad(cost1, param) for param in net1.params]
gparams1 = [T.grad(cost1, param) for param in net1.hiddenLayer.params]
gparams1.append(gparam1_W)
updates1 = [
(param, param - learning_rate * gparam)
for param, gparam in zip(net1.params, gparams1)
]
#gparams1.append(gparam1_b)
if 1: # compute gradient dcost(H1,H2)/dH2
Tval2 = numpy.dot(mat_pow(SigmaHat22), numpy.dot(SigmaHat12.T, mat_pow(SigmaHat11)))
U, D, V, = numpy.linalg.svd(Tval2)
UVT = numpy.dot(U, V.T)
Delta12 = numpy.dot(mat_pow(SigmaHat22), numpy.dot(UVT, mat_pow(SigmaHat11)))
UDUT = numpy.dot(U, numpy.dot(D, U.T))
Delta11 = (-0.5) * numpy.dot(mat_pow(SigmaHat22), numpy.dot(UDUT, mat_pow(SigmaHat11)))
grad_E_to_o = (1.0/m) * (2*numpy.dot(Delta11,H2bar)+numpy.dot(Delta12,H1bar))
##gparam1_W = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * (h1hidden)
gparam2_W = numpy.dot((h2hidden), ((grad_E_to_o) * (h2tmpval*(1-h2tmpval))).T)
##gparam1_b = (grad_E_to_o) * (h1tmpval*(1-h1tmpval)) * theano.shared(numpy.array([1.0],dtype=theano.config.floatX), borrow=True)
gparam2_b = numpy.dot(numpy.ones((1,10000),dtype=theano.config.floatX), ((grad_E_to_o) * (h2tmpval*(1-h2tmpval))).T)
gparam2_W = theano.shared(gparam2_W, borrow=True)
gparam2_b = theano.shared(gparam2_b[0,:], borrow=True)
#gparams1 = [T.grad(cost1, param) for param in net1.params]
gparams2 = [T.grad(cost2, param) for param in net2.hiddenLayer.params]
gparams2.append(gparam2_W)
updates2 = [
(param, param - learning_rate * gparam)
for param, gparam in zip(net2.params, gparams2)
]
#gparams1.append(gparam1_b)
#X_theano = theano.shared(value=X, name='inputs')
#h1tmp = theano.shared( value=h1tmpval, name='hidden1_rep', dtype=theano.config.floatX , borrow=True)
h1tmp = theano.shared(numpy.asarray(H1bar,dtype=theano.config.floatX),
borrow=True)
#h2tmp = theano.shared( value=h2tmpval, name='hidden2_rep', dtype=theano.config.floatX , borrow=True)
h2tmp = theano.shared(numpy.asarray(H2bar,dtype=theano.config.floatX),
borrow=True)
#h1tmp = T.shared( value=net1.output.eval(), name='hidden1_rep' )
#h2tmp = T.shared( net2.output.eval() )
train_model1 = theano.function(
inputs=[],
#outputs=cost1,
updates=updates1,
givens={
#x1: test_set_x,
h1: h1tmp,
h2: h2tmp
}
)
train_model2 = theano.function(
inputs=[],
#outputs=cost2,
updates=updates2,
givens={
#x2: test_set_y,
h1: h1tmp,
h2: h2tmp
}
)
minibatch_avg_cost1 = train_model1()
minibatch_avg_cost2 = train_model2()
#print 'corr1', minibatch_avg_cost1
#print 'corr2', minibatch_avg_cost2
print 'corr', corr
if epoch > 10:
break
end_time = time.clock()
print(('Optimization complete. Best validation score of %f %% '
'obtained at iteration %i, with test performance %f %%') %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
''' Loads the dataset
:type dataset: string
:param dataset: the path to the dataset (here MNIST)
'''
#############
# LOAD DATA #
#############
# Download the MNIST dataset if it is not present
data_dir, data_file = os.path.split(dataset)
if data_dir == "" and not os.path.isfile(dataset):
# Check if dataset is in the data directory.
new_path = os.path.join(
os.path.split(__file__)[0],
"..",
"data",
dataset
)
if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz':
dataset = new_path
if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz':
import urllib
origin = (
'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz'
)
print 'Downloading data from %s' % origin
urllib.urlretrieve(origin, dataset)
print '... loading data'
# Load the dataset
f = gzip.open(dataset, 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
#train_set, valid_set, test_set format: tuple(input, target)
#input is an numpy.ndarray of 2 dimensions (a matrix)
#witch row's correspond to an example. target is a
#numpy.ndarray of 1 dimensions (vector)) that have the same length as
#the number of rows in the input. It should give the target
#target to the example with the same index in the input.
def shared_dataset(data_xy, borrow=True):
""" Function that loads the dataset into shared variables
The reason we store our dataset in shared variables is to allow
Theano to copy it into the GPU memory (when code is run on GPU).
Since copying data into the GPU is slow, copying a minibatch everytime
is needed (the default behaviour if the data is not in a shared
variable) would lead to a large decrease in performance.
"""
#import copy
data_x, data_y = data_xy
#daya_y = copy.deepcopy(data_x)
data_y_new = numpy.zeros((data_y.shape[0], data_y.max()+1))
for i in range(data_y.shape[0]):
data_y_new[i, data_y[i]] = 1
data_y = data_y_new
shared_x = theano.shared(numpy.asarray(data_x,
dtype=theano.config.floatX),
borrow=borrow)
shared_y = theano.shared(numpy.asarray(data_y,
dtype=theano.config.floatX),
borrow=borrow)
# When storing data on the GPU it has to be stored as floats
# therefore we will store the labels as ``floatX`` as well
# (``shared_y`` does exactly that). But during our computations
# we need them as ints (we use labels as index, and if they are
# floats it doesn't make sense) therefore instead of returning
# ``shared_y`` we will have to cast it to int. This little hack
# lets ous get around this issue
return shared_x, shared_y
test_set_x, test_set_y = shared_dataset(test_set)
valid_set_x, valid_set_y = shared_dataset(valid_set)
train_set_x, train_set_y = shared_dataset(train_set)
rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),
(test_set_x, test_set_y)]
return rval
2
Example 11
Project: inasafe Source File: gdal_ogr_tools.py
def polygonize_thresholds(
raster_file_name,
threshold_min=0.0,
threshold_max=float('inf')):
"""
Function to polygonize raster. Areas (pixels) with threshold_min <
pixel_values < threshold_max will be converted to polygons.
:param raster_file_name: Raster file name
:type raster_file_name: string
:param threshold_min: Value that splits raster to
flooded or not flooded.
:type threshold_min: float
:param threshold_max: Value that splits raster to
flooded or not flooded.
:type threshold_max: float
:returns: Polygon shape file name
:rtype: string
"""
# all values that are in the threshold are set to 1, others are set to 0
base_name = unique_filename()
outfile = base_name + '.tif'
indataset = gdal.Open(raster_file_name, gdal.GA_ReadOnly)
out_driver = gdal.GetDriverByName('GTiff')
outdataset = out_driver.Create(
outfile,
indataset.RasterXSize,
indataset.RasterYSize,
indataset.RasterCount,
gdal.GDT_Byte)
gt = indataset.GetGeoTransform()
if gt is not None and gt != (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
outdataset.SetGeoTransform(gt)
prj = indataset.GetProjectionRef()
if prj is not None and len(prj) > 0:
outdataset.SetProjection(prj)
outNoData = 1
for iBand in range(1, indataset.RasterCount + 1):
inband = indataset.GetRasterBand(iBand)
outband = outdataset.GetRasterBand(iBand)
for i in range(inband.YSize - 1, -1, -1):
scanline = inband.ReadAsArray(
0, i, inband.XSize, 1, inband.XSize, 1)
if threshold_min >= 0:
scanline = \
numpy.choose(numpy.less(scanline, float(threshold_min)),
(scanline, 0))
if threshold_max > 0 and threshold_max > threshold_min:
scanline = \
numpy.choose(numpy.greater(scanline, float(threshold_max)),
(scanline, 0))
scanline = numpy.choose(numpy.not_equal(scanline, 0),
(scanline, outNoData))
outband.WriteArray(scanline, 0, i)
# polygonize
spat_ref = osr.SpatialReference()
proj = indataset.GetProjectionRef()
spat_ref.ImportFromWkt(proj)
drv = ogr.GetDriverByName("ESRI Shapefile")
base_name = unique_filename()
out_shape_file = base_name + ".shp"
dst_ds = drv.CreateDataSource(out_shape_file)
# ogr_layer_name = 'polygonized'
ogr_layer_name = os.path.splitext(os.path.split(out_shape_file)[1])[0]
dst_layer = dst_ds.CreateLayer(ogr_layer_name, spat_ref)
# fd = ogr.FieldDefn("DN", ogr.OFTInteger )
fd = ogr.FieldDefn("DN", ogr.OFTReal)
dst_layer.CreateField(fd)
dst_field = 0
# gdal.Polygonize(
# outband, outband, dst_layer, dst_field, [], callback=None)
gdal.Polygonize(outband, None, dst_layer, dst_field, [], callback=None)
# produce in and out polygon layers
base_name = unique_filename()
inside_shape_file = base_name + "_inside.shp"
inside_layer_name = \
os.path.splitext(os.path.split(inside_shape_file)[1])[0]
outside_shape_file = base_name + "_outside.shp"
outside_layer_name = \
os.path.splitext(os.path.split(outside_shape_file)[1])[0]
inside_ds = drv.CreateDataSource(inside_shape_file)
inside_layer = inside_ds.CreateLayer(inside_layer_name, spat_ref)
outside_ds = drv.CreateDataSource(outside_shape_file)
outside_layer = inside_ds.CreateLayer(outside_layer_name, spat_ref)
for feature in dst_layer:
value = feature.GetField("DN")
geom = feature.GetGeometryRef()
if value == 1:
new_feature = ogr.Feature(inside_layer.GetLayerDefn())
new_feature.SetGeometry(geom)
inside_layer.CreateFeature(new_feature)
else:
new_feature = ogr.Feature(outside_layer.GetLayerDefn())
new_feature.SetGeometry(geom)
outside_layer.CreateFeature(new_feature)
inside_ds.Destroy()
outside_ds.Destroy()
dst_ds.Destroy()
return (
inside_shape_file,
inside_layer_name,
outside_shape_file,
outside_layer_name)
2
Example 12
Project: WikiDAT Source File: etl.py
def run(self):
"""
Execute workflow to import logging records of actions on pages and
users from dump file
The data loading workflow is composed of a number of processor
elements, which can be:
- Producer (P): raw input data --> input element queue
- ConsumerProducer (CP): input element queue --> insert db queue
- Consumer (C): insert db queue --> database (MySQL/MariaDB)
In this case, the usual combination is 1:N:1 (P, CP, C)
"""
start = time.time()
print("Starting LoggingETL workflow at %s" % (
time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.localtime())))
# DATA EXTRACTION
xml_reader_name = '-'.join([self.name, 'xml_reader'])
logitem_proc_name = '-'.join([self.name, 'process_logitem'])
logitem_insert_name = '-'.join([self.name, 'insert_logitem'])
# Start subprocess to extract elements from logging dump file
file_path = self.path[0]
dump_file = DumpFile(file_path)
xml_reader = Producer(name=xml_reader_name,
target=process_xml,
kwargs=dict(
dump_file=dump_file),
consumers=self.log_fan,
push_logs_port=self.base_port,
control_port=self.control_port)
xml_reader.start()
print(xml_reader_name, "started")
print(self.name, "Extracting data from XML revision history file:")
print(str(self.path[0]))
# List to keep tracking of logitem workers
workers = []
# Create and start page processes
for worker in range(self.log_fan):
worker_name = '-'.join([logitem_proc_name, str(worker)])
process_logitems = Processor(name=worker_name,
target=logitem_to_file,
producers=1, consumers=1,
pull_port=self.base_port,
push_port=self.base_port+2,
control_port=self.control_port)
process_logitems.start()
workers.append(process_logitems)
print(worker_name, "started")
# Create directory for logging files if it does not exist
log_dir = os.path.join(os.path.split(file_path)[0], 'logs')
tmp_dir = os.path.join(os.getcwd(), os.path.split(file_path)[0], 'tmp')
file_name = os.path.split(file_path)[1]
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
log_file = os.path.join(log_dir, file_name + '.log')
db_log = MySQLDB(host='localhost', port=3306, user=self.db_user,
passwd=self.db_passw, db=self.db_name)
db_log.connect()
logitem_insert_db = Consumer(name=logitem_insert_name,
target=logitem_file_to_db,
kwargs=dict(con=db_log,
log_file=log_file,
tmp_dir=tmp_dir,
file_rows=self.log_cache_size,
etl_prefix=self.name),
producers=self.log_fan,
pull_port=self.base_port+2)
print(logitem_insert_name, "started")
logitem_insert_db.start()
print("Waiting for all processes to finish...")
print()
xml_reader.join()
for w in workers:
w.join()
logitem_insert_db.join()
# All operations finished
end = time.time()
print("All tasks done in %.4f sec." % ((end-start)/1.))
print()
db_log.close()
2
Example 13
Project: plugin.audio.tuneinradio Source File: tunein.py
def tune(self, id):
'''Returns a list of streams associated with the station.
'''
self.log_debug('tune')
self.log_debug('id: %s' % id)
if (not self.is_station_id(id) and not self.is_topic_id(id)):
raise TuneIn.TuneInError(-1, 'Id is not of the correct type.')
params = [{'param': 'id', 'value': id}]
req = urllib2.Request(
self.__add_params_to_url('Tune.ashx', params, addrender=False))
f = urllib2.urlopen(req)
self.log_debug('First pass of streams.')
streams = []
for stream in f:
stream = stream.rsplit()[0]
self.log_debug('stream: %s' % stream)
(filepath, filename) = os.path.split(stream)
(shortname, extension) = os.path.splitext(filename)
filepath = filepath.lower()
filename = filename.lower()
shortname = shortname.lower()
extension = extension.lower()
self.log_debug('filepath: %s' % filepath)
self.log_debug('filename: %s' % filename)
self.log_debug('shortname: %s' % shortname)
self.log_debug('extension: %s' % extension)
if (extension == '.pls'):
self.log_debug('PLS Playlist')
for stream in self.__parse_pls(stream):
streams.append(stream)
elif (extension == '.asx'):
self.log_debug('ASX Playlist')
for stream in self.__parse_asx(stream):
streams.append(stream)
elif (extension == '.m3u'):
self.log_debug('M3U Playlist')
for stream in self.__parse_m3u(stream):
streams.append(stream)
elif (re.search('streamtheworld.com', filepath)):
''' StreamTheWorld Support
'''
self.log_debug('StreamTheWorld stream')
pattern = re.compile('(.*)callsign\=(.*)$')
result = pattern.match(filename)
if (result):
stw = streamtheworld.StreamTheWorld(result.group(2))
stw_url = stw.get_stream_url(result.group(2))
streams.append(stw_url)
elif (stream.find('player.amri.ca') != -1):
''' Astral Radio Support
'''
self.log_debug('Astral Radio stream')
astral = astralradio.AstralRadio(stream)
for stream in astral.get_streams():
streams.append(stream)
else:
self.log_debug('Unknown stream')
try:
request = urllib2.Request(stream)
opener = urllib2.build_opener()
f = opener.open(request)
if f.url != stream:
stream = f.url
streams.append(stream)
elif f.info().gettype() == 'video/x-ms-asf':
try:
for stream in self.__parse_asf(stream):
streams.append(stream)
except:
try:
for stream in self.__parse_asx(stream):
streams.append(stream)
except:
pass
else:
streams.append(stream)
except urllib2.URLError as e:
self.log_debug('Ignoring URLError: %s' % e)
streams.append(stream)
self.log_debug('Second pass of streams.')
tmpstreams = streams
streams = []
for stream in tmpstreams:
stream = stream.rsplit()[0]
self.log_debug('stream: %s' % stream)
(filepath, filename) = os.path.split(stream)
(shortname, extension) = os.path.splitext(filename)
filepath = filepath.lower()
filename = filename.lower()
shortname = shortname.lower()
extension = extension.lower()
self.log_debug('filepath: %s' % filepath)
self.log_debug('filename: %s' % filename)
self.log_debug('shortname: %s' % shortname)
self.log_debug('extension: %s' % extension)
if (extension == '.pls'):
self.log_debug('PLS Playlist')
for stream in self.__parse_pls(stream):
streams.append(stream)
elif (extension == '.asx'):
self.log_debug('ASX Playlist')
for stream in self.__parse_asx(stream):
streams.append(stream)
elif (extension == '.m3u'):
self.log_debug('M3U Playlist')
for stream in self.__parse_m3u(stream):
streams.append(stream)
else:
self.log_debug('Unknown stream')
streams.append(stream)
return streams
2
Example 14
Project: connectomeviewer Source File: plot_directive.py
def plot_directive(name, arguments, options, content, lineno,
content_offset, block_text, state, state_machine):
"""
Handle the plot directive.
"""
formats = setup.config.plot_formats
if type(formats) == str:
formats = eval(formats)
# The user may provide a filename *or* Python code content, but not both
if len(arguments) == 1:
reference = directives.uri(arguments[0])
basedir, fname = os.path.split(reference)
basename, ext = os.path.splitext(fname)
basedir = relpath(basedir, setup.app.builder.srcdir)
if len(content):
raise ValueError("plot directive may not specify both a filename and inline content")
content = None
else:
basedir = "inline"
content = '\n'.join(content)
# Since we don't have a filename, use a hash based on the content
reference = basename = md5(content).hexdigest()[-10:]
fname = None
# Get the directory of the rst file, and determine the relative
# path from the resulting html file to the plot_directive links
# (linkdir). This relative path is used for html links *only*,
# and not the embedded image. That is given an absolute path to
# the temporary directory, and then sphinx moves the file to
# build/html/_images for us later.
rstdir, rstfile = os.path.split(state_machine.docuement.attributes['source'])
reldir = rstdir[len(setup.confdir)+1:]
relparts = [p for p in os.path.split(reldir) if p.strip()]
nparts = len(relparts)
outdir = os.path.join('plot_directive', basedir)
linkdir = ('../' * nparts) + outdir
# tmpdir is where we build all the output files. This way the
# plots won't have to be redone when generating latex after html.
# Prior to Sphinx 0.6, absolute image paths were treated as
# relative to the root of the filesystem. 0.6 and after, they are
# treated as relative to the root of the docuementation tree. We need
# to support both methods here.
tmpdir = os.path.join('build', outdir)
if sphinx_version < (0, 6):
tmpdir = os.path.abspath(tmpdir)
prefix = ''
else:
prefix = '/'
if not os.path.exists(tmpdir):
cbook.mkdirs(tmpdir)
# destdir is the directory within the output to store files
# that we'll be linking to -- not the embedded images.
destdir = os.path.abspath(os.path.join(setup.app.builder.outdir, outdir))
if not os.path.exists(destdir):
cbook.mkdirs(destdir)
# Generate the figures, and return the number of them
num_figs = makefig(reference, content, tmpdir)
if options.has_key('include-source'):
if content is None:
content = open(reference, 'r').read()
lines = ['::', ''] + [' %s'%row.rstrip() for row in content.split('\n')]
del options['include-source']
else:
lines = []
if num_figs > 0:
options = [' :%s: %s' % (key, val) for key, val in
options.items()]
options = "\n".join(options)
if fname is not None:
shutil.copyfile(reference, os.path.join(destdir, fname))
for i in range(num_figs):
if num_figs == 1:
outname = basename
else:
outname = "%s_%02d" % (basename, i)
# Copy the linked-to files to the destination within the build tree,
# and add a link for them
links = []
if fname is not None:
links.append('`source code <%(linkdir)s/%(basename)s.py>`__')
for format in formats[1:]:
shutil.copyfile(os.path.join(tmpdir, outname + "." + format),
os.path.join(destdir, outname + "." + format))
links.append('`%s <%s/%s.%s>`__' % (format, linkdir, outname, format))
links = ', '.join(links) % locals()
# Output the resulting reST
lines.extend((template % locals()).split('\n'))
else:
lines.extend((exception_template % locals()).split('\n'))
if len(lines):
state_machine.insert_input(
lines, state_machine.input_lines.source(0))
return []
2
Example 15
Project: attention-lvcsr Source File: nvcc_compiler.py
@staticmethod
def compile_str(
module_name, src_code,
location=None, include_dirs=[], lib_dirs=[], libs=[], preargs=[],
rpaths=rpath_defaults, py_module=True, hide_symbols=True):
"""
Parameters
----------
module_name: str
This has been embedded in the src_code.
src_code
A complete c or c++ source listing for the module.
location
A pre-existing filesystem directory where the
cpp file and .so will be written.
include_dirs
A list of include directory names (each gets prefixed with -I).
lib_dirs
A list of library search path directory names (each gets
prefixed with -L).
libs
A list of libraries to link with (each gets prefixed with -l).
preargs
A list of extra compiler arguments.
rpaths
List of rpaths to use with Xlinker. Defaults to `rpath_defaults`.
py_module
If False, compile to a shared library, but
do not import as a Python module.
hide_symbols
If True (the default), hide all symbols from the library symbol
table unless explicitely exported.
Returns
-------
module
Dynamically-imported python module of the compiled code.
(unless py_module is False, in that case returns None.)
Notes
-----
On Windows 7 with nvcc 3.1 we need to compile in the real directory
Otherwise nvcc never finish.
"""
# Remove empty string directory
include_dirs = [d for d in include_dirs if d]
lib_dirs = [d for d in lib_dirs if d]
rpaths = list(rpaths)
if sys.platform == "win32":
# Remove some compilation args that cl.exe does not understand.
# cl.exe is the compiler used by nvcc on Windows.
for a in ["-Wno-write-strings", "-Wno-unused-label",
"-Wno-unused-variable", "-fno-math-errno"]:
if a in preargs:
preargs.remove(a)
if preargs is None:
preargs = []
else:
preargs = list(preargs)
if sys.platform != 'win32':
preargs.append('-fPIC')
if config.cmodule.remove_gxx_opt:
preargs = [p for p in preargs if not p.startswith('-O')]
cuda_root = config.cuda.root
# The include dirs gived by the user should have precedence over
# the standards ones.
include_dirs = include_dirs + std_include_dirs()
if os.path.abspath(os.path.split(__file__)[0]) not in include_dirs:
include_dirs.append(os.path.abspath(os.path.split(__file__)[0]))
libs = libs + std_libs()
if 'cudart' not in libs:
libs.append('cudart')
lib_dirs = lib_dirs + std_lib_dirs()
# config.dnn.include_path add this by default for cudnn in the
# new back-end. This should not be used in this back-end. So
# just remove them.
lib_dirs = [ld for ld in lib_dirs if
not(ld == os.path.join(cuda_root, 'lib') or
ld == os.path.join(cuda_root, 'lib64'))]
if sys.platform != 'darwin':
# sometimes, the linker cannot find -lpython so we need to tell it
# explicitly where it is located
# this returns somepath/lib/python2.x
python_lib = distutils.sysconfig.get_python_lib(plat_specific=1,
standard_lib=1)
python_lib = os.path.dirname(python_lib)
if python_lib not in lib_dirs:
lib_dirs.append(python_lib)
cppfilename = os.path.join(location, 'mod.cu')
with open(cppfilename, 'w') as cppfile:
_logger.debug('Writing module C++ code to %s', cppfilename)
cppfile.write(src_code)
lib_filename = os.path.join(location, '%s.%s' %
(module_name, get_lib_extension()))
_logger.debug('Generating shared lib %s', lib_filename)
# TODO: Why do these args cause failure on gtx285 that has 1.3
# compute capability? '--gpu-architecture=compute_13',
# '--gpu-code=compute_13',
# nvcc argument
preargs1 = []
preargs2 = []
for pa in preargs:
if pa.startswith('-Wl,'):
preargs1.append('-Xlinker')
preargs1.append(pa[4:])
continue
for pattern in ['-O', '-arch=', '-ccbin=', '-G', '-g', '-I',
'-L', '--fmad', '--ftz', '--maxrregcount',
'--prec-div', '--prec-sqrt', '--use_fast_math',
'-fmad', '-ftz', '-maxrregcount',
'-prec-div', '-prec-sqrt', '-use_fast_math',
'--use-local-env', '--cl-version=']:
if pa.startswith(pattern):
preargs1.append(pa)
break
else:
preargs2.append(pa)
# Don't put -G by default, as it slow things down.
# We aren't sure if -g slow things down, so we don't put it by default.
cmd = [nvcc_path, '-shared'] + preargs1
if config.nvcc.compiler_bindir:
cmd.extend(['--compiler-bindir', config.nvcc.compiler_bindir])
if sys.platform == 'win32':
# add flags for Microsoft compiler to create .pdb files
preargs2.extend(['/Zi', '/MD'])
cmd.extend(['-Xlinker', '/DEBUG'])
# remove the complaints for the duplication of `double round(double)`
# in both math_functions.h and pymath.h,
# by not including the one in pymath.h
cmd.extend(['-D HAVE_ROUND'])
else:
if hide_symbols:
preargs2.append('-fvisibility=hidden')
if local_bitwidth() == 64:
cmd.append('-m64')
else:
cmd.append('-m32')
if len(preargs2) > 0:
cmd.extend(['-Xcompiler', ','.join(preargs2)])
# We should not use rpath if possible. If the user provided
# provided an cuda.root flag, we need to add one, but
# otherwise, we don't add it. See gh-1540 and
# https://wiki.debian.org/RpathIssue for details.
if (not type(config.cuda).root.is_default and
os.path.exists(os.path.join(config.cuda.root, 'lib'))):
rpaths.append(os.path.join(config.cuda.root, 'lib'))
if sys.platform != 'darwin':
# the CUDA libs are universal (contain both 32-bit and 64-bit)
rpaths.append(os.path.join(config.cuda.root, 'lib64'))
if sys.platform != 'win32':
# the -rpath option is not understood by the Microsoft linker
for rpath in rpaths:
cmd.extend(['-Xlinker', ','.join(['-rpath', rpath])])
cmd.extend('-I%s' % idir for idir in include_dirs)
cmd.extend(['-o', lib_filename])
cmd.append(os.path.split(cppfilename)[-1])
cmd.extend(['-L%s' % ldir for ldir in lib_dirs])
cmd.extend(['-l%s' % l for l in libs])
if sys.platform == 'darwin':
# This tells the compiler to use the already-loaded python
# symbols (which should always be the right ones).
cmd.extend(['-Xcompiler', '-undefined,dynamic_lookup'])
# Remove "-u Symbol" arguments, since they are usually not
# relevant for the new compilation, even if they were used for
# compiling python. If they are necessary, the nvcc syntax is
# "-U Symbol" with a capital U.
done = False
while not done:
try:
indexof = cmd.index('-u')
cmd.pop(indexof) # Remove -u
cmd.pop(indexof) # Remove argument to -u
except ValueError as e:
done = True
# CUDA Toolkit v4.1 Known Issues:
# Host linker on Mac OS 10.7 (and 10.6 for me) passes -no_pie option
# to nvcc this option is not recognized and generates an error
# http://stackoverflow.com/questions/9327265/nvcc-unknown-option-no-pie
# Passing -Xlinker -pie stops -no_pie from getting passed
if sys.platform == 'darwin' and nvcc_version >= '4.1':
cmd.extend(['-Xlinker', '-pie'])
# cmd.append("--ptxas-options=-v") #uncomment this to see
# register and shared-mem requirements
_logger.debug('Running cmd %s', ' '.join(cmd))
orig_dir = os.getcwd()
try:
os.chdir(location)
p = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
nvcc_stdout, nvcc_stderr = decode_iter(p.communicate()[:2])
finally:
os.chdir(orig_dir)
for eline in nvcc_stderr.split('\n'):
if not eline:
continue
if 'skipping incompatible' in eline:
# ld is skipping an incompatible library
continue
if 'declared but never referenced' in eline:
continue
if 'statement is unreachable' in eline:
continue
_logger.info("NVCC: %s", eline)
if p.returncode:
for i, l in enumerate(src_code.split('\n')):
print(i + 1, l, file=sys.stderr)
print('===============================', file=sys.stderr)
# filter the output from the compiler
for l in nvcc_stderr.split('\n'):
if not l:
continue
# filter out the annoying declaration warnings
try:
if l[l.index(':'):].startswith(': warning: variable'):
continue
if l[l.index(':'):].startswith(': warning: label'):
continue
except Exception:
pass
print(l, file=sys.stderr)
print(nvcc_stdout)
print(cmd)
raise Exception('nvcc return status', p.returncode,
'for cmd', ' '.join(cmd))
elif config.cmodule.compilation_warning and nvcc_stdout:
print(nvcc_stdout)
if nvcc_stdout:
# this doesn't happen to my knowledge
print("DEBUG: nvcc STDOUT", nvcc_stdout, file=sys.stderr)
if py_module:
# touch the __init__ file
open(os.path.join(location, "__init__.py"), 'w').close()
return dlimport(lib_filename)
2
Example 16
Project: mysql-utilities Source File: binlog_admin.py
def move_binlogs_from_server(server_cnx_val, destination, options,
bin_basename=None, bin_index=None,
relay_basename=None):
"""Relocate binary logs from the given server to a new location.
This function relocate the binary logs from a MySQL server to the specified
destination directory, attending to the specified options.
server_cnx_val[in] Dictionary with the connection values for the server.
destination[in] Path of the destination directory for the binary log
files.
options[in] Dictionary of options (log_type, skip_flush_binlogs,
modified_before, sequence, verbosity).
bin_basename[in] Base name for the binlog files, i.e., same as the
value for the server option --log-bin. It replaces
the server variable 'log_bin_basename' for versions
< 5.6.2, otherwise it is ignored.
bin_index[in] Path of the binlog index file. It replaces the server
variable 'log_bin_index' for versions < 5.6.4,
otherwise it is ignored.
relay_basename[in] Base name for the relay log files, i.e., filename
without the extension (sequence number). Same as the
value for the server option --relay-log. It replaces
the server variable 'relay_log_basename' for versions
< 5.6.2, otherwise it is ignored.
"""
log_type = options.get('log_type', LOG_TYPE_BIN)
skip_flush = options['skip_flush_binlogs']
verbosity = options['verbosity']
# Connect to server
server_options = {
'conn_info': server_cnx_val,
}
srv = Server(server_options)
srv.connect()
# Check if the server is running locally (not remote server).
if not srv.is_alias('localhost'):
raise UtilError("You are using a remote server. This utility must be "
"run on the local server. It does not support remote "
"access to the binary log files.")
# Check privileges.
_check_privileges_to_move_binlogs(srv, options)
# Process binlog files.
if log_type in (LOG_TYPE_BIN, LOG_TYPE_ALL):
# Get log_bin_basename (available since MySQL 5.6.2).
if srv.check_version_compat(5, 6, 2):
if bin_basename:
print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='bin basename', min_version='5.6.2',
var_name='log_bin_basename'))
binlog_basename = srv.select_variable('log_bin_basename')
if verbosity > 0:
print("#")
print("# log_bin_basename: {0}".format(binlog_basename))
binlog_source, binlog_file = os.path.split(binlog_basename)
# Get log_bin_index (available since MySQL 5.6.4).
if srv.check_version_compat(5, 6, 4):
if bin_index:
print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='bin index', min_version='5.6.4',
var_name='log_bin_index'))
binlog_index = srv.select_variable('log_bin_index')
else:
binlog_index = None
action = _ACTION_SEARCH_INDEX.format(file_type='bin-log')
print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='log_bin_basename', host=srv.host, port=srv.port,
min_version='5.6.4', action=action))
if verbosity > 0:
print("# log_bin_index: {0}".format(binlog_index))
else:
if bin_basename:
binlog_source, binlog_file = os.path.split(bin_basename)
else:
action = _ACTION_DATADIR_USED.format(file_type='bin-log')
print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='log_bin_basename', host=srv.host, port=srv.port,
min_version='5.6.2', action=action))
# Get datadir value.
binlog_source = srv.select_variable('datadir')
binlog_file = None
if verbosity > 0:
print("#")
print("# datadir: {0}".format(binlog_source))
binlog_index = bin_index
# Move binlog files.
num_files = _move_binlogs(
binlog_source, destination, LOG_TYPE_BIN, options,
basename=binlog_file, index_file=binlog_index, skip_latest=True)
print("#")
# Flush binary logs to reload server's cache after move.
if not skip_flush and num_files > 0:
# Note: log_type for FLUSH available since MySQL 5.5.3.
if srv.check_version_compat(5, 5, 3):
print(_INFO_MSG_FLUSH_LOGS.format(log_type='binary'))
srv.flush_logs(log_type='BINARY')
else:
print(_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='binary',
host=srv.host,
port=srv.port))
print("#")
if log_type in (LOG_TYPE_RELAY, LOG_TYPE_ALL):
# Get relay_log_basename (available since MySQL 5.6.2).
if srv.check_version_compat(5, 6, 2):
if relay_basename:
print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='relay basename', min_version='5.6.2',
var_name='relay_log_basename'))
relay_log_basename = srv.select_variable('relay_log_basename')
if verbosity > 0:
print("#")
print("# relay_log_basename: {0}".format(relay_log_basename))
relay_source, relay_file = os.path.split(relay_log_basename)
else:
if relay_basename:
relay_source, relay_file = os.path.split(relay_basename)
else:
action = _ACTION_DATADIR_USED.format(file_type='relay-log')
print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='relay_log_basename', host=srv.host,
port=srv.port, min_version='5.6.2', action=action))
# Get datadir value.
relay_source = srv.select_variable('datadir')
relay_file = None
if verbosity > 0:
print("#")
print("# datadir: {0}".format(relay_source))
# Get relay_log_index (available for all supported versions).
relay_log_index = srv.select_variable('relay_log_index')
if verbosity > 0:
print("# relay_log_index: {0}".format(relay_log_index))
# Move relay log files.
num_files = _move_binlogs(
relay_source, destination, LOG_TYPE_RELAY, options,
basename=relay_file, index_file=relay_log_index, skip_latest=True)
print("#")
# Flush relay logs to reload server's cache after move.
if not skip_flush and num_files > 0:
# Note: log_type for FLUSH available since MySQL 5.5.3.
if srv.check_version_compat(5, 5, 3):
print(_INFO_MSG_FLUSH_LOGS.format(log_type='relay'))
srv.flush_logs(log_type='RELAY')
else:
print(_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='relay',
host=srv.host,
port=srv.port))
print("#")
print("#...done.\n#")
2
Example 17
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
"""A builder for LaTeX files that checks the output in the aux file
and decides how many times to use LaTeXAction, and BibTeXAction."""
global must_rerun_latex
# This routine is called with two actions. In this file for DVI builds
# with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
# set this up now for the case where the user requests a different extension
# for the target filename
if (XXXLaTeXAction == LaTeXAction):
callerSuffix = ".dvi"
else:
callerSuffix = env['PDFSUFFIX']
basename = SCons.Util.splitext(str(source[0]))[0]
basedir = os.path.split(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
abspath = os.path.abspath(basedir)
targetext = os.path.splitext(str(target[0]))[1]
targetdir = os.path.split(str(target[0]))[0]
saved_env = {}
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
saved_env[var] = modify_env_var(env, var, abspath)
# Create base file names with the target directory since the auxiliary files
# will be made there. That's because the *COM variables have the cd
# command in the prolog. We check
# for the existence of files before opening them--even ones like the
# aux file that TeX always creates--to make it possible to write tests
# with stubs that don't necessarily generate all of the same files.
targetbase = os.path.join(targetdir, basefile)
# if there is a \makeindex there will be a .idx and thus
# we have to run makeindex at least once to keep the build
# happy even if there is no index.
# Same for glossaries and nomenclature
src_content = source[0].get_text_contents()
run_makeindex = makeindex_re.search(src_content) and not os.path.exists(targetbase + '.idx')
run_nomenclature = makenomenclature_re.search(src_content) and not os.path.exists(targetbase + '.nlo')
run_glossary = makeglossary_re.search(src_content) and not os.path.exists(targetbase + '.glo')
run_glossaries = makeglossaries_re.search(src_content) and not os.path.exists(targetbase + '.glo')
run_acronyms = makeacronyms_re.search(src_content) and not os.path.exists(targetbase + '.acn')
saved_hashes = {}
suffix_nodes = {}
for suffix in all_suffixes:
theNode = env.fs.File(targetbase + suffix)
suffix_nodes[suffix] = theNode
saved_hashes[suffix] = theNode.get_csig()
if Verbose:
print "hashes: ",saved_hashes
must_rerun_latex = True
#
# routine to update MD5 hash and compare
#
def check_MD5(filenode, suffix):
global must_rerun_latex
# two calls to clear old csig
filenode.clear_memoized_values()
filenode.ninfo = filenode.new_ninfo()
new_md5 = filenode.get_csig()
if saved_hashes[suffix] == new_md5:
if Verbose:
print "file %s not changed" % (targetbase+suffix)
return False # unchanged
saved_hashes[suffix] = new_md5
must_rerun_latex = True
if Verbose:
print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
return True # changed
# generate the file name that latex will generate
resultfilename = targetbase + callerSuffix
count = 0
while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
result = XXXLaTeXAction(target, source, env)
if result != 0:
return result
count = count + 1
must_rerun_latex = False
# Decide if various things need to be run, or run again.
# Read the log file to find warnings/errors
logfilename = targetbase + '.log'
logContent = ''
if os.path.exists(logfilename):
logContent = open(logfilename, "rb").read()
# Read the fls file to find all .aux files
flsfilename = targetbase + '.fls'
flsContent = ''
auxfiles = []
if os.path.exists(flsfilename):
flsContent = open(flsfilename, "rb").read()
auxfiles = openout_aux_re.findall(flsContent)
if Verbose:
print "auxfiles ",auxfiles
# Now decide if bibtex will need to be run.
# The information that bibtex reads from the .aux file is
# pass-independent. If we find (below) that the .bbl file is unchanged,
# then the last latex saw a correct bibliography.
# Therefore only do this on the first pass
if count == 1:
for auxfilename in auxfiles:
target_aux = os.path.join(targetdir, auxfilename)
if os.path.exists(target_aux):
content = open(target_aux, "rb").read()
if content.find("bibdata") != -1:
if Verbose:
print "Need to run bibtex"
bibfile = env.fs.File(targetbase)
result = BibTeXAction(bibfile, bibfile, env)
if result != 0:
check_file_error_message(env['BIBTEX'], 'blg')
return result
must_rerun_latex = check_MD5(suffix_nodes['.bbl'],'.bbl')
break
# Now decide if latex will need to be run again due to index.
if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
# We must run makeindex
if Verbose:
print "Need to run makeindex"
idxfile = suffix_nodes['.idx']
result = MakeIndexAction(idxfile, idxfile, env)
if result != 0:
check_file_error_message(env['MAKEINDEX'], 'ilg')
return result
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# Harder is case is where an action needs to be called -- that should be rare (I hope?)
for index in check_suffixes:
check_MD5(suffix_nodes[index],index)
# Now decide if latex will need to be run again due to nomenclature.
if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
# We must run makeindex
if Verbose:
print "Need to run makeindex for nomenclature"
nclfile = suffix_nodes['.nlo']
result = MakeNclAction(nclfile, nclfile, env)
if result != 0:
check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
'nlg')
#return result
# Now decide if latex will need to be run again due to glossary.
if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
# We must run makeindex
if Verbose:
print "Need to run makeindex for glossary"
glofile = suffix_nodes['.glo']
result = MakeGlossaryAction(glofile, glofile, env)
if result != 0:
check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
'glg')
#return result
# Now decide if latex will need to be run again due to acronyms.
if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
# We must run makeindex
if Verbose:
print "Need to run makeindex for acronyms"
acrfile = suffix_nodes['.acn']
result = MakeAcronymsAction(acrfile, acrfile, env)
if result != 0:
check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
'alg')
return result
# Now decide if latex needs to be run yet again to resolve warnings.
if warning_rerun_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to latex or package rerun warning"
if rerun_citations_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to 'Rerun to get citations correct' warning"
if undefined_references_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to undefined references or citations"
if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop
# rename Latex's output to what the target name is
if not (str(target[0]) == resultfilename and os.path.exists(resultfilename)):
if os.path.exists(resultfilename):
print "move %s to %s" % (resultfilename, str(target[0]), )
shutil.move(resultfilename,str(target[0]))
# Original comment (when TEXPICTS was not restored):
# The TEXPICTS enviroment variable is needed by a dvi -> pdf step
# later on Mac OSX so leave it
#
# It is also used when searching for pictures (implicit dependencies).
# Why not set the variable again in the respective builder instead
# of leaving local modifications in the environment? What if multiple
# latex builds in different directories need different TEXPICTS?
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
if var == 'TEXPICTS':
continue
if saved_env[var] is _null:
try:
del env['ENV'][var]
except KeyError:
pass # was never set
else:
env['ENV'][var] = saved_env[var]
return result
2
Example 18
def process(self, image):
"""
"""
assert image.mode == 'LA'
smotion, mask = image.split()
# Try to get box from info, otherwise from alpha channel
origBox = image.info.get('tracking')
if not origBox:
origBox = mask.getbbox()
expandedBox = (max(0, origBox[0] - self._zonePreExpansionX),
max(0, origBox[1] - self._zonePreExpansionY),
min(image.size[0], origBox[2] + self._zonePreExpansionX),
min(image.size[1], origBox[3] + self._zonePreExpansionY))
imgbox = smotion.crop(expandedBox)
#imgbox = smotion.crop(origBox)
w, h = imgbox.size
imgdata = numpy.array(imgbox.getdata())
imgdata.shape = (h, w)
# Create binary image indicating whether non-zero
# S-Motion exists
salpha = (imgdata > 0).astype(int)
histX = salpha.mean(axis=0)
smoothX = self._smooth(histX, window_len=self._windowX,
window=self._windowType)
#-----------------------------------------------------------------------
# Apply tightening/splitting in horizontal direction
# Pre-compute the minimum length of a strong
# zone that we'll accept.
# This is the max of an absolute length and a
# minimum fraction of the original box.
minZoneLen = max(self._minAbsZoneLenX,
int(round(self._minRelZoneLenX \
* float(len(smoothX)))))
# Minimum length for a weak gap
minWeakLen = max(self._minAbsWeakLenX,
int(round(self._minRelWeakLenX \
* float(len(smoothX)))))
maxX = smoothX.max()
# For now, simple threshold
threshX = self._heightThresh * maxX
strongX = (smoothX >= threshX).astype(int)
# Pre-calculate the minimum peak strength for
# each lobe to avoid being culled
minStrength = maxX * self._secondaryHeightThresh
# Find changes:
# If deltas[k] == 1, then strongX[k+1] is the
# beginning of a new strong block;
# If deltas[k] == -1, then strongX[k+1] is the
# beginning of a new weak block
deltas = strongX[1:] - strongX[:-1]
changes = numpy.where(deltas)[0]
# Form our block lists
strongZonesX = []
if strongX[0]:
curZoneStart = 0
else:
curZoneStart = None
for changeIdx in changes:
strongIdx = changeIdx + 1
changeDir = deltas[changeIdx]
# Start of new strong zone
if changeDir == 1:
assert curZoneStart is None
curZoneStart = strongIdx
# End of existing strong zone
else:
assert changeDir == -1
assert curZoneStart is not None
strongZone = (curZoneStart, strongIdx)
self._acceptOrCull(smoothX, strongZonesX, strongZone, minZoneLen, minStrength)
curZoneStart = None
# Last one
if curZoneStart is not None:
strongZone = (curZoneStart, len(strongX))
self._acceptOrCull(smoothX, strongZonesX, strongZone, minZoneLen, minStrength)
# Remove tiny/thin weak gaps
if len(strongZonesX) > 1:
tempZones = []
lastZone = strongZonesX[0]
for startIdx, endIdx in strongZonesX[1:]:
if startIdx - lastZone[1] >= minWeakLen:
tempZones += [lastZone]
lastZone = (startIdx, endIdx)
else:
lastZone = (lastZone[0], endIdx)
tempZones += [lastZone]
strongZonesX = tempZones
#-----------------------------------------------------------------------
# Apply tightening/splitting in vertical direction (to each strong zone)
strongZonesAll = []
for strongZoneX in strongZonesX:
#histY = salpha.mean(axis=1)
histY = salpha[:,strongZoneX[0]:strongZoneX[1]].mean(axis=1)
smoothY = self._smooth(histY, window_len=self._windowY,
window=self._windowType)
# Pre-compute the minimum length of a strong
# zone that we'll accept.
# This is the max of an absolute length and a
# minimum fraction of the original box.
minZoneLen = max(self._minAbsZoneLenY,
int(round(self._minRelZoneLenY \
* float(len(smoothY)))))
# Minimum length for a weak gap
minWeakLen = max(self._minAbsWeakLenY,
int(round(self._minRelWeakLenY \
* float(len(smoothY)))))
maxY = smoothY.max()
# For now, simple threshold
threshY = self._widthThresh * maxY
strongY = (smoothY >= threshY).astype(int)
# Pre-calculate the minimum peak strength for
# each lobe to avoid being culled
minStrength = maxY * self._secondaryWidthThresh
# Find changes:
deltas = strongY[1:] - strongY[:-1]
changes = numpy.where(deltas)[0]
# Form our block lists
strongZonesY = []
if strongY[0]:
curZoneStart = 0
else:
curZoneStart = None
for changeIdx in changes:
strongIdx = changeIdx + 1
changeDir = deltas[changeIdx]
# Start of new strong zone
if changeDir == 1:
assert curZoneStart is None
curZoneStart = strongIdx
# End of existing strong zone
else:
assert changeDir == -1
assert curZoneStart is not None
strongZoneY = (curZoneStart, strongIdx)
self._acceptOrCull(smoothY, strongZonesY, strongZoneY, minZoneLen, minStrength)
curZoneStart = None
# Last one
if curZoneStart is not None:
strongZoneY = (curZoneStart, len(strongY))
self._acceptOrCull(smoothY, strongZonesY, strongZoneY, minZoneLen, minStrength)
# Remove tiny/thin weak gaps
if len(strongZonesY) > 1:
tempZones = []
lastZone = strongZonesY[0]
for startIdx, endIdx in strongZonesY[1:]:
if startIdx - lastZone[1] >= minWeakLen:
tempZones += [lastZone]
lastZone = (startIdx, endIdx)
else:
lastZone = (lastZone[0], endIdx)
tempZones += [lastZone]
strongZonesY = tempZones
left, right = strongZoneX
#strongZonesAll.extend([(left, top, right, bottom) for (top, bottom) in strongZonesY])
for (top, bottom) in strongZonesY:
expandedZone = (max(0, left - self._zonePostExpansionX),
max(0, top - self._zonePostExpansionY),
min(image.size[0], right + self._zonePostExpansionX),
min(image.size[1], bottom + self._zonePostExpansionY))
if expandedZone[2] < expandedZone[0]:
expandedZone[2] = expandedZone[0]
if expandedZone[3] < expandedZone[1]:
expandedZone[3] = expandedZone[1]
strongZonesAll += [expandedZone]
if False:
# Obtain the videoID and sequenceID
imgDir, imgName = os.path.split(imagePath)
imgPrefix = os.path.split(imgDir)[1]
# Example:
# overlap.0550_sequence10067
match = re.match(r"^(?P<mnemonic>[a-z]+)\.(?P<videoID>[0-9]{4})_sequence(?P<seqID>\-?[0-9]{1,5})$", imgPrefix)
if not match:
match = re.match(r"^vid(?P<videoID>[0-9]{4})_seq(?P<seqID>\-?[0-9]{3,4})$", imgPrefix)
assert match
d = match.groupdict()
videoID = int(d['videoID'])
seqID = int(d['seqID'])
key = (videoID, seqID)
numZones = len(strongZonesAll)
# Debugging (and inefficient)
if self._debugMode:
# Mark up the img box
blank = Image.new('L', image.size, 0)
base = smotion.convert('RGB')
alphaNew = blank.copy()
alphaOrig = blank.copy()
alphaOrig.paste(255, origBox)
#for zoneStart, zoneEnd in strongZones:
for (left, top, right, bottom) in strongZonesAll:
zoneBox = (expandedBox[0] + left, expandedBox[1] + top,
expandedBox[0] + right, expandedBox[1] + bottom)
#zoneBox = (origBox[0] + left, origBox[1] + top,
# origBox[0] + right, origBox[1] + bottom)
alphaNew.paste(255, zoneBox)
blender = Image.merge('RGB', (alphaOrig, alphaNew, blank))
#blender = Image.merge('RGB', (alphaOrig, blank, alphaNew))
blendFraction = 0.5
resultImg = Image.blend(base, blender, blendFraction)
# Dump marked-up images to disk
#imgDir, imgName = os.path.split(imagePath)
#imgPrefix = os.path.split(imgDir)[1]
dstPath = "%06d.png" % self._imgCounter
dstName, dstExt = os.path.splitext(dstPath)
dstName += "__Z%d" % numZones
dstPath = dstName + dstExt
resultImg.save(dstPath)
# Print stats
#(left, top, right, bottom) = strongZonesAll[0]
#print "ORIG (%3d, %3d, %3d, %3d) [%3dx%3d] ==> (%3d, %3d, %3d, %3d) [%3dx%3d]" % \
# (origBox[0], origBox[1], origBox[2], origBox[3],
# origBox[2]-origBox[0], origBox[3]-origBox[1],
# left, top, right, bottom, right-left, bottom-top)
# If there is more than one box, use the biggest
# (this is just a hack heuristic)
tightenedZone = None
if len(strongZonesAll) > 1:
# Take biggest
if self._splitPolicy == 'biggest':
#print "WARNING: multiple (%d) split boxes...choosing biggest one..." % len(strongZonesAll)
bigBoxIdx = None
bigBoxArea = -1
for boxIdx, subBox in enumerate(strongZonesAll):
area = (subBox[2] - subBox[0]) * (subBox[3] - subBox[1])
if area > bigBoxArea:
bigBoxArea = area
bigBoxIdx = boxIdx
tightenedZone = strongZonesAll[bigBoxIdx]
# Take biggest
elif self._splitPolicy == 'union':
#print "WARNING: multiple (%d) split boxes...taking union..." % len(strongZonesAll)
left, top, right, bottom = strongZonesAll[0]
for boxIdx, subBox in enumerate(strongZonesAll[1:]):
left = min(left, subBox[0])
top = min(top, subBox[1])
right = max(right, subBox[2])
bottom = max(bottom, subBox[3])
tightenedZone = (left, top, right, bottom)
elif not strongZonesAll:
#print "WARNING: dissipated box...reverting to original..."
box = origBox
else:
assert len(strongZonesAll) == 1
tightenedZone = strongZonesAll[0]
if strongZonesAll:
subBox = tightenedZone
# 'inference' may be None if box was culled
box = (expandedBox[0] + subBox[0],
expandedBox[1] + subBox[1],
expandedBox[0] + subBox[2],
expandedBox[1] + subBox[3])
# Print stats
(left, top, right, bottom) = box
#print "%06d: (%3d, %3d, %3d, %3d) [%3dx%3d] ==> (%3d, %3d, %3d, %3d) [%3dx%3d]" % \
# (self._imgCounter,
# origBox[0], origBox[1], origBox[2], origBox[3],
# origBox[2]-origBox[0], origBox[3]-origBox[1],
# left, top, right, bottom, right-left, bottom-top)
alphaNew = Image.new('L', smotion.size, 0)
alphaNew.paste(255, box)
dstImage = Image.merge('LA', (smotion, alphaNew))
dstImage.info['tracking'] = box
# TEMP TEMP TEMP - dump box dims (pre and post fix)
if self._debugMode:
if self._imgCounter == 0:
mode = 'w'
self._logBox = open("LOGBOX.txt", mode)
print >>self._logBox, "%d %d %d %d %d %d %d %d %d" % \
(self._imgCounter,
origBox[0], origBox[1], origBox[2], origBox[3],
box[0], box[1], box[2], box[3])
self._imgCounter += 1
return dstImage
2
Example 19
Project: twistranet Source File: bootstrap.py
def bootstrap():
"""
Load initial data if it's not present.
This method is SAFE, ie. it won't destroy any existing data, only add missing stuff`.
This should be called every time twistranet is started!
"""
try:
# Let's log in.
__account__ = SystemAccount.objects.__booster__.get()
except SystemAccount.DoesNotExist:
log.info("No SystemAccount available. That means this instance has never been bootstraped, so let's do it now.")
raise RuntimeError("Please sync your databases with 'manage.py syncdb' before bootstraping.")
except:
# Default fixture probably not installed yet. Don't do anything yet.
log.info("DatabaseError while bootstraping. Your tables are probably not created yet.")
traceback.print_exc()
return
# Now create the bootstrap / default / help fixture objects.
# Import your fixture there, if you don't do so they may not be importable.
from twistranet.fixtures.bootstrap import FIXTURES as BOOTSTRAP_FIXTURES
from twistranet.fixtures.help_en import FIXTURES as HELP_EN_FIXTURES
# XXX TODO: Make a fixture registry? Or fix fixture import someway?
try:
from twistrans.fixtures.help_fr import FIXTURES as HELP_FR_FIXTURES
from twistrans.fixtures.bootstrap_fr import FIXTURES as BOOTSTRAP_FR_FIXTURES
except ImportError:
HELP_FR_FIXTURES = []
BOOTSTRAP_FR_FIXTURES = []
log.info("twistrans not installed, translations are not installed.")
# Load fixtures
for obj in BOOTSTRAP_FIXTURES: obj.apply()
# Special treatment for bootstrap: Set the GlobalCommunity owner = AdminCommunity
glob = GlobalCommunity.objects.get()
admin_cty = AdminCommunity.objects.get()
glob.owner = admin_cty
glob.publisher = glob
glob.save()
admin_cty.publisher = glob
admin_cty.save()
# Create default resources by associating them to the SystemAccount and publishing them on GlobalCommunity.
default_resources_dir = os.path.abspath(
os.path.join(
os.path.split(twistranet.__file__)[0],
'fixtures',
'resources',
)
)
log.debug("Default res. dir: %s" % default_resources_dir)
for root, dirs, files in os.walk(default_resources_dir):
for fname in files:
slug = os.path.splitext(os.path.split(fname)[1])[0]
objects = Resource.objects.filter(slug = slug)
if objects:
if len(objects) > 1:
raise IntegrityError("More than one resource with '%s' slug" % slug)
r = objects[0]
else:
r = Resource()
# Copy file to its actual location with the storage API
source_fn = os.path.join(root, fname)
r.publisher = glob
r.resource_file = File(open(source_fn, "rb"), fname)
r.slug = slugify(slug)
r.save()
break # XXX We don't handle subdirs yet.
# Set SystemAccount picture (which is a way to check if things are working properly).
__account__.picture = Resource.objects.get(slug = "default_tn_picture")
__account__.save()
# Install HELP fixture.
for obj in HELP_EN_FIXTURES: obj.apply()
# Have we got an admin account? If not, we generate one now.
django_admins = UserAccount.objects.filter(user__is_superuser = True)
admin_password = None
if not django_admins.exists():
admin_password = ""
for i in range(6):
admin_password = "%s%s" % (admin_password, random.choice(string.lowercase + string.digits))
admin = User.objects.create(
username = settings.TWISTRANET_DEFAULT_ADMIN_USERNAME,
first_name = settings.TWISTRANET_DEFAULT_ADMIN_FIRSTNAME,
last_name = settings.TWISTRANET_DEFAULT_ADMIN_LASTNAME,
email = settings.TWISTRANET_ADMIN_EMAIL,
is_superuser = True,
)
admin.set_password(admin_password)
admin.save()
# Sample data only imported if asked to in settings.py
if settings.TWISTRANET_IMPORT_SAMPLE_DATA:
from twistranet.fixtures import sample
sample.create_users()
for obj in sample.get_fixtures():
obj.apply()
# Add relations bwn sample users
# A <=> admin
# B => admin
# A = UserAccount.objects.get(slug = "a")
# B = UserAccount.objects.get(slug = "b")
# admin = UserAccount.objects.get(slug = "admin")
# A.follow(admin)
# admin.follow(A)
# B.follow(admin)
# Import COGIP sample (if requested)
if settings.TWISTRANET_IMPORT_COGIP:
fixtures_module = "twistranet.project_templates.cogip.fixtures"
fixtures_module = import_module(fixtures_module)
# We disable email sending
backup_EMAIL_BACKEND = settings.EMAIL_BACKEND
settings.EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'
fixtures_module.load()
settings.EMAIL_BACKEND = backup_EMAIL_BACKEND
# Repair permissions
repair()
# Display admin password
if admin_password is not None:
print "\n\n" \
" You can now run your server with 'manage.py runserver'.\n" \
" Your initial administrator login/password are '%s / %s'" % (settings.TWISTRANET_DEFAULT_ADMIN_USERNAME, admin_password)
2
Example 20
@classmethod
@memoized
def check_compiler(cls, compiler):
def get_output_path(input_path, possible_extensions):
dirname, filename = os.path.split(input_path)
basename = os.path.splitext(filename)[0]
for ext in possible_extensions:
candidate = os.path.join(dirname, basename + ext)
if os.path.exists(candidate):
return candidate
return 'none'
def rm_dest(dest):
if os.path.exists(dest):
try:
os.remove(dest)
except OSError:
# log something
_logger().exception('failed to remove check compiler destination')
return False
return True
from open_cobol_ide.view.dialogs.preferences import DEFAULT_TEMPLATE
working_dir = tempfile.gettempdir()
cbl_path = os.path.join(working_dir, 'test.cbl')
try:
with open(cbl_path, 'w') as f:
f.write(DEFAULT_TEMPLATE)
except OSError as e:
return 'Failed to create %s, error=%r' % (cbl_path, e), -1
dest = os.path.join(tempfile.gettempdir(),
'test' + ('.exe' if system.windows else ''))
_logger().debug('removing executable test if still exists...')
if not rm_dest(dest):
return 'Failed to remove %r before checking if compilation of executable works.\n' \
'Please remove this file before attempting a new compilation check!' % dest, -1
_logger().debug('check compiler')
success1 = False
status, output1 = run_command(compiler, ['-v', '-x', cbl_path], working_dir=working_dir)
dest = get_output_path(cbl_path, possible_extensions=['.exe', '.bat', ''])
if dest:
if os.path.exists(dest):
success1 = True
if status != 0:
_logger().warn('test executable compilation returned a non-zero return code')
# detect default executable extension
GnuCobolCompiler.extensions[0] = os.path.splitext(dest)[1]
try:
os.remove(dest)
except OSError:
_logger().exception('failed to remove destination file: %r' % dest)
dest = os.path.join(tempfile.gettempdir(),
'test' + ('.dll' if system.windows else '.so' if system.linux else '.dylib'))
_logger().debug('removing executable test if still exists...')
if not rm_dest(dest):
return 'Failed to remove %r before checking if compilation of module works.\n' \
'Please remove this file before attempting a new compilation check!' % dest, -1
success2 = False
status, output2 = run_command(compiler, ['-v', cbl_path], working_dir=working_dir)
dest = get_output_path(cbl_path, possible_extensions=['.so', '.dll', '.dylib'])
if dest:
if os.path.exists(dest):
success2 = True
if status != 0:
_logger().warn('test executable compilation returned a non-zero return code')
# detect default executable extension
GnuCobolCompiler.extensions[1] = os.path.splitext(dest)[1]
try:
os.remove(dest)
except OSError:
_logger().exception('failed to remove destination file: %r' % dest)
_logger().info('GnuCOBOL compiler check: %s (%d/%d)',
'success' if success1 and success2 else 'fail',
success1, success2)
if success1:
_logger().info('Executable extension: %s' % GnuCobolCompiler.extensions[0])
if success2:
_logger().info('Module extension: %s' % GnuCobolCompiler.extensions[1])
try:
os.remove(cbl_path)
except OSError:
_logger().exception('failed to remove test file: %r' % dest)
if not success1 or not success2:
status = -1
return '%s\n%s' % (output1, output2), status
2
Example 21
Project: odoo Source File: translate.py
def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, module_name=None, context=None):
"""Populates the ir_translation table."""
if verbose:
_logger.info('loading translation file for language %s', lang)
if context is None:
context = {}
db_name = cr.dbname
registry = openerp.registry(db_name)
lang_obj = registry.get('res.lang')
trans_obj = registry.get('ir.translation')
iso_lang = misc.get_iso_codes(lang)
try:
ids = lang_obj.search(cr, SUPERUSER_ID, [('code','=', lang)])
if not ids:
# lets create the language with locale information
lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name)
# Parse also the POT: it will possibly provide additional targets.
# (Because the POT comments are correct on Launchpad but not the
# PO comments due to a Launchpad limitation. See LP bug 933496.)
pot_reader = []
# now, the serious things: we read the language file
fileobj.seek(0)
if fileformat == 'csv':
reader = csv.reader(fileobj, quotechar='"', delimiter=',')
# read the first line of the file (it contains columns titles)
for row in reader:
fields = row
break
elif fileformat == 'po':
reader = TinyPoFile(fileobj)
fields = ['type', 'name', 'res_id', 'src', 'value', 'comments']
# Make a reader for the POT file and be somewhat defensive for the
# stable branch.
if fileobj.name.endswith('.po'):
try:
# Normally the path looks like /path/to/xxx/i18n/lang.po
# and we try to find the corresponding
# /path/to/xxx/i18n/xxx.pot file.
# (Sometimes we have 'i18n_extra' instead of just 'i18n')
addons_module_i18n, _ = os.path.split(fileobj.name)
addons_module, i18n_dir = os.path.split(addons_module_i18n)
addons, module = os.path.split(addons_module)
pot_handle = misc.file_open(os.path.join(
addons, module, i18n_dir, module + '.pot'))
pot_reader = TinyPoFile(pot_handle)
except:
pass
else:
_logger.error('Bad file format: %s', fileformat)
raise Exception(_('Bad file format'))
# Read the POT references, and keep them indexed by source string.
class Target(object):
def __init__(self):
self.value = None
self.targets = set() # set of (type, name, res_id)
self.comments = None
pot_targets = defaultdict(Target)
for type, name, res_id, src, _, comments in pot_reader:
if type is not None:
target = pot_targets[src]
target.targets.add((type, name, res_id))
target.comments = comments
# read the rest of the file
irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context)
def process_row(row):
"""Process a single PO (or POT) entry."""
# dictionary which holds values for this line of the csv file
# {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
# 'src': ..., 'value': ..., 'module':...}
dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
'comments', 'imd_model', 'imd_name', 'module'))
dic['lang'] = lang
dic.update(zip(fields, row))
# discard the target from the POT targets.
src = dic['src']
if src in pot_targets:
target = pot_targets[src]
target.value = dic['value']
target.targets.discard((dic['type'], dic['name'], dic['res_id']))
# This would skip terms that fail to specify a res_id
res_id = dic['res_id']
if not res_id:
return
if isinstance(res_id, (int, long)) or \
(isinstance(res_id, basestring) and res_id.isdigit()):
dic['res_id'] = int(res_id)
dic['module'] = module_name
else:
# res_id is an xml id
dic['res_id'] = None
dic['imd_model'] = dic['name'].split(',')[0]
if '.' in res_id:
dic['module'], dic['imd_name'] = res_id.split('.', 1)
else:
dic['module'], dic['imd_name'] = False, res_id
irt_cursor.push(dic)
# First process the entries from the PO file (doing so also fills/removes
# the entries from the POT file).
for row in reader:
process_row(row)
# Then process the entries implied by the POT file (which is more
# correct w.r.t. the targets) if some of them remain.
pot_rows = []
for src, target in pot_targets.iteritems():
if target.value:
for type, name, res_id in target.targets:
pot_rows.append((type, name, res_id, src, target.value, target.comments))
pot_targets.clear()
for row in pot_rows:
process_row(row)
irt_cursor.finish()
trans_obj.clear_caches()
if verbose:
_logger.info("translation file loaded succesfully")
except IOError:
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
_logger.exception("couldn't read translation file %s", filename)
2
Example 22
Project: gimp-plugins Source File: saver.py
def save_both(img, drawable, filename, copyname, width, height):
'''Save the image, and also save the copy if appropriate,
doing any duplicating or scaling that might be necessary.
Returns None on success, else a nonempty string error message.
'''
msg = "Saving " + filename
if copyname:
if width and height:
msg += " and a scaled copy, " + copyname
else:
msg += " and a copy, " + copyname
print msg
# Don't use gimp_message -- it can pop up a dialog.
# pdb.gimp_message_set_handler(MESSAGE_BOX)
# pdb.gimp_message(msg)
# Alternately, could use gimp_progress, though that's not ideal.
# If you use gimp_progress, don't forget to call pdb.gimp_progress_end()
#pdb.gimp_progress_set_text(msg)
# Is there any point to pushing and popping the context?
#gimp.context_pop()
# Set up the parasite with information about the copied image,
# so it will be saved with the main XCF image.
if copyname:
if (width and height):
percent = width * 100.0 / img.width
# XXX note that this does not guard against changed aspect ratios.
parastring = '%s\n%d\n%d\n%d' % (copyname, percent, width, height)
else:
parastring = '%s\n100.0\nNone\nNone' % (copyname)
print "Saving parasite", parastring
para = img.attach_new_parasite('export-copy', 1, parastring)
# The second argument is is_persistent
# Also make sure that copyname is a full path (if filename is).
copypath = os.path.split(copyname)
if not copypath[0] or len(copypath) == 1:
filepath = os.path.split(filename)
if len(filepath) > 1 and filepath[0]:
print "Turning", copyname, "into a full path:",
copyname = os.path.join(filepath[0], copyname)
print copyname
# Now the parasite is safely attached, and we can save.
# Alas, we can't attach the JPEG settings parasite until after
# we've saved the copy, so that won't get saved with the main image
# though it will be attached to the image and remembered in
# this session, and the next time we save it should be remembered.
def is_xcf(thefilename):
base, ext = os.path.splitext(thefilename)
ext = ext.lower()
if ext == '.gz' or ext == '.bz2':
base, ext = os.path.splitext(base)
ext = ext.lower()
return (ext == '.xcf')
# First, save the original image.
if is_xcf(filename) or len(img.layers) < 2:
pdb.gimp_file_save(img, drawable, filename, filename)
else:
# It's not XCF and it has multiple layers.
# We need to make a new image and flatten it.
copyimg = pdb.gimp_image_duplicate(img)
# Don't actually flatten since that will prevent saving transparent png.
#copyimg.flatten()
pdb.gimp_image_merge_visible_layers(copyimg, CLIP_TO_IMAGE)
pdb.gimp_file_save(copyimg, copyimg.active_layer, filename, filename)
gimp.delete(copyimg)
# We've done the important part, so mark the image clean.
img.filename = filename
pdb.gimp_image_clean_all(img)
# If we don't have to save a copy, return.
if not copyname:
return None
# We'll need a new image if the copy is non-xcf and we have more
# than one layer, or if we're scaling.
if (width and height):
# We're scaling!
copyimg = pdb.gimp_image_duplicate(img)
copyimg.scale(width, height)
print "Scaling to", width, 'x', height
print "Set parastring to", parastring, "(end of parastring)"
elif len(img.layers) > 1 and not is_xcf(copyname):
# We're not scaling, but we still need to flatten.
copyimg = pdb.gimp_image_duplicate(img)
# copyimg.flatten()
pdb.gimp_image_merge_visible_layers(copyimg, CLIP_TO_IMAGE)
print "Flattening but not scaling"
else:
copyimg = img
print "Not scaling or flattening"
# gimp-file-save insists on being passed a valid layer,
# even if saving to a multilayer format such as XCF. Go figure.
# I don't get it. What's the rule for whether gimp_file_save
# will prompt for jpg parameters? Saving back to an existing
# file doesn't seem to help.
# run_mode=RUN_WITH_LAST_VALS is supposed to prevent prompting,
# but actually seems to do nothing. Copying the -save-options
# parasites is more effective.
try:
pdb.gimp_file_save(copyimg, copyimg.active_layer, copyname, copyname,
run_mode=RUN_WITH_LAST_VALS)
except RuntimeError, e:
gimp.delete(copyimg)
print "Runtime error -- didn't save"
return "Runtime error -- didn't save"
# Find any image type settings parasites (e.g. jpeg settings)
# that got set during save, so we'll be able to use them
# next time.
def copy_settings_parasites(fromimg, toimg):
for pname in fromimg.parasite_list():
if pname[-9:] == '-settings' or pname[-13:] == '-save-options':
para = fromimg.parasite_find(pname)
if para:
toimg.attach_new_parasite(pname, para.flags, para.data)
# Copy any settings parasites we may have saved from previous runs:
copy_settings_parasites(copyimg, img)
gimp.delete(copyimg)
#gimp.Display(copyimg)
return None
2
Example 23
Project: deepcca Source File: dcca.py
def test_dcca_old(learning_rate=0.01, L1_reg=0.0001, L2_reg=0.0001, n_epochs=1000,
dataset='mnist.pkl.gz', batch_size=20, n_hidden=500):
"""
Demonstrate stochastic gradient descent optimization for a multilayer
perceptron
This is demonstrated on MNIST.
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient
:type L1_reg: float
:param L1_reg: L1-norm's weight when added to the cost (see
regularization)
:type L2_reg: float
:param L2_reg: L2-norm's weight when added to the cost (see
regularization)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: the path of the MNIST dataset file from
http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.matrix('y') # the labels are presented as 1D vector of
# [int] labels
rng = numpy.random.RandomState(1234)
# construct the MLP class
if 0:
net1 = MLP(
rng=rng,
input=x,
n_in=28 * 28,
n_hidden=300,
n_out=50
)
net2 = MLP(
rng=rng,
input=y,
n_in=10,
n_hidden=20,
n_out=5
)
net = DCCA(
rng=rng,
x1=x,
x2=y,
n_in1=28 * 28,
n_hidden1=300,
n_out1=50,
n_in2=10,
n_hidden2=20,
n_out2=5
)
# start-snippet-4
# the cost we minimize during training is the negative log likelihood of
# the model plus the regularization terms (L1 and L2); cost is expressed
# here symbolically
cost1 = (
net.correlation(y)
+ L1_reg * net.L11
+ L2_reg * net.L2_sqr1
)
cost2 = (
net.correlation(y)
+ L1_reg * net.L12
+ L2_reg * net.L2_sqr2
)
# end-snippet-4
# compiling a Theano function that computes the mistakes that are made
# by the model on a minibatch
"""
test_model = theano.function(
inputs=[index],
outputs=net1.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
}
)
validate_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
}
)
"""
# start-snippet-5
# compute the gradient of cost with respect to theta (sotred in params)
# the resulting gradients will be stored in a list gparams
gparams1 = [T.grad(cost1, param) for param in net.params1]
gparams2 = [T.grad(cost2, param) for param in net.params2]
# specify how to update the parameters of the model as a list of
# (variable, update expression) pairs
# given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of
# same length, zip generates a list C of same size, where each element
# is a pair formed from the two lists :
# C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)]
updates1 = [
(param, param - learning_rate * gparam)
for param, gparam in zip(net.params1, gparams1)
]
updates2 = [
(param, param - learning_rate * gparam)
for param, gparam in zip(net.params2, gparams2)
]
# compiling a Theano function `train_model` that returns the cost, but
# in the same time updates the parameter of the model based on the rules
# defined in `updates`
train_model1 = theano.function(
inputs=[index],
outputs=cost1,
updates=updates1,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size]
}
)
train_model2 = theano.function(
inputs=[index],
outputs=cost2,
updates=updates2,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size]
}
)
# end-snippet-5
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = time.clock()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i
in xrange(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
print(
'epoch %i, minibatch %i/%i, validation error %f %%' %
(
epoch,
minibatch_index + 1,
n_train_batches,
this_validation_loss * 100.
)
)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [test_model(i) for i
in xrange(n_test_batches)]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(('Optimization complete. Best validation score of %f %% '
'obtained at iteration %i, with test performance %f %%') %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
''' Loads the dataset
:type dataset: string
:param dataset: the path to the dataset (here MNIST)
'''
#############
# LOAD DATA #
#############
# Download the MNIST dataset if it is not present
data_dir, data_file = os.path.split(dataset)
if data_dir == "" and not os.path.isfile(dataset):
# Check if dataset is in the data directory.
new_path = os.path.join(
os.path.split(__file__)[0],
"..",
"data",
dataset
)
if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz':
dataset = new_path
if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz':
import urllib
origin = (
'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz'
)
print 'Downloading data from %s' % origin
urllib.urlretrieve(origin, dataset)
print '... loading data'
# Load the dataset
f = gzip.open(dataset, 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
#train_set, valid_set, test_set format: tuple(input, target)
#input is an numpy.ndarray of 2 dimensions (a matrix)
#witch row's correspond to an example. target is a
#numpy.ndarray of 1 dimensions (vector)) that have the same length as
#the number of rows in the input. It should give the target
#target to the example with the same index in the input.
def shared_dataset(data_xy, borrow=True):
""" Function that loads the dataset into shared variables
The reason we store our dataset in shared variables is to allow
Theano to copy it into the GPU memory (when code is run on GPU).
Since copying data into the GPU is slow, copying a minibatch everytime
is needed (the default behaviour if the data is not in a shared
variable) would lead to a large decrease in performance.
"""
#import copy
data_x, data_y = data_xy
#daya_y = copy.deepcopy(data_x)
data_y_new = numpy.zeros((data_y.shape[0], data_y.max()+1))
for i in range(data_y.shape[0]):
data_y_new[i, data_y[i]] = 1
data_y = data_y_new
shared_x = theano.shared(numpy.asarray(data_x,
dtype=theano.config.floatX),
borrow=borrow)
shared_y = theano.shared(numpy.asarray(data_y,
dtype=theano.config.floatX),
borrow=borrow)
# When storing data on the GPU it has to be stored as floats
# therefore we will store the labels as ``floatX`` as well
# (``shared_y`` does exactly that). But during our computations
# we need them as ints (we use labels as index, and if they are
# floats it doesn't make sense) therefore instead of returning
# ``shared_y`` we will have to cast it to int. This little hack
# lets ous get around this issue
return shared_x, shared_y
test_set_x, test_set_y = shared_dataset(test_set)
valid_set_x, valid_set_y = shared_dataset(valid_set)
train_set_x, train_set_y = shared_dataset(train_set)
rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),
(test_set_x, test_set_y)]
return rval
2
Example 24
Project: WikiDAT Source File: etl.py
def run(self):
"""
Execute workflow to import revision history data from dump files
The data loading workflow is composed of a number of processor
elements, which can be:
- Producer (P): raw input data --> input element queue
- ConsumerProducer (CP): input element queue --> insert db queue
- Consumer (C): insert db queue --> database (MySQL/MariaDB)
In this case, the logical combination is usually N:N:1 (P, CP, C)
"""
start = time.time()
print(self.name, "Starting PageRevisionETL workflow at %s" % (
time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.localtime())))
db_ns = MySQLDB(host='localhost', port=3306, user=self.db_user,
passwd=self.db_passw, db=self.db_name)
db_ns.connect()
db_pages = MySQLDB(host='localhost', port=3306,
user=self.db_user, passwd=self.db_passw,
db=self.db_name)
db_pages.connect()
db_revs = MySQLDB(host='localhost', port=3306, user=self.db_user,
passwd=self.db_passw, db=self.db_name)
db_revs.connect()
# DATA EXTRACTION
# Use consistent naming for all child processes
xml_reader_name = '-'.join([self.name, 'xml_reader'])
page_proc_name = '-'.join([self.name, 'process_page'])
rev_proc_name = '-'.join([self.name, 'process_revision'])
page_insert_name = '-'.join([self.name, 'insert_page'])
rev_insert_name = '-'.join([self.name, 'insert_revision'])
for path in iter(self.paths_queue.get, 'STOP'):
# Start subprocess to extract elements from revision dump file
dump_file = DumpFile(path)
xml_reader = Producer(name=xml_reader_name,
target=process_xml,
kwargs=dict(
dump_file=dump_file),
consumers=self.page_fan + self.rev_fan,
push_pages_port=self.base_port,
push_revs_port=self.base_port+1,
control_port=self.control_port)
xml_reader.start()
print(xml_reader_name, "started")
print(self.name, "Extracting data from XML revision history file:")
print(path)
# List to keep tracking of page and revision workers
workers = []
db_workers_revs = []
# Create and start page processes
for worker in range(self.page_fan):
page_worker_name = '-'.join([page_proc_name, str(worker)])
process_page = Processor(name=page_worker_name,
target=pages_to_file,
producers=1, consumers=1,
pull_port=self.base_port,
push_port=self.base_port+2,
control_port=self.control_port)
process_page.start()
workers.append(process_page)
print(page_worker_name, "started")
# Create and start revision processes
for worker in range(self.rev_fan):
rev_worker_name = '-'.join([rev_proc_name, str(worker)])
db_wrev = MySQLDB(host='localhost', port=3306,
user=self.db_user,
passwd=self.db_passw, db=self.db_name)
db_wrev.connect()
process_revision = Processor(name=rev_worker_name,
target=revs_to_file,
kwargs=dict(
lang=self.lang),
producers=1, consumers=1,
pull_port=self.base_port+1,
push_port=self.base_port+3,
control_port=self.control_port)
process_revision.start()
workers.append(process_revision)
db_workers_revs.append(db_wrev)
print(rev_worker_name, "started")
# Create directory for logging files if it does not exist
log_dir = os.path.join(os.path.split(path)[0], 'logs')
tmp_dir = os.path.join(os.getcwd(), os.path.split(path)[0], 'tmp')
file_name = os.path.split(path)[1]
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
log_file = os.path.join(log_dir, file_name + '.log')
page_insert_db = Consumer(name=page_insert_name,
target=pages_file_to_db,
kwargs=dict(con=db_pages,
log_file=log_file,
tmp_dir=tmp_dir,
file_rows=self.page_cache_size,
etl_prefix=self.name),
producers=self.page_fan,
pull_port=self.base_port+2)
rev_insert_db = Consumer(name=rev_insert_name,
target=revs_file_to_db,
kwargs=dict(con=db_revs,
log_file=log_file,
tmp_dir=tmp_dir,
file_rows=self.rev_cache_size,
etl_prefix=self.name),
producers=self.rev_fan,
pull_port=self.base_port+3)
page_insert_db.start()
print(page_insert_name, "started")
rev_insert_db.start()
print(rev_insert_name, "started")
print(self.name, "Waiting for all processes to finish...")
print()
xml_reader.join()
for w in workers:
w.join()
page_insert_db.join()
rev_insert_db.join()
# Mark this path as done
self.paths_queue.task_done()
# Mark STOP message as processed and finish
self.paths_queue.task_done()
end = time.time()
print(self.name, ": All tasks done in %.4f sec." % ((end-start)/1.))
print()
db_ns.close()
db_pages.close()
db_revs.close()
for dbcon in db_workers_revs:
dbcon.close()
2
Example 25
Project: deepcca Source File: SdA_mapping.py
def test_SdA_regress(finetune_lr=0.05, pretraining_epochs=10,
pretrain_lr=0.1, training_epochs=10000,
dataset='mnist.pkl.gz', batch_size=20):
datasets = load_data_half(dataset)
train_set_x, train_set_y = datasets[0]##
valid_set_x, valid_set_y = datasets[1]##
test_set_x, test_set_y = datasets[2]##
train_set_x=train_set_x.eval()
train_set_y=train_set_y.eval()
import theano
train_set_x_lab=train_set_x[:,:]
train_set_x_unlab=train_set_x[:,:]
train_set_y_lab=train_set_y[:,:]
train_set_y_unlab=train_set_y[:,:]
train_set_x_lab=theano.shared(numpy.asarray(train_set_x_lab,
dtype=theano.config.floatX),
borrow=True)
train_set_y_lab=theano.shared(numpy.asarray(train_set_y_lab,
dtype=theano.config.floatX),
borrow=True)
train_set_x_unlab=theano.shared(numpy.asarray(train_set_x_unlab,
dtype=theano.config.floatX),
borrow=True)
train_set_y_unlab=theano.shared(numpy.asarray(train_set_y_unlab,
dtype=theano.config.floatX),
borrow=True)
# compute number of minibatches for training, validation and testing
n_train_batches_l = train_set_y_lab.eval().shape[0]
n_train_batches_l /= batch_size
n_train_batches_u = train_set_y_unlab.eval().shape[0]
n_train_batches_u /= batch_size
# compute number of minibatches for training, validation and testing
#n_train_batches = train_set_x.get_value(borrow=True).shape[0]
#n_train_batches /= batch_size
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
#from SdA_orig import SdA as SdA_old
hidden_layer_size = 100
SdA_inp = SdA(numpy_rng,
n_ins=392,
hidden_layers_sizes=[hidden_layer_size]
)
SdA_out = SdA(numpy_rng,
n_ins=392,
hidden_layers_sizes=[hidden_layer_size]
)
# PRETRAINING THE MODEL #
if 0 : # pretrain inp ae
print '... getting the pretraining functions for INPUT AE'
pretraining_fns = SdA_inp.pretraining_functions(train_set_x=train_set_x_unlab,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [.1, .2, .3]
for i in xrange(SdA_inp.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches_u):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
if 0 : # pretrain out ae
print '... getting the pretraining functions for OUTPUT AE'
pretraining_fns = SdA_out.pretraining_functions(train_set_x=train_set_y_unlab,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [.5, .2, .3]
for i in xrange(SdA_out.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches_u):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
if 0: # save aes
f=open('aes_shallow_sig_nobias.pkl', 'w+')
import pickle
pickle.dump(SdA_inp, f)
pickle.dump(SdA_out, f)
f.flush()
f.close()
if 0: # load aes
f=open('aes_shallow_sig_nobias.pkl', 'r')
import pickle
SdA_inp=pickle.load(f)
SdA_out=pickle.load(f)
f.close()
if 1: # cca
from dcca_numpy import netCCA_nobias, netCCA, dCCA
from mlp_numpy import expit, logistic_prime, linear, linear_prime, relu, relu_prime, tanh, tanh_prime
train_y1 = train_set_x_lab.eval()
train_y2 = train_set_y_lab.eval()
test_y1 = test_set_x.eval()
test_y2 = test_set_y.eval()
##param1=((train_y1.shape[1],0,0),(2038, relu, relu_prime),(50, relu, relu_prime))
##param2=((train_y2.shape[1],0,0),(1608, relu, relu_prime),(50, relu, relu_prime))
param1=((train_y1.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
param2=((train_y2.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
W1s = []
b1s = []
for i in range(len(SdA_inp.dA_layers)):
W1s.append( SdA_inp.dA_layers[i].W.T.eval() )
##b1s.append( SdA_inp.dA_layers[i].b.eval() )
##b1s[-1] = b1s[-1].reshape((b1s[-1].shape[0], 1))
W2s = []
b2s = []
for i in range(len(SdA_out.dA_layers)):
W2s.append( SdA_out.dA_layers[i].W.T.eval() )
##b2s.append( SdA_out.dA_layers[i].b.eval() )
##b2s[-1] = b2s[-1].reshape((b2s[-1].shape[0], 1))
numpy.random.seed(0)
N1=netCCA_nobias(train_y1,param1, W1s)
N2=netCCA_nobias(train_y2,param2, W2s)
N = dCCA(train_y1, train_y2, N1, N2)
N1.reconstruct(test_set_x.eval()[0,:])
cnt = 0
from dcca_numpy import cca_cost, cca, order_cost, cor_cost
while True:
X=N1.predict(test_set_x.eval())
Y=N2.predict(test_set_y.eval())
_H1 = numpy.dot(X, N.A1)
_H2 = numpy.dot(Y, N.A2)
print 'cuem', cnt, cor_cost(_H1, _H2)
X1_rec = numpy.tanh(X.dot(N1.weights[0]))
X2_rec = numpy.tanh(Y.dot(N2.weights[0]))
param=((hidden_layer_size,0,0),(hidden_layer_size, relu, relu_prime))
from mlp_numpy import NeuralNetwork as NN
lr=NN(X,Y,param)
lr.train(X[:,:],Y[:,:],10, 0.005)
Yh=lr.predict(X[:,:])
X2_reg = N2.fs[-1](numpy.dot(Yh,N2.weights[0]))
#X2_reg = N2.fs[-1](numpy.dot(_H1.dot(numpy.linalg.inv(N.A1)),N2.weights[0]))
print '****', 'mse1:', numpy.mean((X1_rec-test_set_x.eval())**2.0)
print '****', 'mse2:', numpy.mean((X2_rec-test_set_y.eval())**2.0)
print '****', 'mse_map:', numpy.mean((X2_reg-test_set_y.eval())**2.0)
if cnt % 2:
N.train(5, True, 10000.0)
else:
N.train(5, False, 10000.0)
cnt += 1
f=open('netcca.pkl', 'w+')
import pickle
pickle.dump(N, f)
pickle.dump(N, f)
f.flush()
f.close()
if cnt == 200:
break
for i in range(len(SdA_inp.dA_layers)):
SdA_inp.dA_layers[i].W = theano.shared( N1.weights[i].T )
SdA_inp.dA_layers[i].b = theano.shared( N1.biases[i][:,0] )
for i in range(len(SdA_out.dA_layers)):
SdA_out.dA_layers[i].W = theano.shared( N2.weights[i].T )
SdA_out.dA_layers[i].b = theano.shared( N2.weights[i][:,0] )
if 1 : # pretrain middle layer
print '... pre-training MIDDLE layer'
h1 = T.matrix('x') # the data is presented as rasterized images
h2 = T.matrix('y') # the labels are presented as 1D vector of
log_reg = HiddenLayer(numpy_rng, h1, hidden_layer_size, hidden_layer_size)
if 1: # for middle layer
learning_rate = 0.01
fprop_inp = theano.function(
[],
SdA_inp.sigmoid_layers[-1].output,
givens={
SdA_inp.sigmoid_layers[0].input: train_set_x_lab
},
name='fprop_inp'
)
fprop_out = theano.function(
[],
SdA_out.sigmoid_layers[-1].output,
givens={
SdA_out.sigmoid_layers[0].input: train_set_y_lab
},
name='fprop_out'
)
#H11=fprop_inp()
#H21=fprop_out()
##H1=N1.predict(train_set_x.eval())
##H2=N2.predict(train_set_y.eval())
H1=fprop_inp()
H2=fprop_out()
H1=theano.shared(H1)
H2=theano.shared(H2)
# compute the gradients with respect to the model parameters
logreg_cost = log_reg.mse(h2)
gparams = T.grad(logreg_cost, log_reg.params)
# compute list of fine-tuning updates
updates = [
(param, param - gparam * learning_rate)
for param, gparam in zip(log_reg.params, gparams)
]
train_fn_middle = theano.function(
inputs=[],
outputs=logreg_cost,
updates=updates,
givens={
h1: H1,
h2: H2
},
name='train_middle'
)
epoch = 0
while epoch < 10:
print epoch, train_fn_middle()
epoch += 1
sda = SdA_regress(
SdA_inp,
SdA_out,
log_reg,
numpy_rng=numpy_rng,
n_inp=28*28//2,
hidden_layers_sizes_inp=[hidden_layer_size],
hidden_layers_sizes_out=[hidden_layer_size],
n_out=28*28//2
)
# end-snippet-3 start-snippet-4
# end-snippet-4
# FINETUNING THE MODEL #
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
print '... finetunning the model'
# early-stopping parameters
patience = 10 * n_train_batches_l # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches_l, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
fprop = theano.function(
[],
sda.sigmoid_layers[-1].output,
givens={
sda.x: test_set_x
},
name='fprop'
)
while True:
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches_l):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches_l + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches_l,
this_validation_loss ))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches_l,
test_score ))
if patience <= iter:
done_looping = True
#break
if 0: # vis weights
fprop = theano.function(
[],
sda.sigmoid_layers[-1].output,
givens={
sda.x: test_set_x
},
name='fprop'
)
yh=fprop()
yh=yh
end_time = time.clock()
print(
(
'Optimization complete with best validation score of %f %%, '
'on iteration %i, '
'with test performance %f %%'
)
% (best_validation_loss , best_iter + 1, test_score)
)
print >> sys.stderr, ('The training code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
2
Example 26
def tex_emitter_core(target, source, env, graphics_extensions):
"""An emitter for TeX and LaTeX sources.
For LaTeX sources we try and find the common created files that
are needed on subsequent runs of latex to finish tables of contents,
bibliographies, indices, lists of figures, and hyperlink references.
"""
basename = SCons.Util.splitext(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
targetdir = os.path.split(str(target[0]))[0]
targetbase = os.path.join(targetdir, basefile)
basedir = os.path.split(str(source[0]))[0]
abspath = os.path.abspath(basedir)
target[0].attributes.path = abspath
#
# file names we will make use of in searching the sources and log file
#
emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
auxfilename = targetbase + '.aux'
logfilename = targetbase + '.log'
flsfilename = targetbase + '.fls'
env.SideEffect(auxfilename,target[0])
env.SideEffect(logfilename,target[0])
env.SideEffect(flsfilename,target[0])
if Verbose:
print "side effect :",auxfilename,logfilename,flsfilename
env.Clean(target[0],auxfilename)
env.Clean(target[0],logfilename)
env.Clean(target[0],flsfilename)
content = source[0].get_text_contents()
# These variables are no longer used.
#idx_exists = os.path.isfile(targetbase + '.idx')
#nlo_exists = os.path.isfile(targetbase + '.nlo')
#glo_exists = os.path.isfile(targetbase + '.glo')
#acr_exists = os.path.isfile(targetbase + '.acn')
# set up list with the regular expressions
# we use to find features used
file_tests_search = [auxfile_re,
makeindex_re,
bibliography_re,
bibunit_re,
tableofcontents_re,
listoffigures_re,
listoftables_re,
hyperref_re,
makenomenclature_re,
makeglossary_re,
makeglossaries_re,
makeacronyms_re,
beamer_re ]
# set up list with the file suffixes that need emitting
# when a feature is found
file_tests_suff = [['.aux','aux_file'],
['.idx', '.ind', '.ilg','makeindex'],
['.bbl', '.blg','bibliography'],
['.bbl', '.blg','bibunit'],
['.toc','contents'],
['.lof','figures'],
['.lot','tables'],
['.out','hyperref'],
['.nlo', '.nls', '.nlg','nomenclature'],
['.glo', '.gls', '.glg','glossary'],
['.glo', '.gls', '.glg','glossaries'],
['.acn', '.acr', '.alg','acronyms'],
['.nav', '.snm', '.out', '.toc','beamer'] ]
# build the list of lists
file_tests = []
for i in range(len(file_tests_search)):
file_tests.append( [None, file_tests_suff[i]] )
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
paths = env['ENV']['TEXINPUTS']
if SCons.Util.is_List(paths):
pass
else:
# Split at os.pathsep to convert into absolute path
paths = paths.split(os.pathsep)
# now that we have the path list restore the env
if savedpath is _null:
try:
del env['ENV']['TEXINPUTS']
except KeyError:
pass # was never set
else:
env['ENV']['TEXINPUTS'] = savedpath
if Verbose:
print "search path ",paths
aux_files = []
file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)
for (theSearch,suffix_list) in file_tests:
# add side effects if feature is present.If file is to be generated,add all side effects
if (theSearch != None) or (not source[0].exists() ):
file_list = [targetbase,]
# for bibunit we need a list of files
if suffix_list[-1] == 'bibunit':
file_basename = os.path.join(targetdir, 'bu*.aux')
file_list = glob.glob(file_basename)
# remove the suffix '.aux'
for i in range(len(file_list)):
file_list[i] = SCons.Util.splitext(file_list[i])[0]
# now define the side effects
for file_name in file_list:
for suffix in suffix_list[:-1]:
env.SideEffect(file_name + suffix,target[0])
if Verbose:
print "side effect :",file_name + suffix
env.Clean(target[0],file_name + suffix)
for aFile in aux_files:
aFile_base = SCons.Util.splitext(aFile)[0]
env.SideEffect(aFile_base + '.aux',target[0])
if Verbose:
print "side effect :",aFile_base + '.aux'
env.Clean(target[0],aFile_base + '.aux')
# read fls file to get all other files that latex creates and will read on the next pass
# remove files from list that we explicitly dealt with above
if os.path.isfile(flsfilename):
content = open(flsfilename, "rb").read()
out_files = openout_re.findall(content)
myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
for filename in out_files[:]:
if filename in myfiles:
out_files.remove(filename)
env.SideEffect(out_files,target[0])
if Verbose:
print "side effect :",out_files
env.Clean(target[0],out_files)
return (target, source)
2
Example 27
Project: gprMax Source File: gprMax.py
def run_model(args, modelrun, numbermodelruns, inputfile, usernamespace):
"""Runs a model - processes the input file; builds the Yee cells; calculates update coefficients; runs main FDTD loop.
Args:
args (dict): Namespace with command line arguments
modelrun (int): Current model run number.
numbermodelruns (int): Total number of model runs.
inputfile (str): Name of the input file to open.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
Returns:
tsolve (int): Length of time (seconds) of main FDTD calculations
"""
# Monitor memory usage
p = psutil.Process()
# Declare variable to hold FDTDGrid class
global G
# Normal model reading/building process; bypassed if geometry information to be reused
if 'G' not in globals():
inputfilestr = '\n--- Model {} of {}, input file: {}'.format(modelrun, numbermodelruns, inputfile)
print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
# Add the current model run to namespace that can be accessed by user in any Python code blocks in input file
usernamespace['current_model_run'] = modelrun
# Read input file and process any Python or include commands
processedlines = process_python_include_code(inputfile, usernamespace)
# Print constants/variables in user-accessable namespace
uservars = ''
for key, value in sorted(usernamespace.items()):
if key != '__builtins__':
uservars += '{}: {}, '.format(key, value)
print('Constants/variables used/available for Python scripting: {{{}}}\n'.format(uservars[:-2]))
# Write a file containing the input commands after Python or include commands have been processed
if args.write_processed:
write_processed_file(inputfile, modelrun, numbermodelruns, processedlines)
# Check validity of command names and that essential commands are present
singlecmds, multicmds, geometry = check_cmd_names(processedlines)
# Initialise an instance of the FDTDGrid class
G = FDTDGrid()
G.inputfilename = os.path.split(inputfile)[1]
G.inputdirectory = os.path.dirname(os.path.abspath(inputfile))
# Create built-in materials
m = Material(0, 'pec')
m.se = float('inf')
m.type = 'builtin'
m.averagable = False
G.materials.append(m)
m = Material(1, 'free_space')
m.type = 'builtin'
G.materials.append(m)
# Process parameters for commands that can only occur once in the model
process_singlecmds(singlecmds, G)
# Process parameters for commands that can occur multiple times in the model
print()
process_multicmds(multicmds, G)
# Initialise an array for volumetric material IDs (solid), boolean arrays for specifying materials not to be averaged (rigid),
# an array for cell edge IDs (ID)
G.initialise_geometry_arrays()
# Initialise arrays for the field components
G.initialise_field_arrays()
# Process geometry commands in the order they were given
process_geometrycmds(geometry, G)
# Build the PMLs and calculate initial coefficients
print()
if all(value == 0 for value in G.pmlthickness.values()):
if G.messages:
print('PML boundaries: switched off')
pass # If all the PMLs are switched off don't need to build anything
else:
if G.messages:
if all(value == G.pmlthickness['xminus'] for value in G.pmlthickness.values()):
pmlinfo = G.pmlthickness['xminus']
else:
pmlinfo = ''
for key, value in G.pmlthickness.items():
pmlinfo += '{}: {}, '.format(key, value)
pmlinfo = pmlinfo[:-2]
print('PML boundaries: {} cells'.format(pmlinfo))
pbar = tqdm(total=sum(1 for value in G.pmlthickness.values() if value > 0), desc='Building PML boundaries', ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
build_pmls(G, pbar)
pbar.close()
# Build the model, i.e. set the material properties (ID) for every edge of every Yee cell
print()
pbar = tqdm(total=2, desc='Building main grid', ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
build_electric_components(G.solid, G.rigidE, G.ID, G)
pbar.update()
build_magnetic_components(G.solid, G.rigidH, G.ID, G)
pbar.update()
pbar.close()
# Process any voltage sources (that have resistance) to create a new material at the source location
for voltagesource in G.voltagesources:
voltagesource.create_material(G)
# Initialise arrays of update coefficients to pass to update functions
G.initialise_std_update_coeff_arrays()
# Initialise arrays of update coefficients and temporary values if there are any dispersive materials
if Material.maxpoles != 0:
G.initialise_dispersive_arrays()
# Process complete list of materials - calculate update coefficients, store in arrays, and build text list of materials/properties
materialsdata = process_materials(G)
if G.messages:
materialstable = AsciiTable(materialsdata)
materialstable.outer_border = False
materialstable.justify_columns[0] = 'right'
print(materialstable.table)
# Check to see if numerical dispersion might be a problem
results = dispersion_analysis(G)
if results['deltavp'] and np.abs(results['deltavp']) > G.maxnumericaldisp:
print(Fore.RED + "\nWARNING: Potentially significant numerical dispersion. Largest physical phase-velocity error is {:.2f}% in material '{}' with wavelength sampled by {} cells (maximum significant frequency {:g}Hz)".format(results['deltavp'], results['material'].ID, round_value(results['N']), results['maxfreq']) + Style.RESET_ALL)
elif results['deltavp']:
print("\nNumerical dispersion analysis: largest physical phase-velocity error is {:.2f}% in material '{}' with wavelength sampled by {} cells (maximum significant frequency {:g}Hz)".format(results['deltavp'], results['material'].ID, round_value(results['N']), results['maxfreq']))
# If geometry information to be reused between model runs
else:
inputfilestr = '\n--- Model {} of {}, input file (not re-processed, i.e. geometry fixed): {}'.format(modelrun, numbermodelruns, inputfile)
print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
# Clear arrays for field components
G.initialise_field_arrays()
# Clear arrays for fields in PML
for pml in G.pmls:
pml.initialise_field_arrays()
# Adjust position of simple sources and receivers if required
if G.srcsteps[0] > 0 or G.srcsteps[1] > 0 or G.srcsteps[2] > 0:
for source in itertools.chain(G.hertziandipoles, G.magneticdipoles):
if modelrun == 1:
if source.xcoord + G.srcsteps[0] * (numbermodelruns - 1) > G.nx or source.ycoord + G.srcsteps[1] * (numbermodelruns - 1) > G.ny or source.zcoord + G.srcsteps[2] * (numbermodelruns - 1) > G.nz:
raise GeneralError('Source(s) will be stepped to a position outside the domain.')
source.xcoord = source.xcoordorigin + (modelrun - 1) * G.srcsteps[0]
source.ycoord = source.ycoordorigin + (modelrun - 1) * G.srcsteps[1]
source.zcoord = source.zcoordorigin + (modelrun - 1) * G.srcsteps[2]
if G.rxsteps[0] > 0 or G.rxsteps[1] > 0 or G.rxsteps[2] > 0:
for receiver in G.rxs:
if modelrun == 1:
if receiver.xcoord + G.rxsteps[0] * (numbermodelruns - 1) > G.nx or receiver.ycoord + G.rxsteps[1] * (numbermodelruns - 1) > G.ny or receiver.zcoord + G.rxsteps[2] * (numbermodelruns - 1) > G.nz:
raise GeneralError('Receiver(s) will be stepped to a position outside the domain.')
receiver.xcoord = receiver.xcoordorigin + (modelrun - 1) * G.rxsteps[0]
receiver.ycoord = receiver.ycoordorigin + (modelrun - 1) * G.rxsteps[1]
receiver.zcoord = receiver.zcoordorigin + (modelrun - 1) * G.rxsteps[2]
# Write files for any geometry views and geometry object outputs
if not (G.geometryviews or G.geometryobjectswrite) and args.geometry_only:
print(Fore.RED + '\nWARNING: No geometry views or geometry objects to output found.' + Style.RESET_ALL)
if G.geometryviews:
print()
for i, geometryview in enumerate(G.geometryviews):
geometryview.set_filename(modelrun, numbermodelruns, G)
pbar = tqdm(total=geometryview.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry view file {} of {}, {}'.format(i + 1, len(G.geometryviews), os.path.split(geometryview.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
geometryview.write_vtk(modelrun, numbermodelruns, G, pbar)
pbar.close()
if G.geometryobjectswrite:
for i, geometryobject in enumerate(G.geometryobjectswrite):
pbar = tqdm(total=geometryobject.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry object file {} of {}, {}'.format(i + 1, len(G.geometryobjectswrite), os.path.split(geometryobject.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
geometryobject.write_hdf5(G, pbar)
pbar.close()
# Run simulation (if not doing geometry only)
if not args.geometry_only:
# Prepare any snapshot files
for snapshot in G.snapshots:
snapshot.prepare_vtk_imagedata(modelrun, numbermodelruns, G)
# Output filename
inputfileparts = os.path.splitext(inputfile)
if numbermodelruns == 1:
outputfile = inputfileparts[0] + '.out'
else:
outputfile = inputfileparts[0] + str(modelrun) + '.out'
print('\nOutput file: {}\n'.format(outputfile))
####################################
# Start - Main FDTD calculations #
####################################
tsolvestart = perf_counter()
# Absolute time
abstime = 0
for timestep in tqdm(range(G.iterations), desc='Running simulation, model ' + str(modelrun) + ' of ' + str(numbermodelruns), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable):
# Store field component values for every receiver and transmission line
store_outputs(timestep, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G)
# Write any snapshots to file
for i, snap in enumerate(G.snapshots):
if snap.time == timestep + 1:
snapiters = 36 * (((snap.xf - snap.xs) / snap.dx) * ((snap.yf - snap.ys) / snap.dy) * ((snap.zf - snap.zs) / snap.dz))
pbar = tqdm(total=snapiters, leave=False, unit='byte', unit_scale=True, desc=' Writing snapshot file {} of {}, {}'.format(i + 1, len(G.snapshots), os.path.split(snap.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=G.tqdmdisable)
snap.write_vtk_imagedata(G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G, pbar)
pbar.close()
# Update electric field components
if Material.maxpoles == 0: # All materials are non-dispersive so do standard update
update_electric(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsE, G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
elif Material.maxpoles == 1: # If there are any dispersive materials do 1st part of dispersive update (it is split into two parts as it requires present and updated electric field values).
update_electric_dispersive_1pole_A(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsE, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
elif Material.maxpoles > 1:
update_electric_dispersive_multipole_A(G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles, G.updatecoeffsE, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
# Update electric field components with the PML correction
for pml in G.pmls:
pml.update_electric(G)
# Update electric field components from sources (update any Hertzian dipole sources last)
for source in G.voltagesources + G.transmissionlines + G.hertziandipoles:
source.update_electric(abstime, G.updatecoeffsE, G.ID, G.Ex, G.Ey, G.Ez, G)
# If there are any dispersive materials do 2nd part of dispersive update (it is split into two parts as it requires present and updated electric field values). Therefore it can only be completely updated after the electric field has been updated by the PML and source updates.
if Material.maxpoles == 1:
update_electric_dispersive_1pole_B(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez)
elif Material.maxpoles > 1:
update_electric_dispersive_multipole_B(G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez)
# Increment absolute time value
abstime += 0.5 * G.dt
# Update magnetic field components
update_magnetic(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsH, G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
# Update magnetic field components with the PML correction
for pml in G.pmls:
pml.update_magnetic(G)
# Update magnetic field components from sources
for source in G.transmissionlines + G.magneticdipoles:
source.update_magnetic(abstime, G.updatecoeffsH, G.ID, G.Hx, G.Hy, G.Hz, G)
# Increment absolute time value
abstime += 0.5 * G.dt
tsolve = int(perf_counter() - tsolvestart)
# Write an output file in HDF5 format
write_hdf5_outputfile(outputfile, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G)
##################################
# End - Main FDTD calculations #
##################################
if G.messages:
print('Memory (RAM) used: ~{}'.format(human_size(p.memory_info().rss)))
# If geometry information to be reused between model runs then FDTDGrid class instance must be global so that it persists
if not args.geometry_fixed:
del G
# Return time to complete solving if in benchmarking mode
if args.benchmark:
return tsolve
2
Example 28
Project: VisTrails Source File: SelectPredictorsLayers.py
def __init__(self, inputMDS, outputMDS, displayJPEG, rPath, parent=None):
#print inputMDS, outputMDS, rPath, modelsPath
self.rPath = rPath
self.selection_name = os.path.split(outputMDS)[1]
self.selection_name = os.path.splitext(self.selection_name)[0]
self.selection_name = self.selection_name.split('_')[-1]
self.displayJPEG = displayJPEG
QtGui.QDialog.__init__(self, parent)
self.inputMDS = inputMDS
self.outputMDS = outputMDS
self.outputDir = os.path.split(outputMDS)[0]
layout = QtGui.QVBoxLayout()
self.setWindowFlags(QtCore.Qt.Window)
##begin autogenerated code from QtDesigner
#
self.horizontalLayout_4 = QtGui.QHBoxLayout()
self.horizontalLayout_4.setSpacing(5)
self.horizontalLayout_4.setMargin(5)
self.horizontalLayout_4.setObjectName(_fromUtf8("horizontalLayout_4"))
self.verticalLayout_3 = QtGui.QVBoxLayout()
self.verticalLayout_3.setObjectName(_fromUtf8("verticalLayout_3"))
self.splitter = QtGui.QSplitter()
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.splitter.sizePolicy().hasHeightForWidth())
self.splitter.setSizePolicy(sizePolicy)
self.splitter.setMinimumSize(QtCore.QSize(0, 0))
self.splitter.setFrameShape(QtGui.QFrame.Box)
self.splitter.setFrameShadow(QtGui.QFrame.Plain)
self.splitter.setLineWidth(1)
self.splitter.setOrientation(QtCore.Qt.Horizontal)
self.splitter.setHandleWidth(10)
self.splitter.setObjectName(_fromUtf8("splitter"))
self.widget = QtGui.QWidget(self.splitter)
self.widget.setObjectName(_fromUtf8("widget"))
self.verticalLayout = QtGui.QVBoxLayout(self.widget)
self.verticalLayout.setSpacing(4)
self.verticalLayout.setSizeConstraint(QtGui.QLayout.SetDefaultConstraint)
self.verticalLayout.setContentsMargins(6, 6, 3, 3)
self.verticalLayout.setObjectName(_fromUtf8("verticalLayout"))
self.label_2 = QtGui.QLabel(self.widget)
self.label_2.setObjectName(_fromUtf8("label_2"))
self.verticalLayout.addWidget(self.label_2)
self.treeview = QtGui.QTreeWidget(self.widget)
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Ignored, QtGui.QSizePolicy.Expanding)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.treeview.sizePolicy().hasHeightForWidth())
self.treeview.setSizePolicy(sizePolicy)
self.treeview.setMinimumSize(QtCore.QSize(75, 0))
self.treeview.setMaximumSize(QtCore.QSize(16777215, 16777215))
self.treeview.setSizeIncrement(QtCore.QSize(100, 0))
self.treeview.setBaseSize(QtCore.QSize(0, 0))
self.treeview.setObjectName(_fromUtf8("treeview"))
self.verticalLayout.addWidget(self.treeview)
self.horizontalLayout_3 = QtGui.QHBoxLayout()
self.horizontalLayout_3.setSizeConstraint(QtGui.QLayout.SetDefaultConstraint)
self.horizontalLayout_3.setObjectName(_fromUtf8("horizontalLayout_3"))
self.btnRunR = QtGui.QPushButton(self.widget)
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.btnRunR.sizePolicy().hasHeightForWidth())
self.btnRunR.setSizePolicy(sizePolicy)
self.btnRunR.setObjectName(_fromUtf8("btnRunR"))
self.horizontalLayout_3.addWidget(self.btnRunR)
spacerItem = QtGui.QSpacerItem(10, 0, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
self.horizontalLayout_3.addItem(spacerItem)
self.label = QtGui.QLabel(self.widget)
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Preferred)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.label.sizePolicy().hasHeightForWidth())
self.label.setSizePolicy(sizePolicy)
self.label.setObjectName(_fromUtf8("label"))
self.horizontalLayout_3.addWidget(self.label)
self.lineEdit = QtGui.QLineEdit(self.widget)
self.lineEdit.setText(_fromUtf8("0.7"))
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.lineEdit.sizePolicy().hasHeightForWidth())
self.lineEdit.setSizePolicy(sizePolicy)
self.lineEdit.setMaximumSize(QtCore.QSize(75, 16777215))
self.lineEdit.setObjectName(_fromUtf8("lineEdit"))
self.horizontalLayout_3.addWidget(self.lineEdit)
self.chkPresence = QtGui.QCheckBox(self.widget)
self.chkPresence.setChecked(True)
self.chkPresence.setText(_fromUtf8("Include presence/count points"))
self.chkPresence.setObjectName(_fromUtf8("chkPresence"))
self.verticalLayout.addWidget(self.chkPresence)
self.chkAbsence = QtGui.QCheckBox(self.widget)
self.chkAbsence.setChecked(True)
self.chkAbsence.setText(_fromUtf8("Include absence points"))
self.chkAbsence.setObjectName(_fromUtf8("chkAbsence"))
self.verticalLayout.addWidget(self.chkAbsence)
self.chkBackground = QtGui.QCheckBox(self.widget)
self.chkBackground.setChecked(True)
self.chkBackground.setText(_fromUtf8("Include background points"))
self.chkBackground.setObjectName(_fromUtf8("chkBackground"))
self.verticalLayout.addWidget(self.chkBackground)
self.verticalLayout.addLayout(self.horizontalLayout_3)
self.scene = QtGui.QGraphicsScene()
self.view = QtGui.QGraphicsView(self.scene)
#self.view = customGraphicsView(self.scene)
self.view.setDragMode(QtGui.QGraphicsView.ScrollHandDrag)
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
sizePolicy.setHorizontalStretch(4)
sizePolicy.setVerticalStretch(0)
#sizePolicy.setHeightForWidth(self.view.sizePolicy().hasHeightForWidth())
self.view.setSizePolicy(sizePolicy)
self.view.setResizeAnchor(QtGui.QGraphicsView.AnchorUnderMouse)
self.view.setObjectName(_fromUtf8("view"))
self.splitter.addWidget(self.view)
self.verticalLayout_3.addWidget(self.splitter)
self.horizontalLayout = QtGui.QHBoxLayout()
self.horizontalLayout.setContentsMargins(-1, 3, -1, 3)
self.horizontalLayout.setObjectName(_fromUtf8("horizontalLayout"))
self.btnOK = QtGui.QPushButton()
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.btnOK.sizePolicy().hasHeightForWidth())
self.btnOK.setSizePolicy(sizePolicy)
self.btnOK.setBaseSize(QtCore.QSize(100, 0))
self.btnOK.setToolTip(_fromUtf8(""))
self.btnOK.setObjectName(_fromUtf8("btnOK"))
self.horizontalLayout.addWidget(self.btnOK)
self.btnCancel = QtGui.QPushButton()
sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.btnCancel.sizePolicy().hasHeightForWidth())
self.btnCancel.setSizePolicy(sizePolicy)
self.btnCancel.setBaseSize(QtCore.QSize(100, 0))
self.btnCancel.setObjectName(_fromUtf8("btnCancel"))
self.horizontalLayout.addWidget(self.btnCancel)
self.verticalLayout_3.addLayout(self.horizontalLayout)
self.horizontalLayout_4.addLayout(self.verticalLayout_3)
self.setWindowTitle(_fromUtf8("Covariate Coorelation viewer"))
self.label_2.setText(_fromUtf8("Covariates"))
self.btnRunR.setText(_fromUtf8("Update"))
self.label.setText(_fromUtf8("Threshold"))
self.btnOK.setText(_fromUtf8("OK"))
self.btnCancel.setText(_fromUtf8("Cancel"))
self.resize(1100, 800)
self.view.resize(800, 750)
##End autogenerated code from QtDesigner
layout.addLayout(self.horizontalLayout_4)
self.btnCancel.setShortcut('Esc')
self.connect(self.btnOK, QtCore.SIGNAL('clicked(bool)'),
self.okTriggered)
self.connect(self.btnCancel, QtCore.SIGNAL('clicked(bool)'),
self.cancel)
self.connect(self.btnRunR, QtCore.SIGNAL('clicked(bool)'),
self.updateROutput)
# self.connect(self.view, QtCore.SIGNAL('resize()'),
# self.resize)
self.scene.wheelEvent = self.wheel_event
# self.resizeEvent = self.resize_event
#code to populate the treeview with the contents of our MDS
self.PopulateTreeview()
self.setLayout(layout)
self.repaint()
#utils.breakpoint()
#code to add in pictureviewer stuff
outputPic = self.runR(self.inputMDS)
# self.setup_view()
self.load_picture(outputPic)
2
Example 29
def __call__(self):
self.log.info("Starting sub command %s", self.command_name)
endpoint = self._endpoint(self.args)
manifest = self._load_manifest_by_name(endpoint,
self.args.backup_name)
# We put the files that need to be restored in there.
work_queue = Queue.Queue()
# We put the results in here
# So we can say what was copied to where.
result_queue = Queue.Queue()
# Fill the queue with components we want to restore.
components = []
for component in manifest.iter_components():
msg = (
json.dumps(manifest.serialise()),
json.dumps(component.serialise())
)
work_queue.put(msg)
components.append(component)
self.log.info("Queued components for restore: %s",
", ".join(str(c) for c in components))
# Make worker threads to do the work.
workers = [
self._create_worker_thread(i, work_queue, result_queue)
for i in range(self.args.threads)
]
for worker in workers:
worker.start()
if self.args.report_interval_secs > 0:
reporter = SlurpReporterThread(work_queue,
self.args.report_interval_secs)
reporter.start()
else:
self.log.info("Progress reporting is disabled.")
# Wait for the work queue to empty.
self.log.info("Waiting on workers.")
work_queue.join()
self.log.info("Finished sub command %s", self.command_name)
# Make a pretty message
op_results = []
# result_queue has BackupFile's
while not result_queue.empty():
op_results.append(RestoreResult.deserialise(json.loads(
result_queue.get_nowait())))
str_bld = []
failed_ops = [
op_result
for op_result in op_results
if op_result.failed_reason
]
skipped_ops = [
op_result
for op_result in op_results
if not op_result.should_restore
]
success_ops = [
op_result
for op_result in op_results
if not op_result.failed_reason and op_result.should_restore
]
corrupt_files = [
op_result
for op_result in op_results
if op_result.corrupt_path
]
def gen_groups(lst, key):
lst.sort(key=key)
for k, i in itertools.groupby(lst, key):
yield k, i
if corrupt_files:
str_bld.append("Moved corrupt existing files:")
for op_result in corrupt_files:
str_bld.append("%s -> %s" % (
op_result.restore_path, op_result.corrupt_path))
str_bld.append("")
if failed_ops:
for k, ops in gen_groups(failed_ops, lambda x: x.failed_reason):
str_bld.append("Failed transfers - %s:" % (k,))
for op_result in ops:
_, name = os.path.split(op_result.source_path)
str_bld.append("%s -> %s \tbecause %s" % (
name, op_result.restore_path, op_result.restore_reason))
str_bld.append("")
if skipped_ops:
str_bld.append("Skipped files:")
for op_result in skipped_ops:
_, name = os.path.split(op_result.source_path)
str_bld.append("%s -> %s \tbecause %s" %(
name, op_result.restore_path, op_result.restore_reason))
str_bld.append("")
if success_ops:
str_bld.append("Restored files:")
for op_result in success_ops:
_, name = os.path.split(op_result.source_path)
str_bld.append("%s -> %s \tbecause %s" %(
name, op_result.restore_path, op_result.restore_reason))
str_bld.append("")
return (0, "\n".join(str_bld))
2
Example 30
def tex_emitter_core(target, source, env, graphics_extensions):
"""An emitter for TeX and LaTeX sources.
For LaTeX sources we try and find the common created files that
are needed on subsequent runs of latex to finish tables of contents,
bibliographies, indices, lists of figures, and hyperlink references.
"""
basename = SCons.Util.splitext(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
targetdir = os.path.split(str(target[0]))[0]
targetbase = os.path.join(targetdir, basefile)
basedir = os.path.split(str(source[0]))[0]
abspath = os.path.abspath(basedir)
target[0].attributes.path = abspath
#
# file names we will make use of in searching the sources and log file
#
emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
auxfilename = targetbase + '.aux'
logfilename = targetbase + '.log'
flsfilename = targetbase + '.fls'
env.SideEffect(auxfilename,target[0])
env.SideEffect(logfilename,target[0])
env.SideEffect(flsfilename,target[0])
if Verbose:
print "side effect :",auxfilename,logfilename,flsfilename
env.Clean(target[0],auxfilename)
env.Clean(target[0],logfilename)
env.Clean(target[0],flsfilename)
content = source[0].get_text_contents()
idx_exists = os.path.exists(targetbase + '.idx')
nlo_exists = os.path.exists(targetbase + '.nlo')
glo_exists = os.path.exists(targetbase + '.glo')
acr_exists = os.path.exists(targetbase + '.acn')
# set up list with the regular expressions
# we use to find features used
file_tests_search = [auxfile_re,
makeindex_re,
bibliography_re,
bibunit_re,
tableofcontents_re,
listoffigures_re,
listoftables_re,
hyperref_re,
makenomenclature_re,
makeglossary_re,
makeglossaries_re,
makeacronyms_re,
beamer_re ]
# set up list with the file suffixes that need emitting
# when a feature is found
file_tests_suff = [['.aux','aux_file'],
['.idx', '.ind', '.ilg','makeindex'],
['.bbl', '.blg','bibliography'],
['.bbl', '.blg','bibunit'],
['.toc','contents'],
['.lof','figures'],
['.lot','tables'],
['.out','hyperref'],
['.nlo', '.nls', '.nlg','nomenclature'],
['.glo', '.gls', '.glg','glossary'],
['.glo', '.gls', '.glg','glossaries'],
['.acn', '.acr', '.alg','acronyms'],
['.nav', '.snm', '.out', '.toc','beamer'] ]
# build the list of lists
file_tests = []
for i in range(len(file_tests_search)):
file_tests.append( [None, file_tests_suff[i]] )
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
paths = env['ENV']['TEXINPUTS']
if SCons.Util.is_List(paths):
pass
else:
# Split at os.pathsep to convert into absolute path
paths = paths.split(os.pathsep)
# now that we have the path list restore the env
if savedpath is _null:
try:
del env['ENV']['TEXINPUTS']
except KeyError:
pass # was never set
else:
env['ENV']['TEXINPUTS'] = savedpath
if Verbose:
print "search path ",paths
aux_files = []
file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)
for (theSearch,suffix_list) in file_tests:
# add side effects if feature is present.If file is to be generated,add all side effects
if (theSearch != None) or (not source[0].exists() ):
file_list = [targetbase,]
# for bibunit we need a list of files
if suffix_list[-1] == 'bibunit':
file_basename = os.path.join(targetdir, 'bu*.aux')
file_list = glob.glob(file_basename)
# remove the suffix '.aux'
for i in range(len(file_list)):
file_list[i] = SCons.Util.splitext(file_list[i])[0]
# now define the side effects
for file_name in file_list:
for suffix in suffix_list[:-1]:
env.SideEffect(file_name + suffix,target[0])
if Verbose:
print "side effect :",file_name + suffix
env.Clean(target[0],file_name + suffix)
for aFile in aux_files:
aFile_base = SCons.Util.splitext(aFile)[0]
env.SideEffect(aFile_base + '.aux',target[0])
if Verbose:
print "side effect :",aFile_base + '.aux'
env.Clean(target[0],aFile_base + '.aux')
# read fls file to get all other files that latex creates and will read on the next pass
# remove files from list that we explicitly dealt with above
if os.path.exists(flsfilename):
content = open(flsfilename, "rb").read()
out_files = openout_re.findall(content)
myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
for filename in out_files[:]:
if filename in myfiles:
out_files.remove(filename)
env.SideEffect(out_files,target[0])
if Verbose:
print "side effect :",out_files
env.Clean(target[0],out_files)
return (target, source)
2
Example 31
def tex_emitter_core(target, source, env, graphics_extensions):
"""An emitter for TeX and LaTeX sources.
For LaTeX sources we try and find the common created files that
are needed on subsequent runs of latex to finish tables of contents,
bibliographies, indices, lists of figures, and hyperlink references.
"""
basename = SCons.Util.splitext(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
targetdir = os.path.split(str(target[0]))[0]
targetbase = os.path.join(targetdir, basefile)
basedir = os.path.split(str(source[0]))[0]
abspath = os.path.abspath(basedir)
target[0].attributes.path = abspath
#
# file names we will make use of in searching the sources and log file
#
emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
auxfilename = targetbase + '.aux'
logfilename = targetbase + '.log'
flsfilename = targetbase + '.fls'
syncfilename = targetbase + '.synctex.gz'
env.SideEffect(auxfilename,target[0])
env.SideEffect(logfilename,target[0])
env.SideEffect(flsfilename,target[0])
env.SideEffect(syncfilename,target[0])
if Verbose:
print "side effect :",auxfilename,logfilename,flsfilename,syncfilename
env.Clean(target[0],auxfilename)
env.Clean(target[0],logfilename)
env.Clean(target[0],flsfilename)
env.Clean(target[0],syncfilename)
content = source[0].get_text_contents()
# These variables are no longer used.
#idx_exists = os.path.isfile(targetbase + '.idx')
#nlo_exists = os.path.isfile(targetbase + '.nlo')
#glo_exists = os.path.isfile(targetbase + '.glo')
#acr_exists = os.path.isfile(targetbase + '.acn')
# set up list with the regular expressions
# we use to find features used
file_tests_search = [auxfile_re,
makeindex_re,
bibliography_re,
bibunit_re,
multibib_re,
addbibresource_re,
tableofcontents_re,
listoffigures_re,
listoftables_re,
hyperref_re,
makenomenclature_re,
makeglossary_re,
makeglossaries_re,
makeacronyms_re,
beamer_re,
newglossary_re,
biblatex_re ]
# set up list with the file suffixes that need emitting
# when a feature is found
file_tests_suff = [['.aux','aux_file'],
['.idx', '.ind', '.ilg','makeindex'],
['.bbl', '.blg','bibliography'],
['.bbl', '.blg','bibunit'],
['.bbl', '.blg','multibib'],
['.bbl', '.blg','.bcf','addbibresource'],
['.toc','contents'],
['.lof','figures'],
['.lot','tables'],
['.out','hyperref'],
['.nlo', '.nls', '.nlg','nomenclature'],
['.glo', '.gls', '.glg','glossary'],
['.glo', '.gls', '.glg','glossaries'],
['.acn', '.acr', '.alg','acronyms'],
['.nav', '.snm', '.out', '.toc','beamer'],
['newglossary',],
['.bcf', '.blg','biblatex'] ]
# for newglossary the suffixes are added as we find the command
# build the list of lists
file_tests = []
for i in range(len(file_tests_search)):
file_tests.append( [None, file_tests_suff[i]] )
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
paths = env['ENV']['TEXINPUTS']
if SCons.Util.is_List(paths):
pass
else:
# Split at os.pathsep to convert into absolute path
paths = paths.split(os.pathsep)
# now that we have the path list restore the env
if savedpath is _null:
try:
del env['ENV']['TEXINPUTS']
except KeyError:
pass # was never set
else:
env['ENV']['TEXINPUTS'] = savedpath
if Verbose:
print "search path ",paths
# scan all sources for side effect files
aux_files = []
file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)
for (theSearch,suffix_list) in file_tests:
# add side effects if feature is present.If file is to be generated,add all side effects
if Verbose and theSearch:
print "check side effects for ",suffix_list[-1]
if (theSearch != None) or (not source[0].exists() ):
file_list = [targetbase,]
# for bibunit we need a list of files
if suffix_list[-1] == 'bibunit':
file_basename = os.path.join(targetdir, 'bu*.aux')
file_list = glob.glob(file_basename)
# remove the suffix '.aux'
for i in range(len(file_list)):
file_list.append(SCons.Util.splitext(file_list[i])[0])
# for multibib we need a list of files
if suffix_list[-1] == 'multibib':
for multibibmatch in multibib_re.finditer(content):
if Verbose:
print "multibib match ",multibibmatch.group(1)
if multibibmatch != None:
baselist = multibibmatch.group(1).split(',')
if Verbose:
print "multibib list ", baselist
for i in range(len(baselist)):
file_list.append(os.path.join(targetdir, baselist[i]))
# now define the side effects
for file_name in file_list:
for suffix in suffix_list[:-1]:
env.SideEffect(file_name + suffix,target[0])
if Verbose:
print "side effect tst :",file_name + suffix, " target is ",str(target[0])
env.Clean(target[0],file_name + suffix)
for aFile in aux_files:
aFile_base = SCons.Util.splitext(aFile)[0]
env.SideEffect(aFile_base + '.aux',target[0])
if Verbose:
print "side effect aux :",aFile_base + '.aux'
env.Clean(target[0],aFile_base + '.aux')
# read fls file to get all other files that latex creates and will read on the next pass
# remove files from list that we explicitly dealt with above
if os.path.isfile(flsfilename):
content = open(flsfilename, "rb").read()
out_files = openout_re.findall(content)
myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
for filename in out_files[:]:
if filename in myfiles:
out_files.remove(filename)
env.SideEffect(out_files,target[0])
if Verbose:
print "side effect fls :",out_files
env.Clean(target[0],out_files)
return (target, source)
2
Example 32
Project: tomato Source File: swf_image_dumper.py
def save_image(self, object_id, v):
# 実際に出力した画像の ID を保存
file_base = self.output_file
if v['tag'] == 'DefineBitsLossLess' and v['format'] == 3:
color_table_length = ord(v['data'][0])+1
decompress_data = zlib.decompress(v['data'][1:])
color_table = []
for i in range(color_table_length):
c = (ord(decompress_data[i*3]),
ord(decompress_data[i*3+1]),
ord(decompress_data[i*3+2]))
color_table.append(c)
color_map = decompress_data[color_table_length*3:]
width = v['width']
height = v['height']
width = width if width % 4 == 0 else (width/4+1)* 4
im = Image.new("RGB", (width, height), "white")
x = y = 0
for i in range(len(color_map)):
im.putpixel((x,y), color_table[ord(color_map[i])])
x += 1
if x == width:
x = 0
y += 1
out_file = "%s_%s.png" % (file_base, object_id)
im.save(out_file)
v['file_name'] = os.path.split(out_file)[1]
elif v['tag'] == 'DefineBitsLossless2' and v['format'] == 3:
color_table_length = ord(v['data'][0])+1
decompress_data = zlib.decompress(v['data'][1:])
color_table = []
for i in range(color_table_length):
c = (ord(decompress_data[i*4+0]),
ord(decompress_data[i*4+1]),
ord(decompress_data[i*4+2]),
ord(decompress_data[i*4+3]))
color_table.append(c)
color_map = decompress_data[color_table_length*4:]
width = v['width']
height = v['height']
width = width if width % 4 == 0 else (width/4+1)* 4
im = Image.new("RGB", (width, height), "white")
x = y = 0
for i in range(len(color_map)):
im.putpixel((x,y), color_table[ord(color_map[i])])
x += 1
if x == width:
x = 0
y += 1
out_file = "%s_%s.png" % (file_base, object_id)
im.save(out_file)
v['file_name'] = os.path.split(out_file)[1]
elif v['tag'] == 'DefineBitsLossLess' and v['format'] == 5:
decompress_data = zlib.decompress(v['data'])
width = v['width']
height = v['height']
im = Image.new("RGB", (width, height), "white")
x = y = 0
for i in range(0, len(decompress_data), 4):
im.putpixel((x,y), (ord(decompress_data[i+1]),
ord(decompress_data[i+2]),
ord(decompress_data[i+3])))
x += 1
if x == width:
x = 0
y += 1
out_file = "%s_%s.png" % (file_base, object_id)
im.save(out_file)
v['file_name'] = os.path.split(out_file)[1]
elif v['tag'] == 'DefineBitsLossless2' and v['format'] == 5:
decompress_data = zlib.decompress(v['data'])
width = v['width']
height = v['height']
im = Image.new("RGBA", (width, height), "white")
x = y = 0
for i in range(0, len(decompress_data), 4):
im.putpixel((x,y), (ord(decompress_data[i+1]),
ord(decompress_data[i+2]),
ord(decompress_data[i+3]),
ord(decompress_data[i+0])))
x += 1
if x == width:
x = 0
y += 1
out_file = "%s_%s.png" % (file_base, object_id)
im.save(out_file)
v['file_name'] = os.path.split(out_file)[1]
elif v['tag'] == 'DefineBitsJPEG2':
data = v['data']
if data[:4] == '\xff\xd9\xff\xd8':
"""
See: http://pwiki.awm.jp/~yoya/?Flash/JPEG
JPEG SOI marker (FF D8) -> \xff\xd8
JPEG EOI marker (FF D9) -> \xff\xd9
Flash に格納されているのは基本的にはそのままの JPEG データ
ただし、データの最初に [EOI] + [SOI] が付けられていることがあり
その場合は除去する必要がある
"""
data = data[4:]
im = Image.open(StringIO(data))
v['width'] = im.size[0]
v['height'] = im.size[1]
out_file = "%s_%s.jpg" % (file_base, object_id)
im.save(out_file)
v['file_name'] = os.path.split(out_file)[1]
2
Example 33
Project: romcollectionbrowser Source File: dialogeditromcollection.py
def updateRomCollectionControls(self):
Logutil.log('updateRomCollectionControls', util.LOG_LEVEL_INFO)
control = self.getControlById(CONTROL_LIST_ROMCOLLECTIONS)
selectedRomCollectionName = str(control.getSelectedItem().getLabel())
Logutil.log('selected rom collection: ' +str(selectedRomCollectionName), util.LOG_LEVEL_INFO)
self.selectedRomCollection = None
for rcId in self.romCollections.keys():
romCollection = self.romCollections[rcId]
if romCollection.name == selectedRomCollectionName:
self.selectedRomCollection = romCollection
break
if(self.selectedRomCollection == None):
return
#Import Games
#HACK: split romPath and fileMask
firstRomPath = ''
fileMask = ''
for romPath in self.selectedRomCollection.romPaths:
pathParts = os.path.split(romPath)
if(firstRomPath == ''):
firstRomPath = pathParts[0]
fileMask = pathParts[1]
elif(firstRomPath == pathParts[0]):
fileMask = fileMask +',' +pathParts[1]
control = self.getControlById(CONTROL_BUTTON_ROMPATH)
util.setLabel(firstRomPath, control)
control = self.getControlById(CONTROL_BUTTON_FILEMASK)
util.setLabel(fileMask, control)
control = self.getControlById(CONTROL_BUTTON_IGNOREONSCAN)
control.setSelected(self.selectedRomCollection.ignoreOnScan)
control = self.getControlById(CONTROL_BUTTON_ALLOWUPDATE)
control.setSelected(self.selectedRomCollection.allowUpdate)
control = self.getControlById(CONTROL_BUTTON_DISKINDICATOR)
util.setLabel(self.selectedRomCollection.diskPrefix, control)
control = self.getControlById(CONTROL_BUTTON_MAXFOLDERDEPTH)
util.setLabel(self.selectedRomCollection.maxFolderDepth, control)
control = self.getControlById(CONTROL_BUTTON_USEFOLDERASGAMENAME)
control.setSelected(self.selectedRomCollection.useFoldernameAsGamename)
#Import Game Data
#Media Types
mediaTypeList = []
firstMediaPath = ''
firstMediaFileMask = ''
for mediaPath in self.selectedRomCollection.mediaPaths:
mediaTypeList.append(mediaPath.fileType.name)
if(firstMediaPath == ''):
pathParts = os.path.split(mediaPath.path)
firstMediaPath = pathParts[0]
firstMediaFileMask = pathParts[1]
self.addItemsToList(CONTROL_LIST_MEDIATYPES, mediaTypeList)
control = self.getControlById(CONTROL_BUTTON_MEDIAPATH)
util.setLabel(firstMediaPath, control)
control = self.getControlById(CONTROL_BUTTON_MEDIAFILEMASK)
util.setLabel(firstMediaFileMask, control)
self.selectScrapersInList(self.selectedRomCollection.scraperSites, self.availableScrapers)
#Browse Games
optionMain = self.selectedRomCollection.imagePlacingMain.name
try:
optionMain = config.imagePlacingDict[optionMain]
except:
pass
self.selectItemInList(optionMain, CONTROL_LIST_IMAGEPLACING_MAIN)
optionInfo = self.selectedRomCollection.imagePlacingInfo.name
try:
optionInfo = config.imagePlacingDict[optionInfo]
except:
pass
self.selectItemInList(optionInfo, CONTROL_LIST_IMAGEPLACING_INFO)
control = self.getControlById(CONTROL_BUTTON_AUTOPLAYVIDEO_MAIN)
if(control != None):
control.setSelected(self.selectedRomCollection.autoplayVideoMain)
control = self.getControlById(CONTROL_BUTTON_AUTOPLAYVIDEO_INFO)
if(control != None):
control.setSelected(self.selectedRomCollection.autoplayVideoInfo)
#Launch Games
control = self.getControlById(CONTROL_BUTTON_USERETROPLAYER)
if(control):
control.setSelected(self.selectedRomCollection.useBuiltinEmulator)
control = self.getControlById(CONTROL_BUTTON_GAMECLIENT)
if(control):
util.setLabel(self.selectedRomCollection.gameclient, control)
control = self.getControlById(CONTROL_BUTTON_EMUCMD)
util.setLabel(self.selectedRomCollection.emulatorCmd, control)
control = self.getControlById(CONTROL_BUTTON_PARAMS)
util.setLabel(self.selectedRomCollection.emulatorParams, control)
control = self.getControlById(CONTROL_BUTTON_USEEMUSOLO)
control.setSelected(self.selectedRomCollection.useEmuSolo)
control = self.getControlById(CONTROL_BUTTON_USEPOPEN)
control.setSelected(self.selectedRomCollection.usePopen)
pathParts = os.path.split(self.selectedRomCollection.saveStatePath)
saveStatePath = pathParts[0]
saveStateFileMask = pathParts[1]
control = self.getControlById(CONTROL_BUTTON_SAVESTATEPATH)
util.setLabel(saveStatePath, control)
control = self.getControlById(CONTROL_BUTTON_SAVESTATEMASK)
util.setLabel(saveStateFileMask, control)
control = self.getControlById(CONTROL_BUTTON_SAVESTATEPARAMS)
util.setLabel(self.selectedRomCollection.saveStateParams, control)
control = self.getControlById(CONTROL_BUTTON_DONTEXTRACTZIP)
control.setSelected(self.selectedRomCollection.doNotExtractZipFiles)
control = self.getControlById(CONTROL_BUTTON_MAKELOCALCOPY)
control.setSelected(self.selectedRomCollection.makeLocalCopy)
control = self.getControlById(CONTROL_BUTTON_PRECMD)
util.setLabel(self.selectedRomCollection.preCmd, control)
control = self.getControlById(CONTROL_BUTTON_POSTCMD)
util.setLabel(self.selectedRomCollection.postCmd, control)
2
Example 34
Project: enigma2 Source File: MovieList.py
def buildMovieListEntry(self, serviceref, info, begin, data):
switch = config.usage.show_icons_in_movielist.value
ext = config.movielist.useextlist.value
width = self.l.getItemSize().width()
pathName = serviceref.getPath()
res = [ None ]
if ext != '0':
ih = self.itemHeight / 2
else:
ih = self.itemHeight
if self.screenwidth and self.screenwidth == 1920:
listBeginX = 3
listEndX = 3
listMarginX = 12
pathIconSize = 29
dataIconSize = 25
progressIconSize = 25
progressBarSize = 72
textPosY = 2
else:
listBeginX = 2
listEndX = 2
listMarginX = 8
pathIconSize = 25
dataIconSize = 21
progressIconSize = 21
progressBarSize = 48
textPosY = 1
textPosX = listBeginX + dataIconSize + listMarginX
if serviceref.flags & eServiceReference.mustDescent:
# Directory
iconSize = pathIconSize
iconPosX = listBeginX-1
iconPosY = ih/2-iconSize/2
if iconPosY < iconPosX:
iconPosY = iconPosX
# Name is full path name
if info is None:
# Special case: "parent"
txt = ".."
else:
p = os.path.split(pathName)
if not p[1]:
# if path ends in '/', p is blank.
p = os.path.split(p[0])
txt = p[1]
if txt == ".Trash":
dateSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), _("Trashcan")).width()
res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=self.iconTrash))
res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = _("Deleted items")))
res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, textPosY), size=(dateSize, self.itemHeight), font=1, flags=RT_HALIGN_RIGHT|RT_VALIGN_CENTER, text=_("Trashcan")))
return res
dateSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), _("Directory")).width()
res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=self.iconFolder))
res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = txt))
res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, textPosY), size=(dateSize, self.itemHeight), font=1, flags=RT_HALIGN_RIGHT|RT_VALIGN_CENTER, text=_("Directory")))
return res
if (data == -1) or (data is None):
data = MovieListData()
cur_idx = self.l.getCurrentSelectionIndex()
x = self.list[cur_idx] # x = ref,info,begin,...
data.len = 0 #dont recalc movielist to speedup loading the list
self.list[cur_idx] = (x[0], x[1], x[2], data) #update entry in list... so next time we don't need to recalc
data.txt = info.getName(serviceref)
if config.movielist.hide_extensions.value:
fileName, fileExtension = os.path.splitext(data.txt)
if fileExtension in KNOWN_EXTENSIONS:
data.txt = fileName
data.icon = None
data.part = None
if os.path.split(pathName)[1] in self.runningTimers:
if switch == 'i':
if (self.playInBackground or self.playInForeground) and serviceref == (self.playInBackground or self.playInForeground):
data.icon = self.iconMoviePlayRec
else:
data.icon = self.iconMovieRec
elif switch == 'p' or switch == 's':
data.part = 100
if (self.playInBackground or self.playInForeground) and serviceref == (self.playInBackground or self.playInForeground):
data.partcol = 0xffc71d
else:
data.partcol = 0xff001d
elif (self.playInBackground or self.playInForeground) and serviceref == (self.playInBackground or self.playInForeground):
data.icon = self.iconMoviePlay
else:
data.part = moviePlayState(pathName + '.cuts', serviceref, data.len)
if switch == 'i':
if data.part is not None and data.part > 0:
data.icon = self.iconPart[data.part // 25]
else:
if config.usage.movielist_unseen.value:
data.icon = self.iconUnwatched
elif switch == 'p' or switch == 's':
if data.part is not None and data.part > 0:
data.partcol = 0xffc71d
else:
if config.usage.movielist_unseen.value:
data.part = 100
data.partcol = 0x206333
len = data.len
if len > 0:
len = "%d:%02d" % (len / 60, len % 60)
else:
len = ""
iconSize = 0
if switch == 'i':
iconSize = dataIconSize
iconPosX = listBeginX
iconPosY = ih/2-iconSize/2
if iconPosY < iconPosX:
iconPosY = iconPosX
res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=data.icon))
elif switch == 'p':
if data.part is not None and data.part > 0:
iconSize = progressBarSize
iconPosX = listBeginX
iconPosY = ih/2-iconSize/8
if iconPosY < iconPosX:
iconPosY = iconPosX
res.append(MultiContentEntryProgress(pos=(iconPosX,iconPosY), size=(iconSize,iconSize/4), percent=data.part, borderWidth=2, foreColor=data.partcol, foreColorSelected=None, backColor=None, backColorSelected=None))
else:
iconSize = dataIconSize
iconPosX = listBeginX
iconPosY = ih/2-iconSize/2
if iconPosY < iconPosX:
iconPosY = iconPosX
res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=data.icon))
elif switch == 's':
iconSize = progressIconSize
iconPosX = listBeginX
iconPosY = ih/2-iconSize/2
if iconPosY < iconPosX:
iconPosY = iconPosX
if data.part is not None and data.part > 0:
res.append(MultiContentEntryProgress(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), percent=data.part, borderWidth=2, foreColor=data.partcol, foreColorSelected=None, backColor=None, backColorSelected=None))
else:
res.append(MultiContentEntryPixmapAlphaBlend(pos=(iconPosX,iconPosY), size=(iconSize,iconSize), png=data.icon))
begin_string = ""
if begin > 0:
begin_string = ', '.join(FuzzyTime(begin, inPast = True))
dateSize = serviceSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), begin_string).width()
if iconSize:
textPosX = listBeginX + iconSize + listMarginX
else:
textPosX = listBeginX
if ext != '0':
getrec = info.getName(serviceref)
fileName, fileExtension = os.path.splitext(getrec)
desc = None
picon = None
service = None
try:
serviceHandler = eServiceCenter.getInstance()
info = serviceHandler.info(serviceref)
desc = info.getInfoString(serviceref, iServiceInformation.sDescription) # get description
ref = info.getInfoString(serviceref, iServiceInformation.sServiceref) # get reference
service = ServiceReference(ref).getServiceName() # get service name
serviceSize = getTextBoundarySize(self.instance, self.dateFont, self.l.getItemSize(), service).width()
except Exception, e:
print('[MovieList] load extended infos get failed: ', e)
if ext == '2':
try:
picon = getPiconName(ref)
picon = loadPNG(picon)
except Exception, e:
print('[MovieList] load picon get failed: ', e)
if fileExtension in RECORD_EXTENSIONS:
if ext == '1':
res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-serviceSize-listMarginX-listEndX, ih), font=0, flags=RT_HALIGN_LEFT|RT_VALIGN_CENTER, text=data.txt))
res.append(MultiContentEntryText(pos=(width-serviceSize-listEndX, textPosY), size=(serviceSize, ih), font=1, flags=RT_HALIGN_RIGHT | RT_VALIGN_CENTER, text=service))
if ext == '2':
piconSize = ih * 1.667
res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font=0, flags=RT_HALIGN_LEFT|RT_VALIGN_CENTER, text=data.txt))
res.append(MultiContentEntryPixmapAlphaTest(pos=(width-piconSize-listEndX, listEndX), size=(piconSize, ih), png=picon, flags = BT_SCALE | BT_KEEP_ASPECT_RATIO))
res.append(MultiContentEntryText(pos=(listBeginX, ih+textPosY), size=(width-listBeginX-dateSize-listMarginX-listEndX, ih), font=1, flags=RT_HALIGN_LEFT, text=desc))
res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, ih+textPosY), size=(dateSize, ih), font=1, flags=RT_HALIGN_RIGHT, text=begin_string))
return res
else:
res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = data.txt))
res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, ih), size=(dateSize, ih), font=1, flags=RT_HALIGN_RIGHT, text=begin_string))
return res
else:
res.append(MultiContentEntryText(pos=(textPosX, 0), size=(width-textPosX-dateSize-listMarginX-listEndX, ih), font = 0, flags = RT_HALIGN_LEFT|RT_VALIGN_CENTER, text = data.txt))
res.append(MultiContentEntryText(pos=(width-dateSize-listEndX, textPosY), size=(dateSize, ih), font=1, flags=RT_HALIGN_RIGHT|RT_VALIGN_CENTER, text=begin_string))
return res
2
Example 35
def extract_data(stdout):
"""Extract data from youtube-dl stdout.
Args:
stdout (string): String that contains the youtube-dl stdout.
Returns:
Python dictionary. The returned dictionary can be empty if there are
no data to extract else it may contain one or more of the
following keys:
'status' : Contains the status of the download process.
'path' : Destination path.
'extension' : The file extension.
'filename' : The filename without the extension.
'percent' : The percentage of the video being downloaded.
'eta' : Estimated time for the completion of the download process.
'speed' : Download speed.
'filesize' : The size of the video file being downloaded.
'playlist_index' : The playlist index of the current video file being downloaded.
'playlist_size' : The number of videos in the playlist.
"""
data_dictionary = {}
if not stdout:
return data_dictionary
stdout = [string for string in stdout.split(' ') if string != '']
stdout[0] = stdout[0].lstrip('\r')
if stdout[0] == '[download]':
data_dictionary['status'] = 'Downloading'
# Get path, filename & extension
if stdout[1] == 'Destination:':
path, fullname = os.path.split(' '.join(stdout[2:]))
filename, extension = os.path.splitext(fullname)
data_dictionary['path'] = path
data_dictionary['extension'] = extension
# remove the format that youtube-dl adds during the merging
# process at end of the filename
data_dictionary['filename'] = re.sub('\.f[0-9]{1,3}$', '', filename)
# Get progress info
if '%' in stdout[1]:
if stdout[1] == '100%':
data_dictionary['speed'] = ''
data_dictionary['eta'] = ''
data_dictionary['percent'] = '100%'
data_dictionary['filesize'] = stdout[3]
else:
data_dictionary['percent'] = stdout[1]
data_dictionary['filesize'] = stdout[3]
data_dictionary['speed'] = stdout[5]
data_dictionary['eta'] = stdout[7]
# Get playlist info
if stdout[1] == 'Downloading' and stdout[2] == 'video':
data_dictionary['playlist_index'] = stdout[3]
data_dictionary['playlist_size'] = stdout[5]
# Get file already downloaded status
if stdout[-1] == 'downloaded':
data_dictionary['status'] = 'Already Downloaded'
# Get filesize abort status
if stdout[-1] == 'Aborting.':
data_dictionary['status'] = 'Filesize Abort'
elif stdout[0] == '[hlsnative]':
# native hls extractor
# see: https://github.com/rg3/youtube-dl/blob/master/youtube_dl/downloader/hls.py#L54
data_dictionary['status'] = 'Downloading'
if len(stdout) == 7:
segment_no = float(stdout[6])
current_segment = float(stdout[4])
# Get the percentage
percent = '{0:.1f}%'.format(current_segment / segment_no * 100)
data_dictionary['percent'] = percent
elif stdout[0] == '[ffmpeg]':
data_dictionary['status'] = 'Post Processing'
# Get final extension after merging process
if stdout[1] == 'Merging':
data_dictionary['extension'] = os.path.splitext(' '.join(stdout[4:])[:-1])[1]
else:
data_dictionary['status'] = 'Pre Processing'
return data_dictionary
2
Example 36
Project: nzbToMedia Source File: transcoder.py
def buildCommands(file, newDir, movieName, bitbucket):
if isinstance(file, str):
inputFile = file
if '"concat:' in file:
file = file.split('|')[0].replace('concat:', '')
video_details, result = getVideoDetails(file)
dir, name = os.path.split(file)
name, ext = os.path.splitext(name)
check = re.match("VTS_([0-9][0-9])_[0-9]+", name)
if check and core.CONCAT:
name = movieName
elif check:
name = ('{0}.cd{1}'.format(movieName, check.groups()[0]))
elif core.CONCAT and re.match("(.+)[cC][dD][0-9]", name):
name = re.sub("([\ \.\-\_\=\:]+[cC][dD][0-9])", "", name)
if ext == core.VEXTENSION and newDir == dir: # we need to change the name to prevent overwriting itself.
core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext'
else:
img, data = iteritems(file).next()
name = data['name']
video_details, result = getVideoDetails(data['files'][0], img, bitbucket)
inputFile = '-'
file = '-'
newfilePath = os.path.normpath(os.path.join(newDir, name) + core.VEXTENSION)
map_cmd = []
video_cmd = []
audio_cmd = []
audio_cmd2 = []
sub_cmd = []
meta_cmd = []
other_cmd = []
if not video_details or not video_details.get(
"streams"): # we couldn't read streams with ffprobe. Set defaults to try transcoding.
videoStreams = []
audioStreams = []
subStreams = []
map_cmd.extend(['-map', '0'])
if core.VCODEC:
video_cmd.extend(['-c:v', core.VCODEC])
if core.VCODEC == 'libx264' and core.VPRESET:
video_cmd.extend(['-pre', core.VPRESET])
else:
video_cmd.extend(['-c:v', 'copy'])
if core.VFRAMERATE:
video_cmd.extend(['-r', str(core.VFRAMERATE)])
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VRESOLUTION:
video_cmd.extend(['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)])
if core.VPRESET:
video_cmd.extend(['-preset', core.VPRESET])
if core.VCRF:
video_cmd.extend(['-crf', str(core.VCRF)])
if core.VLEVEL:
video_cmd.extend(['-level', str(core.VLEVEL)])
if core.ACODEC:
audio_cmd.extend(['-c:a', core.ACODEC])
if core.ACODEC in ['aac',
'dts']: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg
audio_cmd.extend(['-strict', '-2'])
else:
audio_cmd.extend(['-c:a', 'copy'])
if core.ACHANNELS:
audio_cmd.extend(['-ac', str(core.ACHANNELS)])
if core.ABITRATE:
audio_cmd.extend(['-b:a', str(core.ABITRATE)])
if core.OUTPUTQUALITYPERCENT:
audio_cmd.extend(['-q:a', str(core.OUTPUTQUALITYPERCENT)])
if core.SCODEC and core.ALLOWSUBS:
sub_cmd.extend(['-c:s', core.SCODEC])
elif core.ALLOWSUBS: # Not every subtitle codec can be used for every video container format!
sub_cmd.extend(['-c:s', 'copy'])
else: # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options
sub_cmd.extend(['-sn']) # Don't copy the subtitles over
if core.OUTPUTFASTSTART:
other_cmd.extend(['-movflags', '+faststart'])
else:
videoStreams = [item for item in video_details["streams"] if item["codec_type"] == "video"]
audioStreams = [item for item in video_details["streams"] if item["codec_type"] == "audio"]
subStreams = [item for item in video_details["streams"] if item["codec_type"] == "subtitle"]
if core.VEXTENSION not in ['.mkv', '.mpegts']:
subStreams = [item for item in video_details["streams"] if
item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item[
"codec_name"] != "pgssub"]
for video in videoStreams:
codec = video["codec_name"]
fr = video.get("avg_frame_rate", 0)
width = video.get("width", 0)
height = video.get("height", 0)
scale = core.VRESOLUTION
if codec in core.VCODEC_ALLOW or not core.VCODEC:
video_cmd.extend(['-c:v', 'copy'])
else:
video_cmd.extend(['-c:v', core.VCODEC])
if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001):
video_cmd.extend(['-r', str(core.VFRAMERATE)])
if scale:
w_scale = width / float(scale.split(':')[0])
h_scale = height / float(scale.split(':')[1])
if w_scale > h_scale: # widescreen, Scale by width only.
scale = "{width}:{height}".format(
width=scale.split(':')[0],
height=int((height / w_scale) / 2) * 2,
)
if w_scale > 1:
video_cmd.extend(['-vf', 'scale={width}'.format(width=scale)])
else: # lower or matching ratio, scale by height only.
scale = "{width}:{height}".format(
width=int((width / h_scale) / 2) * 2,
height=scale.split(':')[1],
)
if h_scale > 1:
video_cmd.extend(['-vf', 'scale={height}'.format(height=scale)])
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VPRESET:
video_cmd.extend(['-preset', core.VPRESET])
if core.VCRF:
video_cmd.extend(['-crf', str(core.VCRF)])
if core.VLEVEL:
video_cmd.extend(['-level', str(core.VLEVEL)])
no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v']
if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy):
video_cmd[1] = core.VCODEC
if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding.
video_cmd = ['-c:v', 'copy']
map_cmd.extend(['-map', '0:{index}'.format(index=video["index"])])
break # Only one video needed
used_audio = 0
a_mapped = []
if audioStreams:
try:
audio1 = [item for item in audioStreams if item["tags"]["language"] == core.ALANGUAGE]
except: # no language tags. Assume only 1 language.
audio1 = audioStreams
try:
audio2 = [item for item in audio1 if item["codec_name"] in core.ACODEC_ALLOW]
except:
audio2 = []
try:
audio3 = [item for item in audioStreams if item["tags"]["language"] != core.ALANGUAGE]
except:
audio3 = []
try:
audio4 = [item for item in audio3 if item["codec_name"] in core.ACODEC_ALLOW]
except:
audio4 = []
if audio2: # right (or only) language and codec...
map_cmd.extend(['-map', '0:{index}'.format(index=audio2[0]["index"])])
a_mapped.extend([audio2[0]["index"]])
bitrate = int(float(audio2[0].get("bit_rate", 0))) / 1000
channels = int(float(audio2[0].get("channels", 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right (or only) language, wrong codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])])
a_mapped.extend([audio1[0]["index"]])
bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000
channels = int(float(audio1[0].get("channels", 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])
elif audio4: # wrong language, right codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio4[0]["index"])])
a_mapped.extend([audio4[0]["index"]])
bitrate = int(float(audio4[0].get("bit_rate", 0))) / 1000
channels = int(float(audio4[0].get("channels", 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio3: # wrong language, wrong codec. just pick the default audio track
map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])])
a_mapped.extend([audio3[0]["index"]])
bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000
channels = int(float(audio3[0].get("channels", 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])
if core.ACHANNELS and channels and channels > core.ACHANNELS:
audio_cmd.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1):
audio_cmd.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.OUTPUTQUALITYPERCENT:
audio_cmd.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if audio_cmd[1] in ['aac', 'dts']:
audio_cmd[2:2] = ['-strict', '-2']
if core.ACODEC2_ALLOW:
used_audio += 1
try:
audio5 = [item for item in audio1 if item["codec_name"] in core.ACODEC2_ALLOW]
except:
audio5 = []
try:
audio6 = [item for item in audio3 if item["codec_name"] in core.ACODEC2_ALLOW]
except:
audio6 = []
if audio5: # right language and codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio5[0]["index"])])
a_mapped.extend([audio5[0]["index"]])
bitrate = int(float(audio5[0].get("bit_rate", 0))) / 1000
channels = int(float(audio5[0].get("channels", 0)))
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right language wrong codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])])
a_mapped.extend([audio1[0]["index"]])
bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000
channels = int(float(audio1[0].get("channels", 0)))
if core.ACODEC2:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio6: # wrong language, right codec
map_cmd.extend(['-map', '0:{index}'.format(index=audio6[0]["index"])])
a_mapped.extend([audio6[0]["index"]])
bitrate = int(float(audio6[0].get("bit_rate", 0))) / 1000
channels = int(float(audio6[0].get("channels", 0)))
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio3: # wrong language, wrong codec just pick the default audio track
map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])])
a_mapped.extend([audio3[0]["index"]])
bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000
channels = int(float(audio3[0].get("channels", 0)))
if core.ACODEC2:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS2 and channels and channels > core.ACHANNELS2:
audio_cmd2.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1):
audio_cmd2.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.OUTPUTQUALITYPERCENT:
audio_cmd2.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if audio_cmd2[1] in ['aac', 'dts']:
audio_cmd2[2:2] = ['-strict', '-2']
audio_cmd.extend(audio_cmd2)
if core.AINCLUDE and core.ACODEC3:
for audio in audioStreams:
if audio["index"] in a_mapped:
continue
used_audio += 1
map_cmd.extend(['-map', '0:{index}'.format(index=audio["index"])])
audio_cmd3 = []
bitrate = int(float(audio.get("bit_rate", 0))) / 1000
channels = int(float(audio.get("channels", 0)))
if audio["codec_name"] in core.ACODEC3_ALLOW:
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
else:
if core.ACODEC3:
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), core.ACODEC3])
else:
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS3 and channels and channels > core.ACHANNELS3:
audio_cmd3.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1):
audio_cmd3.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.OUTPUTQUALITYPERCENT > 0:
audio_cmd3.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if audio_cmd3[1] in ['aac', 'dts']:
audio_cmd3[2:2] = ['-strict', '-2']
audio_cmd.extend(audio_cmd3)
s_mapped = []
burnt = 0
n = 0
for lan in core.SLANGUAGES:
try:
subs1 = [item for item in subStreams if item["tags"]["language"] == lan]
except:
subs1 = []
if core.BURN and not subs1 and not burnt and os.path.isfile(file):
for subfile in get_subs(file):
if lan in os.path.split(subfile)[1]:
video_cmd.extend(['-vf', 'subtitles={subs}'.format(subs=subfile)])
burnt = 1
for sub in subs1:
if core.BURN and not burnt and os.path.isfile(inputFile):
subloc = 0
for index in range(len(subStreams)):
if subStreams[index]["index"] == sub["index"]:
subloc = index
break
video_cmd.extend(['-vf', 'subtitles={sub}:si={loc}'.format(sub=inputFile, loc=subloc)])
burnt = 1
if not core.ALLOWSUBS:
break
if sub["codec_name"] in ["dvd_subtitle", "VobSub"] and core.SCODEC == "mov_text": # We can't convert these.
continue
map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])])
s_mapped.extend([sub["index"]])
if core.SINCLUDE:
for sub in subStreams:
if not core.ALLOWSUBS:
break
if sub["index"] in s_mapped:
continue
if sub["codec_name"] in ["dvd_subtitle", "VobSub"] and core.SCODEC == "mov_text": # We can't convert these.
continue
map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])])
s_mapped.extend([sub["index"]])
if core.OUTPUTFASTSTART:
other_cmd.extend(['-movflags', '+faststart'])
command = [core.FFMPEG, '-loglevel', 'warning']
if core.HWACCEL:
command.extend(['-hwaccel', 'auto'])
if core.GENERALOPTS:
command.extend(core.GENERALOPTS)
command.extend(['-i', inputFile])
if core.SEMBED and os.path.isfile(file):
for subfile in get_subs(file):
sub_details, result = getVideoDetails(subfile)
if not sub_details or not sub_details.get("streams"):
continue
command.extend(['-i', subfile])
lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0]
metlan = None
try:
if len(lan) == 3:
metlan = Language(lan)
if len(lan) == 2:
metlan = Language.fromalpha2(lan)
except: pass
if metlan:
meta_cmd.extend(['-metadata:s:s:{x}'.format(x=len(s_mapped) + n),
'language={lang}'.format(lang=metlan.alpha3)])
n += 1
map_cmd.extend(['-map', '{x}:0'.format(x=n)])
if not core.ALLOWSUBS or (not s_mapped and not n):
sub_cmd.extend(['-sn'])
else:
if core.SCODEC:
sub_cmd.extend(['-c:s', core.SCODEC])
else:
sub_cmd.extend(['-c:s', 'copy'])
command.extend(map_cmd)
command.extend(video_cmd)
command.extend(audio_cmd)
command.extend(sub_cmd)
command.extend(meta_cmd)
command.extend(other_cmd)
command.append(newfilePath)
if platform.system() != 'Windows':
command = core.NICENESS + command
return command
2
Example 37
Project: mysql-utilities Source File: diskusage.py
def show_log_usage(server, datadir, options):
"""Show binary or relay log disk space usage.
Display binary log file information if binlog turned on if log_type =
'binary log' (default) or show relay log file information is server is
a slave and relay log is engaged.
server[in] Connected server to operate against
datadir[in] The datadir for the server
options[in] Required options for operation: format, no_headers.
log_type
return True or raise exception on error
"""
log_type = options.get("log_type", "binary log")
have_read = options.get("have_read", False)
is_remote = options.get("is_remote", False)
quiet = options.get("quiet", False)
# Check privileges to execute required queries: SUPER or REPLICATION CLIENT
user_inst = User(server, "{0}@{1}".format(server.user, server.host))
has_super = user_inst.has_privilege("*", "*", "SUPER")
has_rpl_client = user_inst.has_privilege("*", "*", "REPLICATION CLIENT")
# Verify necessary permissions (access to filesystem) and privileges
# (execute queries) to get logs usage information.
if log_type == 'binary log':
# Check for binlog ON first.
res = server.show_server_variable('log_bin')
if res and res[0][1].upper() == 'OFF':
print("# Binary logging is turned off on the server.")
return True
# Check required privileges according to the access to the datadir.
if not is_remote and have_read:
# Requires SUPER or REPLICATION CLIENT to execute:
# SHOW MASTER STATUS.
if not has_super and not has_rpl_client:
print("# {0} information not accessible. User must have the "
"SUPER or REPLICATION CLIENT "
"privilege.".format(log_type.capitalize()))
return True
else:
# Requires SUPER for server < 5.6.6 or also REPLICATION CLIENT for
# server >= 5.6.6 to execute: SHOW BINARY LOGS.
if (server.check_version_compat(5, 6, 6)
and not has_super and not has_rpl_client):
print("# {0} information not accessible. User must have the "
"SUPER or REPLICATION CLIENT "
"privilege.".format(log_type.capitalize()))
return True
elif not has_super:
print("# {0} information not accessible. User must have the "
"SUPER "
"privilege.".format(log_type.capitalize()))
return True
else: # relay log
# Requires SUPER or REPLICATION CLIENT to execute SHOW SLAVE STATUS.
if not has_super and not has_rpl_client:
print("# {0} information not accessible. User must have the "
"SUPER or REPLICATION CLIENT "
"privilege.".format(log_type.capitalize()))
return True
# Can only retrieve usage information from the localhost filesystem.
if is_remote:
print("# {0} information not accessible from a remote host."
"".format(log_type.capitalize()))
return True
elif not have_read:
print("# {0} information not accessible. Check your permissions "
"to {1}.".format(log_type.capitalize(), datadir))
return True
# Check server status and availability of specified log file type.
if log_type == 'binary log':
try:
res = server.exec_query("SHOW MASTER STATUS")
if res:
current_log = res[0][0]
else:
print("# Cannot access files - no binary log information")
return True
except:
raise UtilError("Cannot get {0} information.".format(log_type))
else:
try:
res = server.exec_query("SHOW SLAVE STATUS")
if res:
current_log = res[0][7]
else:
print("# Server is not an active slave - no relay log "
"information.")
return True
except:
raise UtilError("Cannot get {0} information.".format(log_type))
# Enough permissions and privileges, get the usage information.
if not quiet:
print("# {0} information:".format(log_type.capitalize()))
print("Current {0} file = {1}".format(log_type, current_log))
if log_type == 'binary log' and (is_remote or not have_read):
# Retrieve binlog usage info from SHOW BINARY LOGS.
try:
logs = server.exec_query("SHOW BINARY LOGS")
if logs:
# Calculate total size.
total = sum([int(item[1]) for item in logs])
else:
print("# No binary logs data available.")
return True
except:
raise UtilError("Cannot get {0} information.".format(log_type))
else:
# Retrieve usage info from localhost filesystem.
# Note: as of 5.6.2, users can specify location of binlog and relaylog.
if server.check_version_compat(5, 6, 2):
if log_type == 'binary log':
res = server.show_server_variable("log_bin_basename")[0]
else:
res = server.show_server_variable("relay_log_basename")[0]
log_path, log_prefix = os.path.split(res[1])
# In case log_path and log_prefix are '' (not defined) set them
# to the default value.
if not log_path:
log_path = datadir
if not log_prefix:
log_prefix = os.path.splitext(current_log)[0]
else:
log_path = datadir
log_prefix = os.path.splitext(current_log)[0]
logs, total = _build_log_list(log_path, log_prefix)
if not logs:
raise UtilError("The {0}s are missing.".format(log_type))
# Print logs usage information.
_print_logs(logs, total, options)
return True
2
Example 38
def __init__(self, __genparams):
'''The constructor for the GenerateListForBzip2 class'''
__arcpath = __genparams['arcpath']
__delimiter = __genparams['delimiter']
self._genfull = __genparams['genfull']
self.__listoutput = __genparams['listoutput']
self.__confoutput = __genparams['confoutput']
self.__fulloutput = __genparams['fulloutput']
self.__getallhashes = __genparams['getallhashes']
self.__hashtype = __genparams['hashtype']
self.__parsingexceptions = __genparams['parsingexceptions']
self.__confname = __genparams['confname']
__listoffiles = ['[files]\n']
__filetype = 'f'
__filehash = get_hash(bz2.BZ2File(__arcpath, 'r'), 'md5')
if self.__getallhashes:
if not self.__hashtype:
__onelinewithhash = '{value}{delimiter} type{delimiter}{value} md5{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
else:
__onelinewithhash = '{value}{delimiter} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__hashtype, delimiter=__delimiter)
__listoffiles.append(__onelinewithhash.format(
os.path.split(__arcpath)[-1][:-4],
__filetype,
__filehash))
else:
if self.__parsingexceptions :
for __file in self.__parsingexceptions:
if fnmatch.fnmatch(os.path.split(__arcpath)[-1][:-4], __file):
__filehash = get_hash(bz2.BZ2File(__arcpath, 'r'), self.__parsingexceptions[__file])
__onelinewithhash = '{value}{delimiter} type{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__parsingexceptions[__file], delimiter=__delimiter)
__listoffiles.append(__onelinewithhash.format(
os.path.split(__arcpath)[-1][:-4],
__filetype,
__filehash))
else:
__onelinewithouthash = '{value}{delimiter} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
__listoffiles.append(__onelinewithouthash.format(
os.path.split(__arcpath)[-1][:-4],
__filetype))
else:
__onelinewithouthash = '{value}{delimiter} type{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
__listoffiles.append(__onelinewithouthash.format(
os.path.split(__arcpath)[-1][:-4],
__filetype))
# define the flexible file list path
__arcwithext = os.path.split(''.join([__arcpath[:-3], 'list']))[1]
if self.__listoutput:
if self.__confname:
# --gen-list and --output-list-dir and --configuration-name
__arclistpath = os.path.join(self.__listoutput, '.'.join([self.__confname, 'list']))
else:
# --gen-list and --output-list-dir
__arclistpath = os.path.join(self.__listoutput, __arcwithext)
elif self.__fulloutput:
if self.__confname:
# --gen-list and --output-conf-and-list-dir and --configuration-name
__arclistpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'list']))
else:
# --gen-list and --output-conf-and-list-dir
__arclistpath = os.path.join(self.__fulloutput, __arcwithext)
else:
# --gen-list only
if self.__confname:
__arc = os.path.dirname(__arcpath)
__arclistpath = os.path.join(__arc, '.'.join([self.__confname, 'list']))
else:
__arclistpath = ''.join([__arcpath[:-3], 'list'])
# call the method to write information in a file
__listconfinfo = {'arclistpath': __arclistpath,
'listoffiles': __listoffiles}
self.__lci = __listconfinfo
self._generate_list(__listconfinfo)
# call the method to write the configuration file if --gen-full was required
if self._genfull:
# generate the hash sum of the list of files
__listhashsum = self._get_list_hash(__listconfinfo['arclistpath'])
# define the flexible configuration file path
__arcwithext = os.path.split(''.join([__arcpath[:-3], 'conf']))[1]
if self.__confoutput:
if self.__confname:
# --gen-full and --output-conf-dir and --configuration-name
__arcconfpath = os.path.join(self.__confoutput, '.'.join([self.__confname, 'conf']))
else:
# --gen-full and --output-conf-dir
__arcconfpath = os.path.join(self.__confoutput, __arcwithext)
elif self.__fulloutput:
if self.__confname:
# --gen-full and --output-conf-and-list-dir and --configuration-name
__arcconfpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'conf']))
else:
# --gen-full and --output-conf-and-list-dir
__arcconfpath = os.path.join(self.__fulloutput, __arcwithext)
else:
# --gen-full only
if self.__confname:
__arc = os.path.dirname(__arcpath)
__arcconfpath = os.path.join(__arc, '.'.join([self.__confname, 'conf']))
else:
__arcconfpath = ''.join([__arcpath[:-3], 'conf'])
# user-define name of the archive
if self.__confname:
__arcname = self.__confname
else:
__arcname = os.path.basename(__arcpath[:-4])
__confinfo = {'arcname': __arcname,
'arcpath': __arcpath,
'arcconfpath': __arcconfpath,
'arclistpath': __listconfinfo['arclistpath'],
'arctype': 'archive',
'sha512': __listhashsum}
self.__ci = __confinfo
self._generate_conf(__confinfo)
2
Example 39
Project: yap Source File: yap_postprocess.py
def run_postprocess(
postprocess_cmd_arr,
file_basecount_dict,
workflow_prov,
err_log,
stat_log):
'''
Prepare postprocess command with input/output paths according to sample name,
pass commands to yap_tee or subprocess for execution.
'''
if wd.regroup_output =='yes':
workflow_output_path = wd.workflow_output_path + "/regroup_output"
else:
workflow_output_path= wd.workflow_output_path
for zz in range(0, len(postprocess_cmd_arr)):
postprocess_tee_arr = []
postprocess_nontee_arr = []
initial_pipe_commands = []
postprocess_temp_arr = []
cmd_type = postprocess_cmd_arr[zz][0]
cmd_meta_data = postprocess_cmd_arr[zz][1]
postprocess_temp_arr = postprocess_cmd_arr[zz][2]
input_file_extension = ''
pipe1=''
pipe2=''
#set default input directory for postprocess stage as aligner_output
#user can specify "postprocess_output" through configuration file
input_directory = "aligner_output"
for kk in range(0, len(cmd_meta_data)):
input_meta_data = cmd_meta_data[kk].split(" ")
if input_meta_data:
if re.search('input_file_type', input_meta_data[0]) is not None:
input_file_extension = input_meta_data[1]
if re.search('input_directory', input_meta_data[0]) is not None:
input_directory = input_meta_data[1]
'''iterate over filename and barcode, fetch files from the source directory,
file extensions and python glob module'''
for filename_key in file_basecount_dict.iterkeys():
#fetch original input file pair for this sample
for tmp_arr in wd.inp_files_list:
if tmp_arr[2] == filename_key:
#store it in variables pipe1 and pipe2
pipe1 = tmp_arr[0]
pipe2 = tmp_arr[1]
break
postprocess_input_file_arr = []
path, file_name = os.path.split(filename_key)
if wd.run_preprocess_analysis == "no" and wd.run_reference_alignment == "no" and wd.run_postprocess_analysis == "yes":
file_name, extension = os.path.splitext(file_name)
file_name, extension = os.path.splitext(file_name)
file_basecount_dict[filename_key] = {
'no_barcode_specified': []}
for barcode in file_basecount_dict[filename_key]:
barcode_value = yap_tools.rename_barcode(barcode)
aligner_dir_path = ''
postprocess_dir_path = ''
aligner_dir_path = workflow_output_path + "/" + file_name + "/" + barcode + "/" + input_directory
postprocess_input = aligner_dir_path + "/" + "*" + input_file_extension + "*"
postprocess_input_file_arr = glob.glob(postprocess_input)
if wd.run_preprocess_analysis == "no" and wd.run_reference_alignment == "no" and wd.run_postprocess_analysis == "yes":
if input_directory == "aligner_output":
aligner_dir_path = path
postprocess_input = filename_key
temp_arr = glob.glob(aligner_dir_path + "/" + "*" + input_file_extension + "*")
if temp_arr > 0:
for k in temp_arr:
if k == postprocess_input:
postprocess_input_file_arr = [postprocess_input]
if input_file_extension == '':
postprocess_input_file_arr = []
postprocess_dir_path = workflow_output_path + "/" + file_name + "/" + barcode + "/" + "postprocess_output"
postprocess_input_file_arr.sort()
if (len(postprocess_input_file_arr) > 0):
if wd.run_preprocess_analysis == "no" and wd.run_reference_alignment == "no" and wd.run_postprocess_analysis == "yes":
if input_directory == "aligner_output":
if postprocess_input_file_arr[0] == filename_key:
pass
else:
break
for k in range(0, len(postprocess_temp_arr)):
postprocess_cmd = postprocess_temp_arr[k][1]
postprocess_cmd_name = postprocess_temp_arr[k][0]
if file_name == '':
if barcode_value != '':
postprocess_outfile = postprocess_dir_path + "/" + \
barcode_value + "_" + postprocess_cmd_name
else:
postprocess_outfile = postprocess_dir_path + \
"/" + postprocess_cmd_name
else:
if barcode_value != '':
postprocess_outfile = postprocess_dir_path + "/" + \
file_name + "_" + barcode_value + \
"_" + postprocess_cmd_name
else:
postprocess_outfile = postprocess_dir_path + \
"/" + file_name + "_" + \
postprocess_cmd_name
#replace generic keywords with appropriate file path variables
postprocess_cmd = postprocess_cmd.replace(
'input_directory', '')
postprocess_cmd = postprocess_cmd.replace(
'input_file_type' + ' ' + input_file_extension, '')
postprocess_cmd = postprocess_cmd.replace(
"aligner_output", '')
postprocess_cmd = postprocess_cmd.replace(
"postprocess_output", '')
postprocess_cmd = postprocess_cmd.replace(
'output_file', postprocess_outfile)
postprocess_cmd = postprocess_cmd.replace(
'output_directory', postprocess_dir_path)
postprocess_cmd = postprocess_cmd.replace(' =', '=')
postprocess_cmd = postprocess_cmd.replace(
'sample_name', file_name)
if re.search("file_based_input", postprocess_cmd) is not None:
postprocess_cmd = postprocess_cmd.replace(
'file_based_input',
postprocess_input_file_arr[0])
postprocess_nontee_arr = [
postprocess_cmd_name, postprocess_cmd]
elif re.search("directory_based_input", postprocess_cmd) is not None:
postprocess_cmd = postprocess_cmd.replace(
'directory_based_input', aligner_dir_path)
postprocess_nontee_arr = [
postprocess_cmd_name, postprocess_cmd]
else:
postprocess_tee_arr.append(postprocess_cmd)
workflow_prov.append(
"INPUT: " + postprocess_input_file_arr[0])
for kk in postprocess_tee_arr:
if kk != '':
workflow_prov.append(kk)
if len(postprocess_tee_arr) != 0:
#pass commands to yap_tee function
yap_tools.yap_tee(
initial_pipe_commands,
postprocess_tee_arr,
postprocess_input_file_arr[0],
err_log,
stat_log)
if len(postprocess_nontee_arr) != 0:
#pass commands to non_tee function which uses subproces
run_postprocess_nontee(
postprocess_nontee_arr,
workflow_prov,
err_log,
stat_log)
else:
if file_name == '':
print "Warning: No aligner output for barcode = ", barcode, " ...skipping the postprocess step for command : \n", postprocess_temp_arr, "\n"
else:
print "Warning: No aligner output for filename = ", file_name, " barcode = ", barcode, " ...skipping the postprocess step for command: \n", postprocess_temp_arr, "\n"
return workflow_prov
2
Example 40
def tex_emitter_core(target, source, env, graphics_extensions):
"""An emitter for TeX and LaTeX sources.
For LaTeX sources we try and find the common created files that
are needed on subsequent runs of latex to finish tables of contents,
bibliographies, indices, lists of figures, and hyperlink references.
"""
basename = SCons.Util.splitext(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
targetdir = os.path.split(str(target[0]))[0]
targetbase = os.path.join(targetdir, basefile)
basedir = os.path.split(str(source[0]))[0]
abspath = os.path.abspath(basedir)
target[0].attributes.path = abspath
#
# file names we will make use of in searching the sources and log file
#
emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
auxfilename = targetbase + '.aux'
logfilename = targetbase + '.log'
flsfilename = targetbase + '.fls'
env.SideEffect(auxfilename,target[0])
env.SideEffect(logfilename,target[0])
env.SideEffect(flsfilename,target[0])
if Verbose:
print "side effect :",auxfilename,logfilename,flsfilename
env.Clean(target[0],auxfilename)
env.Clean(target[0],logfilename)
env.Clean(target[0],flsfilename)
content = source[0].get_text_contents()
idx_exists = os.path.exists(targetbase + '.idx')
nlo_exists = os.path.exists(targetbase + '.nlo')
glo_exists = os.path.exists(targetbase + '.glo')
acr_exists = os.path.exists(targetbase + '.acn')
# set up list with the regular expressions
# we use to find features used
file_tests_search = [auxfile_re,
makeindex_re,
bibliography_re,
tableofcontents_re,
listoffigures_re,
listoftables_re,
hyperref_re,
makenomenclature_re,
makeglossary_re,
makeglossaries_re,
makeacronyms_re,
beamer_re ]
# set up list with the file suffixes that need emitting
# when a feature is found
file_tests_suff = [['.aux'],
['.idx', '.ind', '.ilg'],
['.bbl', '.blg'],
['.toc'],
['.lof'],
['.lot'],
['.out'],
['.nlo', '.nls', '.nlg'],
['.glo', '.gls', '.glg'],
['.glo', '.gls', '.glg'],
['.acn', '.acr', '.alg'],
['.nav', '.snm', '.out', '.toc'] ]
# build the list of lists
file_tests = []
for i in range(len(file_tests_search)):
file_tests.append( [None, file_tests_suff[i]] )
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
paths = env['ENV']['TEXINPUTS']
if SCons.Util.is_List(paths):
pass
else:
# Split at os.pathsep to convert into absolute path
paths = paths.split(os.pathsep)
# now that we have the path list restore the env
if savedpath is _null:
try:
del env['ENV']['TEXINPUTS']
except KeyError:
pass # was never set
else:
env['ENV']['TEXINPUTS'] = savedpath
if Verbose:
print "search path ",paths
aux_files = []
file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir, aux_files)
for (theSearch,suffix_list) in file_tests:
if theSearch:
for suffix in suffix_list:
env.SideEffect(targetbase + suffix,target[0])
if Verbose:
print "side effect :",targetbase + suffix
env.Clean(target[0],targetbase + suffix)
for aFile in aux_files:
aFile_base = SCons.Util.splitext(aFile)[0]
env.SideEffect(aFile_base + '.aux',target[0])
if Verbose:
print "side effect :",aFile_base + '.aux'
env.Clean(target[0],aFile_base + '.aux')
# read fls file to get all other files that latex creates and will read on the next pass
# remove files from list that we explicitly dealt with above
if os.path.exists(flsfilename):
content = open(flsfilename, "rb").read()
out_files = openout_re.findall(content)
myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
for filename in out_files[:]:
if filename in myfiles:
out_files.remove(filename)
env.SideEffect(out_files,target[0])
if Verbose:
print "side effect :",out_files
env.Clean(target[0],out_files)
return (target, source)
2
Example 41
def tex_emitter_core(target, source, env, graphics_extensions):
"""An emitter for TeX and LaTeX sources.
For LaTeX sources we try and find the common created files that
are needed on subsequent runs of latex to finish tables of contents,
bibliographies, indices, lists of figures, and hyperlink references.
"""
basename = SCons.Util.splitext(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
targetdir = os.path.split(str(target[0]))[0]
targetbase = os.path.join(targetdir, basefile)
basedir = os.path.split(str(source[0]))[0]
abspath = os.path.abspath(basedir)
target[0].attributes.path = abspath
#
# file names we will make use of in searching the sources and log file
#
emit_suffixes = ['.aux', '.log', '.ilg', '.blg', '.nls', '.nlg', '.gls', '.glg', '.alg'] + all_suffixes
auxfilename = targetbase + '.aux'
logfilename = targetbase + '.log'
flsfilename = targetbase + '.fls'
env.SideEffect(auxfilename,target[0])
env.SideEffect(logfilename,target[0])
env.SideEffect(flsfilename,target[0])
if Verbose:
print "side effect :",auxfilename,logfilename,flsfilename
env.Clean(target[0],auxfilename)
env.Clean(target[0],logfilename)
env.Clean(target[0],flsfilename)
content = source[0].get_text_contents()
idx_exists = os.path.exists(targetbase + '.idx')
nlo_exists = os.path.exists(targetbase + '.nlo')
glo_exists = os.path.exists(targetbase + '.glo')
acr_exists = os.path.exists(targetbase + '.acn')
# set up list with the regular expressions
# we use to find features used
file_tests_search = [auxfile_re,
makeindex_re,
bibliography_re,
tableofcontents_re,
listoffigures_re,
listoftables_re,
hyperref_re,
makenomenclature_re,
makeglossary_re,
makeglossaries_re,
makeacronyms_re,
beamer_re ]
# set up list with the file suffixes that need emitting
# when a feature is found
file_tests_suff = [['.aux'],
['.idx', '.ind', '.ilg'],
['.bbl', '.blg'],
['.toc'],
['.lof'],
['.lot'],
['.out'],
['.nlo', '.nls', '.nlg'],
['.glo', '.gls', '.glg'],
['.glo', '.gls', '.glg'],
['.acn', '.acr', '.alg'],
['.nav', '.snm', '.out', '.toc'] ]
# build the list of lists
file_tests = []
for i in range(len(file_tests_search)):
file_tests.append( [None, file_tests_suff[i]] )
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS']
savedpath = modify_env_var(env, 'TEXINPUTS', abspath)
paths = env['ENV']['TEXINPUTS']
if SCons.Util.is_List(paths):
pass
else:
# Split at os.pathsep to convert into absolute path
# TODO(1.5)
#paths = paths.split(os.pathsep)
paths = string.split(paths, os.pathsep)
# now that we have the path list restore the env
if savedpath is _null:
try:
del env['ENV']['TEXINPUTS']
except KeyError:
pass # was never set
else:
env['ENV']['TEXINPUTS'] = savedpath
if Verbose:
print "search path ",paths
file_tests = ScanFiles(source[0], target, paths, file_tests, file_tests_search, env, graphics_extensions, targetdir)
for (theSearch,suffix_list) in file_tests:
if theSearch:
for suffix in suffix_list:
env.SideEffect(targetbase + suffix,target[0])
if Verbose:
print "side effect :",targetbase + suffix
env.Clean(target[0],targetbase + suffix)
# read fls file to get all other files that latex creates and will read on the next pass
# remove files from list that we explicitly dealt with above
if os.path.exists(flsfilename):
content = open(flsfilename, "rb").read()
out_files = openout_re.findall(content)
myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
for filename in out_files[:]:
if filename in myfiles:
out_files.remove(filename)
env.SideEffect(out_files,target[0])
if Verbose:
print "side effect :",out_files
env.Clean(target[0],out_files)
return (target, source)
2
Example 42
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
"""A builder for LaTeX files that checks the output in the aux file
and decides how many times to use LaTeXAction, and BibTeXAction."""
global must_rerun_latex
# This routine is called with two actions. In this file for DVI builds
# with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
# set this up now for the case where the user requests a different extension
# for the target filename
if (XXXLaTeXAction == LaTeXAction):
callerSuffix = ".dvi"
else:
callerSuffix = env['PDFSUFFIX']
basename = SCons.Util.splitext(str(source[0]))[0]
basedir = os.path.split(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
abspath = os.path.abspath(basedir)
targetext = os.path.splitext(str(target[0]))[1]
targetdir = os.path.split(str(target[0]))[0]
saved_env = {}
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
saved_env[var] = modify_env_var(env, var, abspath)
# Create base file names with the target directory since the auxiliary files
# will be made there. That's because the *COM variables have the cd
# command in the prolog. We check
# for the existence of files before opening them--even ones like the
# aux file that TeX always creates--to make it possible to write tests
# with stubs that don't necessarily generate all of the same files.
targetbase = os.path.join(targetdir, basefile)
# if there is a \makeindex there will be a .idx and thus
# we have to run makeindex at least once to keep the build
# happy even if there is no index.
# Same for glossaries and nomenclature
src_content = source[0].get_text_contents()
run_makeindex = makeindex_re.search(src_content) and not os.path.exists(targetbase + '.idx')
run_nomenclature = makenomenclature_re.search(src_content) and not os.path.exists(targetbase + '.nlo')
run_glossary = makeglossary_re.search(src_content) and not os.path.exists(targetbase + '.glo')
run_glossaries = makeglossaries_re.search(src_content) and not os.path.exists(targetbase + '.glo')
run_acronyms = makeacronyms_re.search(src_content) and not os.path.exists(targetbase + '.acn')
saved_hashes = {}
suffix_nodes = {}
for suffix in all_suffixes:
theNode = env.fs.File(targetbase + suffix)
suffix_nodes[suffix] = theNode
saved_hashes[suffix] = theNode.get_csig()
if Verbose:
print "hashes: ",saved_hashes
must_rerun_latex = True
#
# routine to update MD5 hash and compare
#
def check_MD5(filenode, suffix):
global must_rerun_latex
# two calls to clear old csig
filenode.clear_memoized_values()
filenode.ninfo = filenode.new_ninfo()
new_md5 = filenode.get_csig()
if saved_hashes[suffix] == new_md5:
if Verbose:
print "file %s not changed" % (targetbase+suffix)
return False # unchanged
saved_hashes[suffix] = new_md5
must_rerun_latex = True
if Verbose:
print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
return True # changed
# generate the file name that latex will generate
resultfilename = targetbase + callerSuffix
count = 0
while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
result = XXXLaTeXAction(target, source, env)
if result != 0:
return result
count = count + 1
must_rerun_latex = False
# Decide if various things need to be run, or run again.
# Read the log file to find warnings/errors
logfilename = targetbase + '.log'
logContent = ''
if os.path.exists(logfilename):
logContent = open(logfilename, "rb").read()
# Read the fls file to find all .aux files
flsfilename = targetbase + '.fls'
flsContent = ''
auxfiles = []
if os.path.exists(flsfilename):
flsContent = open(flsfilename, "rb").read()
auxfiles = openout_aux_re.findall(flsContent)
# remove duplicates
dups = {}
for x in auxfiles:
dups[x] = 1
auxfiles = list(dups.keys())
if Verbose:
print "auxfiles ",auxfiles
# Now decide if bibtex will need to be run.
# The information that bibtex reads from the .aux file is
# pass-independent. If we find (below) that the .bbl file is unchanged,
# then the last latex saw a correct bibliography.
# Therefore only do this on the first pass
if count == 1:
for auxfilename in auxfiles:
target_aux = os.path.join(targetdir, auxfilename)
if os.path.exists(target_aux):
content = open(target_aux, "rb").read()
if content.find("bibdata") != -1:
if Verbose:
print "Need to run bibtex"
bibfile = env.fs.File(SCons.Util.splitext(target_aux)[0])
result = BibTeXAction(bibfile, bibfile, env)
if result != 0:
check_file_error_message(env['BIBTEX'], 'blg')
must_rerun_latex = must_rerun_latex or check_MD5(suffix_nodes['.bbl'],'.bbl')
# Now decide if latex will need to be run again due to index.
if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
# We must run makeindex
if Verbose:
print "Need to run makeindex"
idxfile = suffix_nodes['.idx']
result = MakeIndexAction(idxfile, idxfile, env)
if result != 0:
check_file_error_message(env['MAKEINDEX'], 'ilg')
return result
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# Harder is case is where an action needs to be called -- that should be rare (I hope?)
for index in check_suffixes:
check_MD5(suffix_nodes[index],index)
# Now decide if latex will need to be run again due to nomenclature.
if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
# We must run makeindex
if Verbose:
print "Need to run makeindex for nomenclature"
nclfile = suffix_nodes['.nlo']
result = MakeNclAction(nclfile, nclfile, env)
if result != 0:
check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
'nlg')
#return result
# Now decide if latex will need to be run again due to glossary.
if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
# We must run makeindex
if Verbose:
print "Need to run makeindex for glossary"
glofile = suffix_nodes['.glo']
result = MakeGlossaryAction(glofile, glofile, env)
if result != 0:
check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
'glg')
#return result
# Now decide if latex will need to be run again due to acronyms.
if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
# We must run makeindex
if Verbose:
print "Need to run makeindex for acronyms"
acrfile = suffix_nodes['.acn']
result = MakeAcronymsAction(acrfile, acrfile, env)
if result != 0:
check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
'alg')
return result
# Now decide if latex needs to be run yet again to resolve warnings.
if warning_rerun_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to latex or package rerun warning"
if rerun_citations_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to 'Rerun to get citations correct' warning"
if undefined_references_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to undefined references or citations"
if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop
# rename Latex's output to what the target name is
if not (str(target[0]) == resultfilename and os.path.exists(resultfilename)):
if os.path.exists(resultfilename):
print "move %s to %s" % (resultfilename, str(target[0]), )
shutil.move(resultfilename,str(target[0]))
# Original comment (when TEXPICTS was not restored):
# The TEXPICTS enviroment variable is needed by a dvi -> pdf step
# later on Mac OSX so leave it
#
# It is also used when searching for pictures (implicit dependencies).
# Why not set the variable again in the respective builder instead
# of leaving local modifications in the environment? What if multiple
# latex builds in different directories need different TEXPICTS?
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
if var == 'TEXPICTS':
continue
if saved_env[var] is _null:
try:
del env['ENV'][var]
except KeyError:
pass # was never set
else:
env['ENV'][var] = saved_env[var]
return result
2
Example 43
def processCSV(self):
self.validateArgs()
if self.verbose:
self.writetolog("Starting on Field Data Query for " + os.path.split(self.csv)[1])
self.writetolog(" using template " + os.path.split(self.template)[1])
templatename = os.path.split(self.template)[1]
if templatename == 'hdr.adf':
templatename = os.path.split(os.path.split(self.template)[0])[1]
csvfile = open(self.csv, "r")
#dialect = csv.Sniffer().sniff(csvfile.read(1024))
reader = csv.reader(csvfile)
usedPixels = {}
header = reader.next()
if header[2].lower() == 'responsecount':
self.countdata = True
#Commented this out because it is causing an error
#to be thrown by the java, uncomment out when the
#java has been replaced
header.append("frequency")
header.append("numPresence")
header.append("pixelColumn")
header.append("pixelRow")
header.append(os.path.abspath(self.template))
header.append(os.path.abspath(self.csv))
#loop through each row (observation) and
#if that particular pixel hasn't been encountered before
#add it to a dictionary containing a key of the pixel X,Y
#and values of each row encountered for that pixel
#if pixel
lineCount = linesInFile(self.csv)
extraPoints = []
pointCount = 0
pcntDone = 0
for row in reader:
if self.pointInTemplate(row[0], row[1]):
pixelColumn = int(floor((float(row[0]) - self.templateParams["ulx"])
/ self.templateParams["xScale"]))
pixelRow = int(floor((float(row[1]) - self.templateParams["uly"])
/ self.templateParams["yScale"]))
pixel = "".join(["X:",str(pixelColumn),":Y:",str(pixelRow)])
#if verbose == True:
if not pixel in usedPixels:
usedPixels[pixel] = [row]
#usedPixels[pixel] = usedPixels[pixel].append(row)
else:
curVal = usedPixels[pixel]
curVal.append(row)
usedPixels[pixel] = curVal
else:
extraPoints.append([row[0], row[1], row[2]])
pointCount += 1
if self.verbose:
if float(pointCount)/lineCount > float(pcntDone)/100:
pcntDone += 10
if self.verbose:
print str(pcntDone) + "...",
#Open up and write to an output file
oFile = open(self.output, 'wb')
fOut = csv.writer(oFile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
fOut.writerow(header)
#Add each used pixel to the output file
for k in usedPixels:
v = usedPixels[k]
outputLine = v[0]
pixelColumn = int(k.rsplit(':')[1])
pixelRow = int(k.rsplit(':')[3])
outPixelX = (self.templateParams["ulx"] + (self.templateParams["xScale"] * pixelColumn) +
self.templateParams["xScale"]/2)
outPixelY = (self.templateParams["uly"] + (self.templateParams["yScale"] * pixelRow) +
self.templateParams["yScale"]/2)
frequency = len(v)
#loop though the 'points' in each pixel
#for count data the value is the sum of the 'points'
# if there were any 'hits' not equal to 0
# otherwise if there were any absences the value is 0
# what's left is background points
#for presense/absense data
# The value is 1 if there were any 1s in our points
# Or 0 if there were any zeros (absenses)
# Otherwise it's a background pixel.
total = 0
numAbsense = 0
for i in range (frequency):
if int(float(v[i][2])) > 0:
total += int(float(v[i][2]))
if int(float(v[i][2])) == 0:
numAbsense += 1
outputLine[0] = outPixelX
outputLine[1] = outPixelY
if self.countdata and total > 0:
outputLine[2] = total
elif total > 0:
outputLine[2] = 1
elif numAbsense > 0:
outputLine[2] = 0
else:
outputLine[2] = -9999
outputLine.append(frequency)
outputLine.append(total)
outputLine.append(pixelColumn)
outputLine.append(pixelRow)
fOut.writerow(outputLine)
oFile.close
if self.verbose:
self.writetolog("Done\nFinished creating field data query output.\n")
if len(extraPoints) > 0:
self.writetolog (" WARNING: " + str(len(extraPoints)) + " points" +
" out of " + str(pointCount) + " total points in the " +
"original CSV were outside the template extent and WERE NOT " +
"INCLUDED IN THE FDQ OUTPUT.")
else:
pass
2
Example 44
def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None):
"""A builder for LaTeX files that checks the output in the aux file
and decides how many times to use LaTeXAction, and BibTeXAction."""
global must_rerun_latex
# This routine is called with two actions. In this file for DVI builds
# with LaTeXAction and from the pdflatex.py with PDFLaTeXAction
# set this up now for the case where the user requests a different extension
# for the target filename
if (XXXLaTeXAction == LaTeXAction):
callerSuffix = ".dvi"
else:
callerSuffix = env['PDFSUFFIX']
basename = SCons.Util.splitext(str(source[0]))[0]
basedir = os.path.split(str(source[0]))[0]
basefile = os.path.split(str(basename))[1]
abspath = os.path.abspath(basedir)
targetext = os.path.splitext(str(target[0]))[1]
targetdir = os.path.split(str(target[0]))[0]
saved_env = {}
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
saved_env[var] = modify_env_var(env, var, abspath)
# Create base file names with the target directory since the auxiliary files
# will be made there. That's because the *COM variables have the cd
# command in the prolog. We check
# for the existence of files before opening them--even ones like the
# aux file that TeX always creates--to make it possible to write tests
# with stubs that don't necessarily generate all of the same files.
targetbase = os.path.join(targetdir, basefile)
# if there is a \makeindex there will be a .idx and thus
# we have to run makeindex at least once to keep the build
# happy even if there is no index.
# Same for glossaries, nomenclature, and acronyms
src_content = source[0].get_text_contents()
run_makeindex = makeindex_re.search(src_content) and not os.path.isfile(targetbase + '.idx')
run_nomenclature = makenomenclature_re.search(src_content) and not os.path.isfile(targetbase + '.nlo')
run_glossary = makeglossary_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
run_glossaries = makeglossaries_re.search(src_content) and not os.path.isfile(targetbase + '.glo')
run_acronyms = makeacronyms_re.search(src_content) and not os.path.isfile(targetbase + '.acn')
saved_hashes = {}
suffix_nodes = {}
for suffix in all_suffixes:
theNode = env.fs.File(targetbase + suffix)
suffix_nodes[suffix] = theNode
saved_hashes[suffix] = theNode.get_csig()
if Verbose:
print "hashes: ",saved_hashes
must_rerun_latex = True
#
# routine to update MD5 hash and compare
#
def check_MD5(filenode, suffix):
global must_rerun_latex
# two calls to clear old csig
filenode.clear_memoized_values()
filenode.ninfo = filenode.new_ninfo()
new_md5 = filenode.get_csig()
if saved_hashes[suffix] == new_md5:
if Verbose:
print "file %s not changed" % (targetbase+suffix)
return False # unchanged
saved_hashes[suffix] = new_md5
must_rerun_latex = True
if Verbose:
print "file %s changed, rerunning Latex, new hash = " % (targetbase+suffix), new_md5
return True # changed
# generate the file name that latex will generate
resultfilename = targetbase + callerSuffix
count = 0
while (must_rerun_latex and count < int(env.subst('$LATEXRETRIES'))) :
result = XXXLaTeXAction(target, source, env)
if result != 0:
return result
count = count + 1
must_rerun_latex = False
# Decide if various things need to be run, or run again.
# Read the log file to find warnings/errors
logfilename = targetbase + '.log'
logContent = ''
if os.path.isfile(logfilename):
logContent = open(logfilename, "rb").read()
# Read the fls file to find all .aux files
flsfilename = targetbase + '.fls'
flsContent = ''
auxfiles = []
if os.path.isfile(flsfilename):
flsContent = open(flsfilename, "rb").read()
auxfiles = openout_aux_re.findall(flsContent)
# remove duplicates
dups = {}
for x in auxfiles:
dups[x] = 1
auxfiles = list(dups.keys())
if Verbose:
print "auxfiles ",auxfiles
# Now decide if bibtex will need to be run.
# The information that bibtex reads from the .aux file is
# pass-independent. If we find (below) that the .bbl file is unchanged,
# then the last latex saw a correct bibliography.
# Therefore only do this on the first pass
if count == 1:
for auxfilename in auxfiles:
target_aux = os.path.join(targetdir, auxfilename)
if os.path.isfile(target_aux):
content = open(target_aux, "rb").read()
if content.find("bibdata") != -1:
if Verbose:
print "Need to run bibtex"
bibfile = env.fs.File(SCons.Util.splitext(target_aux)[0])
result = BibTeXAction(bibfile, bibfile, env)
if result != 0:
check_file_error_message(env['BIBTEX'], 'blg')
must_rerun_latex = must_rerun_latex or check_MD5(suffix_nodes['.bbl'],'.bbl')
# Now decide if latex will need to be run again due to index.
if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex):
# We must run makeindex
if Verbose:
print "Need to run makeindex"
idxfile = suffix_nodes['.idx']
result = MakeIndexAction(idxfile, idxfile, env)
if result != 0:
check_file_error_message(env['MAKEINDEX'], 'ilg')
return result
# TO-DO: need to add a way for the user to extend this list for whatever
# auxiliary files they create in other (or their own) packages
# Harder is case is where an action needs to be called -- that should be rare (I hope?)
for index in check_suffixes:
check_MD5(suffix_nodes[index],index)
# Now decide if latex will need to be run again due to nomenclature.
if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature):
# We must run makeindex
if Verbose:
print "Need to run makeindex for nomenclature"
nclfile = suffix_nodes['.nlo']
result = MakeNclAction(nclfile, nclfile, env)
if result != 0:
check_file_error_message('%s (nomenclature)' % env['MAKENCL'],
'nlg')
#return result
# Now decide if latex will need to be run again due to glossary.
if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary):
# We must run makeindex
if Verbose:
print "Need to run makeindex for glossary"
glofile = suffix_nodes['.glo']
result = MakeGlossaryAction(glofile, glofile, env)
if result != 0:
check_file_error_message('%s (glossary)' % env['MAKEGLOSSARY'],
'glg')
#return result
# Now decide if latex will need to be run again due to acronyms.
if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms):
# We must run makeindex
if Verbose:
print "Need to run makeindex for acronyms"
acrfile = suffix_nodes['.acn']
result = MakeAcronymsAction(acrfile, acrfile, env)
if result != 0:
check_file_error_message('%s (acronyms)' % env['MAKEACRONYMS'],
'alg')
return result
# Now decide if latex needs to be run yet again to resolve warnings.
if warning_rerun_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to latex or package rerun warning"
if rerun_citations_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to 'Rerun to get citations correct' warning"
if undefined_references_re.search(logContent):
must_rerun_latex = True
if Verbose:
print "rerun Latex due to undefined references or citations"
if (count >= int(env.subst('$LATEXRETRIES')) and must_rerun_latex):
print "reached max number of retries on Latex ,",int(env.subst('$LATEXRETRIES'))
# end of while loop
# rename Latex's output to what the target name is
if not (str(target[0]) == resultfilename and os.path.isfile(resultfilename)):
if os.path.isfile(resultfilename):
print "move %s to %s" % (resultfilename, str(target[0]), )
shutil.move(resultfilename,str(target[0]))
# Original comment (when TEXPICTS was not restored):
# The TEXPICTS enviroment variable is needed by a dvi -> pdf step
# later on Mac OSX so leave it
#
# It is also used when searching for pictures (implicit dependencies).
# Why not set the variable again in the respective builder instead
# of leaving local modifications in the environment? What if multiple
# latex builds in different directories need different TEXPICTS?
for var in SCons.Scanner.LaTeX.LaTeX.env_variables:
if var == 'TEXPICTS':
continue
if saved_env[var] is _null:
try:
del env['ENV'][var]
except KeyError:
pass # was never set
else:
env['ENV'][var] = saved_env[var]
return result
0
Example 45
def which_files(file, mode=os.F_OK | os.X_OK, path=None, pathext=None):
""" Generate full paths, where the file*is accesible under mode
and is located in the directory passed as a part of the file name,
or in any directory on path if a base file name is passed.
The mode matches an existing executable file by default.
The path defaults to the PATH environment variable,
or to os.defpath if the PATH variable is not set.
On Windows, a current directory is searched before directories in the PATH variable,
but not before directories in an explicitly passed path string or iterable.
The pathext is used to match files with any of the extensions appended to file.
On Windows, it defaults to the ``PATHEXT`` environment variable.
If the PATHEXT variable is not set, then the default pathext value is hardcoded
for different Windows versions, to match the actual search performed on command execution.
On Windows <= 4.x, ie. NT and older, it defaults to '.COM;.EXE;.BAT;.CMD'.
On Windows 5.x, ie. 2k/XP/2003, the extensions '.VBS;.VBE;.JS;.JSE;.WSF;.WSH' are appended,
On Windows >= 6.x, ie. Vista/2008/7, the extension '.MSC' is further appended.
The actual search on command execution may differ under Wine,
which may use a different default value, that is not treated specially here.
In each directory, the file is first searched without any additional extension,
even when a pathext string or iterable is explicitly passed.
>>> def test(expected, *args, **argd):
... result = list(which_files(*args, **argd))
... assert result == expected, 'which_files: %s != %s' % (result, expected)
...
... try:
... result = [ which(*args, **argd) ]
... except IOError:
... result = []
... assert result[:1] == expected[:1], 'which: %s != %s' % (result[:1], expected[:1])
>>> ### Set up
>>> import stat, tempfile
>>> dir = tempfile.mkdtemp(prefix='test-')
>>> ext = '.ext'
>>> tmp = tempfile.NamedTemporaryFile(prefix='command-', suffix=ext, dir=dir)
>>> name = tmp.name
>>> file = os.path.basename(name)
>>> here = os.path.join(os.curdir, file)
>>> nonexistent = '%s-nonexistent' % name
>>> path = os.pathsep.join([ nonexistent, name, dir, dir ])
... # Test also that duplicates are removed, and non-existent objects
... # or non-directories in path do not trigger any exceptions.
>>> ### Test permissions
>>> test(_windows and [name] or [], file, path=path)
>>> test(_windows and [name] or [], file, mode=os.X_OK, path=path)
... # executable flag is not needed on Windows
>>> test([name], file, mode=os.F_OK, path=path)
>>> test([name], file, mode=os.R_OK, path=path)
>>> test([name], file, mode=os.W_OK, path=path)
>>> test([name], file, mode=os.R_OK|os.W_OK, path=path)
>>> os.chmod(name, stat.S_IRWXU)
>>> test([name], file, mode=os.R_OK|os.W_OK|os.X_OK, path=path)
>>> ### Test paths
>>> _save_path = os.environ.get('PATH', '')
>>> cwd = os.getcwd()
>>> test([], file, path='')
>>> test([], file, path=nonexistent)
>>> test([], nonexistent, path=path)
>>> test([name], file, path=path)
>>> test([name], name, path=path)
>>> test([name], name, path='')
>>> test([name], name, path=nonexistent)
>>> os.chdir(dir)
>>> test([name], file, path=path)
>>> test([here], file, path=os.curdir)
>>> test([name], name, path=os.curdir)
>>> test([], file, path='')
>>> test([], file, path=nonexistent)
>>> os.environ['PATH'] = path
>>> test(_windows and [here] or [name], file)
... # current directory is always searched first on Windows
>>> os.environ['PATH'] = os.curdir
>>> test([here], file)
>>> test([name], name)
>>> os.environ['PATH'] = ''
>>> test(_windows and [here] or [], file)
>>> os.environ['PATH'] = nonexistent
>>> test(_windows and [here] or [], file)
>>> os.chdir(cwd)
>>> os.environ['PATH'] = path
>>> test([name], file)
>>> os.environ['PATH'] = _save_path
>>> ### Test extensions
>>> test([], file[:-4], path=path, pathext='')
>>> test([], file[:-4], path=path, pathext=nonexistent)
>>> test([name], file[:-4], path=path, pathext=ext)
>>> test([name], file, path=path, pathext=ext)
>>> test([name], file, path=path, pathext='')
>>> test([name], file, path=path, pathext=nonexistent)
>>> ### Tear down
>>> tmp.close()
>>> os.rmdir(dir)
"""
filepath, file = os.path.split(file)
if filepath:
path = (filepath, )
elif path is None:
path = os.environ.get('PATH', os.defpath).split(os.pathsep)
if _windows and os.curdir not in path:
path.insert(
0, os.curdir
) # current directory is always searched first on Windows
elif isinstance(path, basestring):
path = path.split(os.pathsep)
if pathext is None:
pathext = ['']
if _windows:
pathext += (os.environ.get('PATHEXT', '') or
_getwinpathext()).lower().split(os.pathsep)
elif isinstance(pathext, basestring):
pathext = pathext.split(os.pathsep)
if '' not in pathext:
pathext.insert(
0, ''
) # always check command without extension, even for an explicitly passed pathext
seen = set()
for dir in path:
if dir: # only non-empty directories are searched
id = os.path.normcase(os.path.abspath(dir))
if id not in seen: # each directory is searched only once
seen.add(id)
woex = os.path.join(dir, file)
for ext in pathext:
name = woex + ext
if os.path.isfile(name) and os.access(name, mode):
return name
# yield name
return None
0
Example 46
Project: OwnTube Source File: btmakemetafile.py
def makeinfo(file, piece_length, encoding, flag, progress, progress_percent=1, gethash = None):
if gethash is None:
gethash = {}
if not 'md5' in gethash:
gethash['md5'] = False
if not 'crc32' in gethash:
gethash['crc32'] = False
if not 'sha1' in gethash:
gethash['sha1'] = False
file = abspath(file)
if isdir(file):
subs = subfiles(file)
subs.sort()
pieces = []
sh = sha()
done = 0L
fs = []
totalsize = 0.0
totalhashed = 0L
for p, f in subs:
totalsize += getsize(f)
for p, f in subs:
pos = 0L
size = getsize(f)
h = open(f, 'rb')
if gethash['md5']:
hash_md5 = md5.new()
if gethash['sha1']:
hash_sha1 = sha()
if gethash['crc32']:
hash_crc32 = zlib.crc32('')
while pos < size:
a = min(size - pos, piece_length - done)
readpiece = h.read(a)
# See if the user cancelled
if flag.isSet():
return
sh.update(readpiece)
# See if the user cancelled
if flag.isSet():
return
if gethash['md5']:
# Update MD5
hash_md5.update(readpiece)
# See if the user cancelled
if flag.isSet():
return
if gethash['crc32']:
# Update CRC32
hash_crc32 = zlib.crc32(readpiece, hash_crc32)
# See if the user cancelled
if flag.isSet():
return
if gethash['sha1']:
# Update SHA1
hash_sha1.update(readpiece)
# See if the user cancelled
if flag.isSet():
return
done += a
pos += a
totalhashed += a
if done == piece_length:
pieces.append(sh.digest())
done = 0
sh = sha()
if progress_percent:
progress(totalhashed / totalsize)
else:
progress(a)
newdict = {'length': size,
'path': uniconvertl(p, encoding) }
if gethash['md5']:
newdict['md5sum'] = hash_md5.hexdigest()
if gethash['crc32']:
newdict['crc32'] = "%08X" % hash_crc32
if gethash['sha1']:
newdict['sha1'] = hash_sha1.digest()
fs.append(newdict)
h.close()
if done > 0:
pieces.append(sh.digest())
return {'pieces': ''.join(pieces),
'piece length': piece_length,
'files': fs,
'name': uniconvert(split(file)[1], encoding) }
else:
size = getsize(file)
pieces = []
p = 0L
h = open(file, 'rb')
if gethash['md5']:
hash_md5 = md5.new()
if gethash['crc32']:
hash_crc32 = zlib.crc32('')
if gethash['sha1']:
hash_sha1 = sha()
while p < size:
x = h.read(min(piece_length, size - p))
# See if the user cancelled
if flag.isSet():
return
if gethash['md5']:
# Update MD5
hash_md5.update(x)
# See if the user cancelled
if flag.isSet():
return
if gethash['crc32']:
# Update CRC32
hash_crc32 = zlib.crc32(x, hash_crc32)
# See if the user cancelled
if flag.isSet():
return
if gethash['sha1']:
# Update SHA-1
hash_sha1.update(x)
# See if the user cancelled
if flag.isSet():
return
pieces.append(sha(x).digest())
# See if the user cancelled
if flag.isSet():
return
p += piece_length
if p > size:
p = size
if progress_percent:
progress(float(p) / size)
else:
progress(min(piece_length, size - p))
h.close()
newdict = {'pieces': ''.join(pieces),
'piece length': piece_length,
'length': size,
'name': uniconvert(split(file)[1], encoding) }
if gethash['md5']:
newdict['md5sum'] = hash_md5.hexdigest()
if gethash['crc32']:
newdict['crc32'] = "%08X" % hash_crc32
if gethash['sha1']:
newdict['sha1'] = hash_sha1.digest()
0
Example 47
Project: RMG-Py Source File: input.py
def saveInputFile(path, rmg):
"""
Save an RMG input file at `path` on disk from the :class:`RMG` object
`rmg`.
"""
f = open(path, 'w')
# Databases
f.write('database(\n')
#f.write(' "{0}",\n'.format(rmg.databaseDirectory))
f.write(' thermoLibraries = {0!r},\n'.format(rmg.thermoLibraries))
f.write(' reactionLibraries = {0!r},\n'.format(rmg.reactionLibraries))
f.write(' seedMechanisms = {0!r},\n'.format(rmg.seedMechanisms))
f.write(' kineticsDepositories = {0!r},\n'.format(rmg.kineticsDepositories))
f.write(' kineticsFamilies = {0!r},\n'.format(rmg.kineticsFamilies))
f.write(' kineticsEstimator = {0!r},\n'.format(rmg.kineticsEstimator))
f.write(')\n\n')
# Species
for species in rmg.initialSpecies:
f.write('species(\n')
f.write(' label = "{0}",\n'.format(species.label))
f.write(' reactive = {0},\n'.format(species.reactive))
f.write(' structure = adjacencyList(\n')
f.write('"""\n')
f.write(species.molecule[0].toAdjacencyList())
f.write('"""),\n')
f.write(')\n\n')
# Reaction systems
for system in rmg.reactionSystems:
if rmg.solvent:
f.write('liquidReactor(\n')
f.write(' temperature = ({0:g},"{1!s}"),\n'.format(system.T.getValue(),system.T.units))
f.write(' initialConcentrations={\n')
for species, conc in system.initialConcentrations.iteritems():
f.write(' "{0!s}": ({1:g},"{2!s}"),\n'.format(species.label,conc.getValue(),conc.units))
else:
f.write('simpleReactor(\n')
f.write(' temperature = ({0:g},"{1!s}"),\n'.format(system.T.getValue(),system.T.units))
# Convert the pressure from SI pascal units to bar here
# Do something more fancy later for converting to user's desired units for both T and P..
f.write(' pressure = ({0:g},"{1!s}"),\n'.format(system.P.getValue(),system.P.units))
f.write(' initialMoleFractions={\n')
for species, molfrac in system.initialMoleFractions.iteritems():
f.write(' "{0!s}": {1:g},\n'.format(species.label, molfrac))
f.write(' },\n')
# Termination criteria
conversions = ''
for term in system.termination:
if isinstance(term, TerminationTime):
f.write(' terminationTime = ({0:g},"{1!s}"),\n'.format(term.time.getValue(),term.time.units))
else:
conversions += ' "{0:s}": {1:g},\n'.format(term.species.label, term.conversion)
if conversions:
f.write(' terminationConversion = {\n')
f.write(conversions)
f.write(' },\n')
# Sensitivity analysis
if system.sensitiveSpecies:
sensitivity = []
for item in system.sensitiveSpecies:
sensitivity.append(item.label)
f.write(' sensitivity = {0},\n'.format(sensitivity))
f.write(' sensitivityThreshold = {0},\n'.format(system.sensitivityThreshold))
f.write(')\n\n')
if rmg.solvent:
f.write("solvation(\n solvent = '{0!s}'\n)\n\n".format(rmg.solvent))
# Simulator tolerances
f.write('simulator(\n')
f.write(' atol = {0:g},\n'.format(rmg.absoluteTolerance))
f.write(' rtol = {0:g},\n'.format(rmg.relativeTolerance))
f.write(' sens_atol = {0:g},\n'.format(rmg.sensitivityAbsoluteTolerance))
f.write(' sens_rtol = {0:g},\n'.format(rmg.sensitivityRelativeTolerance))
f.write(')\n\n')
# Model
f.write('model(\n')
f.write(' toleranceMoveToCore = {0:g},\n'.format(rmg.fluxToleranceMoveToCore))
f.write(' toleranceKeepInEdge = {0:g},\n'.format(rmg.fluxToleranceKeepInEdge))
f.write(' toleranceInterruptSimulation = {0:g},\n'.format(rmg.fluxToleranceInterrupt))
f.write(' maximumEdgeSpecies = {0:d},\n'.format(rmg.maximumEdgeSpecies))
f.write(' minCoreSizeForPrune = {0:d},\n'.format(rmg.minCoreSizeForPrune))
f.write(' minSpeciesExistIterationsForPrune = {0:d},\n'.format(rmg.minSpeciesExistIterationsForPrune))
f.write(' filterReactions = {0:d},\n'.format(rmg.filterReactions))
f.write(')\n\n')
# Pressure Dependence
if rmg.pressureDependence:
f.write('pressureDependence(\n')
f.write(' method = {0!r},\n'.format(rmg.pressureDependence.method))
f.write(' maximumGrainSize = ({0:g},"{1!s}"),\n'.format(rmg.pressureDependence.grainSize.getValue(),rmg.pressureDependence.grainSize.units))
f.write(' minimumNumberOfGrains = {0},\n'.format(rmg.pressureDependence.grainCount))
f.write(' temperatures = ({0:g},{1:g},"{2!s}",{3:d}),\n'.format(
rmg.pressureDependence.Tmin.getValue(),
rmg.pressureDependence.Tmax.getValue(),
rmg.pressureDependence.Tmax.units,
rmg.pressureDependence.Tcount,
))
f.write(' pressures = ({0:g},{1:g},"{2!s}",{3:d}),\n'.format(
rmg.pressureDependence.Pmin.getValue(),
rmg.pressureDependence.Pmax.getValue(),
rmg.pressureDependence.Pmax.units,
rmg.pressureDependence.Pcount,
))
f.write(' interpolation = {0},\n'.format(rmg.pressureDependence.interpolationModel))
f.write(' maximumAtoms = {0}, \n'.format(rmg.pressureDependence.maximumAtoms))
f.write(')\n\n')
# Quantum Mechanics
if rmg.quantumMechanics:
f.write('quantumMechanics(\n')
f.write(' software = {0!r},\n'.format(rmg.quantumMechanics.settings.software))
f.write(' method = {0!r},\n'.format(rmg.quantumMechanics.settings.method))
# Split paths created by QMSettings
if rmg.quantumMechanics.settings.fileStore:
f.write(' fileStore = {0!r},\n'.format(os.path.split(rmg.quantumMechanics.settings.fileStore)[0]))
else:
f.write(' fileStore = None,\n')
if rmg.quantumMechanics.settings.scratchDirectory:
f.write(' scratchDirectory = {0!r},\n'.format(os.path.split(rmg.quantumMechanics.settings.scratchDirectory)[0]))
else:
f.write(' scratchDirectory = None,\n')
f.write(' onlyCyclics = {0},\n'.format(rmg.quantumMechanics.settings.onlyCyclics))
f.write(' maxRadicalNumber = {0},\n'.format(rmg.quantumMechanics.settings.maxRadicalNumber))
f.write(')\n\n')
# Species Constraints
if rmg.speciesConstraints:
f.write('generatedSpeciesConstraints(\n')
for constraint, value in sorted(rmg.speciesConstraints.items(), key=lambda constraint: constraint[0]):
if value is not None: f.write(' {0} = {1},\n'.format(constraint,value))
f.write(')\n\n')
# Options
f.write('options(\n')
f.write(' units = "{0}",\n'.format(rmg.units))
if rmg.saveRestartPeriod:
f.write(' saveRestartPeriod = ({0},"{1}"),\n'.format(rmg.saveRestartPeriod.getValue(), rmg.saveRestartPeriod.units))
else:
f.write(' saveRestartPeriod = None,\n')
f.write(' generateOutputHTML = {0},\n'.format(rmg.generateOutputHTML))
f.write(' generatePlots = {0},\n'.format(rmg.generatePlots))
f.write(' saveSimulationProfiles = {0},\n'.format(rmg.saveSimulationProfiles))
f.write(' saveEdgeSpecies = {0},\n'.format(rmg.saveEdgeSpecies))
f.write(' keepIrreversible = {0},\n'.format(rmg.keepIrreversible))
f.write(' verboseComments = {0},\n'.format(rmg.verboseComments))
f.write(' wallTime = {0},\n'.format(rmg.wallTime))
f.write(')\n\n')
f.close()
0
Example 48
Project: picrust Source File: predict_metagenomes.py
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if opts.verbose:
print "Loading OTU table: ",opts.input_otu_table
otu_table = load_table(opts.input_otu_table)
ids_to_load = otu_table.ids(axis='observation').tolist()
if opts.verbose:
print "Done loading OTU table containing %i samples and %i OTUs." \
%(len(otu_table.ids()),len(otu_table.ids(axis='observation')))
#Hardcoded loaction of the precalculated datasets for PICRUSt,
#relative to the project directory
precalc_data_dir=join(get_picrust_project_dir(),'picrust','data')
# Load a table of gene counts by OTUs.
#This can be either user-specified or precalculated
genome_table_fp = determine_data_table_fp(precalc_data_dir,\
opts.type_of_prediction,opts.gg_version,\
user_specified_table=opts.input_count_table,verbose=opts.verbose)
if opts.verbose:
print "Loading gene count data from file: %s" %genome_table_fp
genome_table= load_data_table(genome_table_fp,\
load_data_table_in_biom=opts.load_precalc_file_in_biom,\
suppress_subset_loading=opts.suppress_subset_loading,\
ids_to_load=ids_to_load,verbose=opts.verbose,transpose=True)
if opts.verbose:
print "Loaded %i genes across %i OTUs from gene count table" \
%(len(genome_table.ids(axis='observation')),len(genome_table.ids()))
if opts.with_confidence:
if opts.input_variance_table:
variance_table_fp = opts.input_variance_table
else:
variance_table_fp = determine_data_table_fp(precalc_data_dir,\
opts.type_of_prediction,opts.gg_version,\
precalc_file_suffix='precalculated_variances.tab.gz',\
user_specified_table=opts.input_count_table)
if opts.verbose:
print "Loading variance information from table: %s" \
%variance_table_fp
variance_table= load_data_table(variance_table_fp,\
load_data_table_in_biom=opts.load_precalc_file_in_biom,\
suppress_subset_loading=opts.suppress_subset_loading,\
ids_to_load=ids_to_load,transpose=True)
if opts.verbose:
print "Loaded %i genes across %i OTUs from variance table" \
%(len(variance_table.ids(axis='observation')),len(variance_table.ids()))
#Raise an error if the genome table and variance table differ
#in the genomes they contain.
#better to find out now than have something obscure happen latter on
if opts.verbose:
print "Checking that genome table and variance table are consistent"
try:
assert set(variance_table.ids(axis='observation')) == set(genome_table.ids(axis='observation'))
except AssertionError,e:
for var_id in variance_table.ids(axis='observation'):
if var_id not in genome_table.ids(axis='observation'):
print "Variance table ObsId %s not in genome_table ObsIds" %var_id
raise AssertionError("Variance table and genome table contain different gene ids")
try:
assert set(variance_table.ids()) == set(genome_table.ids())
except AssertionError,e:
for var_id in variance_table.ids():
if var_id not in genome_table.ids():
print "Variance table SampleId %s not in genome_table SampleIds" %var_id
raise AssertionError("Variance table and genome table contain different OTU ids")
#sort the ObservationIds and SampleIds to be in the same order
variance_table=variance_table.sort_order(genome_table.ids(axis='observation'), axis='observation')
variance_table=variance_table.sort_order(genome_table.ids(), axis='sample')
make_output_dir_for_file(opts.output_metagenome_table)
if opts.accuracy_metrics:
# Calculate accuracy metrics
weighted_nsti = calc_nsti(otu_table,genome_table,weighted=True)
samples= weighted_nsti[0]
nstis = list(weighted_nsti[1])
samples_and_nstis = zip(samples,nstis)
if opts.verbose:
print "Writing NSTI information to file:", opts.accuracy_metrics
accuracy_output_fh = open(opts.accuracy_metrics,'w')
accuracy_output_fh.write("#Sample\tMetric\tValue\n")
for sample,nsti in samples_and_nstis:
line = "%s\tWeighted NSTI\t%s\n" %(sample,str(nsti))
accuracy_output_fh.write(line)
if opts.with_confidence:
#If we are calculating variance, we get the prediction as part
#of the process
if opts.verbose:
print "Predicting the metagenome, metagenome variance and confidence intervals for the metagenome..."
predicted_metagenomes,predicted_metagenome_variances,\
predicted_metagenomes_lower_CI_95,predicted_metagenomes_upper_CI_95=\
predict_metagenome_variances(otu_table,genome_table,variance_table)
else:
#If we don't need confidence intervals, we can do a faster pure numpy prediction
if opts.verbose:
print "Predicting the metagenome..."
predicted_metagenomes = predict_metagenomes(otu_table,genome_table)
if opts.normalize_by_otu:
#normalize (e.g. divide) the abundances by the sum of the OTUs per sample
if opts.verbose:
print "Normalizing functional abundances by sum of OTUs per sample"
inverse_otu_sums = [1/x for x in otu_table.sum(axis='sample')]
scaling_factors = dict(zip(otu_table.ids(),inverse_otu_sums))
predicted_metagenomes = scale_metagenomes(predicted_metagenomes,scaling_factors)
if opts.normalize_by_function:
#normalize (e.g. divide) the abundances by the sum of the functions per sample
#Sum of functional abundances per sample will equal 1 (e.g. relative abundance).
if opts.verbose:
print "Normalizing functional abundances by sum of functions per sample"
predicted_metagenomes = predicted_metagenomes.norm(axis='sample', inplace=False)
write_metagenome_to_file(predicted_metagenomes,opts.output_metagenome_table,\
opts.format_tab_delimited,"metagenome prediction",verbose=opts.verbose)
if opts.with_confidence:
output_path,output_filename = split(opts.output_metagenome_table)
base_output_filename,ext = splitext(output_filename)
variance_output_fp =\
join(output_path,"%s_variances%s" %(base_output_filename,ext))
upper_CI_95_output_fp =\
join(output_path,"%s_upper_CI_95%s" %(base_output_filename,ext))
lower_CI_95_output_fp =\
join(output_path,"%s_lower_CI_95%s" %(base_output_filename,ext))
write_metagenome_to_file(predicted_metagenome_variances,\
variance_output_fp,opts.format_tab_delimited,\
"metagenome prediction variance",verbose=opts.verbose)
write_metagenome_to_file(predicted_metagenomes_upper_CI_95,\
upper_CI_95_output_fp,opts.format_tab_delimited,\
"metagenome prediction upper 95% confidence interval",\
verbose=opts.verbose)
write_metagenome_to_file(predicted_metagenomes_lower_CI_95,\
lower_CI_95_output_fp,opts.format_tab_delimited,\
"metagenome prediction lower 95% confidence interval",\
verbose=opts.verbose)
0
Example 49
Project: cookiecutter Source File: generate.py
def generate_files(repo_dir, context=None, output_dir='.',
overwrite_if_exists=False):
"""Render the templates and saves them to files.
:param repo_dir: Project template input directory.
:param context: Dict for populating the template's variables.
:param output_dir: Where to output the generated project dir into.
:param overwrite_if_exists: Overwrite the contents of the output directory
if it exists.
"""
template_dir = find_template(repo_dir)
logger.debug('Generating project from {}...'.format(template_dir))
context = context or {}
unrendered_dir = os.path.split(template_dir)[1]
ensure_dir_is_templated(unrendered_dir)
env = StrictEnvironment(
context=context,
keep_trailing_newline=True,
)
try:
project_dir = render_and_create_dir(
unrendered_dir,
context,
output_dir,
env,
overwrite_if_exists
)
except UndefinedError as err:
msg = "Unable to create project directory '{}'".format(unrendered_dir)
raise UndefinedVariableInTemplate(msg, err, context)
# We want the Jinja path and the OS paths to match. Consequently, we'll:
# + CD to the template folder
# + Set Jinja's path to '.'
#
# In order to build our files to the correct folder(s), we'll use an
# absolute path for the target folder (project_dir)
project_dir = os.path.abspath(project_dir)
logger.debug('Project directory is {}'.format(project_dir))
_run_hook_from_repo_dir(repo_dir, 'pre_gen_project', project_dir, context)
with work_in(template_dir):
env.loader = FileSystemLoader('.')
for root, dirs, files in os.walk('.'):
# We must separate the two types of dirs into different lists.
# The reason is that we don't want ``os.walk`` to go through the
# unrendered directories, since they will just be copied.
copy_dirs = []
render_dirs = []
for d in dirs:
d_ = os.path.normpath(os.path.join(root, d))
# We check the full path, because that's how it can be
# specified in the ``_copy_without_render`` setting, but
# we store just the dir name
if is_copy_only_path(d_, context):
copy_dirs.append(d)
else:
render_dirs.append(d)
for copy_dir in copy_dirs:
indir = os.path.normpath(os.path.join(root, copy_dir))
outdir = os.path.normpath(os.path.join(project_dir, indir))
logger.debug(
'Copying dir {} to {} without rendering'
''.format(indir, outdir)
)
shutil.copytree(indir, outdir)
# We mutate ``dirs``, because we only want to go through these dirs
# recursively
dirs[:] = render_dirs
for d in dirs:
unrendered_dir = os.path.join(project_dir, root, d)
try:
render_and_create_dir(
unrendered_dir,
context,
output_dir,
env,
overwrite_if_exists
)
except UndefinedError as err:
rmtree(project_dir)
_dir = os.path.relpath(unrendered_dir, output_dir)
msg = "Unable to create directory '{}'".format(_dir)
raise UndefinedVariableInTemplate(msg, err, context)
for f in files:
infile = os.path.normpath(os.path.join(root, f))
if is_copy_only_path(infile, context):
outfile_tmpl = env.from_string(infile)
outfile_rendered = outfile_tmpl.render(**context)
outfile = os.path.join(project_dir, outfile_rendered)
logger.debug(
'Copying file {} to {} without rendering'
''.format(infile, outfile)
)
shutil.copyfile(infile, outfile)
shutil.copymode(infile, outfile)
continue
try:
generate_file(project_dir, infile, context, env)
except UndefinedError as err:
rmtree(project_dir)
msg = "Unable to create file '{}'".format(infile)
raise UndefinedVariableInTemplate(msg, err, context)
_run_hook_from_repo_dir(repo_dir, 'post_gen_project', project_dir, context)
return project_dir
0
Example 50
def command(encoder, musicSource, musicDest, albumPath, xldProfile):
"""
Encode a given music file with a certain encoder. Returns True on success,
or False otherwise.
"""
startMusicTime = time.time()
cmd = []
if xldProfile:
xldDestDir = os.path.split(musicDest)[0]
cmd = [encoder]
cmd.extend([musicSource])
cmd.extend(['--profile'])
cmd.extend([xldProfile])
cmd.extend(['-o'])
cmd.extend([xldDestDir])
# Lame
elif headphones.CONFIG.ENCODER == 'lame':
cmd = [encoder]
opts = []
if not headphones.CONFIG.ADVANCEDENCODER:
opts.extend(['-h'])
if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
opts.extend(['--resample', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-b',
str(headphones.CONFIG.BITRATE)])
elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
opts.extend(['-v', str(headphones.CONFIG.ENCODERQUALITY)])
else:
advanced = (headphones.CONFIG.ADVANCEDENCODER.split())
for tok in advanced:
opts.extend([tok.encode(headphones.SYS_ENCODING)])
opts.extend([musicSource])
opts.extend([musicDest])
cmd.extend(opts)
# FFmpeg
elif headphones.CONFIG.ENCODER == 'ffmpeg':
cmd = [encoder, '-i', musicSource]
opts = []
if not headphones.CONFIG.ADVANCEDENCODER:
if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg':
opts.extend(['-acodec', 'libvorbis'])
if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a':
opts.extend(['-strict', 'experimental'])
if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab',
str(headphones.CONFIG.BITRATE) + 'k'])
elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)])
opts.extend(['-y', '-ac', '2', '-vn'])
else:
advanced = (headphones.CONFIG.ADVANCEDENCODER.split())
for tok in advanced:
opts.extend([tok.encode(headphones.SYS_ENCODING)])
opts.extend([musicDest])
cmd.extend(opts)
# Libav
elif headphones.CONFIG.ENCODER == "libav":
cmd = [encoder, '-i', musicSource]
opts = []
if not headphones.CONFIG.ADVANCEDENCODER:
if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg':
opts.extend(['-acodec', 'libvorbis'])
if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a':
opts.extend(['-strict', 'experimental'])
if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab',
str(headphones.CONFIG.BITRATE) + 'k'])
elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)])
opts.extend(['-y', '-ac', '2', '-vn'])
else:
advanced = (headphones.CONFIG.ADVANCEDENCODER.split())
for tok in advanced:
opts.extend([tok.encode(headphones.SYS_ENCODING)])
opts.extend([musicDest])
cmd.extend(opts)
# Prevent Windows from opening a terminal window
startupinfo = None
if headphones.SYS_PLATFORM == "win32":
startupinfo = subprocess.STARTUPINFO()
try:
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
except AttributeError:
startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
# Encode
logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
logger.debug(subprocess.list2cmdline(cmd))
process = subprocess.Popen(cmd, startupinfo=startupinfo,
stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate(headphones.CONFIG.ENCODER)
# Error if return code not zero
if process.returncode:
logger.error(
'Encoding failed for %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
out = stdout if stdout else stderr
out = out.decode(headphones.SYS_ENCODING, 'replace')
outlast2lines = '\n'.join(out.splitlines()[-2:])
logger.error('%s error details: %s' % (headphones.CONFIG.ENCODER, outlast2lines))
out = out.rstrip("\n")
logger.debug(out)
encoded = False
else:
logger.info('%s encoded in %s', musicSource, getTimeEncode(startMusicTime))
encoded = True
return encoded