Here are the examples of the python api sys.stderr taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
143 Examples
3
Example 1
Project: quickrelease Source File: command.py
def Run(self):
"""
Launch the external command specified by this
L{RunShellCommand<quickrelease.command.RunShellCommand>} object.
@raise RunShellCommandError: if C{raiseErrors} was set in the
constructor and the external command either returns with a failure
value or times out, a RunShellCommandError will be raised.
"""
if self._verbose:
timeoutStr = ""
if self.timeout is not None and gUsingKillableProcess:
secondsStr = "seconds"
if self.timeout == 1:
secondsStr = "second"
timeoutStr = " with timeout %d %s" % (self.timeout, secondsStr)
print >> sys.stderr, ("RunShellCommand(): Running %s in directory "
"%s%s." % (str(self), self.workdir, timeoutStr))
# Make sure all the output streams are flushed before we start; this
# really only ever seems to have caused a problem on Win32
sys.stderr.flush()
sys.stdout.flush()
commandLaunched = False
try:
logDescs = []
if self._logfile:
if self._appendLogfile:
logHandle = _OpenLog(self._logfile, 'a')
else:
logHandle = _OpenLog(self._logfile, 'w')
logDescs.append(_LogHandleDesc(logHandle, _PIPE_STDOUT))
if self._combineOutput:
logDescs.append(_LogHandleDesc(logHandle, _PIPE_STDERR))
if not self._combineOutput and self._errorLogfile is not None:
if self._appendErrorLogfile:
errorLogHandle = _OpenLog(self._errorLogfile, 'a')
else:
errorLogHandle = _OpenLog(self._errorLogfile, 'w')
logDescs.append(_LogHandleDesc(errorLogHandle, _PIPE_STDERR))
outputQueue = Queue()
stdinArg = None
if self._input is not None:
stdinArg = PIPE
self._startTime = time.time()
process = Popen(self._execArray, stdin=stdinArg, stdout=PIPE,
stderr=PIPE, cwd=self.workdir, bufsize=0)
commandLaunched = True
stdinWriter = None
if self._stdin is not None:
#print >> sys.stderr, "Starting stdinWriter"
stdinWriter = Thread(target=_WriteInput,
name="RunShellCommand() stdin writer",
args=(self._stdin, process.stdin))
stdinWriter.start()
stdoutReader = Thread(target=_EnqueueOutput,
name="RunShellCommand() stdout reader",
args=(process.stdout, outputQueue, _PIPE_STDOUT))
stderrReader = Thread(target=_EnqueueOutput,
name="RunShellCommand() stderr reader",
args=(process.stderr, outputQueue, _PIPE_STDERR))
self._outputMonitor = _OutputQueueReader(queue=outputQueue,
logHandleDescriptors=logDescs, printOutput=self._printOutput)
stdoutReader.start()
stderrReader.start()
self._outputMonitor.start()
try:
# If you're not using killable process, you theoretically have
# something else (buildbot) that's implementing a timeout for
# you; so, all timeouts here are ignored... ...
if self.timeout is not None and gUsingKillableProcess:
process.wait(self.timeout)
else:
process.wait()
except KeyboardInterrupt, ex:
process.kill()
self._processWasKilled = True
raise ex
except OSError, ex:
if ex.errno == errno.ENOENT:
raise RunShellCommandError(self, "Invalid command or working "
"dir: %s, working dir: %s" % (self, self.workdir))
raise ReleaseFrameworkError("OSError: %s" % str(ex), details=ex)
#except Exception, ex:
# print "EX: %s" % (ex)
# raise ex
finally:
if commandLaunched:
procEndTime = time.time()
#print >> sys.stderr, "Joining stderrReader"
stderrReader.join()
#print >> sys.stderr, "Joining stdoutReader"
stdoutReader.join()
#print >> sys.stderr, "Joining outputMonitor"
self._outputMonitor.join()
#print >> sys.stderr, "Joining q"
outputQueue.join()
if stdinWriter is not None:
#print >> sys.stderr, "Joining stdinWriter"
stdinWriter.join()
for h in logDescs:
h.handle.close()
# Assume if the runtime was up to/beyond the timeout, that it
# was killed, due to timeout.
if commandLaunched and self.runningtime >= self.timeout:
self._processWasKilled = True
self._processTimedOut = True
#om = outputMonitor.collectedOutput[_PIPE_STDOUT]
#for i in range(om):
# print "STDOUT line %d (%d): %s" % (i, om[i].time,
# om[i].content)
#
#om = outputMonitor.collectedOutput[_PIPE_STDERR]
# for i in range(om):
# print "STDERR line %d (%d): %s" % (i, om[i].time,
# om[i].content)
#self._stdout = outputMonitor.GetOutput(_PIPE_STDOUT)
#self._rawstdout = outputMonitor.GetOutput(_PIPE_STDOUT,
# raw=True)
#self._stderr = outputMonitor.GetOutput(_PIPE_STDERR)
self._endTime = procEndTime
self._returncode = process.returncode
if self._input is not None and type(self._input) is str:
#print >> sys.stderr, "Closing stdin file."
self._stdin.close()
if self._raiseErrors and self.returncode:
raise RunShellCommandError(self)
2
Example 2
def convert(pctxt, infile, outfile, base=''):
global docuement, keywords, keywordsCount, chapters, keyword_conflicts
if len(base) > 0 and base[:-1] != '/':
base += '/'
hasSummary = False
data = []
fd = file(infile,"r")
for line in fd:
line.replace("\t", " " * 8)
line = line.rstrip()
data.append(line)
fd.close()
parsers = init_parsers(pctxt)
pctxt.context = {
'headers': {},
'docuement': "",
'base': base,
}
sections = []
currentSection = {
"details": getTitleDetails(""),
"content": "",
}
chapters = {}
keywords = {}
keywordsCount = {}
specialSections = {
"default": {
"hasKeywords": True,
},
"4.1": {
"hasKeywords": True,
},
}
pctxt.keywords = keywords
pctxt.keywordsCount = keywordsCount
pctxt.chapters = chapters
print >> sys.stderr, "Importing %s..." % infile
nblines = len(data)
i = j = 0
while i < nblines:
line = data[i].rstrip()
if i < nblines - 1:
next = data[i + 1].rstrip()
else:
next = ""
if (line == "Summary" or re.match("^[0-9].*", line)) and (len(next) > 0) and (next[0] == '-') \
and ("-" * len(line)).startswith(next): # Fuzzy underline length detection
sections.append(currentSection)
currentSection = {
"details": getTitleDetails(line),
"content": "",
}
j = 0
i += 1 # Skip underline
while not data[i + 1].rstrip():
i += 1 # Skip empty lines
else:
if len(line) > 80:
print >> sys.stderr, "Line `%i' exceeds 80 columns" % (i + 1)
currentSection["content"] = currentSection["content"] + line + "\n"
j += 1
if currentSection["details"]["title"] == "Summary" and line != "":
hasSummary = True
# Learn chapters from the summary
details = getTitleDetails(line)
if details["chapter"]:
chapters[details["chapter"]] = details
i += 1
sections.append(currentSection)
chapterIndexes = sorted(chapters.keys(), key=lambda chapter: map(int, chapter.split('.')))
docuement = ""
# Complete the summary
for section in sections:
details = section["details"]
title = details["title"]
if title:
fulltitle = title
if details["chapter"]:
#docuementAppend("<a name=\"%s\"></a>" % details["chapter"])
fulltitle = details["chapter"] + ". " + title
if not details["chapter"] in chapters:
print >> sys.stderr, "Adding '%s' to the summary" % details["title"]
chapters[details["chapter"]] = details
chapterIndexes = sorted(chapters.keys())
for section in sections:
details = section["details"]
pctxt.details = details
level = details["level"]
title = details["title"]
content = section["content"].rstrip()
print >> sys.stderr, "Parsing chapter %s..." % title
if (title == "Summary") or (title and not hasSummary):
summaryTemplate = pctxt.templates.get_template('summary.html')
docuementAppend(summaryTemplate.render(
pctxt = pctxt,
chapters = chapters,
chapterIndexes = chapterIndexes,
))
if title and not hasSummary:
hasSummary = True
else:
continue
if title:
docuementAppend('<a class="anchor" id="%s" name="%s"></a>' % (details["chapter"], details["chapter"]))
if level == 1:
docuementAppend("<div class=\"page-header\">", False)
docuementAppend('<h%d id="chapter-%s" data-target="%s"><small><a class="small" href="#%s">%s.</a></small> %s</h%d>' % (level, details["chapter"], details["chapter"], details["chapter"], details["chapter"], cgi.escape(title, True), level))
if level == 1:
docuementAppend("</div>", False)
if content:
if False and title:
# Display a navigation bar
docuementAppend('<ul class="well pager">')
docuementAppend('<li><a href="#top">Top</a></li>', False)
index = chapterIndexes.index(details["chapter"])
if index > 0:
docuementAppend('<li class="previous"><a href="#%s">Previous</a></li>' % chapterIndexes[index - 1], False)
if index < len(chapterIndexes) - 1:
docuementAppend('<li class="next"><a href="#%s">Next</a></li>' % chapterIndexes[index + 1], False)
docuementAppend('</ul>', False)
content = cgi.escape(content, True)
content = re.sub(r'section ([0-9]+(.[0-9]+)*)', r'<a href="#\1">section \1</a>', content)
pctxt.set_content(content)
if not title:
lines = pctxt.get_lines()
pctxt.context['headers'] = {
'title': '',
'subtitle': '',
'version': '',
'author': '',
'date': ''
}
if re.match("^-+$", pctxt.get_line().strip()):
# Try to analyze the header of the file, assuming it follows
# those rules :
# - it begins with a "separator line" (several '-' chars)
# - then the docuement title
# - an optional subtitle
# - a new separator line
# - the version
# - the author
# - the date
pctxt.next()
pctxt.context['headers']['title'] = pctxt.get_line().strip()
pctxt.next()
subtitle = ""
while not re.match("^-+$", pctxt.get_line().strip()):
subtitle += " " + pctxt.get_line().strip()
pctxt.next()
pctxt.context['headers']['subtitle'] += subtitle.strip()
if not pctxt.context['headers']['subtitle']:
# No subtitle, try to guess one from the title if it
# starts with the word "HAProxy"
if pctxt.context['headers']['title'].startswith('HAProxy '):
pctxt.context['headers']['subtitle'] = pctxt.context['headers']['title'][8:]
pctxt.context['headers']['title'] = 'HAProxy'
pctxt.next()
pctxt.context['headers']['version'] = pctxt.get_line().strip()
pctxt.next()
pctxt.context['headers']['author'] = pctxt.get_line().strip()
pctxt.next()
pctxt.context['headers']['date'] = pctxt.get_line().strip()
pctxt.next()
if HAPROXY_GIT_VERSION:
pctxt.context['headers']['version'] = 'version ' + HAPROXY_GIT_VERSION
# Skip header lines
pctxt.eat_lines()
pctxt.eat_empty_lines()
docuementAppend('<div>', False)
delay = []
while pctxt.has_more_lines():
try:
specialSection = specialSections[details["chapter"]]
except:
specialSection = specialSections["default"]
line = pctxt.get_line()
if i < nblines - 1:
nextline = pctxt.get_line(1)
else:
nextline = ""
oldline = line
pctxt.stop = False
for parser in parsers:
line = parser.parse(line)
if pctxt.stop:
break
if oldline == line:
# nothing has changed,
# delays the rendering
if delay or line != "":
delay.append(line)
pctxt.next()
elif pctxt.stop:
while delay and delay[-1].strip() == "":
del delay[-1]
if delay:
remove_indent(delay)
docuementAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
delay = []
docuementAppend(line, False)
else:
while delay and delay[-1].strip() == "":
del delay[-1]
if delay:
remove_indent(delay)
docuementAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
delay = []
docuementAppend(line, True)
pctxt.next()
while delay and delay[-1].strip() == "":
del delay[-1]
if delay:
remove_indent(delay)
docuementAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
delay = []
docuementAppend('</div>')
if not hasSummary:
summaryTemplate = pctxt.templates.get_template('summary.html')
print chapters
docuement = summaryTemplate.render(
pctxt = pctxt,
chapters = chapters,
chapterIndexes = chapterIndexes,
) + docuement
# Log warnings for keywords defined in several chapters
keyword_conflicts = {}
for keyword in keywords:
keyword_chapters = list(keywords[keyword])
keyword_chapters.sort()
if len(keyword_chapters) > 1:
print >> sys.stderr, 'Multi section keyword : "%s" in chapters %s' % (keyword, list(keyword_chapters))
keyword_conflicts[keyword] = keyword_chapters
keywords = list(keywords)
keywords.sort()
createLinks()
# Add the keywords conflicts to the keywords list to make them available in the search form
# And remove the original keyword which is now useless
for keyword in keyword_conflicts:
sections = keyword_conflicts[keyword]
offset = keywords.index(keyword)
for section in sections:
keywords.insert(offset, "%s (%s)" % (keyword, chapters[section]['title']))
offset += 1
keywords.remove(keyword)
try:
footerTemplate = pctxt.templates.get_template('footer.html')
footer = footerTemplate.render(
pctxt = pctxt,
headers = pctxt.context['headers'],
docuement = docuement,
chapters = chapters,
chapterIndexes = chapterIndexes,
keywords = keywords,
keywordsCount = keywordsCount,
keyword_conflicts = keyword_conflicts,
version = VERSION,
date = datetime.datetime.now().strftime("%Y/%m/%d"),
)
except TopLevelLookupException:
footer = ""
return {
'pctxt': pctxt,
'headers': pctxt.context['headers'],
'base': base,
'docuement': docuement,
'chapters': chapters,
'chapterIndexes': chapterIndexes,
'keywords': keywords,
'keywordsCount': keywordsCount,
'keyword_conflicts': keyword_conflicts,
'version': VERSION,
'date': datetime.datetime.now().strftime("%Y/%m/%d"),
'footer': footer
}
2
Example 3
Project: gramps Source File: argparser.py
def parse_args(self):
"""
Fill in lists with open, exports, imports, and actions options.
Any errors are added to self.errors
"""
try:
options, leftargs = getopt.getopt(self.args[1:],
SHORTOPTS, LONGOPTS)
except getopt.GetoptError as msg:
# Extract the arguments in the list.
# The % operator replaces the list elements
# with repr() of the list elements
# which is OK for latin characters,
# but not for non latin characters in list elements
cliargs = "[ "
for arg in range(len(self.args) - 1):
cliargs += self.args[arg + 1] + " "
cliargs += "]"
# Must first do str() of the msg object.
msg = str(msg)
self.errors += [(_('Error parsing the arguments'),
msg + '\n' +
_("Error parsing the arguments: %s \n"
"Type gramps --help for an overview of "
"commands, or read the manual pages."
) % cliargs)]
return
# Some args can work on a list of databases:
if leftargs:
for opt_ix in range(len(options)):
option, value = options[opt_ix]
if option in ['-L', '-l', '-t']:
self.database_names = leftargs
leftargs = []
if leftargs:
# if there were an argument without option,
# use it as a file to open and return
self.open_gui = leftargs[0]
print(_("Trying to open: %s ..."
) % leftargs[0],
file=sys.stderr)
#see if force open is on
for opt_ix in range(len(options)):
option, value = options[opt_ix]
if option in ('-u', '--force-unlock'):
self.force_unlock = True
break
return
# Go over all given option and place them into appropriate lists
cleandbg = []
need_to_quit = False
for opt_ix in range(len(options)):
option, value = options[opt_ix]
if option in ['-O', '--open']:
self.open = value
elif option in ['-C', '--create']:
self.create = value
elif option in ['-i', '--import']:
family_tree_format = None
if (opt_ix < len(options) - 1
and options[opt_ix + 1][0] in ('-f', '--format')):
family_tree_format = options[opt_ix + 1][1]
self.imports.append((value, family_tree_format))
elif option in ['-r', '--remove']:
self.removes.append(value)
elif option in ['-e', '--export']:
family_tree_format = None
if (opt_ix < len(options) - 1
and options[opt_ix + 1][0] in ('-f', '--format')):
family_tree_format = options[opt_ix + 1][1]
self.exports.append((value, family_tree_format))
elif option in ['-a', '--action']:
action = value
if action not in ('report', 'tool', 'book'):
print(_("Unknown action: %s. Ignoring."
) % action,
file=sys.stderr)
continue
options_str = ""
if (opt_ix < len(options)-1
and options[opt_ix+1][0] in ('-p', '--options')):
options_str = options[opt_ix+1][1]
self.actions.append((action, options_str))
elif option in ['-d', '--debug']:
print(_('setup debugging'), value, file=sys.stderr)
logger = logging.getLogger(value)
logger.setLevel(logging.DEBUG)
cleandbg += [opt_ix]
elif option in ['-l']:
self.list = True
elif option in ['-L']:
self.list_more = True
elif option in ['-t']:
self.list_table = True
elif option in ['-s', '--show']:
print(_("Gramps config settings from %s:"
) % config.filename)
for sect in config.data:
for setting in config.data[sect]:
print("%s.%s=%s" % (sect, setting,
repr(config.data[sect][setting])))
print()
sys.exit(0)
elif option in ['-b', '--databases']:
default = config.data["database"]["backend"]
pmgr = BasePluginManager.get_instance()
pmgr.reg_plugins(PLUGINS_DIR, self, None)
pmgr.reg_plugins(USER_PLUGINS, self, None, load_on_reg=True)
for plugin in pmgr.get_reg_databases():
pdata = pmgr.get_plugin(plugin.id)
mod = pmgr.load_plugin(pdata)
if mod:
database = getattr(mod, pdata.databaseclass)
summary = database.get_class_summary()
print("Database backend ID:",
pdata.id,
"(default)" if pdata.id == default else "")
for key in sorted(summary.keys()):
print(" ", _("%s:") % key, summary[key])
sys.exit(0)
elif option in ['-c', '--config']:
cfg_name = value
set_value = False
if cfg_name:
if ":" in cfg_name:
cfg_name, new_value = cfg_name.split(":", 1)
set_value = True
if config.has_default(cfg_name):
setting_value = config.get(cfg_name)
print(_("Current Gramps config setting: "
"%(name)s:%(value)s"
) % {'name' : cfg_name,
'value' : repr(setting_value)},
file=sys.stderr)
if set_value:
# does a user want the default config value?
if new_value in ("DEFAULT", _("DEFAULT")):
new_value = config.get_default(cfg_name)
else:
converter = get_type_converter(setting_value)
new_value = converter(new_value)
config.set(cfg_name, new_value)
# translators: indent "New" to match "Current"
print(_(" New Gramps config setting: "
"%(name)s:%(value)s"
) % {'name' : cfg_name,
'value' : repr(config.get(cfg_name))},
file=sys.stderr)
else:
need_to_quit = True
else:
print(_("Gramps: no such config setting: '%s'"
) % cfg_name,
file=sys.stderr)
need_to_quit = True
cleandbg += [opt_ix]
elif option in ['-h', '-?', '--help']:
self.help = True
elif option in ['-u', '--force-unlock']:
self.force_unlock = True
elif option in ['--usage']:
self.usage = True
elif option in ['-y', '--yes']:
self.auto_accept = True
elif option in ['-q', '--quiet']:
self.quiet = True
#clean options list
cleandbg.reverse()
for ind in cleandbg:
del options[ind]
if (len(options) > 0
and self.open is None
and self.imports == []
and self.removes == []
and not (self.list
or self.list_more
or self.list_table
or self.help)):
# Extract and convert to unicode the arguments in the list.
# The % operator replaces the list elements with repr() of
# the list elements, which is OK for latin characters
# but not for non-latin characters in list elements
cliargs = "[ "
for arg in range(len(self.args) - 1):
cliargs += self.args[arg + 1] + ' '
cliargs += "]"
self.errors += [(_('Error parsing the arguments'),
_("Error parsing the arguments: %s \n"
"To use in the command-line mode, supply at "
"least one input file to process."
) % cliargs)]
if need_to_quit:
sys.exit(0)
2
Example 4
Project: cupp Source File: cupp3.py
def interactive():
"""Implementation of the -i switch. Interactively question the user and
create a password dictionary file based on the answer."""
print()
print("[+] Insert the information about the victim to make a dictionary")
print("[+] If you don't know all the info, just hit enter when asked! ;)\n")
# We need some information first!
name = input("First Name: ").lower().strip()
while not name:
print("\n[-] You must enter a name at least!", file=sys.stderr)
name = input("Name: ").lower().strip()
surname = input("Surname: ").lower()
nick = input("Nickname: ").lower()
birthdate = input("Birthdate (DDMMYYYY): ").strip()
while len(birthdate) not in (0, 8):
print("\n[-] You must enter 8 digits for birthday!", file=sys.stderr)
birthdate = input("Birthdate (DDMMYYYY): ").strip()
print("\n")
wife = input("Partner's name: ").lower()
wifen = input("Partner's nickname: ").lower()
wifeb = input("Partner's birthdate (DDMMYYYY): ").strip()
while len(wifeb) not in (0, 8):
print("\n[-] You must enter 8 digits for birthday!", file=sys.stderr)
wifeb = input("Partner's birthdate (DDMMYYYY): ").strip()
print("\n")
kid = input("Child's name: ").lower()
kidn = input("Child's nickname: ").lower()
kidb = input("Child's birthdate (DDMMYYYY): ").strip()
while len(kidb) not in (0, 8):
print("\n[-] You must enter 8 digits for birthday!", file=sys.stderr)
kidb = input("Child's birthdate (DDMMYYYY): ").strip()
print("\n")
pet = input("Pet's name: ").lower().strip()
company = input("Company name: ").lower().strip()
print("\n")
prompt = "Do you want to add some key words about the victim? Y/[N]: "
words1 = input(prompt).lower().strip()
words2 = ''
if words1 == 'y':
prompt = ("Please enter the words, comma-separated."
" [i.e. hacker,juice,black], spaces will be removed: ")
words2 = input(prompt).replace(' ', '')
words = words2.split(',')
spechars = []
prompt = "Do you want to add special characters at the end of words? Y/[N]: "
spechars1 = input(prompt).lower()
if spechars1 == "y":
for spec1 in CONFIG['chars']:
spechars.append(spec1)
for spec2 in CONFIG['chars']:
spechars.append(spec1+spec2)
for spec3 in CONFIG['chars']:
spechars.append(spec1+spec2+spec3)
randnum = input("Do you want to add some random numbers at the end of words? Y/[N]: ").lower()
leetmode = input("Leet mode? (i.e. leet = 1337) Y/[N]: ").lower().strip()
print("\n[+] Now making a dictionary...")
# Now me must do some string modifications
# Birthdays first
birthdate_yy, birthdate_yyy = birthdate[-2:], birthdate[-3:]
birthdate_yyyy = birthdate[-4:]
birthdate_xd, birthdate_xm = birthdate[1:2], birthdate[3:4]
birthdate_dd, birthdate_mm = birthdate[:2], birthdate[2:4]
wifeb_yy = wifeb[-2:]
wifeb_yyy = wifeb[-3:]
wifeb_yyyy = wifeb[-4:]
wifeb_xd = wifeb[1:2]
wifeb_xm = wifeb[3:4]
wifeb_dd = wifeb[:2]
wifeb_mm = wifeb[2:4]
kidb_yy = kidb[-2:]
kidb_yyy = kidb[-3:]
kidb_yyyy = kidb[-4:]
kidb_xd = kidb[1:2]
kidb_xm = kidb[3:4]
kidb_dd = kidb[:2]
kidb_mm = kidb[2:4]
# Convert first letters to uppercase...
nameup = name.title()
surnameup = surname.title()
nickup = nick.title()
wifeup = wife.title()
wifenup = wifen.title()
kidup = kid.title()
kidnup = kidn.title()
petup = pet.title()
companyup = company.title()
wordsup = [words1.title() for words1 in words]
word = words+wordsup
# reverse a name
rev_name = name[::-1]
rev_nameup = nameup[::-1]
rev_nick = nick[::-1]
rev_nickup = nickup[::-1]
rev_wife = wife[::-1]
rev_wifeup = wifeup[::-1]
rev_kid = kid[::-1]
rev_kidup = kidup[::-1]
reverse = [rev_name, rev_nameup, rev_nick, rev_nickup, rev_wife,
rev_wifeup, rev_kid, rev_kidup]
rev_n = [rev_name, rev_nameup, rev_nick, rev_nickup]
rev_w = [rev_wife, rev_wifeup]
rev_k = [rev_kid, rev_kidup]
# Let's do some serious work! This will be a mess of code, but who cares? :)
# Birthdays combinations
bds = [birthdate_yy, birthdate_yyy, birthdate_yyyy, birthdate_xd,
birthdate_xm, birthdate_dd, birthdate_mm]
bdss = []
for bds1 in bds:
bdss.append(bds1)
for bds2 in bds:
if bds.index(bds1) != bds.index(bds2):
bdss.append(bds1 + bds2)
for bds3 in bds:
condition = (bds.index(bds1) != bds.index(bds2) and
bds.index(bds2) != bds.index(bds3) and
bds.index(bds1) != bds.index(bds3))
if condition:
bdss.append(bds1+bds2+bds3)
# For a woman...
wbds = [wifeb_yy, wifeb_yyy, wifeb_yyyy, wifeb_xd, wifeb_xm, wifeb_dd, wifeb_mm]
wbdss = []
for wbds1 in wbds:
wbdss.append(wbds1)
for wbds2 in wbds:
if wbds.index(wbds1) != wbds.index(wbds2):
wbdss.append(wbds1+wbds2)
for wbds3 in wbds:
condition = (wbds.index(wbds1) != wbds.index(wbds2) and
wbds.index(wbds2) != wbds.index(wbds3) and
wbds.index(wbds1) != wbds.index(wbds3))
if condition:
wbdss.append(wbds1+wbds2+wbds3)
# and a child...
kbds = [kidb_yy, kidb_yyy, kidb_yyyy, kidb_xd, kidb_xm, kidb_dd, kidb_mm]
kbdss = []
for kbds1 in kbds:
kbdss.append(kbds1)
for kbds2 in kbds:
if kbds.index(kbds1) != kbds.index(kbds2):
kbdss.append(kbds1+kbds2)
for kbds3 in kbds:
condition = (kbds.index(kbds1) != kbds.index(kbds2) and
kbds.index(kbds2) != kbds.index(kbds3) and
kbds.index(kbds1) != kbds.index(kbds3))
if condition:
kbdss.append(kbds1+kbds2+kbds3)
# string combinations
kombinaac = [pet, petup, company, companyup]
kombina = [name, surname, nick, nameup, surnameup, nickup]
kombinaw = [wife, wifen, wifeup, wifenup, surname, surnameup]
kombinak = [kid, kidn, kidup, kidnup, surname, surnameup]
kombinaa = []
for kombina1 in kombina:
kombinaa.append(kombina1)
for kombina2 in kombina:
condition = (kombina.index(kombina1) != kombina.index(kombina2) and
kombina.index(kombina1.title()) != kombina.index(kombina2.title()))
if condition:
kombinaa.append(kombina1+kombina2)
kombinaaw = []
for kombina1 in kombinaw:
kombinaaw.append(kombina1)
for kombina2 in kombinaw:
condition = (kombinaw.index(kombina1) != kombinaw.index(kombina2) and
kombinaw.index(kombina1.title()) != kombinaw.index(kombina2.title()))
if condition:
kombinaaw.append(kombina1+kombina2)
kombinaak = []
for kombina1 in kombinak:
kombinaak.append(kombina1)
for kombina2 in kombinak:
condition = (kombinak.index(kombina1) != kombinak.index(kombina2) and
kombinak.index(kombina1.title()) != kombinak.index(kombina2.title()))
if condition:
kombinaak.append(kombina1+kombina2)
komb1 = list(komb(kombinaa, bdss))
komb2 = list(komb(kombinaaw, wbdss))
komb3 = list(komb(kombinaak, kbdss))
komb4 = list(komb(kombinaa, CONFIG['years']))
komb5 = list(komb(kombinaac, CONFIG['years']))
komb6 = list(komb(kombinaaw, CONFIG['years']))
komb7 = list(komb(kombinaak, CONFIG['years']))
komb8 = list(komb(word, bdss))
komb9 = list(komb(word, wbdss))
komb10 = list(komb(word, kbdss))
komb11 = list(komb(word, CONFIG['years']))
komb12 = komb13 = komb14 = komb15 = komb16 = komb21 = []
if randnum == "y":
komb12 = list(concats(word, CONFIG['numfrom'], CONFIG['numto']))
komb13 = list(concats(kombinaa, CONFIG['numfrom'], CONFIG['numto']))
komb14 = list(concats(kombinaac, CONFIG['numfrom'], CONFIG['numto']))
komb15 = list(concats(kombinaaw, CONFIG['numfrom'], CONFIG['numto']))
komb16 = list(concats(kombinaak, CONFIG['numfrom'], CONFIG['numto']))
komb21 = list(concats(reverse, CONFIG['numfrom'], CONFIG['numto']))
komb17 = list(komb(reverse, CONFIG['years']))
komb18 = list(komb(rev_w, wbdss))
komb19 = list(komb(rev_k, kbdss))
komb20 = list(komb(rev_n, bdss))
komb001 = komb002 = komb003 = komb004 = komb005 = komb006 = []
if spechars1 == "y":
komb001 = list(komb(kombinaa, spechars))
komb002 = list(komb(kombinaac, spechars))
komb003 = list(komb(kombinaaw, spechars))
komb004 = list(komb(kombinaak, spechars))
komb005 = list(komb(word, spechars))
komb006 = list(komb(reverse, spechars))
print("[+] Sorting list and removing duplicates...")
sets = [set(komb1), set(komb2), set(komb3), set(komb4), set(komb5),
set(komb6), set(komb7), set(komb8), set(komb9), set(komb10),
set(komb11), set(komb12), set(komb13), set(komb14), set(komb15),
set(komb16), set(komb17), set(komb18), set(komb19), set(komb20),
set(komb21), set(kombinaa), set(kombinaac), set(kombinaaw),
set(kombinaak), set(word), set(komb001), set(komb002), set(komb003),
set(komb004), set(komb005), set(komb006)]
uniqset = set()
for s in sets:
uniqset.update(s)
uniqlist = bdss + wbdss + kbdss + reverse + list(uniqset)
unique_lista = sorted(set(uniqlist))
unique_leet = []
if leetmode == "y":
for x in unique_lista:
unique_leet.append(leet_replace(x))
unique_list = unique_lista + unique_leet
unique_list_finished = [x for x in unique_list if CONFIG['wcfrom'] < len(x) < CONFIG['wcto']]
unique_list_finished.sort()
with open(name + '.txt', 'w') as f:
f.write(os.linesep.join(unique_list_finished))
with open(name + '.txt') as f:
lines = len(list(f)) # shorter, but possibly more memory expensive
message = ("[+] Saving dictionary to \033[1;31m%s.txt\033[1;m, counting"
" \033[1;31m%i\033[1;m words.")
print(message % (name, lines))
message = ("[+] Now load your pistolero with \033[1;31m%s.txt\033[1;m and"
" shoot! Good luck!")
print(message % name)
sys.exit()
2
Example 5
def series_export_spreadsheet(matrix, matrixAnswers, logo):
######################################################################
#
# Now take the matrix data type and generate a spreadsheet from it
#
######################################################################
import math
try:
import xlwt
except ImportError:
output = s3_rest_controller(module,
resourcename,
rheader=response.s3.survey_series_rheader)
return output
def wrapText(sheet, cell, style):
row = cell.row
col = cell.col
try:
text = unicode(cell.text)
except:
text = cell.text
width = 16
# Wrap text and calculate the row width and height
characters_in_cell = float(width-2)
twips_per_row = 255 #default row height for 10 point font
if cell.merged():
try:
sheet.write_merge(cell.row,
cell.row + cell.mergeV,
cell.col,
cell.col + cell.mergeH,
text,
style
)
except Exception as msg:
print >> sys.stderr, msg
print >> sys.stderr, "row: %s + vert: %s, col: %s + horiz %s" % (cell.row, cell.mergeV, cell.col, cell.mergeH)
posn = "%s,%s"%(cell.row, cell.col)
if matrix.matrix[posn]:
print >> sys.stderr, matrix.matrix[posn]
rows = math.ceil((len(text) / characters_in_cell) / (1 + cell.mergeH))
else:
sheet.write(cell.row,
cell.col,
text,
style
)
rows = math.ceil(len(text) / characters_in_cell)
new_row_height = int(rows * twips_per_row)
new_col_width = width * COL_WIDTH_MULTIPLIER
if sheet.row(row).height < new_row_height:
sheet.row(row).height = new_row_height
if sheet.col(col).width < new_col_width:
sheet.col(col).width = new_col_width
def mergeStyles(listTemplate, styleList):
"""
Take a list of styles and return a single style object with
all the differences from a newly created object added to the
resultant style.
"""
if len(styleList) == 0:
finalStyle = xlwt.XFStyle()
elif len(styleList) == 1:
finalStyle = listTemplate[styleList[0]]
else:
zeroStyle = xlwt.XFStyle()
finalStyle = xlwt.XFStyle()
for i in range(0,len(styleList)):
finalStyle = mergeObjectDiff(finalStyle,
listTemplate[styleList[i]],
zeroStyle)
return finalStyle
def mergeObjectDiff(baseObj, newObj, zeroObj):
"""
function to copy all the elements in newObj that are different from
the zeroObj and place them in the baseObj
"""
elementList = newObj.__dict__
for (element, value) in elementList.items():
try:
baseObj.__dict__[element] = mergeObjectDiff(baseObj.__dict__[element],
value,
zeroObj.__dict__[element])
except:
if zeroObj.__dict__[element] != value:
baseObj.__dict__[element] = value
return baseObj
COL_WIDTH_MULTIPLIER = 240
book = xlwt.Workbook(encoding="utf-8")
output = StringIO()
protection = xlwt.Protection()
protection.cell_locked = 1
noProtection = xlwt.Protection()
noProtection.cell_locked = 0
borders = xlwt.Borders()
borders.left = xlwt.Borders.DOTTED
borders.right = xlwt.Borders.DOTTED
borders.top = xlwt.Borders.DOTTED
borders.bottom = xlwt.Borders.DOTTED
borderT1 = xlwt.Borders()
borderT1.top = xlwt.Borders.THIN
borderT2 = xlwt.Borders()
borderT2.top = xlwt.Borders.MEDIUM
borderL1 = xlwt.Borders()
borderL1.left = xlwt.Borders.THIN
borderL2 = xlwt.Borders()
borderL2.left = xlwt.Borders.MEDIUM
borderR1 = xlwt.Borders()
borderR1.right = xlwt.Borders.THIN
borderR2 = xlwt.Borders()
borderR2.right = xlwt.Borders.MEDIUM
borderB1 = xlwt.Borders()
borderB1.bottom = xlwt.Borders.THIN
borderB2 = xlwt.Borders()
borderB2.bottom = xlwt.Borders.MEDIUM
alignBase = xlwt.Alignment()
alignBase.horz = xlwt.Alignment.HORZ_LEFT
alignBase.vert = xlwt.Alignment.VERT_TOP
alignWrap = xlwt.Alignment()
alignWrap.horz = xlwt.Alignment.HORZ_LEFT
alignWrap.vert = xlwt.Alignment.VERT_TOP
alignWrap.wrap = xlwt.Alignment.WRAP_AT_RIGHT
shadedFill = xlwt.Pattern()
shadedFill.pattern = xlwt.Pattern.SOLID_PATTERN
shadedFill.pattern_fore_colour = 0x16 # 25% Grey
shadedFill.pattern_back_colour = 0x08 # Black
headingFill = xlwt.Pattern()
headingFill.pattern = xlwt.Pattern.SOLID_PATTERN
headingFill.pattern_fore_colour = 0x1F # ice_blue
headingFill.pattern_back_colour = 0x08 # Black
styleTitle = xlwt.XFStyle()
styleTitle.font.height = 0x0140 # 320 twips, 16 points
styleTitle.font.bold = True
styleTitle.alignment = alignBase
styleHeader = xlwt.XFStyle()
styleHeader.font.height = 0x00F0 # 240 twips, 12 points
styleHeader.font.bold = True
styleHeader.alignment = alignBase
styleSubHeader = xlwt.XFStyle()
styleSubHeader.font.bold = True
styleSubHeader.alignment = alignWrap
styleSectionHeading = xlwt.XFStyle()
styleSectionHeading.font.bold = True
styleSectionHeading.alignment = alignWrap
styleSectionHeading.pattern = headingFill
styleHint = xlwt.XFStyle()
styleHint.protection = protection
styleHint.font.height = 160 # 160 twips, 8 points
styleHint.font.italic = True
styleHint.alignment = alignWrap
styleText = xlwt.XFStyle()
styleText.protection = protection
styleText.alignment = alignWrap
styleInstructions = xlwt.XFStyle()
styleInstructions.font.height = 0x00B4 # 180 twips, 9 points
styleInstructions.font.italic = True
styleInstructions.protection = protection
styleInstructions.alignment = alignWrap
styleBox = xlwt.XFStyle()
styleBox.borders = borders
styleBox.protection = noProtection
styleInput = xlwt.XFStyle()
styleInput.borders = borders
styleInput.protection = noProtection
styleInput.pattern = shadedFill
boxL1 = xlwt.XFStyle()
boxL1.borders = borderL1
boxL2 = xlwt.XFStyle()
boxL2.borders = borderL2
boxT1 = xlwt.XFStyle()
boxT1.borders = borderT1
boxT2 = xlwt.XFStyle()
boxT2.borders = borderT2
boxR1 = xlwt.XFStyle()
boxR1.borders = borderR1
boxR2 = xlwt.XFStyle()
boxR2.borders = borderR2
boxB1 = xlwt.XFStyle()
boxB1.borders = borderB1
boxB2 = xlwt.XFStyle()
boxB2.borders = borderB2
styleList = {}
styleList["styleTitle"] = styleTitle
styleList["styleHeader"] = styleHeader
styleList["styleSubHeader"] = styleSubHeader
styleList["styleSectionHeading"] = styleSectionHeading
styleList["styleHint"] = styleHint
styleList["styleText"] = styleText
styleList["styleInstructions"] = styleInstructions
styleList["styleInput"] = styleInput
styleList["boxL1"] = boxL1
styleList["boxL2"] = boxL2
styleList["boxT1"] = boxT1
styleList["boxT2"] = boxT2
styleList["boxR1"] = boxR1
styleList["boxR2"] = boxR2
styleList["boxB1"] = boxB1
styleList["boxB2"] = boxB2
sheet1 = book.add_sheet(T("Assessment"))
sheetA = book.add_sheet(T("Metadata"))
maxCol = 0
for cell in matrix.matrix.values():
if cell.col + cell.mergeH > 255:
print >> sys.stderr, "Cell (%s,%s) - (%s,%s) ignored" % (cell.col, cell.row, cell.col + cell.mergeH, cell.row + cell.mergeV)
continue
if cell.col + cell.mergeH > maxCol:
maxCol = cell.col + cell.mergeH
if cell.joined():
continue
style = mergeStyles(styleList, cell.styleList)
if (style.alignment.wrap == style.alignment.WRAP_AT_RIGHT):
# get all the styles from the joined cells
# and merge these styles in.
joinedStyles = matrix.joinedElementStyles(cell)
joinedStyle = mergeStyles(styleList, joinedStyles)
try:
wrapText(sheet1, cell, joinedStyle)
except:
pass
else:
if cell.merged():
# get all the styles from the joined cells
# and merge these styles in.
joinedStyles = matrix.joinedElementStyles(cell)
joinedStyle = mergeStyles(styleList, joinedStyles)
try:
sheet1.write_merge(cell.row,
cell.row + cell.mergeV,
cell.col,
cell.col + cell.mergeH,
unicode(cell.text),
joinedStyle
)
except Exception as msg:
print >> sys.stderr, msg
print >> sys.stderr, "row: %s + vert: %s, col: %s + horiz %s" % (cell.row, cell.mergeV, cell.col, cell.mergeH)
posn = "%s,%s"%(cell.row, cell.col)
if matrix.matrix[posn]:
print >> sys.stderr, matrix.matrix[posn]
else:
sheet1.write(cell.row,
cell.col,
unicode(cell.text),
style
)
cellWidth = 480 # approximately 2 characters
if maxCol > 255:
maxCol = 255
for col in range(maxCol+1):
sheet1.col(col).width = cellWidth
sheetA.write(0, 0, "Question Code")
sheetA.write(0, 1, "Response Count")
sheetA.write(0, 2, "Values")
sheetA.write(0, 3, "Cell Address")
for cell in matrixAnswers.matrix.values():
style = mergeStyles(styleList, cell.styleList)
sheetA.write(cell.row,
cell.col,
unicode(cell.text),
style
)
if logo != None:
sheet1.insert_bitmap(logo, 0, 0)
sheet1.protect = True
sheetA.protect = True
for i in range(26):
sheetA.col(i).width = 0
sheetA.write(0,
26,
unicode(T("Please do not remove this sheet")),
styleHeader
)
sheetA.col(26).width = 12000
book.save(output)
return output
2
Example 6
Project: simpleflow Source File: command.py
@click.option('--heartbeat',
type=int,
required=False,
default=60,
help='Heartbeat interval in seconds (0 to disable heartbeating).')
@click.option('--nb-workers',
type=int,
required=False,
help='Number of parallel processes handling activity tasks.')
@click.option('--nb-deciders',
type=int,
required=False,
help='Number of parallel processes handling decision tasks.')
@click.option('--input', '-i',
required=False,
help='JSON input of the workflow.')
@click.option('--input-file',
required=False, type=click.File(),
help='JSON file with the input of the workflow.')
@click.option('--tags',
type=comma_separated_list,
required=False,
help='Tags identifying the workflow execution.')
@click.option('--decision-tasks-timeout',
required=False,
help='Decision tasks timeout.')
@click.option('--execution-timeout',
required=False,
help='Timeout for the whole workflow execution.')
@click.option('--workflow-id',
required=False,
help='ID of the workflow execution.')
@click.option('--domain', '-d',
envvar='SWF_DOMAIN',
required=True,
help='SWF Domain.')
@click.option('--display-status',
type=bool, required=False,
help='Display execution status.'
)
@click.option('--repair',
type=str, required=False,
help='Repair a failed workflow execution ("<workflow id>" or "<workflow id> <run id>").'
)
@click.option('--force-activities',
type=str, required=False,
help='Force the re-execution of some activities in when --repair is enabled.'
)
@click.argument('workflow')
@cli.command('standalone', help='Execute a workflow with a single process.')
@click.pass_context
def standalone(context,
workflow,
domain,
workflow_id,
execution_timeout,
tags,
decision_tasks_timeout,
input,
input_file,
nb_workers,
nb_deciders,
heartbeat,
display_status,
repair,
force_activities,
):
"""
This command spawn a decider and an activity worker to execute a workflow
with a single main process.
"""
disable_boto_connection_pooling()
if force_activities and not repair:
raise ValueError(
"You should only use --force-activities with --repair."
)
if not workflow_id:
workflow_id = get_workflow(workflow).name
wf_input = None
if input or input_file:
wf_input = get_or_load_input(input_file, input)
if repair:
repair_run_id = None
if " " in repair:
repair, repair_run_id = repair.split(" ", 1)
# get the previous execution history, it will serve as "default history"
# for activities that succeeded in the previous execution
logger.info(
'retrieving history of previous execution: domain={} '
'workflow_id={} run_id={}'.format(domain, repair, repair_run_id)
)
previous_history = get_workflow_history(domain, repair, run_id=repair_run_id)
previous_history.parse()
# get the previous execution input if none passed
if not input and not input_file:
wf_input = previous_history.events[0].input
else:
previous_history = None
task_list = get_task_list(workflow_id)
logger.info('using task list {}'.format(task_list))
decider_proc = multiprocessing.Process(
target=decider.command.start,
args=(
[workflow],
domain,
task_list,
),
kwargs={
'nb_processes': nb_deciders,
'repair_with': previous_history,
'force_activities': force_activities,
'is_standalone': True,
},
)
decider_proc.start()
worker_proc = multiprocessing.Process(
target=worker.command.start,
args=(
domain,
task_list,
),
kwargs={
'nb_processes': nb_workers,
'heartbeat': heartbeat,
},
)
worker_proc.start()
print('starting workflow {}'.format(workflow), file=sys.stderr)
ex = start_workflow.callback(
workflow,
domain,
workflow_id,
task_list,
execution_timeout,
tags,
decision_tasks_timeout,
json_dumps(wf_input),
None,
local=False,
)
while True:
time.sleep(2)
ex = helpers.get_workflow_execution(
domain,
ex.workflow_id,
ex.run_id,
)
if display_status:
print('status: {}'.format(ex.status), file=sys.stderr)
if ex.status == ex.STATUS_CLOSED:
print('execution {} finished'.format(ex.workflow_id), file=sys.stderr)
break
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
2
Example 7
def __init__(self, gctx):
self.gctx = gctx
self.db = gctx.db
gctx.vim = False
# After exiting the visual mode we copy last addresses we have visited.
# Then we can just enter 'v' and we go where we were.
self.last_entry = None
self.last_stack = []
self.last_saved_stack = []
# A hack to allow window resizing
os.environ['LINES']="blah"
del os.environ['LINES']
os.environ['COLUMNS']="blah"
del os.environ['COLUMNS']
self.COMMANDS = {
"analyzer": Command(
0,
self.__exec_analyzer,
None,
[
"",
"Analyzer status.",
]
),
"push_analyze_symbols": Command(
0,
self.push_analyze_symbols,
None,
[
"",
"Force to analyze the entry point, symbols and a memory scan will be done.",
]
),
"help": Command(
0,
self.__exec_help,
None,
[
"",
"Display this help."
]
),
"history": Command(
0,
self.__exec_history,
None,
[
"",
"Display the command history.",
]
),
"save": Command(
0,
self.__exec_save,
None,
[
"",
"Save the database.",
]
),
"x": Command(
1,
self.__exec_x,
self.__complete_x,
[
"[SYMBOL|0xXXXX|EP]",
"Decompile and print on stdout. By default it will be main.",
"The decompilation is forced, it dosn't check if addresses",
"are defined as code."
]
),
"v": Command(
1,
self.__exec_v,
self.__complete_x,
[
"[SYMBOL|0xXXXX|EP]",
"Visual mode: if no address or symbol is given, you go at the",
"previous address before exiting the visual mode.",
"",
"Main shortcuts:",
"c create code",
"b/w/d/Q create byte/word/dword/qword",
"a create ascii string",
"p create function",
"o set [d|q]word as an offset",
"* create an array",
"x show xrefs",
"r rename",
"space highlight current word (ctrl-k to clear)",
"; edit inline comment (enter/escape to validate/cancel)",
"U undefine",
"",
"Options:",
"I switch to traditional instruction string output (3 modes)",
"M show/hide mangling",
"B show/hide bytes",
"",
"Navigation:",
"/ binary search: if the first char is ! you can put an",
" hexa string example: /!ab 13 42",
" the search is case sensitive.",
"n/N next/previous search occurence",
"g top",
"G bottom",
"z set current line on the middle",
"% goto next bracket",
"{ } previous/next paragraph",
"tab switch between dump/decompilation",
"enter follow address",
"escape go back",
"u re-enter",
"q quit",
]
),
"hexdump": Command(
2,
self.__exec_hexdump,
self.__complete_x,
[
"SYMBOL|0xXXXX|EP [NB_LINES]",
"Dump memory in hexa."
]
),
# by default it will be gctx.nb_lines
"dump": Command(
2,
self.__exec_dump,
self.__complete_x,
[
"SYMBOL|0xXXXX|EP [NB_LINES]",
"Print contents at the specified address.",
]
),
"sym": Command(
3,
self.__exec_sym,
self.__complete_x,
[
"[SYMBOL 0xXXXX] [| FILTER]",
"Print all symbols or set a new symbol.",
"You can filter symbols by searching the word FILTER.",
"If FILTER starts with -, the match is inversed."
]
),
"rename": Command(
2,
self.__exec_rename,
self.__complete_x,
[
"OLD_SYM NEW_SYM",
"Rename a symbol."
]
),
"exit": Command(
0,
self.__exec_exit,
None,
[
"",
"Exit"
]
),
"sections": Command(
0,
self.__exec_sections,
None,
[
"",
"Print all sections.",
]
),
"info": Command(
0,
self.__exec_info,
None,
[
"",
"Information about the current binary."
]
),
"jmptable": Command(
4,
self.__exec_jmptable,
None,
[
"INST_ADDR TABLE_ADDR NB_ENTRIES SIZE_ENTRY",
"Create a jump table referenced at TABLE_ADDR and called",
"from INST_ADDR."
]
),
"py": Command(
-1,
self.__exec_py,
self.__complete_file,
[
"[!][FILE]",
"Run an interactive python shell or execute a script.",
"Global variables api and args will be passed to the script.",
"The character ! is an alias to the scripts directory."
]
),
"mips_set_gp": Command(
1,
self.__exec_mips_set_gp,
None,
[
"ADDR",
"Set the register $gp to a fixed value. Note that it will",
"erase all defined memory."
]
),
"functions": Command(
1,
self.__exec_functions,
None,
[
"",
"Print the function list."
]
),
"xrefs": Command(
1,
self.__exec_xrefs,
self.__complete_x,
[
"SYMBOL|0xXXXX|EP",
"Print cross references to the specified address."
]
),
"memmap": Command(
0,
self.__exec_memmap,
None,
[
"",
"Open a qt window to display the memory."
]
),
}
if gctx.dis.is_x86:
import plasma.lib.arch.x86.analyzer as arch_analyzer
elif gctx.dis.is_mips:
import plasma.lib.arch.mips.analyzer as arch_analyzer
elif gctx.dis.is_arm:
import plasma.lib.arch.arm.analyzer as arch_analyzer
self.analyzer = Analyzer()
self.analyzer.init()
self.analyzer.start()
self.api = Api(gctx, self.analyzer)
gctx.api = self.api
self.analyzer.set(gctx, arch_analyzer)
self.gctx.dis.binary.api = self.api
if gctx.dis.is_mips and not gctx.dis.mips_gp:
print("please run first these commands :")
print("mips_set_gp 0xADDRESS")
print("push_analyze_symbols")
else:
# If false it means that the first analysis was already done
if gctx.autoanalyzer and len(self.db.mem) == 0:
self.push_analyze_symbols(None)
print("new feature: an instruction preceded by ! means that the analyzer", file=sys.stderr)
print("has computed an immediate value. In the visual mode, use the shortcut", file=sys.stderr)
print("I to show original instructions.", file=sys.stderr)
self.comp = Completer(self)
self.comp.set_history(self.db.history)
while 1:
self.comp.loop()
if SHOULD_EXIT:
break
if not self.check_db_modified():
break
self.analyzer.msg.put("exit")
2
Example 8
Project: deepcca Source File: SdA_mapping.py
def test_SdA_regress(finetune_lr=0.05, pretraining_epochs=10,
pretrain_lr=0.1, training_epochs=10000,
dataset='mnist.pkl.gz', batch_size=20):
datasets = load_data_half(dataset)
train_set_x, train_set_y = datasets[0]##
valid_set_x, valid_set_y = datasets[1]##
test_set_x, test_set_y = datasets[2]##
train_set_x=train_set_x.eval()
train_set_y=train_set_y.eval()
import theano
train_set_x_lab=train_set_x[:,:]
train_set_x_unlab=train_set_x[:,:]
train_set_y_lab=train_set_y[:,:]
train_set_y_unlab=train_set_y[:,:]
train_set_x_lab=theano.shared(numpy.asarray(train_set_x_lab,
dtype=theano.config.floatX),
borrow=True)
train_set_y_lab=theano.shared(numpy.asarray(train_set_y_lab,
dtype=theano.config.floatX),
borrow=True)
train_set_x_unlab=theano.shared(numpy.asarray(train_set_x_unlab,
dtype=theano.config.floatX),
borrow=True)
train_set_y_unlab=theano.shared(numpy.asarray(train_set_y_unlab,
dtype=theano.config.floatX),
borrow=True)
# compute number of minibatches for training, validation and testing
n_train_batches_l = train_set_y_lab.eval().shape[0]
n_train_batches_l /= batch_size
n_train_batches_u = train_set_y_unlab.eval().shape[0]
n_train_batches_u /= batch_size
# compute number of minibatches for training, validation and testing
#n_train_batches = train_set_x.get_value(borrow=True).shape[0]
#n_train_batches /= batch_size
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
#from SdA_orig import SdA as SdA_old
hidden_layer_size = 100
SdA_inp = SdA(numpy_rng,
n_ins=392,
hidden_layers_sizes=[hidden_layer_size]
)
SdA_out = SdA(numpy_rng,
n_ins=392,
hidden_layers_sizes=[hidden_layer_size]
)
# PRETRAINING THE MODEL #
if 0 : # pretrain inp ae
print '... getting the pretraining functions for INPUT AE'
pretraining_fns = SdA_inp.pretraining_functions(train_set_x=train_set_x_unlab,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [.1, .2, .3]
for i in xrange(SdA_inp.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches_u):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
if 0 : # pretrain out ae
print '... getting the pretraining functions for OUTPUT AE'
pretraining_fns = SdA_out.pretraining_functions(train_set_x=train_set_y_unlab,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [.5, .2, .3]
for i in xrange(SdA_out.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches_u):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
if 0: # save aes
f=open('aes_shallow_sig_nobias.pkl', 'w+')
import pickle
pickle.dump(SdA_inp, f)
pickle.dump(SdA_out, f)
f.flush()
f.close()
if 0: # load aes
f=open('aes_shallow_sig_nobias.pkl', 'r')
import pickle
SdA_inp=pickle.load(f)
SdA_out=pickle.load(f)
f.close()
if 1: # cca
from dcca_numpy import netCCA_nobias, netCCA, dCCA
from mlp_numpy import expit, logistic_prime, linear, linear_prime, relu, relu_prime, tanh, tanh_prime
train_y1 = train_set_x_lab.eval()
train_y2 = train_set_y_lab.eval()
test_y1 = test_set_x.eval()
test_y2 = test_set_y.eval()
##param1=((train_y1.shape[1],0,0),(2038, relu, relu_prime),(50, relu, relu_prime))
##param2=((train_y2.shape[1],0,0),(1608, relu, relu_prime),(50, relu, relu_prime))
param1=((train_y1.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
param2=((train_y2.shape[1],0,0),(hidden_layer_size, expit, logistic_prime))
W1s = []
b1s = []
for i in range(len(SdA_inp.dA_layers)):
W1s.append( SdA_inp.dA_layers[i].W.T.eval() )
##b1s.append( SdA_inp.dA_layers[i].b.eval() )
##b1s[-1] = b1s[-1].reshape((b1s[-1].shape[0], 1))
W2s = []
b2s = []
for i in range(len(SdA_out.dA_layers)):
W2s.append( SdA_out.dA_layers[i].W.T.eval() )
##b2s.append( SdA_out.dA_layers[i].b.eval() )
##b2s[-1] = b2s[-1].reshape((b2s[-1].shape[0], 1))
numpy.random.seed(0)
N1=netCCA_nobias(train_y1,param1, W1s)
N2=netCCA_nobias(train_y2,param2, W2s)
N = dCCA(train_y1, train_y2, N1, N2)
N1.reconstruct(test_set_x.eval()[0,:])
cnt = 0
from dcca_numpy import cca_cost, cca, order_cost, cor_cost
while True:
X=N1.predict(test_set_x.eval())
Y=N2.predict(test_set_y.eval())
_H1 = numpy.dot(X, N.A1)
_H2 = numpy.dot(Y, N.A2)
print 'cuem', cnt, cor_cost(_H1, _H2)
X1_rec = numpy.tanh(X.dot(N1.weights[0]))
X2_rec = numpy.tanh(Y.dot(N2.weights[0]))
param=((hidden_layer_size,0,0),(hidden_layer_size, relu, relu_prime))
from mlp_numpy import NeuralNetwork as NN
lr=NN(X,Y,param)
lr.train(X[:,:],Y[:,:],10, 0.005)
Yh=lr.predict(X[:,:])
X2_reg = N2.fs[-1](numpy.dot(Yh,N2.weights[0]))
#X2_reg = N2.fs[-1](numpy.dot(_H1.dot(numpy.linalg.inv(N.A1)),N2.weights[0]))
print '****', 'mse1:', numpy.mean((X1_rec-test_set_x.eval())**2.0)
print '****', 'mse2:', numpy.mean((X2_rec-test_set_y.eval())**2.0)
print '****', 'mse_map:', numpy.mean((X2_reg-test_set_y.eval())**2.0)
if cnt % 2:
N.train(5, True, 10000.0)
else:
N.train(5, False, 10000.0)
cnt += 1
f=open('netcca.pkl', 'w+')
import pickle
pickle.dump(N, f)
pickle.dump(N, f)
f.flush()
f.close()
if cnt == 200:
break
for i in range(len(SdA_inp.dA_layers)):
SdA_inp.dA_layers[i].W = theano.shared( N1.weights[i].T )
SdA_inp.dA_layers[i].b = theano.shared( N1.biases[i][:,0] )
for i in range(len(SdA_out.dA_layers)):
SdA_out.dA_layers[i].W = theano.shared( N2.weights[i].T )
SdA_out.dA_layers[i].b = theano.shared( N2.weights[i][:,0] )
if 1 : # pretrain middle layer
print '... pre-training MIDDLE layer'
h1 = T.matrix('x') # the data is presented as rasterized images
h2 = T.matrix('y') # the labels are presented as 1D vector of
log_reg = HiddenLayer(numpy_rng, h1, hidden_layer_size, hidden_layer_size)
if 1: # for middle layer
learning_rate = 0.01
fprop_inp = theano.function(
[],
SdA_inp.sigmoid_layers[-1].output,
givens={
SdA_inp.sigmoid_layers[0].input: train_set_x_lab
},
name='fprop_inp'
)
fprop_out = theano.function(
[],
SdA_out.sigmoid_layers[-1].output,
givens={
SdA_out.sigmoid_layers[0].input: train_set_y_lab
},
name='fprop_out'
)
#H11=fprop_inp()
#H21=fprop_out()
##H1=N1.predict(train_set_x.eval())
##H2=N2.predict(train_set_y.eval())
H1=fprop_inp()
H2=fprop_out()
H1=theano.shared(H1)
H2=theano.shared(H2)
# compute the gradients with respect to the model parameters
logreg_cost = log_reg.mse(h2)
gparams = T.grad(logreg_cost, log_reg.params)
# compute list of fine-tuning updates
updates = [
(param, param - gparam * learning_rate)
for param, gparam in zip(log_reg.params, gparams)
]
train_fn_middle = theano.function(
inputs=[],
outputs=logreg_cost,
updates=updates,
givens={
h1: H1,
h2: H2
},
name='train_middle'
)
epoch = 0
while epoch < 10:
print epoch, train_fn_middle()
epoch += 1
sda = SdA_regress(
SdA_inp,
SdA_out,
log_reg,
numpy_rng=numpy_rng,
n_inp=28*28//2,
hidden_layers_sizes_inp=[hidden_layer_size],
hidden_layers_sizes_out=[hidden_layer_size],
n_out=28*28//2
)
# end-snippet-3 start-snippet-4
# end-snippet-4
# FINETUNING THE MODEL #
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
print '... finetunning the model'
# early-stopping parameters
patience = 10 * n_train_batches_l # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches_l, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
fprop = theano.function(
[],
sda.sigmoid_layers[-1].output,
givens={
sda.x: test_set_x
},
name='fprop'
)
while True:
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches_l):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches_l + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches_l,
this_validation_loss ))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches_l,
test_score ))
if patience <= iter:
done_looping = True
#break
if 0: # vis weights
fprop = theano.function(
[],
sda.sigmoid_layers[-1].output,
givens={
sda.x: test_set_x
},
name='fprop'
)
yh=fprop()
yh=yh
end_time = time.clock()
print(
(
'Optimization complete with best validation score of %f %%, '
'on iteration %i, '
'with test performance %f %%'
)
% (best_validation_loss , best_iter + 1, test_score)
)
print >> sys.stderr, ('The training code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
2
Example 9
Project: quora-backup Source File: converter.py
def cleanup_tree(doc, src, dest):
for child in src.childNodes:
if child.nodeType == Node.TEXT_NODE:
# Text nodes can simply be left as-is
dest.appendChild(child.cloneNode(False))
continue
if child.nodeType != Node.ELEMENT_NODE:
# ???
raise ValueError()
# Otherwise, it's an element node.
if child.tagName in ['br', 'hr']:
dest.appendChild(child.cloneNode(False))
elif child.tagName in ['b', 'i', 'u', 'h2', 'ol', 'ul', 'li', 'blockquote', 'wbr']:
# This node doesn't need to be modified but its children might.
new_node = doc.createElement(child.tagName)
cleanup_tree(doc, child, new_node)
dest.appendChild(new_node)
elif child.getAttribute('data-embed') != '':
# This is a video. We want to copy the data-embed value, which is HTML for an iframe node.
# So, we have to parse it into a separate docuement and import the node.
iframe_html = child.getAttribute('data-embed')
parser = HTMLParser(tree=treebuilders.getTreeBuilder('dom'))
iframe_doc = parser.parse(iframe_html)
try:
iframe = iframe_doc.docuementElement.childNodes[1].firstChild
if iframe.tagName != 'iframe':
raise ValueError()
new_node = doc.importNode(iframe, False)
# Quora uses a protocol-relative URL (//youtube.com/...) so let's make sure we rewrite this.
src = new_node.getAttribute('src')
if src.startswith('//'):
new_node.setAttribute('src', 'http:' + src)
# The video will look really bad if we don't explicitly set the dimensions.
new_node.setAttribute('width', '525')
new_node.setAttribute('height', '295')
dest.appendChild(new_node)
except Exception:
print('[WARNING] Failed to parse video embed code', file=sys.stderr)
# Bail out by just copying the original HTML
dest.appendChild(child.cloneNode(True))
elif 'inline_codeblock' in child.getAttribute('class'):
# Inline codeblock. Simply replace this with a <code>.
try:
# div > pre > span > (text)
span = child.firstChild.firstChild
if span.tagName != 'span':
raise ValueError()
code_element = doc.createElement('code')
code_element.appendChild(doc.createTextNode(get_text_content(span)))
dest.appendChild(code_element)
except ValueError:
print('[WARNING] Failed to parse inline codeblock', file=sys.stderr)
# Bail out by just copying the original HTML
dest.appendChild(child.cloneNode(True))
elif 'ContentFooter' in child.getAttribute('class') or 'hidden' in child.getAttribute('class'):
# These are nodes we just want to skip.
continue
elif child.tagName in ['span', 'div']:
# don't insert a span or div; just insert its contents
cleanup_tree(doc, child, dest)
# The remaining cases are: link, image (incl. math), and block code.
elif child.tagName == 'a':
# A link. We only want to copy the href, and pass the rest through.
new_node = doc.createElement('a')
href = child.getAttribute('href')
if href.startswith('/'):
href = 'http://quora.com' + href
new_node.setAttribute('href', href)
dest.appendChild(new_node)
cleanup_tree(doc, child, new_node)
elif child.tagName == 'img':
src = child.getAttribute('master_src')
if src == '':
src = child.getAttribute('src')
new_node = doc.createElement('img')
new_node.setAttribute('src', src)
new_node.setAttribute('alt', child.getAttribute('alt'))
if args.no_download:
dest.appendChild(new_node)
continue
# Save a copy of the image locally.
# If an error occurs, just leave the src pointing to Quora.
try:
m = re.search('/([^/?]+)(\?|$)', src)
if m is None:
raise ValueError()
filename = m.group(1)
if not filename.endswith('.png'):
filename += '.png'
try:
img_fd = os.open(args.output_dir + '/' + filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
except OSError as error:
if error.errno == errno.EEXIST:
log_if_v('Image %s has already been saved; skipping' % filename)
new_node.setAttribute('src', filename)
continue
else:
raise
log_if_v('Downloading image from %s' % src)
closed = False
try:
img = urllib.request.urlopen(src).read()
time.sleep(args.delay)
os.write(img_fd, img)
os.close(img_fd)
closed = True
except Exception:
os.close(img_fd)
closed = True
try:
os.remove(args.output_dir + '/' + filename)
except:
print('[WARNING] Failed to remove incomplete file %s' % filename, file=sys.stderr)
raise
finally:
if not closed:
os.close(img_fd)
# Don't leave the file there; we will retry it next time.
# If everything went according to plan, rewrite the src to the local file.
new_node.setAttribute('src', filename)
except urllib.error.URLError as error:
print('[WARNING] Failed to download image from URL %s (%s)' % (src, error.reason), file=sys.stderr)
except OSError as error:
print('[WARNING] Failed to save image from URL %s to file %s (%s)' % (src, filename, error.strerror), file=sys.stderr)
except ValueError:
print('[WARNING] Failed to determine image name from URL %s' % src, file=sys.stderr)
finally:
dest.appendChild(new_node)
elif 'codeblocktable' in child.getAttribute('class'):
# Block (not inline) code. This should become <pre><code>...</code></pre>
try:
pre_node = doc.createElement('pre')
# Each div inside is a line.
code_node = doc.createElement('code')
divs = child.getElementsByTagName('div')
lines = []
for div in divs:
# All the code is inside spans.
spans = div.getElementsByTagName('span')
line = ''.join([get_text_content(span) for span in spans])
lines.append(line)
text_node = doc.createTextNode('\n'.join(lines))
code_node.appendChild(text_node)
pre_node.appendChild(code_node)
dest.appendChild(pre_node)
except Exception:
print('[WARNING] Failed to parse code block', file=sys.stderr)
dest.appendChild(child.cloneNode(True))
else:
print('[WARNING] Unrecognized node', file=sys.stderr)
# Bail out by just copying the original HTML
dest.appendChild(child.cloneNode(True))
2
Example 10
Project: rail Source File: line.py
def compare_bam_and_variants(sample, working_dir, filter_script, genome,
unique=False, bedtools='bedtools',
samtools='samtools',
bigwigtobedgraph='bigWigToBedGraph'):
""" Compares BAMs, bigWigs, and BEDs for a given sample.
Uses samtools mpileup to obtain mismatches and indels from BAM output
of Rail-RNA as well as bigWigToBedGraph to obtain bedgraph with
mismatches. Gives diff between mpileup indels and indel BEDs from
Rail-RNA and diff between mpileup mismatches and mismatch bedgraph.
Note that indels aren't compared when unique=True because there are
no BEDs recording indels from uniquely aligning reads in Rail's output.
sample: sample name
working_dir: directory containing rail-rna_out after running Rail-RNA
filter_script: location of filter.py
genome: path to faidx-indexed reference fasta
unique: True iff only unique alignments should be included
bedtools: path to bedtools executable
samtools: path to samtools executable
Return value: tuple (True iff no diffs,
file containing diffs between bedgraphs)
"""
print >>sys.stderr, (
'Comparing {}BAM to{} mismatch bigWig with '
'samtools mpileup for sample {}...'
).format('unique alignments from ' if unique else '',
' indel BEDs and', sample)
alignments_basename = os.path.join(
working_dir, 'rail-rna_out',
'alignments', 'alignments'
)
coverage_dir = os.path.join(
working_dir, 'rail-rna_out',
'coverage_bigwigs'
)
# Get chromosome order
first_bam_view = '{samtools} view -H {first_bam}'.format(
samtools=samtools,
first_bam=glob.glob(alignments_basename + '.*.bam')[0]
)
chroms = subprocess.check_output(
('{first_bam_view} '
'| grep "@SQ" | cut -f2 | cut -d":" -f2').format(
first_bam_view=first_bam_view
), executable='/bin/bash', shell=True
)
bams = ['{alignments_basename}.{sample}.{chrom}.bam'.format(
alignments_basename=alignments_basename,
sample=sample,
chrom=chrom
) for chrom in chroms.strip().split('\n')]
# Eliminate chroms to which reads did not align
bams = ' '.join([bam for bam in bams if os.path.exists(bam)])
# Store mismatch tracks from BAM via mpileup
bam_base = {}
pileup_command = ('set -exo pipefail; '
'({first_bam_view};'
' for i in {bams};'
' do {samtools} view $i; done) '
'| {executable} {filter_script} ' +
('--uniques ' if unique else '') +
'| {samtools} mpileup -B -d 2147483647 -A -B -o 0 -F 0 '
'-Q 0 -q 0 -x -f {genome} ' +
('-I ' if unique else '') +
'--ff UNMAP,SECONDARY -').format(
first_bam_view=first_bam_view,
bams=bams,
executable=sys.executable,
filter_script=filter_script,
samtools=samtools,
genome=genome
)
pileup_process = subprocess.Popen(pileup_command,
stdout=subprocess.PIPE,
executable='/bin/bash',
shell=True)
try:
# For getting bedgraph coordinates right for mismatches
stretch, last_chrom, last_pos, last_mismatches = (defaultdict(int),
defaultdict(int),
defaultdict(int),
defaultdict(int))
with open(os.path.join(working_dir, 'from_bam.insertions.bed'), 'w') \
as bam_insertions, \
open(os.path.join(working_dir, 'from_bam.deletions.bed'), 'w') \
as bam_deletions, \
open(os.path.join(working_dir, 'from_bam.A.bedgraph'), 'w') \
as bam_base['A'], \
open(os.path.join(working_dir, 'from_bam.C.bedgraph'), 'w') \
as bam_base['C'], \
open(os.path.join(working_dir, 'from_bam.G.bedgraph'), 'w') \
as bam_base['G'], \
open(os.path.join(working_dir, 'from_bam.T.bedgraph'), 'w') \
as bam_base['T'], \
open(os.path.join(working_dir, 'from_bam.N.bedgraph'), 'w') \
as bam_base['N']:
for line in pileup_process.stdout:
chrom, pos, _, _, bases, _ = line.strip().split('\t')
pos = int(pos) - 1
pos_str = str(pos)
mismatches = []
if not unique:
insertions = defaultdict(int)
for insertion in re.finditer(
r'\+([0-9]+)([ACGTNacgtn]+)', bases
):
insertion_string = insertion.string[
insertion.start(2):insertion.start(2)
+ int(insertion.string[
insertion.start(1):insertion.end(1)
])
].upper()
mismatches.append(
insertion.string[
insertion.start(2):insertion.end(2)
][len(insertion_string):]
)
insertions[insertion_string] += 1
for insertion in insertions:
print >>bam_insertions, '\t'.join(map(str,
[chrom, pos_str, pos_str, insertion,
insertions[insertion_string]]
)
)
deletions = defaultdict(int)
for deletion in re.finditer(
r'\-([0-9]+)([ACGTNacgtn]+)', bases
):
deletion_string = deletion.string[
deletion.start(2):deletion.start(2)
+ int(deletion.string[
deletion.start(1):deletion.end(1)
])
].upper()
mismatches.append(
deletion.string[
deletion.start(2):deletion.end(2)
][len(deletion_string):]
)
deletions[deletion_string] += 1
for deletion in deletions:
print >>bam_deletions, '\t'.join(map(str,
[chrom, pos + 1, pos + 1 + len(deletion),
deletion, deletions[deletion]]
)
)
'''Residual mismatches are preceded by neither deletion
nor insertion'''
for mismatch in re.finditer(
r'(^|,|\.|\^.|<|>|\$)([ACGTNacgtn]+)', bases
):
mismatches.append(mismatch.string[
mismatch.start(2):mismatch.end(2)
].upper())
mismatches = ''.join(mismatches)
mismatches = { base : mismatches.count(base)
for base in 'ATCGN' }
for mismatch in mismatches:
if not mismatches[mismatch]: continue
if stretch[mismatch] and (
chrom != last_chrom[mismatch] or (
pos > last_pos[mismatch] + 1
) or (
mismatches[mismatch]
!= last_mismatches[mismatch]
)
):
# Write output
print >>bam_base[mismatch], '\t'.join(map(str,
[last_chrom[mismatch],
last_pos[mismatch] + 1
- stretch[mismatch],
last_pos[mismatch] + 1,
last_mismatches[mismatch]]
)
)
stretch[mismatch] = 1
else:
stretch[mismatch] += 1
last_mismatches[mismatch] = mismatches[mismatch]
last_pos[mismatch] = pos
last_chrom[mismatch] = chrom
# Last stretch has yet to be printed
for mismatch in last_mismatches:
print >>bam_base[mismatch], '\t'.join(map(str,
[last_chrom[mismatch],
last_pos[mismatch] + 1
- stretch[mismatch],
last_pos[mismatch] + 1,
last_mismatches[mismatch]]
)
)
finally:
pileup_process.stdout.close()
pileup_process.wait()
# Store mismatch tracks from bw via bigWigToBedGraph
unsorted_bedgraph_from_bw = os.path.join(working_dir,
'from_bw.temp.bedgraph')
for base in 'ATCGN':
bedgraph_from_bw = os.path.join(working_dir,
'from_bw.{base}.bedgraph'.format(
base=base
))
bw_to_bedgraph_command = (
'set -exo pipefail; '
'{bigwigtobedgraph} {coverage_bigwig} '
'{unsorted_bedgraph}; '
'sort -k1,1 -k2,2n -k3,3n {unsorted_bedgraph} '
'| awk \'$4 != 0\' >{final_bedgraph}; '
'rm {unsorted_bedgraph};'
).format(
bigwigtobedgraph=bigwigtobedgraph,
coverage_bigwig=os.path.join(
coverage_dir,
'{sample}.{base}{unique}.bw'.format(
sample=sample,
unique=('.unique' if unique else ''),
base=base
)
),
unsorted_bedgraph=unsorted_bedgraph_from_bw,
final_bedgraph=bedgraph_from_bw
)
try:
subprocess.check_call(
bw_to_bedgraph_command, executable='/bin/bash', shell=True
)
except subprocess.CalledProcessError as e:
raise RuntimeError(error_out(e, bw_to_bedgraph_command))
diff_output = { diff_type : os.path.join(
working_dir,
'{diff_type}_diffs.{sample}{unique}.txt'.format(
diff_type=diff_type,
sample=sample,
unique=('.unique' if unique else '')
)
) for diff_type in ['insertions', 'deletions',
'A', 'C', 'G', 'T', 'N'] }
for diff_type, from_rail, from_bam in ([
(indel_which,
os.path.join(working_dir, 'rail-rna_out',
'junctions_and_indels',
indel_which + '.' + sample + '.bed'),
os.path.join(working_dir, 'from_bam.' + indel_which + '.bed'))
for indel_which in ['insertions', 'deletions']
] if not unique else []) + [
(base,
os.path.join(working_dir, 'from_bw.' + base + '.bedgraph'),
os.path.join(working_dir, 'from_bam.' + base + '.bedgraph'))
for base in 'ACGTN'
]:
print >>sys.stderr, 'Comparing {}...'.format(diff_type)
diff_output = os.path.join(
working_dir,
'{diff_type}_diffs.{sample}{unique}.txt'.format(
diff_type=diff_type,
sample=sample,
unique=('.unique' if unique else '')
)
)
diff_command = ('set -exo pipefail; '
'diff <(sort -k1,1 -k2,2n -k3,3n {from_bam}) '
'<(cat {from_rail} |{remove_top_line} '
'sort -k1,1 -k2,2n -k3,3n) '
'>{diffs}').format(
from_bam=from_bam,
from_rail=from_rail,
diffs=diff_output,
# Track line needs removal for indel beds
remove_top_line=(' tail -n +2 |'
if diff_type in ['insertions', 'deletions']
else '')
)
try:
subprocess.check_call(
diff_command, executable='/bin/bash', shell=True
)
except subprocess.CalledProcessError as e:
if e.returncode > 1:
raise RuntimeError(error_out(e, diff_command))
if os.path.getsize(diff_output):
print >>sys.stderr, 'FAIL'
return (False, diff_output)
print >>sys.stderr, 'SUCCESS'
# Don't return a diff filename; no need
return (True, '')
2
Example 11
Project: docker-compose-swarm-mode Source File: docker_compose_swarm_mode.py
def convert(self):
# Based on http://stackoverflow.com/a/8661021
represent_dict_order = lambda _self, data: _self.represent_mapping('tag:yaml.org,2002:map', data.items())
yaml.add_representer(OrderedDict, represent_dict_order)
def project_prefix(value):
return '{}-{}'.format(self.project, value) if self.project else value
if self.networks:
print >> sys.stderr, ('WARNING: unsupported parameter "networks"')
for volume in self.volumes:
print >> sys.stderr, ('WARNING: unsupported parameter "volumes"')
for service in self.filtered_services:
service_config = self.services[service]
service = service.replace('_', '-')
service_result = OrderedDict([
('apiVersion', 'v1'),
('kind', 'Service'),
('metadata', OrderedDict([
('name', project_prefix(service)),
('labels', OrderedDict())
])),
('spec', OrderedDict([
('selector', OrderedDict())
]))
])
deployment_result = OrderedDict([
('apiVersion', 'extensions/v1beta1'),
('kind', 'Deployment'),
('metadata', OrderedDict([
('name', project_prefix(service))
])),
('spec', OrderedDict([
('replicas', 1),
('template', OrderedDict([
('metadata', OrderedDict([
('labels', OrderedDict())
])),
('spec', OrderedDict([
('containers', [OrderedDict([
('name', project_prefix(service)),
])])
]))
]))
]))
])
service_labels = service_result['metadata']['labels']
service_selector = service_result['spec']['selector']
deployment_labels = deployment_result['spec']['template']['metadata']['labels']
deployment_spec = deployment_result['spec']['template']['spec']
container = deployment_result['spec']['template']['spec']['containers'][0]
service_labels['service'] = self.project
service_labels['app'] = service
for parameter in service_config:
value = service_config[parameter]
def restart():
deployment_spec['restartPolicy'] = {'always': 'Always'}[value]
def logging():
pass # unsupported
def mem_limit():
container['resources'] = {'limits': {'memory': value.replace('m', 'Mi').replace('g', 'Gi')}}
def image():
container['image'] = value
def command():
if isinstance(value, list):
container['args'] = value
else:
container['args'] = value.split(' ')
def expose():
service_result['spec']['ports'] = []
container['ports'] = []
for port in value:
port_int = int(port)
service_result['spec']['ports'].append(OrderedDict([('port', port_int), ('targetPort', port_int), ('name', str(port_int))]))
container['ports'].append({'containerPort': port_int})
def container_name():
service_result['metadata']['name'] = value
deployment_result['metadata']['name'] = value
container['name'] = value
service_labels['app'] = value
def hostname():
pass # unsupported
def labels():
pass # TODO
def mode():
pass # TODO
def extra_hosts():
pass # unsupported
def ports():
for port in value:
pass # TODO
def networks():
pass # unsupported
def volumes():
container['volumeMounts'] = []
deployment_spec['volumes'] = []
for volume in value:
splitted_volume = volume.split(':')
src = splitted_volume.pop(0)
dst = splitted_volume.pop(0)
readonly = 0
if splitted_volume and splitted_volume[0] == 'ro':
readonly = 1
if src.startswith('.'):
src = src.replace('.', self.compose_base_dir, 1)
if src.startswith('/'):
volume_name = src.split('/')[-1].replace('.', '')
container['volumeMounts'].append(OrderedDict([('name', volume_name), ('mountPath', dst)]))
deployment_spec['volumes'].append(OrderedDict([('name', volume_name), ('hostPath', {'path': src})]))
# TODO readonly
else:
volume_name = src.replace('_', '-')
container['volumeMounts'].append(OrderedDict([('name', volume_name), ('mountPath', dst)]))
deployment_spec['volumes'].append(
OrderedDict([('name', volume_name), ('hostPath', {'path': '/volumes/' + project_prefix(volume_name)})]))
# TODO readonly
def environment():
if isinstance(value, dict):
container['env'] = []
for k, v in value.items():
container['env'].append(OrderedDict([('name', k), ('value', v)]))
else:
for env in value:
if env.startswith('constraint') or env.startswith('affinity'):
if 'nodeSelector' not in deployment_spec:
deployment_spec['nodeSelector'] = OrderedDict()
constraint = env.split(':', 2)[1]
selector = 'FIX_ME'
if constraint.startswith('node.hostname=='):
selector = 'kubernetes.io/hostname'
constraint = constraint.split('==')[1]
if constraint.startswith('engine.labels.'):
[selector, constraint] = constraint.split('==')
selector = selector.replace('engine.labels.', '')
deployment_spec['nodeSelector'][selector] = constraint
else:
if 'env' not in container:
container['env'] = []
[k, v] = env.split('=')
container['env'].append(OrderedDict([('name', k), ('value', v)]))
def replicas():
deployment_result['spec']['replicas'] = int(value)
def unsupported():
print >> sys.stderr, ('WARNING: unsupported parameter {}'.format(parameter))
locals().get(parameter, unsupported)()
service_selector.update(service_labels)
deployment_labels.update(service_labels)
sys.stdout.write(yaml.dump(service_result, default_flow_style=False))
print('---')
sys.stdout.write(yaml.dump(deployment_result, default_flow_style=False))
print('---')
2
Example 12
Project: inkscape-silhouette Source File: Strategy.py
def process_pyramids_barrier(s, y_slice, max_y, left2right=True):
""" finding the next point involves overshadowing other points.
Our assumption is, that it is save to cut the paper at point A,
whenever there is a triangle sitting on the baseline (where the
transport rollers are) with 2x 45 degree coming from both sides,
meeting at 90 degrees at point A, so that the inside of the
triangle is free of any cuts.
We prefer to cut away from the rollers, if possible, but that is
subordinate rule -- applicable, whenever the cut direction can
be freely chosen. If point A is already part of a cut, then we cut the
path A-B always towards A, never starting at A.
A horizontal barrier Y_bar exists, that limits our downwards movement temporarily.
We assume to be called again with lowered Y_bar (increased max_y, it counts downwards).
Another barrier Xf_bar is a forward slanted 45 degree barrier that is swept sideways.
Points become eligible, if they are above Y_bar and behind Xf_bar.
We start with the sideways barrier from left to right aka increasing x.
In this case 'behind' means to the left of Xf_bar. (Every second sweep
will be the opposite direction, but below only left to right is
discussed).
The very first point that is behind Xf_bar is the starting point A. Then we iterate:
From any previous point A, we prefer to follow a line segment to reach
the next point B. Segments are eligible, if their B is rightward from A,
(B.x greater or equal A.x). We chose the segment with the lowest B.y coordinate
if there is any choice and check the following conditions:
a) B is below Y_bar.
Compute point C as the intersection of Y_bar with [AB]. Replace
the segment [AB] by segments [AC], [CB]. Let B and C swap names.
b) B is 45 degrees or more downwards from A (B.x-A.x < B.y-A.y)
We make an extra check to see if B would overshadow any point in the other
direction. Temporarily apply a backwards slanted barrier Xb_bar in A.
While moving the barrier to B, stop at the first point D to the left of AB
(i.e. ccw(A,B,D) == True) it hits, if any.
If so, position Xb_bar in D, compute point E as the intersection of Xb_bar
with A-B. Replace the segment [AB] by segments [AE], [EB].
If we have a point C remembered from a), then replace segments [EB], [BC] with [EC]
and garbage collect point B and swap back roles B and C.
Let B and E swap names.
c) B is more than 45 degrees upwards from A. This overshadows A. But we ignore
that A may have more segments to be done. We keep that B and consider
the issue with A unsolvable.
Note that 'advancing' from A to B in this case is actually backwards in
Xf_bar's view.
If we now have no B, then we simply move the sideways barrier to reveal our
next A -- very likely a jump rather than a cut. If no todo segments are left in
the old A, drop that old A. Iterate.
But if we have a B and it is not case c), then we tentatively advance Xf_bar
from A to B and record all new points F[] in the order we pass them. We
don't care about them, if they are all 'below' (on the right hand side
of) segment [AB]. For the first point F, that has ccw(A,B,F) == True,
we position Xf_bar in F, if any. If so, we compute point G as the
intersection of Xf_bar with [AB]. Replace the segment [AB] by segments
[AG], [GB]. We cut segment [AG]. We make F our next A - very likely a
jump. If no todo segments are left in the old A, drop that old A.
Iterate.
Exception for all subdivide actions above: if the segment is shorter than
self.min_subdivide, then just keep it as is.
If subdividing produces segments shorter than self.min_segmentlen, then we later
garbage collect such segments. overshoot shoud be more than min_segmentlen to
compensate for this.
If iteration exhausts, we are done with this processing sweep and
report back the lowest remaining min_y coordinate of all points we left
behind with segments todo. The next sweep will go the other direction.
Caller should call us again with direction toggled the other way, and
possibly advancing max_y = min_y + monotone_back_travel. The
variable barrier_increment is not used here, as we compute the
increment.
In the above context, 'cutting' a segment means, to add it to the output
list to deactivate its seg[] entries in the endpoints. Endpoints
without active segments do not contribute to the min_y computation
above, they are dropped.
When all points are dropped, we did our final sweep and return min_y =
None. It is caller's responsibility to check the direction of each cut
in the s.output list with regards to sharp points and cutting-towards-the-rollers.
Assert that we cut at least one segment per sweep or drop at least one
point per sweep. Also the number of added segments and points should
be less than what we eventually output and drop.
If not, the above algorithm may never end.
"""
Xf_bar = Barrier(y_slice, key=lambda a: a[0]+a[1] if a else 0) # forward: / moving ->
Xb_bar = Barrier(y_slice, key=lambda a: -a[0]+a[1] if a else 0) # backwards: \ moving <-
if not left2right: # forward: \ moving <-
Xf_bar,Xb_bar = Xb_bar,Xf_bar # backwards: / moving ->
A = Xf_bar.point()
while True:
if A is None:
Ai = Xf_bar.next()
if Ai is None: break
A = Xf_bar.point()
continue
print >>sys.stderr, "process_pyramids_barrier", left2right, A, A.att()
B = None
a_todo = 0
for Bi in A.seg:
if Bi >= 0: # segment already done
a_todo += 1
pt = s.points[Bi]
if A.y+s.min_segmentlen >= max_y and pt.y > A.y:
continue # Do not look downward when close to max_y.
# This avoids a busyloop after hitting Y_bar:
# ... subdiv, advance, retry, hit, subdiv, ...
if left2right:
if pt.x >= A.x:
if B is None or not ccw(A,B,pt): # find the leftmost of all [AB]
B = pt
else: # not left2right
if pt.x <= A.x:
if B is None or ccw(A,B,pt): # find the rightmost of all [AB]
B = pt
if B is None:
print "no more forward segments", A, a_todo
Xb_bar.find(A, start=0)
if a_todo == 0:
s.points[A.id] = None # drop A
while True:
Ai = Xf_bar.next()
A = None
print >>sys.stderr, "xx next Ai candidate", Ai
if Ai is None: break
A = Xf_bar.point()
print >>sys.stderr, "xx next A candidate", A
if A is None: break
if not Xb_bar.ahead(A):
break
else:
print >>sys.stderr, "process_pyramids_barrier jump: Ignored A, ahead of Xb_bar", A
print >>sys.stderr, "process_pyramids_barrier jump to next A", A
continue # just advance Xf_bar: jump
print "segment to check a), b)", A, B
if False: # fake to trigger check a)
b_id = B.id
B = XY_a((1.3,20-2.1))
B.id = b_id
B.seg = [1,2,A.id,3,4]
print "faked segment to check a), b)", A, B
#
subdividable_ab = bool(dist_sq(A,B) > s.min_subdivide_sq)
C = None
E = None
if subdividable_ab and B.y > max_y: # check a)
C = XY_a((intersect_y(A,B, max_y), max_y))
## same, but more expensive:
# C2 = intersect_lines(A,B,XY_a((0,max_y)),XY_a((.5,max_y)))
print "B below barrier, C=", C
s.subdivide_segment(A,B,C)
Xf_bar.insert(C)
Xb_bar.insert(C)
B,C = C,B
# print A.seg, B.seg, C.seg
#
# All of the following shortens [AB] sufficiently, so that B does not
# cast shadow upwards on any other point: \B/
# Such a point would become our B.
# This asserts that there is no other point, whose pyramid would bury B.
# check b)
if (subdividable_ab and (
(left2right and B.x-A.x < B.y-A.y) or (not left2right and A.x-B.x < B.y-A.y))):
Xb_a_idx = Xb_bar.find(A, start=0) # could also use lookup() here. It does not matter.
Xb_b_idx = Xb_bar.find(B) # could also use lookup() here. It does not matter.
print "check b), moving Xb_bar from A to B", A, B, Xb_a_idx, Xb_b_idx, Xb_bar.key(A), Xb_bar.key(B)
D = None
for n in range(Xb_a_idx, Xb_b_idx+1): # sweep from A to B
pt = Xb_bar.point(n)
if pt.id != A.id and pt.id != B.id and ccw(A,B,pt) == True:
D = pt # found a D that is clearly left of AB.
break
else:
print "backsweep ignoring pt", pt, Xb_bar.key(pt)
#
if D is not None: # compute intersection of Xb_bar with [AB]
E = intersect_lines(D,XY_a((D.x+1,D.y+1)),A,B,limit2=True)
if E is None: raise ValueError("finding a shadowed D failed:", A, B, D)
E = XY_a(E)
s.subdivide_segment(A,B,E)
Xf_bar.insert(E)
Xb_bar.insert(E)
if C is not None:
s.shortcut_segment(E,C,B) # replace segments [EB], [BC] with [EC]
B,C = C,B
B,E = E,B
# tentatively advance Xf_bar from A to B
Xf_a_idx = Xf_bar.pos() # unused, we never move back to A.
Xf_b_idx = Xf_bar.lookup(lambda b: b.id==B.id if b else False)
if Xf_b_idx is None: # Should never happen!
print "Xf_bar.lookup(B)=None. B=",B # Okayish fallback, but find() may return
Xf_b_idx = Xf_bar.find(B) # a different point with the same key().
print "line A,B:", A, B, Xf_a_idx, Xf_b_idx, Xf_bar.point(Xf_b_idx)
F = None
Xf_f_idx = None
for n in range(Xf_a_idx, Xf_b_idx+1): # sweep from A to B (inclusive)
pt = Xf_bar.point(n)
if pt is None: continue
if subdividable_ab and pt.id != A.id and pt.id != B.id and ccw(A,B,pt) == (not left2right):
F = pt # found an F that is clearly right of AB.
Xf_f_idx = n
break
else:
print "forward sweep ignoring pt", n, pt, pt.id, Xf_bar.key(pt)
#
if F is not None: # compute intersection of Xb_bar with [AB]
_F_back = (F.x-1,F.y+1) if left2right else (F.x+1,F.y+1)
G = intersect_lines(F,XY_a(_F_back),A,B,limit2=True)
if G is None: raise ValueError("finding a shadowed G failed:", A, B, F, _F_back)
G = XY_a(G)
s.subdivide_segment(A,B,G)
Xf_bar.insert(G)
Xb_bar.insert(G)
if E is not None:
pass
## FIXME: should s.shortcut_segment(...E) something here.
s.output_add(A,G,cut=True)
s.unlink_segment(A,G)
Xf_bar.pos(Xf_f_idx) # advance to F, further up on the same barrier as G
A = Xf_bar.point()
#
else:
s.output_add(A,B,cut=True)
s.unlink_segment(A,B)
Xf_bar.pos(Xf_b_idx) # advance
A = Xf_bar.point()
print >>sys.stderr, "advanced A to",A
## barrier has moved all the way to the other end.
print >>sys.stderr, "barrier moved all the way", Xf_bar.points, max_y, A.att() if A else None, A.seg if A else None
2
Example 13
Project: OWASP-ZSC Source File: rlmain.py
def read_inputrc(self, #in 2.4 we cannot call expanduser with unicode string
inputrcpath=os.path.expanduser(ensure_str("~/pyreadlineconfig.ini"))):
modes = dict([(x.mode, x) for x in self.editingmodes])
mode = self.editingmodes[0].mode
def setmode(name):
self.mode = modes[name]
def bind_key(key, name):
import types
if callable(name):
modes[mode]._bind_key(key, types.MethodType(name, modes[mode]))
elif hasattr(modes[mode], name):
modes[mode]._bind_key(key, getattr(modes[mode], name))
else:
print("Trying to bind unknown command '%s' to key '%s'" %
(name, key))
def un_bind_key(key):
keyinfo = make_KeyPress_from_keydescr(key).tuple()
if keyinfo in modes[mode].key_dispatch:
del modes[mode].key_dispatch[keyinfo]
def bind_exit_key(key):
modes[mode]._bind_exit_key(key)
def un_bind_exit_key(key):
keyinfo = make_KeyPress_from_keydescr(key).tuple()
if keyinfo in modes[mode].exit_dispatch:
del modes[mode].exit_dispatch[keyinfo]
def setkill_ring_to_clipboard(killring):
import pyreadline.lineeditor.lineobj
pyreadline.lineeditor.lineobj.kill_ring_to_clipboard = killring
def sethistoryfilename(filename):
self.mode._history.history_filename = os.path.expanduser(
ensure_str(filename))
def setbellstyle(mode):
self.bell_style = mode
def disable_readline(mode):
self.disable_readline = mode
def sethistorylength(length):
self.mode._history.history_length = int(length)
def allow_ctrl_c(mode):
log("allow_ctrl_c:%s:%s" % (self.allow_ctrl_c, mode))
self.allow_ctrl_c = mode
def setbellstyle(mode):
self.bell_style = mode
def show_all_if_ambiguous(mode):
self.mode.show_all_if_ambiguous = mode
def ctrl_c_tap_time_interval(mode):
self.ctrl_c_tap_time_interval = mode
def mark_directories(mode):
self.mode.mark_directories = mode
def completer_delims(delims):
self.mode.completer_delims = delims
def complete_filesystem(delims):
self.mode.complete_filesystem = delims.lower()
def enable_ipython_paste_for_paths(boolean):
self.mode.enable_ipython_paste_for_paths = boolean
def debug_output(
on, filename="pyreadline_debug_log.txt"): #Not implemented yet
if on in ["on", "on_nologfile"]:
self.debug = True
if on == "on":
logger.start_file_log(filename)
logger.start_socket_log()
logger.log("STARTING LOG")
elif on == "on_nologfile":
logger.start_socket_log()
logger.log("STARTING LOG")
else:
logger.log("STOPING LOG")
logger.stop_file_log()
logger.stop_socket_log()
_color_trtable = {"black": 0,
"darkred": 4,
"darkgreen": 2,
"darkyellow": 6,
"darkblue": 1,
"darkmagenta": 5,
"darkcyan": 3,
"gray": 7,
"red": 4 + 8,
"green": 2 + 8,
"yellow": 6 + 8,
"blue": 1 + 8,
"magenta": 5 + 8,
"cyan": 3 + 8,
"white": 7 + 8}
def set_prompt_color(color):
self.prompt_color = self._color_trtable.get(color.lower(), 7)
def set_input_color(color):
self.command_color = self._color_trtable.get(color.lower(), 7)
loc = {"branch": release.branch,
"version": release.version,
"mode": mode,
"modes": modes,
"set_mode": setmode,
"bind_key": bind_key,
"disable_readline": disable_readline,
"bind_exit_key": bind_exit_key,
"un_bind_key": un_bind_key,
"un_bind_exit_key": un_bind_exit_key,
"bell_style": setbellstyle,
"mark_directories": mark_directories,
"show_all_if_ambiguous": show_all_if_ambiguous,
"completer_delims": completer_delims,
"complete_filesystem": complete_filesystem,
"debug_output": debug_output,
"history_filename": sethistoryfilename,
"history_length": sethistorylength,
"set_prompt_color": set_prompt_color,
"set_input_color": set_input_color,
"allow_ctrl_c": allow_ctrl_c,
"ctrl_c_tap_time_interval": ctrl_c_tap_time_interval,
"kill_ring_to_clipboard": setkill_ring_to_clipboard,
"enable_ipython_paste_for_paths":
enable_ipython_paste_for_paths, }
if os.path.isfile(inputrcpath):
try:
execfile(inputrcpath, loc, loc)
except Exception as x:
raise
import traceback
print("Error reading .pyinputrc", file=sys.stderr)
filepath, lineno = traceback.extract_tb(sys.exc_traceback)[
1][:2]
print("Line: %s in file %s" % (lineno, filepath),
file=sys.stderr)
print(x, file=sys.stderr)
raise ReadlineError("Error reading .pyinputrc")
2
Example 14
Project: ganeti Source File: daemon.py
def GenericMain(daemon_name, optionparser,
check_fn, prepare_fn, exec_fn,
multithreaded=False, console_logging=False,
default_ssl_cert=None, default_ssl_key=None,
warn_breach=False):
"""Shared main function for daemons.
@type daemon_name: string
@param daemon_name: daemon name
@type optionparser: optparse.OptionParser
@param optionparser: initialized optionparser with daemon-specific options
(common -f -d options will be handled by this module)
@type check_fn: function which accepts (options, args)
@param check_fn: function that checks start conditions and exits if they're
not met
@type prepare_fn: function which accepts (options, args)
@param prepare_fn: function that is run before forking, or None;
it's result will be passed as the third parameter to exec_fn, or
if None was passed in, we will just pass None to exec_fn
@type exec_fn: function which accepts (options, args, prepare_results)
@param exec_fn: function that's executed with the daemon's pid file held, and
runs the daemon itself.
@type multithreaded: bool
@param multithreaded: Whether the daemon uses threads
@type console_logging: boolean
@param console_logging: if True, the daemon will fall back to the system
console if logging fails
@type default_ssl_cert: string
@param default_ssl_cert: Default SSL certificate path
@type default_ssl_key: string
@param default_ssl_key: Default SSL key path
@type warn_breach: bool
@param warn_breach: issue a warning at daemon launch time, before
daemonizing, about the possibility of breaking parameter privacy
invariants through the otherwise helpful debug logging.
"""
optionparser.add_option("-f", "--foreground", dest="fork",
help="Don't detach from the current terminal",
default=True, action="store_false")
optionparser.add_option("-d", "--debug", dest="debug",
help="Enable some debug messages",
default=False, action="store_true")
optionparser.add_option("--syslog", dest="syslog",
help="Enable logging to syslog (except debug"
" messages); one of 'no', 'yes' or 'only' [%s]" %
constants.SYSLOG_USAGE,
default=constants.SYSLOG_USAGE,
choices=["no", "yes", "only"])
family = ssconf.SimpleStore().GetPrimaryIPFamily()
# family will default to AF_INET if there is no ssconf file (e.g. when
# upgrading a cluster from 2.2 -> 2.3. This is intended, as Ganeti clusters
# <= 2.2 can not be AF_INET6
if daemon_name in constants.DAEMONS_PORTS:
default_bind_address = constants.IP4_ADDRESS_ANY
if family == netutils.IP6Address.family:
default_bind_address = constants.IP6_ADDRESS_ANY
default_port = netutils.GetDaemonPort(daemon_name)
# For networked daemons we allow choosing the port and bind address
optionparser.add_option("-p", "--port", dest="port",
help="Network port (default: %s)" % default_port,
default=default_port, type="int")
optionparser.add_option("-b", "--bind", dest="bind_address",
help=("Bind address (default: '%s')" %
default_bind_address),
default=default_bind_address, metavar="ADDRESS")
optionparser.add_option("-i", "--interface", dest="bind_interface",
help=("Bind interface"), metavar="INTERFACE")
if default_ssl_key is not None and default_ssl_cert is not None:
optionparser.add_option("--no-ssl", dest="ssl",
help="Do not secure HTTP protocol with SSL",
default=True, action="store_false")
optionparser.add_option("-K", "--ssl-key", dest="ssl_key",
help=("SSL key path (default: %s)" %
default_ssl_key),
default=default_ssl_key, type="string",
metavar="SSL_KEY_PATH")
optionparser.add_option("-C", "--ssl-cert", dest="ssl_cert",
help=("SSL certificate path (default: %s)" %
default_ssl_cert),
default=default_ssl_cert, type="string",
metavar="SSL_CERT_PATH")
# Disable the use of fork(2) if the daemon uses threads
if multithreaded:
utils.DisableFork()
options, args = optionparser.parse_args()
if getattr(options, "bind_interface", None) is not None:
if options.bind_address != default_bind_address:
msg = ("Can't specify both, bind address (%s) and bind interface (%s)" %
(options.bind_address, options.bind_interface))
print >> sys.stderr, msg
sys.exit(constants.EXIT_FAILURE)
interface_ip_addresses = \
netutils.GetInterfaceIpAddresses(options.bind_interface)
if family == netutils.IP6Address.family:
if_addresses = interface_ip_addresses[constants.IP6_VERSION]
else:
if_addresses = interface_ip_addresses[constants.IP4_VERSION]
if len(if_addresses) < 1:
msg = "Failed to find IP for interface %s" % options.bind_interace
print >> sys.stderr, msg
sys.exit(constants.EXIT_FAILURE)
options.bind_address = if_addresses[0]
if getattr(options, "ssl", False):
ssl_paths = {
"certificate": options.ssl_cert,
"key": options.ssl_key,
}
for name, path in ssl_paths.iteritems():
if not os.path.isfile(path):
print >> sys.stderr, "SSL %s file '%s' was not found" % (name, path)
sys.exit(constants.EXIT_FAILURE)
# TODO: By initiating http.HttpSslParams here we would only read the files
# once and have a proper validation (isfile returns False on directories)
# at the same time.
result, running_uid, expected_uid = _VerifyDaemonUser(daemon_name)
if not result:
msg = ("%s started using wrong user ID (%d), expected %d" %
(daemon_name, running_uid, expected_uid))
print >> sys.stderr, msg
sys.exit(constants.EXIT_FAILURE)
if check_fn is not None:
check_fn(options, args)
log_filename = constants.DAEMONS_LOGFILES[daemon_name]
# node-daemon logging in lib/http/server.py, _HandleServerRequestInner
if options.debug and warn_breach:
sys.stderr.write(constants.DEBUG_MODE_CONFIDENTIALITY_WARNING % daemon_name)
if options.fork:
# Newer GnuTLS versions (>= 3.3.0) use a library constructor for
# initialization and open /dev/urandom on library load time, way before we
# fork(). Closing /dev/urandom causes subsequent ganeti.http.client
# requests to fail and the process to receive a SIGABRT. As we cannot
# reliably detect GnuTLS's socket, we work our way around this by keeping
# all fds referring to /dev/urandom open.
noclose_fds = []
for fd in os.listdir("/proc/self/fd"):
try:
if os.readlink(os.path.join("/proc/self/fd", fd)) == "/dev/urandom":
noclose_fds.append(int(fd))
except EnvironmentError:
# The fd might have disappeared (although it shouldn't as we're running
# single-threaded).
continue
utils.CloseFDs(noclose_fds=noclose_fds)
(wpipe, stdio_reopen_fn) = utils.Daemonize(logfile=log_filename)
else:
(wpipe, stdio_reopen_fn) = (None, None)
log_reopen_fn = \
utils.SetupLogging(log_filename, daemon_name,
debug=options.debug,
stderr_logging=not options.fork,
multithreaded=multithreaded,
syslog=options.syslog,
console_logging=console_logging)
# Reopen log file(s) on SIGHUP
signal.signal(signal.SIGHUP,
compat.partial(_HandleSigHup, [log_reopen_fn, stdio_reopen_fn]))
try:
utils.WritePidFile(utils.DaemonPidFileName(daemon_name))
except errors.PidFileLockError, err:
print >> sys.stderr, "Error while locking PID file:\n%s" % err
sys.exit(constants.EXIT_FAILURE)
try:
try:
logging.info("%s daemon startup", daemon_name)
if callable(prepare_fn):
prep_results = prepare_fn(options, args)
else:
prep_results = None
except Exception, err:
utils.WriteErrorToFD(wpipe, _BeautifyError(err))
raise
if wpipe is not None:
# we're done with the preparation phase, we close the pipe to
# let the parent know it's safe to exit
os.close(wpipe)
exec_fn(options, args, prep_results)
finally:
utils.RemoveFile(utils.DaemonPidFileName(daemon_name))
2
Example 15
Project: ros_buildfarm Source File: release_job.py
def configure_release_jobs(
config_url, rosdistro_name, release_build_name, groovy_script=None,
dry_run=False, whitelist_package_names=None):
"""
Configure all Jenkins release jobs.
L{configure_release_job} will be invoked for every released package and
target which matches the build file criteria.
Additionally a job to import Debian packages into the Debian repository is
created.
"""
config = get_config_index(config_url)
build_files = get_release_build_files(config, rosdistro_name)
build_file = build_files[release_build_name]
index = get_index(config.rosdistro_index_url)
# get targets
platforms = []
for os_name in build_file.targets.keys():
for os_code_name in build_file.targets[os_name].keys():
platforms.append((os_name, os_code_name))
print('The build file contains the following targets:')
for os_name, os_code_name in platforms:
print(' - %s %s: %s' % (os_name, os_code_name, ', '.join(
build_file.targets[os_name][os_code_name])))
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
print('No distribution file matches the build file')
return
pkg_names = dist_file.release_packages.keys()
filtered_pkg_names = build_file.filter_packages(pkg_names)
explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names)
if explicitly_ignored_pkg_names:
print(('The following packages are being %s because of ' +
'white-/blacklisting:') %
('ignored' if build_file.skip_ignored_packages else 'disabled'))
for pkg_name in sorted(explicitly_ignored_pkg_names):
print(' -', pkg_name)
dist_cache = get_distribution_cache(index, rosdistro_name)
if explicitly_ignored_pkg_names:
# get direct dependencies from distro cache for each package
direct_dependencies = {}
for pkg_name in pkg_names:
direct_dependencies[pkg_name] = _get_direct_dependencies(
pkg_name, dist_cache, pkg_names) or set([])
# find recursive downstream deps for all explicitly ignored packages
ignored_pkg_names = set(explicitly_ignored_pkg_names)
while True:
implicitly_ignored_pkg_names = _get_downstream_package_names(
ignored_pkg_names, direct_dependencies)
if implicitly_ignored_pkg_names - ignored_pkg_names:
ignored_pkg_names |= implicitly_ignored_pkg_names
continue
break
implicitly_ignored_pkg_names = \
ignored_pkg_names - explicitly_ignored_pkg_names
if implicitly_ignored_pkg_names:
print(('The following packages are being %s because their ' +
'dependencies are being ignored:') % ('ignored'
if build_file.skip_ignored_packages else 'disabled'))
for pkg_name in sorted(implicitly_ignored_pkg_names):
print(' -', pkg_name)
filtered_pkg_names = \
set(filtered_pkg_names) - implicitly_ignored_pkg_names
# all further configuration will be handled by either the Jenkins API
# or by a generated groovy script
jenkins = connect(config.jenkins_url) if groovy_script is None else False
all_view_configs = {}
all_job_configs = OrderedDict()
job_name, job_config = configure_import_package_job(
config_url, rosdistro_name, release_build_name,
config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
if not jenkins:
all_job_configs[job_name] = job_config
job_name, job_config = configure_sync_packages_to_main_job(
config_url, rosdistro_name, release_build_name,
config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
if not jenkins:
all_job_configs[job_name] = job_config
for os_name, os_code_name in platforms:
for arch in sorted(build_file.targets[os_name][os_code_name]):
job_name, job_config = configure_sync_packages_to_testing_job(
config_url, rosdistro_name, release_build_name,
os_code_name, arch,
config=config, build_file=build_file, jenkins=jenkins,
dry_run=dry_run)
if not jenkins:
all_job_configs[job_name] = job_config
targets = []
for os_name, os_code_name in platforms:
targets.append((os_name, os_code_name, 'source'))
for arch in build_file.targets[os_name][os_code_name]:
targets.append((os_name, os_code_name, arch))
views = configure_release_views(
jenkins, rosdistro_name, release_build_name, targets,
dry_run=dry_run)
if not jenkins:
all_view_configs.update(views)
groovy_data = {
'dry_run': dry_run,
'expected_num_views': len(views),
}
# binary jobs must be generated in topological order
from catkin_pkg.package import parse_package_string
from ros_buildfarm.common import topological_order_packages
pkgs = {}
for pkg_name in pkg_names:
if pkg_name not in dist_cache.release_package_xmls:
print("Skipping package '%s': no released package.xml in cache" %
(pkg_name), file=sys.stderr)
continue
pkg_xml = dist_cache.release_package_xmls[pkg_name]
pkg = parse_package_string(pkg_xml)
pkgs[pkg_name] = pkg
ordered_pkg_tuples = topological_order_packages(pkgs)
other_build_files = [v for k, v in build_files.items() if k != release_build_name]
all_source_job_names = []
all_binary_job_names = []
for pkg_name in [p.name for _, p in ordered_pkg_tuples]:
if whitelist_package_names:
if pkg_name not in whitelist_package_names:
print("Skipping package '%s' not in the explicitly passed list" %
pkg_name, file=sys.stderr)
continue
pkg = dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = dist_file.repositories[repo_name]
is_disabled = pkg_name not in filtered_pkg_names
if is_disabled and build_file.skip_ignored_packages:
print("Skipping ignored package '%s' in repository '%s'" %
(pkg_name, repo_name), file=sys.stderr)
continue
if not repo.release_repository:
print(("Skipping package '%s' in repository '%s': no release " +
"section") % (pkg_name, repo_name), file=sys.stderr)
continue
if not repo.release_repository.version:
print(("Skipping package '%s' in repository '%s': no release " +
"version") % (pkg_name, repo_name), file=sys.stderr)
continue
for os_name, os_code_name in platforms:
other_build_files_same_platform = []
for other_build_file in other_build_files:
if os_name not in other_build_file.targets:
continue
if os_code_name not in other_build_file.targets[os_name]:
continue
other_build_files_same_platform.append(other_build_file)
try:
source_job_names, binary_job_names, job_configs = \
configure_release_job(
config_url, rosdistro_name, release_build_name,
pkg_name, os_name, os_code_name,
config=config, build_file=build_file,
index=index, dist_file=dist_file,
dist_cache=dist_cache,
jenkins=jenkins, views=views,
generate_import_package_job=False,
generate_sync_packages_jobs=False,
is_disabled=is_disabled,
other_build_files_same_platform=other_build_files_same_platform,
groovy_script=groovy_script,
dry_run=dry_run)
all_source_job_names += source_job_names
all_binary_job_names += binary_job_names
if groovy_script is not None:
print('Configuration for jobs: ' +
', '.join(source_job_names + binary_job_names))
for source_job_name in source_job_names:
all_job_configs[source_job_name] = job_configs[source_job_name]
for binary_job_name in binary_job_names:
all_job_configs[binary_job_name] = job_configs[binary_job_name]
except JobValidationError as e:
print(e.message, file=sys.stderr)
groovy_data['expected_num_jobs'] = len(all_job_configs)
groovy_data['job_prefixes_and_names'] = {}
# with an explicit list of packages we don't delete obsolete jobs
if not whitelist_package_names:
# delete obsolete binary jobs
for os_name, os_code_name in platforms:
for arch in build_file.targets[os_name][os_code_name]:
binary_view = get_release_binary_view_name(
rosdistro_name, release_build_name,
os_name, os_code_name, arch)
binary_job_prefix = '%s__' % binary_view
excluded_job_names = set([
j for j in all_binary_job_names
if j.startswith(binary_job_prefix)])
if groovy_script is None:
print("Removing obsolete binary jobs with prefix '%s'" %
binary_job_prefix)
remove_jobs(
jenkins, binary_job_prefix, excluded_job_names,
dry_run=dry_run)
else:
binary_key = 'binary_%s_%s_%s' % \
(os_name, os_code_name, arch)
groovy_data['job_prefixes_and_names'][binary_key] = \
(binary_job_prefix, excluded_job_names)
# delete obsolete source jobs
# requires knowledge about all other release build files
for os_name, os_code_name in platforms:
other_source_job_names = []
# get source job names for all other release build files
for other_release_build_name in [
k for k in build_files.keys() if k != release_build_name]:
other_build_file = build_files[other_release_build_name]
other_dist_file = get_distribution_file(
index, rosdistro_name, other_build_file)
if not other_dist_file:
continue
if os_name not in other_build_file.targets or \
os_code_name not in other_build_file.targets[os_name]:
continue
if other_build_file.skip_ignored_packages:
filtered_pkg_names = other_build_file.filter_packages(
pkg_names)
else:
filtered_pkg_names = pkg_names
for pkg_name in sorted(filtered_pkg_names):
pkg = other_dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = other_dist_file.repositories[repo_name]
if not repo.release_repository:
continue
if not repo.release_repository.version:
continue
other_job_name = get_sourcedeb_job_name(
rosdistro_name, other_release_build_name,
pkg_name, os_name, os_code_name)
other_source_job_names.append(other_job_name)
source_view_prefix = get_release_source_view_name(
rosdistro_name, os_name, os_code_name)
source_job_prefix = '%s__' % source_view_prefix
excluded_job_names = set([
j for j in (all_source_job_names + other_source_job_names)
if j.startswith(source_job_prefix)])
if groovy_script is None:
print("Removing obsolete source jobs with prefix '%s'" %
source_job_prefix)
remove_jobs(
jenkins, source_job_prefix, excluded_job_names,
dry_run=dry_run)
else:
source_key = 'source_%s_%s' % (os_name, os_code_name)
groovy_data['job_prefixes_and_names'][source_key] = (
source_job_prefix, excluded_job_names)
if groovy_script is not None:
print(
"Writing groovy script '%s' to reconfigure %d views and %d jobs" %
(groovy_script, len(all_view_configs), len(all_job_configs)))
content = expand_template(
'snippet/reconfigure_jobs.groovy.em', groovy_data)
write_groovy_script_and_configs(
groovy_script, content, all_job_configs,
view_configs=all_view_configs)
2
Example 16
Project: ns-3 Source File: ns3modulescan-modular.py
def __call__(self, module_parser,
pygccxml_definition,
global_annotations,
parameter_annotations):
try:
ns3_header = get_ns3_relative_path(pygccxml_definition.location.file_name)
except ValueError: # the header is not from ns3
return # ignore the definition, it's not ns-3 def.
definition_module = self.headers_map[ns3_header]
## Note: we don't include line numbers in the comments because
## those numbers are very likely to change frequently, which would
## cause needless changes, since the generated python files are
## kept under version control.
#global_annotations['pygen_comment'] = "%s:%i: %s" % \
# (ns3_header, pygccxml_definition.location.line, pygccxml_definition)
global_annotations['pygen_comment'] = "%s (module %r): %s" % \
(ns3_header, definition_module, pygccxml_definition)
## handle ns3::Object::GetObject (left to its own devices,
## pybindgen will generate a mangled name containing the template
## argument type name).
if isinstance(pygccxml_definition, member_function_t) \
and pygccxml_definition.parent.name == 'Object' \
and pygccxml_definition.name == 'GetObject':
template_args = templates.args(pygccxml_definition.demangled_name)
if template_args == ['ns3::Object']:
global_annotations['template_instance_names'] = 'ns3::Object=>GetObject'
## Don't wrap Simulator::Schedule* (manually wrapped)
if isinstance(pygccxml_definition, member_function_t) \
and pygccxml_definition.parent.name == 'Simulator' \
and pygccxml_definition.name.startswith('Schedule'):
global_annotations['ignore'] = None
# manually wrapped
if isinstance(pygccxml_definition, member_function_t) \
and pygccxml_definition.parent.name == 'Simulator' \
and pygccxml_definition.name == 'Run':
global_annotations['ignore'] = True
## http://www.gccxml.org/Bug/view.php?id=9915
if isinstance(pygccxml_definition, calldef_t):
for arg in pygccxml_definition.arguments:
if arg.default_value is None:
continue
elif arg.default_value == "ns3::MilliSeconds( )":
arg.default_value = "ns3::MilliSeconds(0)"
elif arg.default_value == "ns3::Seconds( )":
arg.default_value = "ns3::Seconds(0)"
## classes
if isinstance(pygccxml_definition, class_t):
print >> sys.stderr, pygccxml_definition
# no need for helper classes to allow subclassing in Python, I think...
#if pygccxml_definition.name.endswith('Helper'):
# global_annotations['allow_subclassing'] = 'false'
#
# If a class is template instantiation, even if the
# template was defined in some other module, if a template
# argument belongs to this module then the template
# instantiation will belong to this module.
#
if templates.is_instantiation(pygccxml_definition.decl_string):
cls_name, template_parameters = templates.split(pygccxml_definition.name)
template_parameters_decls = [find_declaration_from_name(module_parser.global_ns, templ_param)
for templ_param in template_parameters]
#print >> sys.stderr, "cuem****************", cls_name, repr(template_parameters_decls)
template_parameters_modules = []
for templ in template_parameters_decls:
if not hasattr(templ, 'location'):
continue
try:
h = get_ns3_relative_path(templ.location.file_name)
except ValueError:
continue
template_parameters_modules.append(self.headers_map[h])
for templ_mod in template_parameters_modules:
if templ_mod == self.module:
definition_module = templ_mod
break
#print >> sys.stderr, "********************", cls_name, repr(template_parameters_modules)
if definition_module != self.module:
global_annotations['import_from_module'] = 'ns.%s' % (definition_module.replace('-', '_'),)
if pygccxml_definition.decl_string.startswith('::ns3::SimpleRefCount<'):
global_annotations['incref_method'] = 'Ref'
global_annotations['decref_method'] = 'Unref'
global_annotations['peekref_method'] = 'GetReferenceCount'
global_annotations['automatic_type_narrowing'] = 'true'
return
if pygccxml_definition.decl_string.startswith('::ns3::Callback<'):
# manually handled in ns3modulegen_core_customizations.py
global_annotations['ignore'] = None
return
if pygccxml_definition.decl_string.startswith('::ns3::TracedCallback<'):
global_annotations['ignore'] = None
return
if pygccxml_definition.decl_string.startswith('::ns3::Ptr<'):
# handled by pybindgen "type transformation"
global_annotations['ignore'] = None
return
# table driven class customization
try:
annotations = type_annotations[pygccxml_definition.decl_string]
except KeyError:
pass
else:
global_annotations.update(annotations)
## enums
if isinstance(pygccxml_definition, enumeration_t):
if definition_module != self.module:
global_annotations['import_from_module'] = 'ns.%s' % definition_module
## free functions
if isinstance(pygccxml_definition, free_function_t):
if definition_module != self.module:
global_annotations['ignore'] = None
return
if pygccxml_definition.name == 'PeekPointer':
global_annotations['ignore'] = None
return
## table driven methods/constructors/functions customization
if isinstance(pygccxml_definition, (free_function_t, member_function_t, constructor_t)):
try:
annotations = type_annotations[str(pygccxml_definition)]
except KeyError:
pass
else:
for key,value in annotations.items():
if key == 'params':
parameter_annotations.update (value)
del annotations['params']
global_annotations.update(annotations)
2
Example 17
Project: camr Source File: amr_parsing.py
def main():
'''
usage = "Usage:%prog [options] amr_file"
opt = OptionParser(usage=usage)
opt.add_option("-v",action="store",dest="verbose",type='int',
default=0,help="set up verbose level")
opt.add_option("-a",action="store_true",dest="align",
default=False,help="do alignment between sentence and amr")
opt.add_option("-b",action="store",dest="begin",type='int',
default=0,help="for debugging"
"When do alignment, where the alignment begins"
"When test oracle, where to begin")
opt.add_option("-s",action="store",dest="start_step",type='int',
default=0,help="where the step begins,for testing oracle")
opt.add_option("-o",action="store",dest="sentfilep",
help="output sentences to file and parse the sentence into dependency graph")
opt.add_option("-i",action="store",dest="parsedfilep",
help="read parsed dependency graph from file")
opt.add_option("-g",action="store",dest="userActfile",
help="read user input action sequences as guide")
opt.add_option("-d",action="store",dest="oracle",type='int',default=0,\
help="test the output actions of deterministic oracle: "
"1: tree oracle 2: list-based oracle")
'''
arg_parser = argparse.ArgumentParser(description="Brandeis transition-based AMR parser 1.0")
arg_parser.add_argument('-v','--verbose',type=int,default=0,help='set up verbose level for debug')
arg_parser.add_argument('-b','--begin',type=int,default=0,help='specify which sentence to begin the alignment or oracle testing for debug')
arg_parser.add_argument('-s','--start_step',type=int,default=0,help='specify which step to begin oracle testing;for debug')
#arg_parser.add_argument('-i','--input_file',help='the input: preprocessed data instances file for aligner or training')
arg_parser.add_argument('-d','--dev',help='development file')
arg_parser.add_argument('-a','--add',help='additional training file')
arg_parser.add_argument('-as','--actionset',choices=['basic'],default='basic',help='choose different action set')
arg_parser.add_argument('-m','--mode',choices=['preprocess','test_gold_graph','align','userGuide','oracleGuide','train','parse','eval'],help="preprocess:generate pos tag, dependency tree, ner\n" "align:do alignment between AMR graph and sentence string")
arg_parser.add_argument('-dp','--depparser',choices=['stanford','stanfordConvert','stdconv+charniak','clear','mate','turbo'],default='stdconv+charniak',help='choose the dependency parser')
arg_parser.add_argument('--coref',action='store_true',help='flag to enable coreference information')
arg_parser.add_argument('--prop',action='store_true',help='flag to enable semantic role labeling information')
arg_parser.add_argument('--rne',action='store_true',help='flag to enable rich name entity')
arg_parser.add_argument('--verblist',action='store_true',help='flag to enable verbalization list')
#arg_parser.add_argument('--onto',action='store_true',help='flag to enable charniak parse result trained on ontonotes')
arg_parser.add_argument('--onto',choices=['onto','onto+bolt','wsj'],default='wsj',help='choose which charniak parse result trained on ontonotes')
arg_parser.add_argument('--model',help='specify the model file')
arg_parser.add_argument('--feat',help='feature template file')
arg_parser.add_argument('-iter','--iterations',default=1,type=int,help='training iterations')
arg_parser.add_argument('amr_file',nargs='?',help='amr annotation file/input sentence file for parsing')
arg_parser.add_argument('--amrfmt',choices=['sent','amr','amreval'],default='sent',help='specifying the input file format')
arg_parser.add_argument('--smatcheval',action='store_true',help='give evaluation score using smatch')
arg_parser.add_argument('-e','--eval',nargs=2,help='Error Analysis: give parsed AMR file and gold AMR file')
arg_parser.add_argument('--section',choices=['proxy','all'],default='all',help='choose section of the corpus. Only works for LDC2014T12 dataset.')
args = arg_parser.parse_args()
amr_file = args.amr_file
instances = None
train_instance = None
constants.FLAG_COREF=args.coref
constants.FLAG_PROP=args.prop
constants.FLAG_RNE=args.rne
constants.FLAG_VERB=args.verblist
constants.FLAG_ONTO=args.onto
constants.FLAG_DEPPARSER=args.depparser
# using corenlp to preprocess the sentences
if args.mode == 'preprocess':
instances = preprocess(amr_file,START_SNLP=True,INPUT_AMR=args.amrfmt)
print "Done preprocessing!"
# preprocess the JAMR aligned amr
elif args.mode == 'test_gold_graph':
instances = preprocess(amr_file,False)
#instances = pickle.load(open('data/gold_edge_graph.pkl','rb'))
gold_amr = []
for inst in instances:
GraphState.sent = inst.tokens
gold_amr.append(GraphState.get_parsed_amr(inst.gold_graph))
#pseudo_gold_amr = [GraphState.get_parsed_amr(inst.gold_graph) for inst in instances]
write_parsed_amr(gold_amr,instances,amr_file,'abt.gold')
#instances = preprocess_aligned(amr_file)
print "Done output AMR!"
# do alignment
elif args.mode == 'align':
if args.input_file:
instances = pickle.load(open(args.input_file,'rb'))
else:
raise ValueError("Missing data file! specify it using --input or using preprocessing!")
gold_instances_file = args.input_file.split('.')[0]+'_gold.p'
print >> log, "Doing alignment..."
if LOGGED:
saveerr = sys.stderr
sys.stderr = open('./log/alignment.log','w')
amr_aligner = Aligner(verbose=args.verbose)
ref_graphs = []
begin = args.begin
counter = 1
#for snt, amr in zip(snts[begin:],amrs[begin:]):
for i in range(len(instances)):
snt = instances[i].text
amr = instances[i].amr
if args.verbose > 1:
print >> log, counter
print >> log, "Sentence:"
print >> log, snt+'\n'
print >> log, "AMR:"
print >> log, amr.to_amr_string()
alresult = amr_aligner.apply_align(snt,amr)
ref_amr_graph = SpanGraph.init_ref_graph(amr,alresult)
#ref_graphs.append(ref_amr_graph)
instances[i].addGoldGraph(ref_amr_graph)
if args.verbose > 1:
#print >> log, "Reference tuples:"
#print >> log, ref_depGraph.print_tuples()
print >> log, amr_aligner.print_align_result(alresult,amr)
#raw_input('ENTER to continue')
counter += 1
pickle.dump(instances,open(gold_instances_file,'wb'),pickle.HIGHEST_PROTOCOL)
#pickle.dump(ref_graphs,open('./data/ref_graph.p','wb'),pickle.HIGHEST_PROTOCOL)
if LOGGED:
sys.stderr.close()
sys.stderr = saveerr
print >> log, "Done alignment and gold graph generation."
sys.exit()
# test user guide actions
elif args.mode == 'userGuide':
print 'Read in training instances...'
train_instances = preprocess(amr_file,False)
sentID = int(raw_input("Input the sent ID:"))
amr_parser = Parser()
amr_parser.testUserGuide(train_instances[sentID])
sys.exit()
# test deterministic oracle
elif args.mode == 'oracleGuide':
train_instances = preprocess(amr_file,START_SNLP=False)
try:
hand_alignments = load_hand_alignments(amr_file+str('.hand_aligned'))
except IOError:
hand_alignments = []
start_step = args.start_step
begin = args.begin
amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT,verbose=args.verbose)
#ref_graphs = pickle.load(open('./data/ref_graph.p','rb'))
n_correct_total = .0
n_parsed_total = .0
n_gold_total = .0
pseudo_gold_amr = []
n_correct_tag_total = .0
n_parsed_tag_total = 0.
n_gold_tag_total = .0
gold_amr = []
aligned_instances = []
#print "shuffling training instances"
#random.shuffle(train_instances)
for instance in train_instances[begin:]:
if hand_alignments and instance.comment['id'] not in hand_alignments: continue
state = amr_parser.testOracleGuide(instance,start_step)
n_correct_arc,n1,n_parsed_arc, n_gold_arc,n_correct_tag,n_parsed_tag,n_gold_tag = state.evaluate()
#assert n_correct_arc == n1
if n_correct_arc != n1:
import pdb
pdb.set_trace()
n_correct_total += n_correct_arc
n_parsed_total += n_parsed_arc
n_gold_total += n_gold_arc
p = n_correct_arc/n_parsed_arc if n_parsed_arc else .0
r = n_correct_arc/n_gold_arc if n_gold_arc else .0
indicator = 'PROBLEM!' if p < 0.5 else ''
if args.verbose > 2: print >> sys.stderr, "Precision: %s Recall: %s %s\n" % (p,r,indicator)
n_correct_tag_total += n_correct_tag
n_parsed_tag_total += n_parsed_tag
n_gold_tag_total += n_gold_tag
p1 = n_correct_tag/n_parsed_tag if n_parsed_tag else .0
r1 = n_correct_tag/n_gold_tag if n_gold_tag else .0
if args.verbose > 2: print >> sys.stderr,"Tagging Precision:%s Recall:%s" % (p1,r1)
instance.comment['alignments'] += ''.join(' %s-%s|%s'%(idx-1,idx,instance.amr.get_pid(state.A.abt_node_table[idx])) for idx in state.A.abt_node_table if isinstance(idx,int))
aligned_instances.append(instance)
pseudo_gold_amr.append(GraphState.get_parsed_amr(state.A))
#gold_amr.append(instance.amr)
#assert set(state.A.tuples()) == set(instance.gold_graph.tuples())
pt = n_correct_total/n_parsed_total if n_parsed_total != .0 else .0
rt = n_correct_total/n_gold_total if n_gold_total !=.0 else .0
ft = 2*pt*rt/(pt+rt) if pt+rt != .0 else .0
write_parsed_amr(pseudo_gold_amr,aligned_instances,amr_file,'pseudo-gold',hand_alignments)
print "Total Accuracy: %s, Recall: %s, F-1: %s" % (pt,rt,ft)
tp = n_correct_tag_total/n_parsed_tag_total if n_parsed_tag_total != .0 else .0
tr = n_correct_tag_total/n_gold_tag_total if n_gold_tag_total != .0 else .0
print "Tagging Precision:%s Recall:%s" % (tp,tr)
#amr_parser.record_actions('data/action_set.txt')
elif args.mode == 'train': # training
print "Parser Config:"
print "Incorporate Coref Information: %s"%(constants.FLAG_COREF)
print "Incorporate SRL Information: %s"%(constants.FLAG_PROP)
print "Substitue the normal name entity tag with rich name entity tag: %s"%(constants.FLAG_RNE)
print "Using verbalization list: %s"%(constants.FLAG_VERB)
print "Using charniak parser trained on ontonotes: %s"%(constants.FLAG_ONTO)
print "Dependency parser used: %s"%(constants.FLAG_DEPPARSER)
train_instances = preprocess(amr_file,START_SNLP=False)
if args.add: train_instances = train_instances + preprocess(args.add,START_SNLP=False)
if args.dev: dev_instances = preprocess(args.dev,START_SNLP=False)
if args.section != 'all':
print "Choosing corpus section: %s"%(args.section)
tcr = constants.get_corpus_range(args.section,'train')
train_instances = train_instances[tcr[0]:tcr[1]]
if args.dev:
dcr = constants.get_corpus_range(args.section,'dev')
dev_instances = dev_instances[dcr[0]:dcr[1]]
feat_template = args.feat if args.feat else None
model = Model(elog=experiment_log)
#model.output_feature_generator()
parser = Parser(model=model,oracle_type=DET_T2G_ORACLE_ABT,action_type=args.actionset,verbose=args.verbose,elog=experiment_log)
model.setup(action_type=args.actionset,instances=train_instances,parser=parser,feature_templates_file=feat_template)
print >> experiment_log, "BEGIN TRAINING!"
best_fscore = 0.0
best_pscore = 0.0
best_rscore = 0.0
best_model = None
best_iter = 1
for iter in xrange(1,args.iterations+1):
print >> experiment_log, "shuffling training instances"
random.shuffle(train_instances)
print >> experiment_log, "Iteration:",iter
begin_updates = parser.perceptron.get_num_updates()
parser.parse_corpus_train(train_instances)
parser.perceptron.average_weight()
if args.dev:
print >> experiment_log ,"Result on develop set:"
_,parsed_amr = parser.parse_corpus_test(dev_instances)
parsed_suffix = args.section+'.'+args.model.split('.')[-1]+'.'+str(iter)+'.parsed'
write_parsed_amr(parsed_amr,dev_instances,args.dev,parsed_suffix)
if args.smatcheval:
smatch_path = "./smatch_2.0.2/smatch.py"
python_path = 'python'
options = '--pr -f'
parsed_filename = args.dev+'.'+parsed_suffix
command = '%s %s %s %s %s' % (python_path, smatch_path, options, parsed_filename, args.dev)
print 'Evaluation using command: ' + (command)
#print subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
eval_output = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
print eval_output
pscore = float(eval_output.split('\n')[0].split(':')[1].rstrip())
rscore = float(eval_output.split('\n')[1].split(':')[1].rstrip())
fscore = float(eval_output.split('\n')[2].split(':')[1].rstrip())
if fscore > best_fscore:
best_model = model
best_iter = iter
best_fscore = fscore
best_pscore = pscore
best_rscore = rscore
if best_model is not None:
print >> experiment_log, "Best result on iteration %d:\n Precision: %f\n Recall: %f\n F-score: %f" % (best_iter, best_pscore, best_rscore, best_fscore)
best_model.save_model(args.model+'.m')
print >> experiment_log ,"DONE TRAINING!"
elif args.mode == 'parse': # actual parsing
test_instances = preprocess(amr_file,START_SNLP=False,INPUT_AMR=args.amrfmt)
if args.section != 'all':
print "Choosing corpus section: %s"%(args.section)
tcr = constants.get_corpus_range(args.section,'test')
test_instances = test_instances[tcr[0]:tcr[1]]
#random.shuffle(test_instances)
print >> experiment_log, "Loading model: ", args.model
model = Model.load_model(args.model)
parser = Parser(model=model,oracle_type=DET_T2G_ORACLE_ABT,action_type=args.actionset,verbose=args.verbose,elog=experiment_log)
print >> experiment_log ,"BEGIN PARSING"
span_graph_pairs,results = parser.parse_corpus_test(test_instances)
parsed_suffix = '%s.%s.parsed'%(args.section,args.model.split('.')[-2])
write_parsed_amr(results,test_instances,amr_file,suffix=parsed_suffix)
#write_span_graph(span_graph_pairs,test_instances,amr_file,suffix='spg.50')
################
# for eval #
################
#pickle.dump(span_graph_pairs,open('data/eval/%s_spg_pair.pkl'%(amr_file),'wb'),pickle.HIGHEST_PROTOCOL)
#pickle.dump(test_instances,open('data/eval/%s_instances.pkl'%(amr_file),'wb'),pickle.HIGHEST_PROTOCOL)
print >> experiment_log ,"DONE PARSING"
if args.smatcheval:
smatch_path = "./smatch_2.0.2/smatch.py"
python_path = 'python'
options = '--pr -f'
parsed_filename = amr_file+'.'+parsed_suffix
command = '%s %s %s %s %s' % (python_path,smatch_path,options,parsed_filename, amr_file)
print 'Evaluation using command: ' + (command)
print subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
#plt.hist(results)
#plt.savefig('result.png')
elif args.mode == 'eval':
'''break down error analysis'''
# TODO: here use pickled file, replace it with parsed AMR and gold AMR
span_graph_pairs = pickle.load(open(args.eval[0],'rb'))
instances = pickle.load(open(args.eval[1],'rb'))
amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT,verbose=args.verbose)
error_stat = defaultdict(lambda:defaultdict(lambda:defaultdict(list)))
for spg_pair,instance in zip(span_graph_pairs,instances):
amr_parser.errorAnalyze(spg_pair[0],spg_pair[1],instance,error_stat)
else:
arg_parser.print_help()
2
Example 18
Project: calibre Source File: cli.py
def main():
from calibre.utils.terminal import geometry
cols = geometry()[0]
parser = OptionParser(usage="usage: %prog [options] command args\n\ncommand "+
"is one of: info, books, df, ls, cp, mkdir, touch, cat, rm, eject, test_file\n\n"+
"For help on a particular command: %prog command", version=__appname__+" version: " + __version__)
parser.add_option("--log-packets", help="print out packet stream to stdout. "+
"The numbers in the left column are byte offsets that allow the packet size to be read off easily.",
dest="log_packets", action="store_true", default=False)
parser.remove_option("-h")
parser.disable_interspersed_args() # Allow unrecognized options
options, args = parser.parse_args()
if len(args) < 1:
parser.print_help()
return 1
command = args[0]
args = args[1:]
dev = None
scanner = DeviceScanner()
scanner.scan()
connected_devices = []
for d in device_plugins():
try:
d.startup()
except:
print ('Startup failed for device plugin: %s'%d)
if d.MANAGES_DEVICE_PRESENCE:
cd = d.detect_managed_devices(scanner.devices)
if cd is not None:
connected_devices.append((cd, d))
dev = d
break
continue
ok, det = scanner.is_device_connected(d)
if ok:
dev = d
dev.reset(log_packets=options.log_packets, detected_device=det)
connected_devices.append((det, dev))
if dev is None:
print >>sys.stderr, 'Unable to find a connected ebook reader.'
shutdown_plugins()
return 1
for det, d in connected_devices:
try:
d.open(det, None)
except:
continue
else:
dev = d
d.specialize_global_preferences(device_prefs)
break
try:
if command == "df":
total = dev.total_space(end_session=False)
free = dev.free_space()
where = ("Memory", "Card A", "Card B")
print "Filesystem\tSize \tUsed \tAvail \tUse%"
for i in range(3):
print "%-10s\t%s\t%s\t%s\t%s"%(where[i], human_readable(total[i]), human_readable(total[i]-free[i]), human_readable(free[i]),
str(0 if total[i]==0 else int(100*(total[i]-free[i])/(total[i]*1.)))+"%")
elif command == 'eject':
dev.eject()
elif command == "books":
print "Books in main memory:"
for book in dev.books():
print book
print "\nBooks on storage carda:"
for book in dev.books(oncard='carda'):
print book
print "\nBooks on storage cardb:"
for book in dev.books(oncard='cardb'):
print book
elif command == "mkdir":
parser = OptionParser(usage="usage: %prog mkdir [options] path\nCreate a directory on the device\n\npath must begin with / or card:/")
if len(args) != 1:
parser.print_help()
sys.exit(1)
dev.mkdir(args[0])
elif command == "ls":
parser = OptionParser(usage="usage: %prog ls [options] path\nList files on the device\n\npath must begin with / or card:/")
parser.add_option(
"-l", help="In addition to the name of each file, print the file type, permissions, and timestamp (the modification time, in the local timezone). Times are local.", # noqa
dest="ll", action="store_true", default=False)
parser.add_option("-R", help="Recursively list subdirectories encountered. /dev and /proc are omitted",
dest="recurse", action="store_true", default=False)
parser.remove_option("-h")
parser.add_option("-h", "--human-readable", help="show sizes in human readable format", dest="hrs", action="store_true", default=False)
options, args = parser.parse_args(args)
if len(args) != 1:
parser.print_help()
return 1
print ls(dev, args[0], recurse=options.recurse, ll=options.ll, human_readable_size=options.hrs, cols=cols),
elif command == "info":
info(dev)
elif command == "cp":
usage="usage: %prog cp [options] source destination\nCopy files to/from the device\n\n"+\
"One of source or destination must be a path on the device. \n\nDevice paths have the form\n"+\
"dev:mountpoint/my/path\n"+\
"where mountpoint is one of / or carda: or cardb:/\n\n"+\
"source must point to a file for which you have read permissions\n"+\
"destination must point to a file or directory for which you have write permissions"
parser = OptionParser(usage=usage)
parser.add_option('-f', '--force', dest='force', action='store_true', default=False,
help='Overwrite the destination file if it exists already.')
options, args = parser.parse_args(args)
if len(args) != 2:
parser.print_help()
return 1
if args[0].startswith("dev:"):
outfile = args[1]
path = args[0][4:]
if path.endswith("/"):
path = path[:-1]
if os.path.isdir(outfile):
outfile = os.path.join(outfile, path[path.rfind("/")+1:])
try:
outfile = lopen(outfile, "wb")
except IOError as e:
print >> sys.stderr, e
parser.print_help()
return 1
dev.get_file(path, outfile)
fsync(outfile)
outfile.close()
elif args[1].startswith("dev:"):
try:
infile = lopen(args[0], "rb")
except IOError as e:
print >> sys.stderr, e
parser.print_help()
return 1
dev.put_file(infile, args[1][4:], replace_file=options.force)
infile.close()
else:
parser.print_help()
return 1
elif command == "cat":
outfile = sys.stdout
parser = OptionParser(
usage="usage: %prog cat path\nShow file on the device\n\npath should point to a file on the device and must begin with /,a:/ or b:/")
options, args = parser.parse_args(args)
if len(args) != 1:
parser.print_help()
return 1
if args[0].endswith("/"):
path = args[0][:-1]
else:
path = args[0]
outfile = sys.stdout
dev.get_file(path, outfile)
elif command == "rm":
parser = OptionParser(usage="usage: %prog rm path\nDelete files from the device\n\npath should point to a file or empty directory on the device "+
"and must begin with / or card:/\n\n"+
"rm will DELETE the file. Be very CAREFUL")
options, args = parser.parse_args(args)
if len(args) != 1:
parser.print_help()
return 1
dev.rm(args[0])
elif command == "touch":
parser = OptionParser(usage="usage: %prog touch path\nCreate an empty file on the device\n\npath should point to a file on the device and must begin with /,a:/ or b:/\n\n"+ # noqa
"Unfortunately, I cant figure out how to update file times on the device, so if path already exists, touch does nothing")
options, args = parser.parse_args(args)
if len(args) != 1:
parser.print_help()
return 1
dev.touch(args[0])
elif command == 'test_file':
parser = OptionParser(usage=("usage: %prog test_file path\n"
'Open device, copy file specified by path to device and '
'then eject device.'))
options, args = parser.parse_args(args)
if len(args) != 1:
parser.print_help()
return 1
path = args[0]
from calibre.ebooks.metadata.meta import get_metadata
mi = get_metadata(lopen(path, 'rb'), path.rpartition('.')[-1].lower())
print dev.upload_books([args[0]], [os.path.basename(args[0])],
end_session=False, metadata=[mi])
dev.eject()
else:
parser.print_help()
if getattr(dev, 'handle', False):
dev.close()
return 1
except DeviceLocked:
print >> sys.stderr, "The device is locked. Use the --unlock option"
except (ArgumentError, DeviceError) as e:
print >>sys.stderr, e
return 1
finally:
shutdown_plugins()
return 0
2
Example 19
Project: rivulet Source File: __init__.py
def session_thread(receive_queue, queue_poll_interval, torrents_by_hash, config, session_obj, session):
# TODO make sure the torrent_handles are hashable
def alert_to_torrent(alert):
for torrent_handle, torrent in torrents.items():
if torrent_handle == alert.handle:
return torrent
raise LibtorrentErrorException('Got alert for unknown torrent')
def send_events(streamer_infos, alert):
# Timeout
# TODO: this code is unreachable
if isinstance(alert, type(None)):
for torrent in torrents:
for si in streamer_infos:
si.time_waited += queue_poll_interval
if si.time_waited >= si.timeout:
err = "TorrentStream %d timed out waiting for data" % my_id
si.put(('kill', TorrentTimeoutException(err)))
# Error
elif alert.category() == lt.alert.category_t.error_notification:
if isinstance(alert, lt.torrent_alert):
torrent = alert_to_torrent(alert)
# kill all streamers for this torrent
for si in torrent.streamers.items().values():
si.put(('kill', LibtorrentErrorException(alert)))
else:
raise LibtorrentErrorException(alert)
# Metadata
elif isinstance(alert, lt.metadata_received_alert):
torrent = alert_to_torrent(alert)
for si in torrent.streamers:
metadata = get_torrent_info(torrent.torrent_handle)
print >> sys.stderr, 'XXXXXXXXMETADATA', metadata
torrents_by_hash[str(metadata.info_hash())] = torrent.torrent_handle
si.put(('metadata', metadata))
# Data
elif isinstance(alert, lt.read_piece_alert):
print >> sys.stderr, 'received data', alert
torrent = alert_to_torrent(alert)
print >> sys.stderr, 'found streamers', torrent.streamers
# TODO: figure out which streamer requested the piece and send the data only to that streamer
for si in torrent.streamers:
print >> sys.stderr, 'found streamer', si
si.time_waited = 0
# very important! duplicate the alert! Otherwise accessing the alert from another thread segfaults
si.put(('data', {'piece': alert.piece, 'size': alert.size, 'buffer': alert.buffer[:]}))
else:
print >> sys.stderr, 'unknown alert', alert, type(alert), alert.category()
root_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", os.path.expanduser(config.get('private_data_path', DEFAULT_PRIVATE_DATA_PATH)))
resume_data_path = os.path.join(root_path, os.path.expanduser(config.get('resume_data_path', DEFAULT_RESUME_DATA_PATH)))
mkdir_p(os.path.dirname(resume_data_path))
session.listen_on(6881, 6891)
session.set_alert_mask(lt.alert.category_t.status_notification)
session.add_dht_router("dht.transmissionbt.com", 6881)
session.add_dht_router("router.bittorrent.com", 6881)
session.add_dht_router("router.utorrent.com", 6881)
torrents = {}
streamer_infos = {}
# load resume data for each torrent if available
try:
with open(resume_data_path, 'r') as f:
resume_data = json.load(f)
for info_hash, resume_data in resume_data.items():
params = session_obj.torrent_params_from_info_hash(info_hash)
params['resume_data'] = base64.b64decode(resume_data)
th = session.add_torrent(params)
th.set_sequential_download(True)
torrents_by_hash[str(get_torrent_info(th).info_hash())] = th
except IOError:
pass
while True:
# alert processing loop forever
alert = session.wait_for_alert(queue_poll_interval)
if alert:
print >> sys.stderr, 'alert:', alert, type(alert), alert.category()
send_events(streamer_infos, alert)
session.pop_alert()
#TODO: check if any recipients are done and remove them from the list
# every X seconds or every alert check for requests from new callers
if not receive_queue.empty():
action, caller, data = receive_queue.get()
print >> sys.stderr, 'action:', action, caller, data
if action == 'subscribe_streamer':
torrent_params, file_index_or_name, streamer_info = data
streamer_infos[caller] = streamer_info
#TODO check if this is safe to do on another thread so we can get rid of the queue
torrent_handle = session.add_torrent(torrent_params)
torrent_handle.set_sequential_download(True)
# TODO: make this check work even if the metadata is not already available
# if this torrent already exists, use the existing Torrent object instead of creating a new one
torrent = None
for th, t in torrents.items():
if torrent_handle == th:
torrent = t
# we have to assume at first that this is a new torrent; if it turns out that it is not,
# when we receive the alert telling us this, we merge it with the existing torrent
if torrent is None:
torrent = Torrent(torrent_handle)
torrents[torrent.torrent_handle] = torrent
# this is done here so that metadata can be delivered to this streamer; the streamer removes itself
torrent.add_streamer(streamer_info)
# if the metadata is already available, there will be no event, so we have to do this here:
torrent_info = None
try:
torrent_info = get_torrent_info(torrent_handle)
torrents_by_hash[str(torrent_info.info_hash())] = torrent.torrent_handle
except:
# and if the metadata is not available, it will come via an event
pass
streamer_info.torrent = torrent
streamer_info.put((torrent, torrent_info))
elif action == 'unsubscribe_streamer':
streamer_info = data
last_user = streamer_info.torrent.remove_streamer(streamer_info)
if last_user == True:
print >> sys.stderr, 'Torrent not needed any more, maybe we should remove it?'
del torrents_by_hash[str(get_torrent_info(streamer_info.torrent.torrent_handle).info_hash())]
streamer_info.torrent = None
elif action == 'quit':
print 'shutting down gracefully and saving resume data'
# go through each torrent and save their resume_data
outstanding = 0
for th, t in torrents.items():
if not th.is_valid():
continue
if not th.status().has_metadata:
continue
if not th.need_save_resume_data():
continue
th.save_resume_data(3) # TODO: use the proper named flags
outstanding += 1
save = {}
try:
with open(resume_data_path, 'r') as old_f:
save = json.load(old_f)
except IOError:
pass
while outstanding > 0:
alert = session.wait_for_alert(10000) #wait extra long in this case
if not alert:
# give up if it times out
print 'timed out on shutdown'
break
if isinstance(alert, lt.save_resume_data_failed_alert):
print 'failed to save resume data'
elif isinstance(alert, lt.save_resume_data_alert):
th = alert.handle
hsh = str(get_torrent_info(th).info_hash())
# Sorry. If you feel like improving this, be my guest, but I didn't want to deal with it
save[hsh] = base64.b64encode(lt.bencode(alert.resume_data))
session.pop_alert()
outstanding -= 1
print 'dumping resume_data'
with open(resume_data_path, 'w') as f:
json.dump(save, f)
print 'dumped resume_data'
return
else:
raise Exception('WTF')
2
Example 20
Project: morfessor Source File: cmd.py
def main(args):
if args.verbose >= 2:
loglevel = logging.DEBUG
elif args.verbose >= 1:
loglevel = logging.INFO
else:
loglevel = logging.WARNING
logging_format = '%(asctime)s - %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
default_formatter = logging.Formatter(logging_format, date_format)
plain_formatter = logging.Formatter('%(message)s')
logging.basicConfig(level=loglevel)
_logger.propagate = False # do not forward messages to the root logger
# Basic settings for logging to the error stream
ch = logging.StreamHandler()
ch.setLevel(loglevel)
ch.setFormatter(plain_formatter)
_logger.addHandler(ch)
# Settings for when log_file is present
if args.log_file is not None:
fh = logging.FileHandler(args.log_file, 'w')
fh.setLevel(loglevel)
fh.setFormatter(default_formatter)
_logger.addHandler(fh)
# If logging to a file, make INFO the highest level for the
# error stream
ch.setLevel(max(loglevel, logging.INFO))
# If debug messages are printed to screen or if stderr is not a tty (but
# a pipe or a file), don't show the progressbar
global show_progress_bar
if (ch.level > logging.INFO or
(hasattr(sys.stderr, 'isatty') and not sys.stderr.isatty())):
show_progress_bar = False
if args.progress:
show_progress_bar = True
ch.setLevel(min(ch.level, logging.INFO))
if (args.loadfile is None and
args.loadsegfile is None and
len(args.trainfiles) == 0):
raise ArgumentException("either model file or training data should "
"be defined")
if args.randseed is not None:
random.seed(args.randseed)
io = MorfessorIO(encoding=args.encoding,
compound_separator=args.cseparator,
atom_separator=args.separator,
lowercase=args.lowercase)
# Load exisiting model or create a new one
if args.loadfile is not None:
model = io.read_binary_model_file(args.loadfile)
else:
model = BaselineModel(forcesplit_list=args.forcesplit,
corpusweight=args.corpusweight,
use_skips=args.skips,
nosplit_re=args.nosplit)
if args.loadsegfile is not None:
model.load_segmentations(io.read_segmentation_file(args.loadsegfile))
analysis_sep = (args.analysisseparator
if args.analysisseparator != 'NONE' else None)
if args.annofile is not None:
annotations = io.read_annotations_file(args.annofile,
analysis_sep=analysis_sep)
model.set_annotations(annotations, args.annotationweight)
if args.develfile is not None:
develannots = io.read_annotations_file(args.develfile,
analysis_sep=analysis_sep)
updater = AnnotationCorpusWeight(develannots, args.threshold)
model.set_corpus_weight_updater(updater)
if args.morphlength is not None:
updater = MorphLengthCorpusWeight(args.morphlength, args.threshold)
model.set_corpus_weight_updater(updater)
if args.morphtypes is not None:
updater = NumMorphCorpusWeight(args.morphtypes, args.threshold)
model.set_corpus_weight_updater(updater)
start_corpus_weight = model.get_corpus_coding_weight()
# Set frequency dampening function
if args.dampening == 'none':
dampfunc = None
elif args.dampening == 'log':
dampfunc = lambda x: int(round(math.log(x + 1, 2)))
elif args.dampening == 'ones':
dampfunc = lambda x: 1
else:
raise ArgumentException("unknown dampening type '%s'" % args.dampening)
# Set algorithm parameters
if args.algorithm == 'viterbi':
algparams = (args.viterbismooth, args.viterbimaxlen)
else:
algparams = ()
# Train model
if args.trainmode == 'none':
pass
elif args.trainmode == 'batch':
if len(model.get_compounds()) == 0:
_logger.warning("Model contains no compounds for batch training."
" Use 'init+batch' mode to add new data.")
else:
if len(args.trainfiles) > 0:
_logger.warning("Training mode 'batch' ignores new data "
"files. Use 'init+batch' or 'online' to "
"add new compounds.")
ts = time.time()
e, c = model.train_batch(args.algorithm, algparams,
args.finish_threshold, args.maxepochs)
te = time.time()
_logger.info("Epochs: %s" % e)
_logger.info("Final cost: %s" % c)
_logger.info("Training time: %.3fs" % (te - ts))
elif len(args.trainfiles) > 0:
ts = time.time()
if args.trainmode == 'init':
if args.list:
data = io.read_corpus_list_files(args.trainfiles)
else:
data = io.read_corpus_files(args.trainfiles)
c = model.load_data(data, args.freqthreshold, dampfunc,
args.splitprob)
elif args.trainmode == 'init+batch':
if args.list:
data = io.read_corpus_list_files(args.trainfiles)
else:
data = io.read_corpus_files(args.trainfiles)
c = model.load_data(data, args.freqthreshold, dampfunc,
args.splitprob)
e, c = model.train_batch(args.algorithm, algparams,
args.finish_threshold, args.maxepochs)
_logger.info("Epochs: %s" % e)
if args.fullretrain:
if abs(model.get_corpus_coding_weight() - start_corpus_weight) > 0.1:
model.set_corpus_weight_updater(
FixedCorpusWeight(model.get_corpus_coding_weight()))
model.clear_segmentation()
e, c = model.train_batch(args.algorithm, algparams,
args.finish_threshold,
args.maxepochs)
_logger.info("Retrain Epochs: %s" % e)
elif args.trainmode == 'online':
data = io.read_corpus_files(args.trainfiles)
e, c = model.train_online(data, dampfunc, args.epochinterval,
args.algorithm, algparams,
args.splitprob, args.maxepochs)
_logger.info("Epochs: %s" % e)
elif args.trainmode == 'online+batch':
data = io.read_corpus_files(args.trainfiles)
e, c = model.train_online(data, dampfunc, args.epochinterval,
args.algorithm, algparams,
args.splitprob, args.maxepochs)
e, c = model.train_batch(args.algorithm, algparams,
args.finish_threshold, args.maxepochs - e)
_logger.info("Epochs: %s" % e)
if args.fullretrain:
if abs(model.get_corpus_coding_weight() -
start_corpus_weight) > 0.1:
model.clear_segmentation()
e, c = model.train_batch(args.algorithm, algparams,
args.finish_threshold,
args.maxepochs)
_logger.info("Retrain Epochs: %s" % e)
else:
raise ArgumentException("unknown training mode '%s'"
% args.trainmode)
te = time.time()
_logger.info("Final cost: %s" % c)
_logger.info("Training time: %.3fs" % (te - ts))
else:
_logger.warning("No training data files specified.")
# Save model
if args.savefile is not None:
io.write_binary_model_file(args.savefile, model)
if args.savesegfile is not None:
io.write_segmentation_file(args.savesegfile, model.get_segmentations())
# Output lexicon
if args.lexfile is not None:
io.write_lexicon_file(args.lexfile, model.get_constructions())
if args.savereduced is not None:
model.make_segment_only()
io.write_binary_model_file(args.savereduced, model)
# Segment test data
if len(args.testfiles) > 0:
_logger.info("Segmenting test data...")
outformat = args.outputformat
csep = args.outputformatseparator
outformat = outformat.replace(r"\n", "\n")
outformat = outformat.replace(r"\t", "\t")
keywords = [x[1] for x in string.Formatter().parse(outformat)]
with io._open_text_file_write(args.outfile) as fobj:
testdata = io.read_corpus_files(args.testfiles)
i = 0
for count, compound, atoms in testdata:
if len(atoms) == 0:
# Newline in corpus
if args.outputnewlines:
fobj.write("\n")
continue
if "clogprob" in keywords:
clogprob = model.forward_logprob(atoms)
else:
clogprob = 0
if args.nbest > 1:
nbestlist = model.viterbi_nbest(atoms, args.nbest,
args.viterbismooth,
args.viterbimaxlen)
for constructions, logp in nbestlist:
analysis = io.format_constructions(constructions,
csep=csep)
fobj.write(outformat.format(analysis=analysis,
compound=compound,
count=count, logprob=logp,
clogprob=clogprob))
else:
constructions, logp = model.viterbi_segment(
atoms, args.viterbismooth, args.viterbimaxlen)
analysis = io.format_constructions(constructions, csep=csep)
fobj.write(outformat.format(analysis=analysis,
compound=compound,
count=count, logprob=logp,
clogprob=clogprob))
i += 1
if i % 10000 == 0:
sys.stderr.write(".")
sys.stderr.write("\n")
_logger.info("Done.")
if args.goldstandard is not None:
_logger.info("Evaluating Model")
e = MorfessorEvaluation(io.read_annotations_file(args.goldstandard))
result = e.evaluate_model(model, meta_data={'name': 'MODEL'})
print(result.format(FORMAT_STRINGS['default']))
_logger.info("Done")
2
Example 21
def Start(parser=None,
argv=sys.argv,
quiet=False,
add_pipe_options=True,
return_parser=False):
"""set up an experiment.
The :py:func:`Start` method will set up a file logger and add some
default and some optional options to the command line parser. It
will then parse the command line and set up input/output
redirection and start a timer for benchmarking purposes.
The default options added by this method are:
``-v/--verbose``
the :term:`loglevel`
``timeit``
turn on benchmarking information and save to file
``timeit-name``
name to use for timing information,
``timeit-header``
output header for timing information.
``seed``
the random seed. If given, the python random
number generator will be initialized with this
seed.
Optional options added are:
Arguments
---------
param parser : :py:class:`U.OptionParser`
instance with command line options.
argv : list
command line options to parse. Defaults to
:py:data:`sys.argv`
quiet : bool
set :term:`loglevel` to 0 - no logging
return_parser : bool
return the parser object, no parsing. Useful for inspecting
the command line options of a script without running it.
add_pipe_options : bool
add common options for redirecting input/output
Returns
-------
tuple
(:py:class:`U.OptionParser` object, list of positional
arguments)
"""
if not parser:
parser = OptionParser(
version="%prog version: $Id$")
global global_options, global_args, global_starting_time
# save default values given by user
user_defaults = copy.copy(parser.defaults)
global_starting_time = time.time()
group = OptionGroup(parser, "Script timing options")
group.add_option("--timeit", dest='timeit_file', type="string",
help="store timeing information in file [%default].")
group.add_option("--timeit-name", dest='timeit_name', type="string",
help="name in timing file for this class of jobs "
"[%default].")
group.add_option("--timeit-header", dest='timeit_header',
action="store_true",
help="add header for timing information [%default].")
parser.add_option_group(group)
group = OptionGroup(parser, "Common options")
group.add_option("--random-seed", dest='random_seed', type="int",
help="random seed to initialize number generator "
"with [%default].")
group.add_option("-v", "--verbose", dest="loglevel", type="int",
help="loglevel [%default]. The higher, the more output.")
group.add_option("-?", dest="short_help", action="callback",
callback=callbackShortHelp,
help="output short help (command line options only.")
parser.add_option_group(group)
if quiet:
parser.set_defaults(loglevel=0)
else:
parser.set_defaults(loglevel=1)
parser.set_defaults(
timeit_file=None,
timeit_name='all',
timeit_header=None,
random_seed=None,
)
if add_pipe_options:
group = OptionGroup(parser, "Input/output options")
group.add_option("-I", "--stdin", dest="stdin", type="string",
help="file to read stdin from [default = stdin].",
metavar="FILE")
group.add_option("-L", "--log", dest="stdlog", type="string",
help="file with logging information "
"[default = stdout].",
metavar="FILE")
group.add_option("-E", "--error", dest="stderr", type="string",
help="file with error information "
"[default = stderr].",
metavar="FILE")
group.add_option("-S", "--stdout", dest="stdout", type="string",
help="file where output is to go "
"[default = stdout].",
metavar="FILE")
group.add_option("--log2stderr", dest="log2stderr",
action="store_true", help="send logging information"
" to stderr [default = False].")
parser.set_defaults(stderr=sys.stderr)
parser.set_defaults(stdout=sys.stdout)
parser.set_defaults(stdlog=sys.stdout)
parser.set_defaults(stdin=sys.stdin)
parser.set_defaults(log2stderr=False)
parser.add_option_group(group)
# restore user defaults
parser.defaults.update(user_defaults)
if return_parser:
return parser
global_options, global_args = parser.parse_args(argv[1:])
if global_options.random_seed is not None:
random.seed(global_options.random_seed)
if add_pipe_options:
if global_options.stdout != sys.stdout:
global_options.stdout = openFile(global_options.stdout, "w")
if global_options.stderr != sys.stderr:
if global_options.stderr == "stderr":
global_options.stderr = global_options.stderr
else:
global_options.stderr = openFile(global_options.stderr, "w")
if global_options.stdlog != sys.stdout:
global_options.stdlog = openFile(global_options.stdlog, "a")
elif global_options.log2stderr:
global_options.stdlog = global_options.stderr
if global_options.stdin != sys.stdin:
global_options.stdin = openFile(global_options.stdin, "r")
else:
global_options.stderr = sys.stderr
global_options.stdout = sys.stdout
global_options.stdin = sys.stdin
if global_options.log2stderr:
global_options.stdlog = sys.stderr
else:
global_options.stdlog = sys.stdout
if global_options.loglevel >= 1:
global_options.stdlog.write(getHeader() + "\n")
global_options.stdlog.write(getParams(global_options) + "\n")
global_options.stdlog.flush()
# configure logging
# map from 0-10 to logging scale
# 0: quiet
# 1: little verbositiy
# >1: increased verbosity
if global_options.loglevel == 0:
lvl = logging.ERROR
elif global_options.loglevel == 1:
lvl = logging.INFO
else:
lvl = logging.DEBUG
if global_options.stdout == global_options.stdlog:
format = '# %(asctime)s %(levelname)s %(message)s'
else:
format = '%(asctime)s %(levelname)s %(message)s'
logging.basicConfig(
level=lvl,
format=format,
stream=global_options.stdlog)
# set up multi-line logging
# Note that .handlers is not part of the API, might change
# Solution is to configure handlers explicitely.
for handler in logging.getLogger().handlers:
handler.setFormatter(MultiLineFormatter(format))
return global_options, global_args
2
Example 22
def migrate(fromdir, todir):
os.makedirs(todir)
dbdir = os.path.join(todir, 'db')
if not os.path.exists(dbdir):
os.makedirs(dbdir)
data = None
config = ConfigParser()
config.read(os.path.join(fromdir, 'include', 'defaults.ini'))
for filename in os.listdir(fromdir):
src = os.path.join(fromdir, filename)
if filename == 'grufti-responses.txt':
copy(src, os.path.join(todir, 'response.grufti'))
elif filename == 'data':
data = src
elif filename == 'madcow.ini':
config.read(src)
# migrate settings
nicks = set()
nicks.add(config.get('irc', 'nick'))
nicks.add(config.get('silcplugin', 'nick'))
settings.PROTOCOL = config.get('main', 'module')
settings.IRC_HOST = config.get('irc', 'host')
settings.IRC_PORT = config.getint('irc', 'port')
settings.IRC_SSL = config.get('irc', 'ssl') == 'yes'
settings.IRC_PASSWORD = config.get('irc', 'password')
settings.IRC_RECONNECT = config.get('irc', 'reconnect') == 'yes'
settings.IRC_RECONNECT_WAIT = config.getint('irc', 'reconnectWait')
settings.IRC_REJOIN = config.get('irc', 'rejoin') == 'yes'
settings.IRC_REJOIN_WAIT = config.getint('irc', 'rejoinWait')
settings.IRC_REJOIN_MESSAGE = config.get('irc', 'rejoinReply')
settings.IRC_QUIT_MESSAGE = config.get('irc', 'quitMessage')
settings.IRC_OPER = config.get('irc', 'oper') == 'yes'
settings.IRC_OPER_PASS = config.get('irc', 'operPass')
settings.IRC_OPER_USER = config.get('irc', 'operUser')
settings.IRC_NICKSERV_PASS = config.get('irc', 'nickServPass')
settings.IRC_NICKSERV_USER = config.get('irc', 'nickServUser')
settings.IRC_FORCE_WRAP = config.getint('irc', 'wrapSize')
settings.IRC_DELAY_LINES = config.getint('irc', 'delay')
settings.IRC_IDENTIFY_NICKSERV = bool(settings.IRC_NICKSERV_PASS and settings.IRC_NICKSERV_USER)
settings.IRC_KEEPALIVE = config.get('irc', 'keepalive') == 'yes'
settings.IRC_KEEPALIVE_FREQ = config.getint('irc', 'keepalive_freq')
settings.IRC_KEEPALIVE_TIMEOUT = config.getint('irc', 'keepalive_timeout')
settings.IRC_RECONNECT_MESSAGE = settings.IRC_REJOIN_MESSAGE
settings.MODULES = []
for module in os.listdir(os.path.join(madcow.PREFIX, 'modules')):
if module.endswith('.py') and module != '__init__.py':
module = module.replace('.py', '')
try:
try:
test = RENAMED_MODULES[module]
except KeyError:
test = module
if config.get('modules', test) == 'yes':
settings.MODULES.append(module)
else:
print >> sys.stderr, 'WARN: %r module is disabled' % module
except NoOptionError:
print >> sys.stderr, 'WARN: unknown module: %r' % module
if config.get('steam', 'enabled') == 'yes':
settings.MODULES.append('steam')
settings.STEAM_GROUP = config.get('steam', 'group')
settings.STEAM_SHOW_ONLINE = config.get('steam', 'online') == 'yes'
settings.AIM_AUTOJOIN_CHAT = config.get('aim', 'autojoin') == 'yes'
settings.AIM_PASSWORD = config.get('aim', 'password')
settings.AIM_PROFILE = config.get('aim', 'profile')
settings.AIM_USERNAME = config.get('aim', 'username')
nicks.add(settings.AIM_USERNAME)
settings.PRIVATE_MODULES = config.get('modules', 'private').split(',')
settings.DETACH = config.get('main', 'detach') == 'yes'
settings.WORKERS = config.getint('main', 'workers')
settings.YELP_DEFAULT_LOCATION = config.get('yelp', 'default_location')
settings.TASKS = []
if config.get('twitter', 'enabled') == 'yes':
settings.TASKS.append('tweets')
settings.TWITTER_CONSUMER_KEY = config.get('twitter', 'username')
settings.TWITTER_CONSUMER_SECRET = config.get('twitter', 'password')
try:
settings.TWITTER_TOKEN_KEY = config.get('twitter', 'token_key')
settings.TWITTER_TOKEN_SECRET = config.get('twitter', 'token_secret')
except NoOptionError:
settings.TWITTER_TOKEN_KEY = None
settings.TWITTER_TOKEN_SECRET = None
settings.TWITTER_UPDATE_FREQ = config.getint('twitter', 'updatefreq')
if config.get('ircops', 'enabled') == 'yes':
settings.TASKS.append('ircops')
settings.IRC_GIVE_OPS_FREQ = config.getint('ircops', 'updatefreq')
settings.HTTP_AGENT = config.get('http', 'agent')
settings.HTTP_COOKIES = config.get('http', 'cookies') == 'yes'
settings.HTTP_TIMEOUT = config.getint('http', 'timeout')
settings.PIDFILE = config.get('main', 'pidfile')
channels = set()
channels.update(config.get('irc', 'channels').split(','))
channels.update(config.get('silcplugin', 'channels').split(','))
channels.add(config.get('gateway', 'channel'))
channels.add(config.get('twitter', 'channel'))
channels = [channel for channel in channels if channel]
channels.remove('#madcow')
if len(channels) > 1:
channels = validate_list('channels', channels)
settings.IRC_CHANNELS = channels
settings.SILC_CHANNELS = channels
settings.TWITTER_CHANNELS = 'ALL'
settings.GATEWAY_CHANNELS = 'ALL'
settings.LOG_PUBLIC = config.get('main', 'logpublic') == 'yes'
settings.IGNORE_NICKS = config.get('main', 'ignoreList').split(',')
settings.LOGGING_LEVEL = config.get('main', 'loglevel')
settings.LOGGING_ENCODING = settings.ENCODING = config.get('main', 'charset')
settings.OWNER_NICK = config.get('main', 'owner')
settings.ALIASES = config.get('main', 'aliases').split(',')
settings.SILC_DELAY = config.getint('silcplugin', 'delay')
settings.SILC_HOST = config.get('silcplugin', 'host')
settings.SILC_PORT = config.getint('silcplugin', 'port')
settings.SILC_RECONNECT = config.get('silcplugin', 'reconnect') == 'yes'
settings.SILC_RECONNECT_WAIT = config.getint('silcplugin', 'reconnectWait')
settings.SMTP_FROM = config.get('smtp', 'sender')
settings.SMTP_PASS = config.get('smtp', 'password')
settings.SMTP_SERVER = config.get('smtp', 'server')
settings.SMTP_USER = config.get('smtp', 'user')
settings.GATEWAY_ADDR = config.get('gateway', 'bind')
settings.GATEWAY_ENABLED = config.get('gateway', 'enabled') == 'yes'
settings.GATEWAY_IMAGE_PATH = config.get('gateway', 'imagePath')
settings.GATEWAY_IMAGE_URL = config.get('gateway', 'imageURL')
settings.GATEWAY_PORT = config.getint('gateway', 'port')
settings.GATEWAY_SAVE_IMAGES = bool(settings.GATEWAY_IMAGE_URL and settings.GATEWAY_IMAGE_PATH)
settings.ADMIN_ENABLED = config.get('admin', 'enabled') == 'yes'
settings.ALLOW_REGISTRATION = config.get('admin', 'allowRegistration') == 'yes'
settings.DEFAULT_FLAGS = config.get('admin', 'defaultFlags')
settings.DELICIOUS_USERNAME = config.get('delicious', 'username')
settings.DELICIOUS_PASSWORD = config.get('delicious', 'password')
nicks = [nick for nick in set(nicks) if nick]
if len(nicks) > 1:
print 'Multiple nicks found, pick one to use:'
for i, nick in enumerate(nicks):
print '%d %s' % (i + 1, nick)
while True:
i = raw_input('>>> ')
try:
nick = nicks[int(i) - 1]
break
except:
pass
else:
nick = nicks[0]
settings.BOTNAME = nick
with open(os.path.join(todir, 'settings.py'), 'wb') as fp:
for key in sorted(dir(settings)):
if key.isupper():
val = getattr(settings, key)
if val == '':
val = None
out = None
if isinstance(val, list):
val = [_ for _ in val if _]
if val:
out = dumplist(val, key, 80)
if not out:
out = '%s = %r' % (key, val)
print >> fp, out
print 'wrote settings.py'
dbdir = os.path.join(todir, 'db')
namespace = config.get('modules', 'dbNamespace')
def get_dbfile(file):
valid_namespace = False
name = None
try:
basedir, filename = os.path.split(file)
filename = filename.replace('.db', '')
key, this_namespace, name = filename.split('-')
if this_namespace != namespace:
raise ValueError
valid_namespace = True
for ext in '', '.db':
test = os.path.join(basedir, filename + ext)
try:
db = dbm.open(test, 'r')
except:
continue
db.close()
return test, name, 'dbm'
except:
pass
if valid_namespace and name:
return file, name, 'txt'
if data is not None:
for filename in os.listdir(data):
src = os.path.join(data, filename)
if os.path.isfile(src):
try:
src, name, type = get_dbfile(src)
except ValueError:
continue
if type == 'dbm':
dst = os.path.join(dbdir, name)
db = dbm.open(src, 'r')
try:
new = dbm.open(dst, 'c', 0640)
try:
print 'copying %d items for %s database' % (len(db), name)
for key in db.keys():
new[key] = db[key]
finally:
new.close()
finally:
db.close()
elif type == 'txt':
dst = os.path.join(dbdir, name)
shutil.copy(src, dst)
print 'copied %s' % name
elif os.path.isdir(src) and filename == 'megahal':
seen = set()
megahal_from = src
megahal_to = os.path.join(dbdir, filename)
for basedir, subdirs, filenames in os.walk(megahal_from):
for filename in filenames:
if filename in BRAIN_FILES:
src = os.path.join(basedir, filename)
dstpath = src.replace(megahal_from + os.sep, '')
dst = os.path.join(megahal_to, dstpath)
dstdir, dstfile = os.path.split(dst)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
shutil.copy(src, dst)
print 'copied %s' % dstpath
# memebot needs migration too since we ditched sqlobject
try:
migrate_memebot(config, dbdir, fromdir)
except Exception, error:
print >> sys.stderr, "couldn't migrate memebot: %s" % error
src = os.path.join(fromdir, 'logs')
if os.path.isdir(src):
print 'copying logs'
logdir = os.path.join(todir, 'log')
os.makedirs(logdir)
dst = os.path.join(logdir, 'public')
main_log = []
name = 'madcow.log'
count = [0]
def check(basedir, filenames):
count[0] += len(filenames)
if name in filenames:
main_log.append(os.path.join(basedir, name))
count[0] -= 1
return [name]
return []
shutil.copytree(src, dst, ignore=check)
print 'copied %d public logs' % count[0]
if main_log:
copy(main_log[0], os.path.join(logdir, name))
2
Example 23
Project: rlpy Source File: game.py
def run(self):
"""
Main control loop for game play.
"""
self.display.initialize(self.state.data)
self.numMoves = 0
# self.display.initialize(self.state.makeObservation(1).data)
# inform learning agents of the game start
for i in range(len(self.agents)):
agent = self.agents[i]
if not agent:
self.mute(i)
# this is a null agent, meaning it failed to load
# the other team wins
print >>sys.stderr, "Agent %d failed to load" % i
self.unmute()
self._agentCrash(i, quiet=True)
return
if ("registerInitialState" in dir(agent)):
self.mute(i)
if self.catchExceptions:
try:
timed_func = TimeoutFunction(
agent.registerInitialState,
int(self.rules.getMaxStartupTime(i)))
try:
start_time = time.time()
timed_func(self.state.deepCopy())
time_taken = time.time() - start_time
self.totalAgentTimes[i] += time_taken
except TimeoutFunctionException:
print >>sys.stderr, "Agent %d ran out of time on startup!" % i
self.unmute()
self.agentTimeout = True
self._agentCrash(i, quiet=True)
return
except Exception as data:
self._agentCrash(i, quiet=False)
self.unmute()
return
else:
agent.registerInitialState(self.state.deepCopy())
# TODO: could this exceed the total time
self.unmute()
agentIndex = self.startingIndex
numAgents = len(self.agents)
while not self.gameOver:
# Fetch the next agent
agent = self.agents[agentIndex]
move_time = 0
skip_action = False
# Generate an observation of the state
if 'observationFunction' in dir(agent):
self.mute(agentIndex)
if self.catchExceptions:
try:
timed_func = TimeoutFunction(
agent.observationFunction,
int(self.rules.getMoveTimeout(agentIndex)))
try:
start_time = time.time()
observation = timed_func(self.state.deepCopy())
except TimeoutFunctionException:
skip_action = True
move_time += time.time() - start_time
self.unmute()
except Exception as data:
self._agentCrash(agentIndex, quiet=False)
self.unmute()
return
else:
observation = agent.observationFunction(
self.state.deepCopy())
self.unmute()
else:
observation = self.state.deepCopy()
# Solicit an action
action = None
self.mute(agentIndex)
if self.catchExceptions:
try:
timed_func = TimeoutFunction(
agent.getAction,
int(self.rules.getMoveTimeout(agentIndex)) - int(move_time))
try:
start_time = time.time()
if skip_action:
raise TimeoutFunctionException()
action = timed_func(observation)
except TimeoutFunctionException:
print >>sys.stderr, "Agent %d timed out on a single move!" % agentIndex
self.agentTimeout = True
self._agentCrash(agentIndex, quiet=True)
self.unmute()
return
move_time += time.time() - start_time
if move_time > self.rules.getMoveWarningTime(agentIndex):
self.totalAgentTimeWarnings[agentIndex] += 1
print >>sys.stderr, "Agent %d took too long to make a move! This is warning %d" % (
agentIndex, self.totalAgentTimeWarnings[agentIndex])
if self.totalAgentTimeWarnings[agentIndex] > self.rules.getMaxTimeWarnings(agentIndex):
print >>sys.stderr, "Agent %d exceeded the maximum number of warnings: %d" % (
agentIndex, self.totalAgentTimeWarnings[agentIndex])
self.agentTimeout = True
self._agentCrash(agentIndex, quiet=True)
self.unmute()
return
self.totalAgentTimes[agentIndex] += move_time
# print "Agent: %d, time: %f, total: %f" % (agentIndex,
# move_time, self.totalAgentTimes[agentIndex])
if self.totalAgentTimes[agentIndex] > self.rules.getMaxTotalTime(agentIndex):
print >>sys.stderr, "Agent %d ran out of time! (time: %1.2f)" % (
agentIndex, self.totalAgentTimes[agentIndex])
self.agentTimeout = True
self._agentCrash(agentIndex, quiet=True)
self.unmute()
return
self.unmute()
except Exception as data:
self._agentCrash(agentIndex)
self.unmute()
return
else:
action = agent.getAction(observation)
self.unmute()
# Execute the action
self.moveHistory.append((agentIndex, action))
if self.catchExceptions:
try:
self.state = self.state.generateSuccessor(
agentIndex,
action)
except Exception as data:
self.mute(agentIndex)
self._agentCrash(agentIndex)
self.unmute()
return
else:
self.state = self.state.generateSuccessor(agentIndex, action)
# Change the display
self.display.update(self.state.data)
###idx = agentIndex - agentIndex % 2 + 1
###self.display.update( self.state.makeObservation(idx).data )
# Allow for game specific conditions (winning, losing, etc.)
self.rules.process(self.state, self)
# Track progress
if agentIndex == numAgents + 1:
self.numMoves += 1
# Next agent
agentIndex = (agentIndex + 1) % numAgents
if _BOINC_ENABLED:
boinc.set_fraction_done(self.getProgress())
# inform a learning agent of the game result
for agentIndex, agent in enumerate(self.agents):
if "final" in dir(agent):
try:
self.mute(agentIndex)
agent.final(self.state)
self.unmute()
except Exception as data:
if not self.catchExceptions:
raise
self._agentCrash(agentIndex)
self.unmute()
return
self.display.finish()
2
Example 24
Project: FIDDLE Source File: data4trainng.py
def main():
usage = 'usage: %prog [options] <assembly> <annotation_file> <out_file>'
parser = OptionParser(usage)
parser.add_option('-d', dest='rootDir', type='str', default='.', help='Root directory of the project [Default: %default]')
parser.add_option('-b', dest='chunkSize', default=1000, type='int', help='Align sizes with batch size')
parser.add_option('-e', dest='width', type='int', default=500, help='Extend all sequences to this length [Default: %default]')
parser.add_option('-r', dest='stride', default=20, type='int', help='Stride sequences [Default: %default]')
(options,args) = parser.parse_args()
if len(args) !=3 :
print(args)
print(options)
print(len(args))
parser.error('Must provide assembly, annotation file and an output name')
else:
assembly = args[0]
annot_file = args[1]
out_file = args[2]
# read in the annotation file
annot = pd.read_table(annot_file,sep=',')
# Make directory for the project
directory = "../../data/hdf5datasets/"
if not os.path.exists(directory):
os.makedirs(directory)
# Parameters
x1 = 500 #upstream
x2 = 500 #downstream
# open the hdf5 file to write
f = h5py.File(os.path.join(directory,out_file), "w")
trainSize = np.floor(0.90*len(annot)*(x1+x2-options.width)/options.stride)
testSize = np.floor(0.05*len(annot)*(x1+x2-options.width)/options.stride)
# make sure that the sizes are integer multiple of chunk size
trainSize -= (trainSize % options.chunkSize)
testSize -= (testSize % options.chunkSize)
trainSize = int(trainSize)
testSize = int(testSize)
print >> sys.stderr, '%d training sequences\n %d test sequences ' % (trainSize,testSize)
# note that we have 1 channel and 4xoptions.width matrices for dna sequence.
NStrainData = f.create_dataset("NStrainInp", (trainSize,2,1,options.width))
MStrainData = f.create_dataset("MStrainInp", (trainSize,2,1,options.width))
DStrainData = f.create_dataset("DStrainInp", (trainSize,4,1,options.width))
RStrainData = f.create_dataset("RStrainInp", (trainSize,1,1,options.width))
TFtrainData = f.create_dataset("TFtrainInp", (trainSize,2,1,options.width))
trainTarget = f.create_dataset("trainOut", (trainSize,1,1,options.width))
# note that we have 1 channel and 4xoptions.width matrices for dna sequence.
NStestData = f.create_dataset("NStestInp", (testSize,2,1,options.width))
MStestData = f.create_dataset("MStestInp", (testSize,2,1,options.width))
DStestData = f.create_dataset("DStestInp", (testSize,4,1,options.width))
RStestData = f.create_dataset("RStestInp", (testSize,1,1,options.width))
TFtestData = f.create_dataset("TFtestInp", (testSize,2,1,options.width))
testTarget = f.create_dataset("testOut", (testSize,1,1,options.width))
info = f.create_dataset("info", (trainSize+testSize,4)) # chromosome no, strand, index of the annotation, genomic position
chrRange = annot['chr'].unique()
# Use 4 channel and 1xoptions.width
NSdata = np.zeros([options.chunkSize,2,1,options.width])
MSdata = np.zeros([options.chunkSize,2,1,options.width])
DSdata = np.zeros([options.chunkSize,4,1,options.width])
RSdata = np.zeros([options.chunkSize,1,1,options.width])
TFdata = np.zeros([options.chunkSize,2,1,options.width])
target = np.zeros([options.chunkSize,1,1,options.width])
infodata = np.zeros([options.chunkSize,4])
qq=0;
cc=0;
ps =0;
nestVar = 0;
debugMode = True
# if options.species in ['YJ167','YJ168','YJ169','Scer','YSC001']:
# assembly = 'sacCer3'
# elif options.species in ['YJ160']:
# assembly = 'Klla'
# elif options.species in ['YJ177']:
# assembly = 'DeHa2'
# else:
# raise('unknown species')
print assembly
gdb = genome.db.GenomeDB(path='/Users/umut/Projects/genome/data/share/genome_db',assembly=assembly)
NSpos = gdb.open_track('NSpos')
NSneg = gdb.open_track('NSneg')
MSpos = gdb.open_track('MSpos')
MSneg = gdb.open_track('MSneg')
TFpos = gdb.open_track('TFpos')
TFneg = gdb.open_track('TFneg')
RS = gdb.open_track('RS')
TSpos = gdb.open_track('TSpos')
TSneg = gdb.open_track('TSneg')
seq = gdb.open_track("seq")
for chname in chrRange:
if nestVar:
break
cc +=1
tf = annot.chr==chname
print 'doing %s' % (chname)
for i in range(sum(tf)):
if nestVar:
break
tss = annot[tf].tss.iloc[i]
if annot[tf].strand.iloc[i]=="-":
xran = range(tss-x2,tss+x1-options.width,options.stride)
else:
if tss<1000:
continue
xran = range(tss-x1,tss+x2-options.width,options.stride)
annotIdx = annot[tf].index[i]
for pos in xran:
if nestVar:
break
seqVec = seq.get_seq_str(chname,pos+1,(pos+options.width))
dsdata = vectorizeSequence(seqVec.lower())
nsP = NSpos.get_nparray(chname,pos+1,(pos+options.width))
nsN = NSneg.get_nparray(chname,pos+1,(pos+options.width))
msP = MSpos.get_nparray(chname,pos+1,(pos+options.width))
msN = MSneg.get_nparray(chname,pos+1,(pos+options.width))
tfP = TFpos.get_nparray(chname,pos+1,(pos+options.width))
tfN = TFneg.get_nparray(chname,pos+1,(pos+options.width))
rs = RS.get_nparray(chname,pos+1,(pos+options.width))
tsP = TSpos.get_nparray(chname,pos+1,(pos+options.width))
tsN = TSneg.get_nparray(chname,pos+1,(pos+options.width))
if debugMode:
if not checkData(np.r_[nsP,nsN,msP,msN,rs,tsP,tsN,tfP,tfN]):
print('NaN detected in chr' + chname + ' and at the position:' + str(pos))
# print nsmstsrsdata
# nestVar = 1;
continue
if annot[tf].strand.iloc[i]=="+":
NSdata[qq,0,0,:] = nsP.T
NSdata[qq,1,0,:] = nsN.T
MSdata[qq,0,0,:] =msP.T
MSdata[qq,1,0,:] = msN.T
DSdata[qq,:,0,:] = dsdata.T
RSdata[qq,0,0,:] = rs.T
TFdata[qq,0,0,:] =tfP.T
TFdata[qq,1,0,:] = tfN.T
if sum(tsP)==0:
tsP = tsP + 1/np.float(options.width)
else:
tsP = tsP/sum(tsP+1e-5)
target[qq,0,0,:] =tsP.T
infodata[qq,:] = [cc, 1,annotIdx,pos]
else:
NSdata[qq,0,0,:] = np.flipud(nsN).T
NSdata[qq,1,0,:] = np.flipud(nsP).T
MSdata[qq,0,0,:] = np.flipud(msN).T
MSdata[qq,1,0,:] = np.flipud(msP).T
RSdata[qq,0,0,:] = np.flipud(rs).T
DSdata[qq,:,0,:] = np.flipud(np.fliplr(dsdata)).T
TFdata[qq,0,0,:] = np.flipud(tfN).T
TFdata[qq,1,0,:] = np.flipud(tfP).T
if sum(tsN)==0:
tsN = tsN + 1/np.float(options.width)
else:
tsN = tsN/sum(tsN+1e-5)
target[qq,0,0,:] =np.flipud(tsN).T
infodata[qq,:] = [cc, -1,annotIdx,pos]
qq+=1
if ((ps+options.chunkSize) <= trainSize) and (qq>=options.chunkSize):
stp = options.chunkSize
NStrainData[range(ps,ps+stp),:,:,:] = NSdata
MStrainData[range(ps,ps+stp),:,:,:] = MSdata
DStrainData[range(ps,ps+stp),:,:,:] = DSdata
RStrainData[range(ps,ps+stp),:,:,:] = RSdata
TFtrainData[range(ps,ps+stp),:,:,:] = TFdata
trainTarget[range(ps,ps+stp),:,:,:] = target
info[range(ps,ps+stp),:] = infodata
NSdata = np.zeros([options.chunkSize,2,1,options.width])
MSdata = np.zeros([options.chunkSize,2,1,options.width])
DSdata = np.zeros([options.chunkSize,4,1,options.width])
RSdata = np.zeros([options.chunkSize,1,1,options.width])
infodata = np.zeros([options.chunkSize,4])
ps+=stp
qq=0
print >> sys.stderr, '%d training chunk saved ' % ps
if (ps >= trainSize) & (ps < (trainSize + testSize)) and (qq>=options.chunkSize):
NStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = NSdata
MStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = MSdata
DStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = DSdata
RStestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = RSdata
TFtestData[range(ps-trainSize,ps-trainSize+stp),:,:,:] = TFdata
testTarget[range(ps-trainSize,ps-trainSize+stp),:,:,:] = target
info[range(ps,ps+stp),:] = infodata
rt = ps-trainSize
ps+=stp
qq=0
print >> sys.stderr, '%d testing chunk saved ' % rt
if ps >=(trainSize+testSize):
nestVar = 1;
break
print ps
f.close()
NSpos.close()
NSneg.close()
MSpos.close()
MSneg.close()
RS.close()
TFpos.close()
TFneg.close()
TSpos.close()
TSneg.close()
seq.close()
2
Example 25
Project: dx-toolkit Source File: app_unbuilder.py
def dump_executable(executable, destination_directory, omit_resources=False, describe_output=[]):
"""
Reconstitutes executable into a directory that would create a
functionally identical executable if "dx build" were run on it.
destination_directory will be the root source directory for the
applet.
:param executable: executable, i.e. app or applet, to be dumped
:type executable: DXExecutable (only DXApp or DXApplet now)
:param destination_directory: an existing, empty, and writable directory
:type destination_directory: str
"""
old_cwd = os.getcwd()
os.chdir(destination_directory)
try:
info = executable.get()
if info["runSpec"]["interpreter"] == "bash":
suffix = "sh"
elif info["runSpec"]["interpreter"] == "python2.7":
suffix = "py"
else:
print('Sorry, I don\'t know how to get executables with interpreter ' +
info["runSpec"]["interpreter"] + '\n', file=sys.stderr)
sys.exit(1)
# Entry point script
script = "src/code.%s" % (suffix,)
os.mkdir("src")
with open(script, "w") as f:
f.write(info["runSpec"]["code"])
# Get all the asset bundles
asset_depends = []
deps_to_remove = []
# When an applet is built bundledDepends are added in the following order:
# 1. bundledDepends explicitly specified in the dxapp.json
# 2. resources (contents of resources directory added as bundledDepends)
# 3. assetDepends (translated into bundledDepends)
#
# Therefore while translating bundledDepends to assetDepends, we are traversing the
# list in reverse order and exiting when we can't find the "AssetBundle" property
# with the tarball file.
#
# NOTE: If last item (and contiguous earlier items) of bundledDepends (#1 above) refers to an
# AssetBundle tarball, those items will be converted to assetDepends.
#
# TODO: The bundledDepends should be annotated with another field called {"asset": true}
# to distinguish it from non assets. It will be needed to annotate the bundleDepends,
# when the wrapper record object is no more accessible.
for dep in reversed(info["runSpec"]["bundledDepends"]):
file_handle = get_handler(dep["id"])
if isinstance(file_handle, dxpy.DXFile):
asset_record_id = file_handle.get_properties().get("AssetBundle")
asset_record = None
if asset_record_id:
asset_record = dxpy.DXRecord(asset_record_id)
if asset_record:
try:
asset_depends.append({"name": asset_record.describe().get("name"),
"project": asset_record.get_proj_id(),
"folder": asset_record.describe().get("folder"),
"version": asset_record.describe(fields={"properties": True}
)["properties"]["version"]
})
deps_to_remove.append(dep)
except DXError:
print("Describe failed on the assetDepends record object with ID - " +
asset_record_id + "\n", file=sys.stderr)
pass
else:
break
# Reversing the order of the asset_depends[] so that original order is maintained
asset_depends.reverse()
# resources/ directory
created_resources_directory = False
if not omit_resources:
for dep in info["runSpec"]["bundledDepends"]:
if dep in deps_to_remove:
continue
handler = get_handler(dep["id"])
if isinstance(handler, dxpy.DXFile):
if not created_resources_directory:
os.mkdir("resources")
created_resources_directory = True
handler_id = handler.get_id()
fname = "resources/%s.tar.gz" % (handler_id)
download_dxfile(handler_id, fname)
print("Unpacking resources", file=sys.stderr)
tar = tarfile.open(fname)
tar.extractall("resources")
tar.close()
os.unlink(fname)
deps_to_remove.append(dep)
# TODO: if output directory is not the same as executable name we
# should print a warning and/or offer to rewrite the "name"
# field in the 'dxapp.json'
dxapp_json = collections.OrderedDict()
all_keys = executable._get_required_keys() + executable._get_optional_keys()
for key in all_keys:
if key in executable._get_describe_output_keys() and key in describe_output:
dxapp_json[key] = describe_output[key]
if key in info:
dxapp_json[key] = info[key]
if info.get("hidden", False):
dxapp_json["hidden"] = True
# TODO: inputSpec and outputSpec elements should have their keys
# printed in a sensible (or at least consistent) order too
# Un-inline code
del dxapp_json["runSpec"]["code"]
dxapp_json["runSpec"]["file"] = script
# Remove resources from bundledDepends
for dep in deps_to_remove:
dxapp_json["runSpec"]["bundledDepends"].remove(dep)
# Add assetDepends to dxapp.json
if len(asset_depends) > 0:
dxapp_json["runSpec"]["assetDepends"] = asset_depends
# Ordering input/output spec keys
ordered_spec_keys = ("name", "label", "help", "class", "type", "patterns", "optional", "default", "choices",
"suggestions", "group")
for spec_key in "inputSpec", "outputSpec":
if spec_key not in dxapp_json.keys():
continue
for i, spec in enumerate(dxapp_json[spec_key]):
ordered_spec = collections.OrderedDict()
# Adding keys, for which the ordering is defined
for key in ordered_spec_keys:
if key in spec.keys():
ordered_spec[key] = spec[key]
# Adding the rest of the keys
for key in spec.keys():
if key not in ordered_spec_keys:
ordered_spec[key] = spec[key]
dxapp_json[spec_key][i] = ordered_spec
# Remove dx-toolkit from execDepends
dx_toolkit = {"name": "dx-toolkit", "package_manager": "apt"}
if dx_toolkit in dxapp_json["runSpec"]["execDepends"]:
dxapp_json["runSpec"]["execDepends"].remove(dx_toolkit)
# Remove "bundledDependsByRegion" field from "runSpec". This utility
# will reconstruct the resources directory based on the
# "bundledDepends" field, which should be equivalent to
# "bundledDependsByRegion".
dxapp_json["runSpec"].pop("bundledDependsByRegion", None)
# Cleanup of empty elements. Be careful not to let this step
# introduce any semantic changes to the app specification. For
# example, an empty input (output) spec is not equivalent to a
# missing input (output) spec.
if 'runSpec' in dxapp_json:
_recursive_cleanup(dxapp_json['runSpec'])
if 'access' in dxapp_json:
_recursive_cleanup(dxapp_json['access'])
for key in executable._get_cleanup_keys():
if key in dxapp_json and not dxapp_json[key]:
del dxapp_json[key]
readme = info.get("description", "")
devnotes = info.get("developerNotes", "")
# Write dxapp.json, Readme.md, and Readme.developer.md
with open("dxapp.json", "w") as f:
f.write(flatten_json_array(json.dumps(dxapp_json, indent=2, separators=(',', ': ')), "patterns"))
f.write('\n')
if readme:
with open("Readme.md", "w") as f:
f.write(readme)
if devnotes:
with open("Readme.developer.md", "w") as f:
f.write(devnotes)
except:
err_exit()
finally:
os.chdir(old_cwd)
2
Example 26
Project: dx-toolkit Source File: __init__.py
def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
timeout=DEFAULT_TIMEOUT,
use_compression=None, jsonify_data=True, want_full_response=False,
decode_response_body=True, prepend_srv=True, session_handler=None,
max_retries=DEFAULT_RETRIES, always_retry=False,
**kwargs):
'''
:param resource: API server route, e.g. "/record/new". If *prepend_srv* is False, a fully qualified URL is expected. If this argument is a callable, it will be called just before each request attempt, and expected to return a tuple (URL, headers). Headers returned by the callback are updated with *headers* (including headers set by this method).
:type resource: string
:param data: Content of the request body
:type data: list or dict, if *jsonify_data* is True; or string or file-like object, otherwise
:param headers: Names and values of HTTP headers to submit with the request (in addition to those needed for authentication, compression, or other options specified with the call).
:type headers: dict
:param auth:
Controls the ``Authentication`` header or other means of authentication supplied with the request. If ``True``
(default), a token is obtained from the ``DX_SECURITY_CONTEXT``. If the value evaluates to false, no action is
taken to prepare authentication for the request. Otherwise, the value is assumed to be callable, and called with
three arguments (method, url, headers) and expected to prepare the authentication headers by reference.
:type auth: tuple, object, True (default), or None
:param timeout: HTTP request timeout, in seconds
:type timeout: float
:param config: *config* value to pass through to :meth:`requests.request`
:type config: dict
:param use_compression: Deprecated
:type use_compression: string or None
:param jsonify_data: If True, *data* is converted from a Python list or dict to a JSON string
:type jsonify_data: boolean
:param want_full_response: If True, the full :class:`requests.Response` object is returned (otherwise, only the content of the response body is returned)
:type want_full_response: boolean
:param decode_response_body: If True (and *want_full_response* is False), the response body is decoded and, if it is a JSON string, deserialized. Otherwise, the response body is uncompressed if transport compression is on, and returned raw.
:type decode_response_body: boolean
:param prepend_srv: If True, prepends the API server location to the URL
:type prepend_srv: boolean
:param session_handler: Deprecated.
:param max_retries: Maximum number of retries to perform for a request. A "failed" request is retried if any of the following is true:
- A response is received from the server, and the content length received does not match the "Content-Length" header.
- A response is received from the server, and the response has an HTTP status code in 5xx range.
- A response is received from the server, the "Content-Length" header is not set, and the response JSON cannot be parsed.
- No response is received from the server, and either *always_retry* is True or the request *method* is "GET".
:type max_retries: int
:param always_retry: If True, indicates that it is safe to retry a request on failure
- Note: It is not guaranteed that the request will *always* be retried on failure; rather, this is an indication to the function that it would be safe to do so.
:type always_retry: boolean
:returns: Response from API server in the format indicated by *want_full_response* and *decode_response_body*.
:raises: :exc:`exceptions.DXAPIError` or a subclass if the server returned a non-200 status code; :exc:`requests.exceptions.HTTPError` if an invalid response was received from the server; or :exc:`requests.exceptions.ConnectionError` if a connection cannot be established.
Wrapper around :meth:`requests.request()` that makes an HTTP
request, inserting authentication headers and (by default)
converting *data* to JSON.
.. note:: Bindings methods that make API calls make the underlying
HTTP request(s) using :func:`DXHTTPRequest`, and most of them
will pass any unrecognized keyword arguments you have supplied
through to :func:`DXHTTPRequest`.
'''
if headers is None:
headers = {}
global _UPGRADE_NOTIFY
seq_num = _get_sequence_number()
url = APISERVER + resource if prepend_srv else resource
method = method.upper() # Convert method name to uppercase, to ease string comparisons later
if auth is True:
auth = AUTH_HELPER
if auth:
auth(_RequestForAuth(method, url, headers))
pool_args = {arg: kwargs.pop(arg, None) for arg in ("verify", "cert_file", "key_file")}
test_retry = kwargs.pop("_test_retry_http_request", False)
if _DEBUG >= 2:
if isinstance(data, basestring) or isinstance(data, mmap.mmap):
if len(data) == 0:
formatted_data = '""'
else:
formatted_data = "<file data>"
else:
try:
if _DEBUG >= 3:
formatted_data = json.dumps(data, indent=2)
else:
formatted_data = json.dumps(data)
except (UnicodeDecodeError, TypeError):
formatted_data = "<binary data>"
if jsonify_data:
data = json.dumps(data)
if 'Content-Type' not in headers and method == 'POST':
headers['Content-Type'] = 'application/json'
# If the input is a buffer, its data gets consumed by
# requests.request (moving the read position). Record the initial
# buffer position so that we can return to it if the request fails
# and needs to be retried.
rewind_input_buffer_offset = None
if hasattr(data, 'seek') and hasattr(data, 'tell'):
rewind_input_buffer_offset = data.tell()
try_index = 0
retried_responses = []
while True:
success, time_started = True, None
response = None
req_id = None
try:
time_started = time.time()
_method, _url, _headers = _process_method_url_headers(method, url, headers)
if _DEBUG >= 2:
maybe_headers = ''
if 'Range' in _headers:
maybe_headers = " " + json.dumps({"Range": _headers["Range"]})
print("%s [%f] %s %s%s => %s\n" % (YELLOW(BOLD(">%d" % seq_num)),
time_started,
BLUE(method),
_url,
maybe_headers,
formatted_data),
file=sys.stderr,
end="")
elif _DEBUG > 0:
from repr import Repr
print("%s [%f] %s %s => %s\n" % (YELLOW(BOLD(">%d" % seq_num)),
time_started,
BLUE(method),
_url,
Repr().repr(data)),
file=sys.stderr,
end="")
body = _maybe_trucate_request(_url, try_index, data)
# throws BadStatusLine if the server returns nothing
try:
pool_manager = _get_pool_manager(**pool_args)
def unicode2str(s):
if isinstance(s, unicode):
return s.encode('ascii')
else:
return s
_headers['User-Agent'] = USER_AGENT
_headers['DNAnexus-API'] = API_VERSION
# Converted Unicode headers to ASCII and throw an error if not possible
_headers = {unicode2str(k): unicode2str(v) for k, v in _headers.items()}
response = pool_manager.request(_method, _url, headers=_headers, body=body,
timeout=timeout, retries=False, **kwargs)
except urllib3.exceptions.ClosedPoolError:
# If another thread closed the pool before the request was
# started, will throw ClosedPoolError
raise exceptions.UrllibInternalError("ClosedPoolError")
_raise_error_for_testing(try_index, method)
req_id = response.headers.get("x-request-id", "unavailable")
if _UPGRADE_NOTIFY and response.headers.get('x-upgrade-info', '').startswith('A recommended update is available') and '_ARGCOMPLETE' not in os.environ:
logger.info(response.headers['x-upgrade-info'])
try:
with file(_UPGRADE_NOTIFY, 'a'):
os.utime(_UPGRADE_NOTIFY, None)
except:
pass
_UPGRADE_NOTIFY = False
# If an HTTP code that is not in the 200 series is received and the content is JSON, parse it and throw the
# appropriate error. Otherwise, raise the usual exception.
if response.status // 100 != 2:
# response.headers key lookup is case-insensitive
if response.headers.get('content-type', '').startswith('application/json'):
try:
content = response.data.decode('utf-8')
except AttributeError:
raise exceptions.UrllibInternalError("Content is none", response.status)
try:
content = json.loads(content)
except ValueError:
# The JSON is not parsable, but we should be able to retry.
raise exceptions.BadJSONInReply("Invalid JSON received from server", response.status)
try:
error_class = getattr(exceptions, content["error"]["type"], exceptions.DXAPIError)
except (KeyError, AttributeError, TypeError):
raise exceptions.HTTPError(response.status, content)
raise error_class(content, response.status, time_started, req_id)
else:
try:
content = response.data.decode('utf-8')
except AttributeError:
raise exceptions.UrllibInternalError("Content is none", response.status)
raise exceptions.HTTPError("{} {} [Time={} RequestID={}]\n{}".format(response.status,
response.reason,
time_started,
req_id,
content))
if want_full_response:
return response
else:
if 'content-length' in response.headers:
if int(response.headers['content-length']) != len(response.data):
range_str = (' (%s)' % (headers['Range'],)) if 'Range' in headers else ''
raise exceptions.ContentLengthError(
"Received response with content-length header set to %s but content length is %d%s. " +
"[Time=%f RequestID=%s]" %
(response.headers['content-length'], len(response.data), range_str, time_started, req_id)
)
content = response.data
content_to_print = "(%d bytes)" % len(content) if len(content) > 0 else ''
if decode_response_body:
content = content.decode('utf-8')
if response.headers.get('content-type', '').startswith('application/json'):
try:
content = json.loads(content)
except ValueError:
# The JSON is not parsable, but we should be able to retry.
raise exceptions.BadJSONInReply("Invalid JSON received from server", response.status)
if _DEBUG >= 3:
content_to_print = "\n " + json.dumps(content, indent=2).replace("\n", "\n ")
elif _DEBUG == 2:
content_to_print = json.dumps(content)
elif _DEBUG > 0:
content_to_print = Repr().repr(content)
if _DEBUG > 0:
t = int((time.time() - time_started) * 1000)
req_id = response.headers.get('x-request-id') or "--"
code_format = GREEN if (200 <= response.status < 300) else RED
print(" " + YELLOW(BOLD("<%d" % seq_num)),
"[%f]" % time_started,
BLUE(method),
req_id,
_url,
"<=",
code_format(str(response.status)),
WHITE(BOLD("(%dms)" % t)),
content_to_print,
file=sys.stderr)
if test_retry:
retried_responses.append(content)
if len(retried_responses) == 1:
continue
else:
_set_retry_response(retried_responses[0])
return retried_responses[1]
return content
raise AssertionError('Should never reach this line: expected a result to have been returned by now')
except Exception as e:
# Avoid reusing connections in the pool, since they may be
# in an inconsistent state (observed as "ResponseNotReady"
# errors).
_get_pool_manager(**pool_args).clear()
success = False
exception_msg = _extract_msg_from_last_exception()
if isinstance(e, _expected_exceptions):
if response is not None and response.status == 503:
seconds_to_wait = _extract_retry_after_timeout(response)
logger.warn("%s %s: %s. Request Time=[%f] RequestID=[%s] Waiting %d seconds due to server unavailability...",
method, url, exception_msg, time_started, req_id, seconds_to_wait)
time.sleep(seconds_to_wait)
# Note, we escape the "except" block here without
# incrementing try_index because 503 responses with
# Retry-After should not count against the number of
# permitted retries.
continue
# Total number of allowed tries is the initial try + up to
# (max_retries) subsequent retries.
total_allowed_tries = max_retries + 1
ok_to_retry = False
is_retryable = always_retry or (method == 'GET') or _is_retryable_exception(e)
# Because try_index is not incremented until we escape this
# iteration of the loop, try_index is equal to the number of
# tries that have failed so far, minus one. Test whether we
# have exhausted all retries.
#
# BadStatusLine --- server did not return anything
# BadJSONInReply --- server returned JSON that didn't parse properly
if try_index + 1 < total_allowed_tries:
if response is None or \
isinstance(e, (exceptions.ContentLengthError, BadStatusLine, exceptions.BadJSONInReply, \
urllib3.exceptions.ProtocolError, exceptions.UrllibInternalError)):
ok_to_retry = is_retryable
else:
ok_to_retry = 500 <= response.status < 600
# The server has closed the connection prematurely
if response is not None and \
response.status == 400 and is_retryable and method == 'PUT' and \
isinstance(e, requests.exceptions.HTTPError):
if '<Code>RequestTimeout</Code>' in exception_msg:
logger.info("Retrying 400 HTTP error, due to slow data transfer. " +
"Request Time=[%f] RequestID=[%s]", time_started, req_id)
else:
logger.info("400 HTTP error, of unknown origin, exception_msg=[%s]. " +
"Request Time=[%f] RequestID=[%s]", exception_msg, time_started, req_id)
ok_to_retry = True
if ok_to_retry:
if rewind_input_buffer_offset is not None:
data.seek(rewind_input_buffer_offset)
delay = min(2 ** try_index, DEFAULT_TIMEOUT)
range_str = (' (range=%s)' % (headers['Range'],)) if 'Range' in headers else ''
logger.warn("[%s] %s %s: %s. Waiting %d seconds before retry %d of %d... %s",
time.ctime(), method, url, exception_msg, delay, try_index + 1, max_retries, range_str)
time.sleep(delay)
try_index += 1
continue
# All retries have been exhausted OR the error is deemed not
# retryable. Print the latest error and propagate it back to the caller.
if not isinstance(e, exceptions.DXAPIError):
logger.error("[%s] %s %s: %s.", time.ctime(), method, url, exception_msg)
# Retries have been exhausted, and we are unable to get a full
# buffer from the data source. Raise a special exception.
if isinstance(e, urllib3.exceptions.ProtocolError) and \
'Connection broken: IncompleteRead' in exception_msg:
raise exceptions.DXIncompleteReadsError(exception_msg)
raise
finally:
if success and try_index > 0:
logger.info("[%s] %s %s: Recovered after %d retries", time.ctime(), method, url, try_index)
raise AssertionError('Should never reach this line: should have attempted a retry or reraised by now')
raise AssertionError('Should never reach this line: should never break out of loop')
2
Example 27
Project: spinoff Source File: win32fix.py
def fix_unicode_on_win32():
from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int
from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID
original_stderr = sys.stderr
# If any exception occurs in this code, we'll probably try to print it on stderr,
# which makes for frustrating debugging if stderr is directed to our wrapper.
# So be paranoid about catching errors and reporting them to original_stderr,
# so that we can at least see them.
def _complain(message):
print(original_stderr, message if isinstance(message, str) else repr(message))
# Work around <http://bugs.python.org/issue6058>.
codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
# Make Unicode console output work independently of the current code page.
# This also fixes <http://bugs.python.org/issue1602>.
# Credit to Michael Kaplan <http://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx>
# and TZOmegaTZIOY
# <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>.
try:
# <http://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx>
# HANDLE WINAPI GetStdHandle(DWORD nStdHandle);
# returns INVALID_HANDLE_VALUE, NULL, or a valid handle
#
# <http://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx>
# DWORD WINAPI GetFileType(DWORD hFile);
#
# <http://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx>
# BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode);
GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(("GetStdHandle", windll.kernel32))
STD_OUTPUT_HANDLE = DWORD(-11)
STD_ERROR_HANDLE = DWORD(-12)
GetFileType = WINFUNCTYPE(DWORD, DWORD)(("GetFileType", windll.kernel32))
FILE_TYPE_CHAR = 0x0002
FILE_TYPE_REMOTE = 0x8000
GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))(("GetConsoleMode", windll.kernel32))
INVALID_HANDLE_VALUE = DWORD(-1).value
def not_a_console(handle):
if handle == INVALID_HANDLE_VALUE or handle is None:
return True
return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
or GetConsoleMode(handle, byref(DWORD())) == 0)
old_stdout_fileno = None
old_stderr_fileno = None
if hasattr(sys.stdout, 'fileno'):
old_stdout_fileno = sys.stdout.fileno()
if hasattr(sys.stderr, 'fileno'):
old_stderr_fileno = sys.stderr.fileno()
STDOUT_FILENO = 1
STDERR_FILENO = 2
real_stdout = (old_stdout_fileno == STDOUT_FILENO)
real_stderr = (old_stderr_fileno == STDERR_FILENO)
if real_stdout:
hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
if not_a_console(hStdout):
real_stdout = False
if real_stderr:
hStderr = GetStdHandle(STD_ERROR_HANDLE)
if not_a_console(hStderr):
real_stderr = False
if real_stdout or real_stderr:
# BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, LPDWORD lpCharsWritten, LPVOID lpReserved);
WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)(("WriteConsoleW", windll.kernel32))
class UnicodeOutput:
def __init__(self, hConsole, stream, fileno, name):
self._hConsole = hConsole
self._stream = stream
self._fileno = fileno
self.closed = False
self.softspace = False
self.mode = 'w'
self.encoding = 'utf-8'
self.name = name
self.flush()
def isatty(self):
return False
def close(self):
# don't really close the handle, that would only cause problems
self.closed = True
def fileno(self):
return self._fileno
def flush(self):
if self._hConsole is None:
try:
self._stream.flush()
except Exception as e:
_complain("%s.flush: %r from %r" % (self.name, e, self._stream))
raise
def write(self, text):
try:
if self._hConsole is None:
if isinstance(text, unicode):
text = text.encode('utf-8')
self._stream.write(text)
else:
if not isinstance(text, unicode):
text = str(text).decode('utf-8')
remaining = len(text)
while remaining > 0:
n = DWORD(0)
# There is a shorter-than-docuemented limitation on the
# length of the string passed to WriteConsoleW (see
# <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>.
retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None)
if retval == 0 or n.value == 0:
raise IOError("WriteConsoleW returned %r, n.value = %r" % (retval, n.value))
remaining -= n.value
if remaining == 0:
break
text = text[n.value:]
except Exception as e:
_complain("%s.write: %r" % (self.name, e))
raise
def writelines(self, lines):
try:
for line in lines:
self.write(line)
except Exception as e:
_complain("%s.writelines: %r" % (self.name, e))
raise
def __repr__(self):
return self.name
sys.stdout = (UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>')
if real_stdout else
UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>'))
sys.stderr = (UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>')
if real_stderr else
UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>'))
except Exception as e:
_complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,))
if UNMANGLE_CMDLINE_ARGS:
# This works around <http://bugs.python.org/issue2128>.
GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(("CommandLineToArgvW", windll.shell32))
argc = c_int(0)
argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
argv = [argv_unicode[i].encode('utf-8') for i in xrange(0, argc.value)]
if not hasattr(sys, 'frozen'):
# If this is an executable produced by py2exe or bbfreeze, then it will
# have been invoked directly. Otherwise, unicode_argv[0] is the Python
# interpreter, so skip that.
argv = argv[1:]
# Also skip option arguments to the Python interpreter.
while len(argv) > 0:
arg = argv[0]
if not arg.startswith(u"-") or arg == u"-":
break
argv = argv[1:]
if arg == u'-m':
# sys.argv[0] should really be the absolute path of the module source,
# but never mind
break
if arg == u'-c':
argv[0] = u'-c'
break
# if you like:
sys.argv = argv
2
Example 28
Project: netgrph Source File: path.py
def get_full_path(src, dst, popt, rtype="NGTREE"):
""" Gets the full path (switch->rt->VRF->rt->switch)
Requires NetDB for switchpath
"""
rtypes = ('CSV', 'TREE', 'JSON', 'YAML', 'NGTREE', 'QTREE')
if rtype in rtypes:
if 'onepath' not in popt:
popt['onepath'] = True
if 'l2path' not in popt:
popt['l2path'] = True
if 'verbose' not in popt:
popt['verbose'] = False
if nglib.verbose:
popt['verbose'] = True
if 'depth' not in popt:
popt['depth'] = '20'
logger.info("Query: Finding Full Path (%s --> %s) for %s",
src, dst, nglib.user)
net1, net2 = src, dst
n1tree, n2tree = None, None
# Translate IPs to CIDRs
n1tree = nglib.query.net.get_net(net1, rtype="NGTREE", verbose=popt['verbose'])
if n1tree:
net1 = n1tree['data'][0]['Name']
n2tree = nglib.query.net.get_net(net2, rtype="NGTREE", verbose=popt['verbose'])
if n2tree:
net2 = n2tree['data'][0]['Name']
if not n1tree or not n2tree:
errort = nglib.ngtree.get_ngtree("Path Error", tree_type="L3-PATH")
errort['Error'] = 'Network Lookup Error'
errort["Lx Path"] = src + " -> " + dst
return errort
srctree, dsttree, srcswp, dstswp = None, None, None, None
if 'vrfcidr' not in n1tree['data'][0]:
print("Warning: Could not locate", src, file=sys.stderr)
return
if 'vrfcidr' not in n2tree['data'][0]:
print("Warning: Could not locate", dst, file=sys.stderr)
return
# Routing Check
routing = True
if n1tree['data'][0]['vrfcidr'] == n2tree['data'][0]['vrfcidr']:
routing = False
if nglib.use_netdb:
srctree = nglib.netdb.ip.get_netdb_ip(src)
dsttree = nglib.netdb.ip.get_netdb_ip(dst)
# Find Switched Path from Source to Router
if srctree:
router = n1tree['data'][0]['Router']
if 'StandbyRouter' in n1tree['data'][0]:
router = router + '|' + n1tree['data'][0]['StandbyRouter']
if 'Switch' in srctree and srctree['Switch']:
srcswp = get_switched_path(srctree['Switch'], router, popt)
else:
srctree = None
print("Warning: Could not find source switch data in NetDB:", src, file=sys.stderr)
# Find Switched Path from Router to Destination
if dsttree:
router = n2tree['data'][0]['Router']
if 'StandbyRouter' in n2tree['data'][0]:
router = router + '|' + n2tree['data'][0]['StandbyRouter']
if 'Switch' in dsttree and dsttree['Switch']:
dstswp = get_switched_path(router, dsttree['Switch'], popt)
else:
dsttree = None
print("Warning: Could not find destination switch data in NetDB", \
dst, file=sys.stderr)
# If only switching, update srcswp to show switched path
if not routing and srctree and dsttree:
srcswp = get_switched_path(srctree['Switch'], dsttree['Switch'], popt)
# Same switch/vlan check
switching = True
if srctree and dsttree:
if srctree['Switch'] == dsttree['Switch'] and \
srctree['VLAN'] == dsttree['VLAN']:
switching = False
## Create Parent Data Structure
ngtree = nglib.ngtree.get_ngtree("L2-L4", tree_type="PATHs")
# Populate Overall Paths
if n1tree['data'][0]['Name'] != n2tree['data'][0]['Name']:
ngtree["L3 Path"] = net1 + " -> " + net2
ngtree["Lx Path"] = src + " -> " + dst
if srctree and dsttree:
ngtree["L2 Path"] = srctree['Switch'] + " (" + srctree['SwitchPort'] \
+ ") -> " + dsttree['Switch'] + " (" + dsttree['SwitchPort'] + ")"
if popt['onepath']:
ngtree["Traversal Type"] = 'Single Path'
else:
ngtree["Traversal Type"] = 'All Paths'
## Add the SRC Data
if len(n1tree['data']) > 1:
n1tree['data'][1]['_type'] = "SRC"
if 'SwitchPort' in n1tree['data'][1]:
n1tree['data'][1]['Name'] = src + ' ' + n1tree['data'][1]['MAC'] \
+ ' ' + str(n1tree['data'][1]['Switch']) + '(' \
+ str(n1tree['data'][1]['SwitchPort']) \
+ ') [vid:' + str(n1tree['data'][1]['VLAN']) + ']'
nglib.ngtree.add_child_ngtree(ngtree, n1tree['data'][1])
# Add L2 path if not switching
if not switching and len(n1tree['data']) > 1:
nglib.ngtree.add_child_ngtree(n1tree, n2tree['data'][1])
n1tree['_type'] = "L2-PATH"
n1tree['Name'] = src + ' -> ' + dst
# If there's src switched data add it
if switching and srcswp and srcswp['data']:
nglib.ngtree.add_child_ngtree(ngtree, srcswp)
# Add L3 Gateway
if switching and routing:
n1tree['data'][0]['_type'] = "L3-GW"
n1tree['data'][0]['Name'] = n1tree['data'][0]['Name'] \
+ ' ' + n1tree['data'][0]['Router']
if 'StandbyRouter' in n1tree['data'][0]:
n1tree['data'][0]['Name'] = n1tree['data'][0]['Name'] \
+ '|' + n1tree['data'][0]['StandbyRouter']
nglib.ngtree.add_child_ngtree(ngtree, n1tree['data'][0])
## Check for routed paths (inter/intra VRF)
rtree = get_full_routed_path(src, dst, popt)
if rtree and 'PATH' in rtree['_type']:
# Breakdown L4 Path
if rtree['_type'] == 'L4-PATH':
ngtree['L4 Path'] = rtree['Name']
for p in rtree['data']:
nglib.ngtree.add_child_ngtree(ngtree, p)
else:
ngtree['L4 Path'] = 'VRF:' + n1tree['data'][0]['VRF']
nglib.ngtree.add_child_ngtree(ngtree, rtree)
# Add the DST L3-GW Data
if switching and routing:
n2tree['data'][0]['_type'] = "L3-GW"
n2tree['data'][0]['Name'] = n2tree['data'][0]['Name'] \
+ ' ' + n2tree['data'][0]['Router']
if 'StandbyRouter' in n2tree['data'][0]:
n2tree['data'][0]['Name'] = n2tree['data'][0]['Name'] \
+ '|' + n2tree['data'][0]['StandbyRouter']
nglib.ngtree.add_child_ngtree(ngtree, n2tree['data'][0])
# Destination Switch Data
if switching and routing and dstswp and dstswp['data']:
nglib.ngtree.add_child_ngtree(ngtree, dstswp)
# NetDB Destination Data
if len(n2tree['data']) > 1:
n2tree['data'][1]['_type'] = "DST"
if 'SwitchPort' in n2tree['data'][1]:
n2tree['data'][1]['Name'] = dst + ' ' + n2tree['data'][1]['MAC'] \
+ ' ' + str(n2tree['data'][1]['Switch']) + '(' \
+ str(n2tree['data'][1]['SwitchPort']) \
+ ') [vid:' + str(n2tree['data'][1]['VLAN']) + ']'
nglib.ngtree.add_child_ngtree(ngtree, n2tree['data'][1])
# Export NGTree
if 'Lx Path' in ngtree:
ngtree = nglib.query.exp_ngtree(ngtree, rtype)
return ngtree
else:
raise ResultError("No Path Results", "Could not find a path from %s -> %s" % (src, dst))
2
Example 29
Project: p2ptv-pi Source File: WebUI.py
def doget(self, urlpath):
if not urlpath.startswith(URLPATH_WEBIF_PREFIX):
return streaminfo404()
else:
self.lastreqtime = time.time()
try:
fakeurl = 'http://127.0.0.1' + urlpath[len(URLPATH_WEBIF_PREFIX):]
if DEBUG:
log('webui::doget: fakeurl', fakeurl)
request_url = urlparse.urlparse(fakeurl)
except:
print_exc()
return
path = request_url[2]
query_string = request_url[4]
query_params = urlparse.parse_qs(query_string)
if DEBUG:
log('webui::doget: urlpath', urlpath, 'request_url', request_url, 'path', path, 'query_params', query_params)
if len(path) == 0:
if DEBUG:
log('webui::doget: show status page')
page = self.statusPage()
pageStream = StringIO(page)
return {'statuscode': 200,
'mimetype': 'text/html',
'stream': pageStream,
'length': len(page)}
if path == 'permid.js':
try:
permid = encodestring(self.bgApp.s.get_permid()).replace('\n', '')
txt = "var permid = '%s';" % permid
dataStream = StringIO(txt)
except:
print_exc()
return {'statuscode': 500,
'statusmsg': 'Bad permid'}
return {'statuscode': 200,
'mimetype': 'text/javascript',
'stream': dataStream,
'length': len(txt)}
if path == '/createstream':
if DEBUG:
log('webui::doget: show create stream page')
page = self.createStreamPage()
pageStream = StringIO(page)
return {'statuscode': 200,
'mimetype': 'text/html',
'stream': pageStream,
'length': len(page)}
if path == '/dispatch':
if 'url' not in query_params:
if DEBUG:
log('webui::doget:dispatch: missing url')
return streaminfo404()
url = query_params['url'][0]
redirect_url = 'http://127.0.0.1:6878/webui/' + url
params = []
for name, val in query_params.iteritems():
if name != 'url':
params.append(urllib.quote_plus(name) + '=' + urllib.quote_plus(val[0]))
if len(params):
redirect_url += '?' + '&'.join(params)
if DEBUG:
log('webui::doget:dispatch: redirect_url', redirect_url)
page = '<!DOCTYPE html><html><head><script type="text/javascript">'
page += 'parent.location.href = "' + redirect_url + '";'
page += '</script></head><body></body></html>'
pageStream = StringIO(page)
return {'statuscode': 200,
'mimetype': 'text/html',
'stream': pageStream,
'length': len(page)}
if path.startswith('/player/') and query_params.has_key('a') and query_params['a'][0] == 'check':
player_id = path.split('/')[2]
redirect_url = 'http://127.0.0.1:6878/webui/player/' + player_id
params = []
for name, val in query_params.iteritems():
if name != 'a':
params.append(urllib.quote_plus(name) + '=' + urllib.quote_plus(val[0]))
if len(params):
redirect_url += '?' + '&'.join(params)
if DEBUG:
log('webui::doget:dispatch: redirect_url', redirect_url)
page = '<!DOCTYPE html><html><head><script type="text/javascript">'
page += 'parent.location.href = "' + redirect_url + '";'
page += '</script></head><body></body></html>'
pageStream = StringIO(page)
return {'statuscode': 200,
'mimetype': 'text/html',
'stream': pageStream,
'length': len(page)}
if path.startswith('/player/'):
player_id = path.split('/')[2]
if DEBUG:
log('webui::doget: show player page: id', player_id)
params = {}
for name, val in query_params.iteritems():
params[name] = val[0]
page = self.playerPage(player_id, params)
pageStream = StringIO(page)
return {'statuscode': 200,
'mimetype': 'text/html',
'stream': pageStream,
'length': len(page)}
static_path = None
json_query = None
if path.startswith('/json/'):
json_query = request_url[4]
else:
static_path = os.path.join(self.webUIPath, path[1:])
if DEBUG:
log('webui::doget: request parsed: static_path', static_path, 'json_query', json_query)
if static_path is not None:
if not os.path.isfile(static_path):
if DEBUG:
log('webui::doget: file not found:', static_path)
return streaminfo404()
extension = os.path.splitext(static_path)[1]
if extension in self.binaryExtensions:
mode = 'rb'
else:
mode = 'r'
fp = open(static_path, mode)
data = fp.read()
fp.close()
dataStream = StringIO(data)
return {'statuscode': 200,
'mimetype': self.getContentType(extension),
'stream': dataStream,
'length': len(data)}
if json_query is not None:
params = {}
for s in json_query.split('&'):
name, value = s.split('=')
params[name] = value
if DEBUG:
log('webui:doget: got json request:', json_query, 'params', params)
if 'q' not in params:
return
try:
req = urllib.unquote(params['q'])
if DEBUG:
log('webui::doget: parse json: req', req)
jreq = json.loads(req)
if DEBUG:
log('webui::doget: parse json done: jreq', jreq)
except:
print_exc()
return
try:
method = jreq['method']
except:
return {'statuscode': 504,
'statusmsg': 'Json request in wrong format! At least a method has to be specified!'}
try:
args = jreq['arguments']
if DEBUG:
print >> sys.stderr, 'webUI: Got JSON request: ', jreq, '; method: ', method, '; arguments: ', args
except:
args = None
if DEBUG:
print >> sys.stderr, 'webUI: Got JSON request: ', jreq, '; method: ', method
if args is None:
data = self.process_json_request(method)
if DEBUG:
print >> sys.stderr, 'WebUI: response to JSON ', method, ' request: ', data
else:
data = self.process_json_request(method, args)
if DEBUG:
print >> sys.stderr, 'WebUI: response to JSON ', method, ' request: ', data, ' arguments: ', args
if data == 'Args missing':
return {'statuscode': 504,
'statusmsg': 'Json request in wrong format! Arguments have to be specified!'}
dataStream = StringIO(data)
return {'statuscode': 200,
'mimetype': 'application/json',
'stream': dataStream,
'length': len(data)}
if DEBUG:
log('webui::doget: unknow request format: request_url', request_url)
return streaminfo404()
2
Example 30
Project: ros_buildfarm Source File: devel_job.py
def configure_devel_jobs(
config_url, rosdistro_name, source_build_name, groovy_script=None,
dry_run=False, whitelist_repository_names=None):
"""
Configure all Jenkins devel jobs.
L{configure_release_job} will be invoked for source repository and target
which matches the build file criteria.
"""
config = get_config_index(config_url)
build_files = get_source_build_files(config, rosdistro_name)
build_file = build_files[source_build_name]
index = get_index(config.rosdistro_index_url)
dist_cache = None
if build_file.notify_maintainers:
dist_cache = get_distribution_cache(index, rosdistro_name)
# get targets
targets = []
for os_name in build_file.targets.keys():
for os_code_name in build_file.targets[os_name].keys():
for arch in build_file.targets[os_name][os_code_name]:
targets.append((os_name, os_code_name, arch))
print('The build file contains the following targets:')
for os_name, os_code_name, arch in targets:
print(' -', os_name, os_code_name, arch)
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
print('No distribution file matches the build file')
return
devel_view_name = get_devel_view_name(
rosdistro_name, source_build_name, pull_request=False)
pull_request_view_name = get_devel_view_name(
rosdistro_name, source_build_name, pull_request=True)
# all further configuration will be handled by either the Jenkins API
# or by a generated groovy script
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url) if groovy_script is None else False
view_configs = {}
views = {}
if build_file.test_commits_force is not False:
views[devel_view_name] = configure_devel_view(
jenkins, devel_view_name, dry_run=dry_run)
if build_file.test_pull_requests_force is not False:
views[pull_request_view_name] = configure_devel_view(
jenkins, pull_request_view_name, dry_run=dry_run)
if not jenkins:
view_configs.update(views)
groovy_data = {
'dry_run': dry_run,
'expected_num_views': len(view_configs),
}
repo_names = dist_file.repositories.keys()
filtered_repo_names = build_file.filter_repositories(repo_names)
devel_job_names = []
pull_request_job_names = []
job_configs = OrderedDict()
for repo_name in sorted(repo_names):
if whitelist_repository_names:
if repo_name not in whitelist_repository_names:
print("Skipping repository '%s' not in the explicitly passed list" %
repo_name, file=sys.stderr)
continue
is_disabled = repo_name not in filtered_repo_names
if is_disabled and build_file.skip_ignored_repositories:
print("Skipping ignored repository '%s'" % repo_name,
file=sys.stderr)
continue
repo = dist_file.repositories[repo_name]
if not repo.source_repository:
print("Skipping repository '%s': no source section" % repo_name)
continue
if not repo.source_repository.version:
print("Skipping repository '%s': no source version" % repo_name)
continue
job_types = []
# check for testing commits
if build_file.test_commits_force is False:
print(("Skipping repository '%s': 'test_commits' is forced to " +
"false in the build file") % repo_name)
elif repo.source_repository.test_commits is False:
print(("Skipping repository '%s': 'test_commits' of the " +
"repository set to false") % repo_name)
elif repo.source_repository.test_commits is None and \
not build_file.test_commits_default:
print(("Skipping repository '%s': 'test_commits' defaults to " +
"false in the build file") % repo_name)
else:
job_types.append('commit')
if not is_disabled:
# check for testing pull requests
if build_file.test_pull_requests_force is False:
# print(("Skipping repository '%s': 'test_pull_requests' " +
# "is forced to false in the build file") % repo_name)
pass
elif repo.source_repository.test_pull_requests is False:
# print(("Skipping repository '%s': 'test_pull_requests' of " +
# "the repository set to false") % repo_name)
pass
elif repo.source_repository.test_pull_requests is None and \
not build_file.test_pull_requests_default:
# print(("Skipping repository '%s': 'test_pull_requests' " +
# "defaults to false in the build file") % repo_name)
pass
else:
print("Pull request job for repository '%s'" % repo_name)
job_types.append('pull_request')
for job_type in job_types:
pull_request = job_type == 'pull_request'
for os_name, os_code_name, arch in targets:
try:
job_name, job_config = configure_devel_job(
config_url, rosdistro_name, source_build_name,
repo_name, os_name, os_code_name, arch, pull_request,
config=config, build_file=build_file,
index=index, dist_file=dist_file,
dist_cache=dist_cache, jenkins=jenkins, views=views,
is_disabled=is_disabled,
groovy_script=groovy_script,
dry_run=dry_run)
if not pull_request:
devel_job_names.append(job_name)
else:
pull_request_job_names.append(job_name)
if groovy_script is not None:
print("Configuration for job '%s'" % job_name)
job_configs[job_name] = job_config
except JobValidationError as e:
print(e.message, file=sys.stderr)
groovy_data['expected_num_jobs'] = len(job_configs)
groovy_data['job_prefixes_and_names'] = {}
devel_job_prefix = '%s__' % devel_view_name
pull_request_job_prefix = '%s__' % pull_request_view_name
if not whitelist_repository_names:
groovy_data['job_prefixes_and_names']['devel'] = \
(devel_job_prefix, devel_job_names)
groovy_data['job_prefixes_and_names']['pull_request'] = \
(pull_request_job_prefix, pull_request_job_names)
if groovy_script is None:
# delete obsolete jobs in these views
from ros_buildfarm.jenkins import remove_jobs
print('Removing obsolete devel jobs')
remove_jobs(
jenkins, devel_job_prefix, devel_job_names, dry_run=dry_run)
print('Removing obsolete pull request jobs')
remove_jobs(
jenkins, pull_request_job_prefix, pull_request_job_names,
dry_run=dry_run)
if groovy_script is not None:
print(
"Writing groovy script '%s' to reconfigure %d views and %d jobs" %
(groovy_script, len(view_configs), len(job_configs)))
content = expand_template(
'snippet/reconfigure_jobs.groovy.em', groovy_data)
write_groovy_script_and_configs(
groovy_script, content, job_configs, view_configs=view_configs)
2
Example 31
Project: bedup Source File: __main__.py
def main(argv):
progname = 'bedup' if is_in_path('bedup') else 'python3 -m bedup'
io_enc = codecs.lookup(locale.getpreferredencoding()).name
if io_enc == 'ascii':
print(
'bedup will abort because Python was configured to use ASCII '
'for console I/O.\nSee https://git.io/vnzk6 which '
'explains how to use a UTF-8 locale.', file=sys.stderr)
return 1
parser = argparse.ArgumentParser(prog=progname)
parser.add_argument(
'--debug', action='store_true', help=argparse.SUPPRESS)
commands = parser.add_subparsers(dest='command', metavar='command')
sp_scan_vol = commands.add_parser(
'scan', help='Scan', description="""
Scans volumes to keep track of potentially duplicated files.""")
sp_scan_vol.set_defaults(action=vol_cmd)
scan_flags(sp_scan_vol)
# In Python 3.2+ we can add aliases here.
# Hidden aliases doesn't seem supported though.
sp_dedup_vol = commands.add_parser(
'dedup', help='Scan and deduplicate', description="""
Runs scan, then deduplicates identical files.""")
sp_dedup_vol.set_defaults(action=vol_cmd)
scan_flags(sp_dedup_vol)
sp_dedup_vol.add_argument(
'--defrag', action='store_true',
help='Defragment files that are going to be deduplicated')
# An alias so as not to break btrfs-time-machine.
# help='' is unset, which should make it (mostly) invisible.
sp_dedup_vol_compat = commands.add_parser(
'dedup-vol', description="""
A deprecated alias for the 'dedup' command.""")
sp_dedup_vol_compat.set_defaults(action=vol_cmd)
scan_flags(sp_dedup_vol_compat)
sp_reset_vol = commands.add_parser(
'reset', help='Reset tracking metadata', description="""
Reset tracking data for the listed volumes. Mostly useful for testing.""")
sp_reset_vol.set_defaults(action=vol_cmd)
vol_flags(sp_reset_vol)
sp_show_vols = commands.add_parser(
'show', help='Show metadata overview', description="""
Shows filesystems and volumes with their tracking status.""")
sp_show_vols.set_defaults(action=cmd_show_vols)
sp_show_vols.add_argument('fsuuid_or_device', nargs='?')
sp_show_vols.add_argument(
'--show-deleted', dest='show_deleted', action='store_true',
help='Show volumes that have been deleted')
sql_flags(sp_show_vols)
sp_find_new = commands.add_parser(
'find-new', help='List changed files', description="""
lists changes to volume since generation
This is a reimplementation of btrfs find-new,
modified to include directories as well.""")
sp_find_new.set_defaults(action=cmd_find_new)
sp_find_new.add_argument(
'-0|--zero-terminated', dest='zero_terminated', action='store_true',
help='Use a NUL character as the line separator')
sp_find_new.add_argument(
'--terse', dest='terse', action='store_true', help='Print names only')
sp_find_new.add_argument('volume', help='Volume to search')
sp_find_new.add_argument(
'generation', type=int, nargs='?', default=0,
help='Only show items modified at generation or a newer transaction')
sp_forget_fs = commands.add_parser(
'forget-fs', help='Wipe all metadata', description="""
Wipe all metadata for the listed filesystems.
Useful if the filesystems don't exist anymore.""")
sp_forget_fs.set_defaults(action=cmd_forget_fs)
sp_forget_fs.add_argument('uuid', nargs='+', help='Btrfs filesystem uuids')
sql_flags(sp_forget_fs)
sp_dedup_files = commands.add_parser(
'dedup-files', help='Deduplicate listed', description="""
Freezes listed files, checks them for being identical,
and projects the extents of the first file onto the other files.
The effects are visible with filefrag -v (apt:e2fsprogs),
which displays the extent map of files.
""".strip())
sp_dedup_files.set_defaults(action=cmd_dedup_files)
sp_dedup_files.add_argument('source', metavar='SRC', help='Source file')
sp_dedup_files.add_argument(
'dests', metavar='DEST', nargs='+', help='Dest files')
# Don't forget to also set new options in the dedup-vol test in vol_cmd
sp_dedup_files.add_argument(
'--defrag', action='store_true',
help='Defragment the source file first')
sp_generation = commands.add_parser(
'generation', help='Display volume generation', description="""
Display the btrfs generation of VOLUME.""")
sp_generation.set_defaults(action=cmd_generation)
sp_generation.add_argument('volume', help='Btrfs volume')
sp_generation.add_argument(
'--flush', action='store_true', dest='flush',
help='Flush outstanding data using syncfs before lookup')
sp_size_lookup = commands.add_parser(
'size-lookup', help='Look up inodes by size', description="""
List tracked inodes with a given size.""")
sp_size_lookup.set_defaults(action=cmd_size_lookup)
sp_size_lookup.add_argument('size', type=int)
sp_size_lookup.add_argument(
'-0|--zero-terminated', dest='zero_terminated', action='store_true',
help='Use a NUL character as the line separator')
sql_flags(sp_size_lookup)
sp_shell = commands.add_parser(
'shell', description="""
Run an interactive shell (useful for prototyping).""")
sp_shell.set_defaults(action=cmd_shell)
sql_flags(sp_shell)
sp_fake_updates = commands.add_parser(
'fake-updates', description="""
Fake inode updates from the latest dedup events (useful for benchmarking).""")
sp_fake_updates.set_defaults(action=cmd_fake_updates)
sp_fake_updates.add_argument('max_events', type=int)
sql_flags(sp_fake_updates)
# Give help when no subcommand is given
if not argv[1:]:
parser.print_help()
return
args = parser.parse_args(argv[1:])
if args.debug:
try:
from ipdb import launch_ipdb_on_exception
except ImportError:
sys.stderr.write(
'Please install bedup[interactive] for this feature\n')
return 1
with launch_ipdb_on_exception():
# Handle all warnings as errors.
# Overrides the default filter that ignores deprecations
# and prints the rest.
warnings.simplefilter('error')
warnings.filterwarnings('ignore', module='IPython\..*')
warnings.filterwarnings('ignore', module='alembic\..*')
return args.action(args)
else:
try:
return args.action(args)
except IOError as err:
if err.errno == errno.EPERM:
sys.stderr.write(
"You need to run this command as root.\n")
return 1
raise
2
Example 32
Project: WASP Source File: combined_test.py
def main():
options = parse_options()
if options.pc_file:
pc_matrix = load_covariates(options.pc_file)
num_pcs = options.num_pcs
else:
pc_matrix = []
num_pcs = 0
if options.out_file.endswith(".gz"):
outfile = gzip.open(options.out_file, "wb")
else:
outfile = open(options.out_file, 'w')
if options.benchmark:
if options.benchmark == "-":
bench_file = sys.stderr
else:
bench_file = open(options.benchmark, "w")
bench_file.write("TEST.TYPE TIME\n")
write_header(outfile)
# read list of input files (one for each individual)
infiles = open_input_files(options.infile_list)
# read dispersion parameters for each individual
bnb_sigmas = read_bnb_sigmas(options, infiles)
as_sigmas = read_as_sigmas(options, infiles)
# add first row of each input file to snpinfo list
snpinfo = []
for f in infiles:
snpinfo.append(f.readline().strip().split())
row_count = 0
finished=False
options.dup_snp_warn = True
while not finished:
try:
test_snps = []
# parse test SNP and associated info from input file row
for i in range(len(infiles)):
test_snps.append(parse_test_snp(snpinfo[i], options))
# rescale totals to put values into reasonable range for
# alpha and beta parameter estimation
rescale_totals(test_snps)
# how many allele-specific reads are there across all
# linked SNPs and and individuals?
ref_as_counts = sum([np.sum(x.AS_target_ref) for x in test_snps])
alt_as_counts = sum([np.sum(x.AS_target_alt) for x in test_snps])
tot_as_counts = ref_as_counts + alt_as_counts
all_counts = sum([test_snps[i].counts for i in range(len(test_snps))])
if tot_as_counts < options.min_as_counts:
if options.verbose:
sys.stderr.write("-----\nskipping SNP %s because "
"total AS counts %d <= %d\n" %
(test_snps[0].name, tot_as_counts, options.min_as_counts))
# skip, not enough allele-specific counts
for i in range(len(infiles)):
line = infiles[i].readline().strip()
if line:
snpinfo[i] = line.split()
else:
# out of lines from at least one file, assume we are finished
finished = True
continue
if options.verbose:
sys.stderr.write("-----\ntesting SNP %s\n" % test_snps[0].name)
row_count+=1
old_genos = [test_snps[y].geno_hap1 + test_snps[y].geno_hap2
for y in range(len(test_snps))]
if options.shuffle:
# permute genotypes
perm = range(len(test_snps))
random.shuffle(perm)
geno1temp = [test_snps[y].geno_hap1 for y in perm]
geno2temp = [test_snps[y].geno_hap2 for y in perm]
for i in range(len(test_snps)):
test_snps[i].geno_hap1 = geno1temp[i]
test_snps[i].geno_hap2 = geno2temp[i]
if options.benchmark:
# start timing test for NULL model
bench_file.write("null %s %s %d %d " % (snpinfo[0][0], snpinfo[0][1],
tot_as_counts, all_counts))
bench_file.flush()
t1 = time.time()
starting_gene = [np.float64(x) for x in [0.1, 0.001]]
maxlike = 10000000000
for start in starting_gene:
starts = [np.float64(0.5), np.float64(start)]
# regress against the covariates and get residuals
#fit_cov(test_snps,cov_table)
# maximize likelihood with alpha = beta (no difference between genotypes)
res = minimize(ll_one, starts, args=(test_snps, True, #options.is_bnb_only,
options.is_as_only,
bnb_sigmas,
as_sigmas,
options.read_error_rate,
[],
pc_matrix),
options={"maxiter" : 50000, "disp" : options.verbose},
method=OPTIMIZER)
new_par = res.x
new_loglike = ll_one(new_par, test_snps, options.is_bnb_only,
options.is_as_only, bnb_sigmas,
as_sigmas, options.read_error_rate,
[], pc_matrix)
if new_loglike < maxlike:
starting_par = new_par
pc_coefs=[]
for pc in range(num_pcs):
res = minimize(ll_pc, [np.float64(0)],
args=(starting_par, test_snps, True, #options.is_bnb_only,
options.is_as_only, bnb_sigmas, as_sigmas,
options.read_error_rate, pc_coefs, pc_matrix),
options={"maxiter" : 50000, "disp" : options.verbose},
method=OPTIMIZER)
new_coef = res.x
pc_coefs = np.concatenate([pc_coefs, new_coef])
res = minimize(ll_one, starting_par,
args=(test_snps, options.is_bnb_only, options.is_as_only, bnb_sigmas,
as_sigmas, options.read_error_rate, pc_coefs, pc_matrix),
options={"maxiter" : 50000, "disp" : options.verbose},
method=OPTIMIZER)
best1par = res.x
time_taken = time.time() - t1
if options.verbose:
sys.stderr.write("null model optimization took %.3fs\n" % time_taken)
if options.benchmark:
bench_file.write("%.3f\n" % time_taken)
bench_file.flush()
loglike1par = ll_one(best1par, test_snps, options.is_bnb_only,
options.is_as_only, bnb_sigmas,
as_sigmas, options.read_error_rate,
pc_coefs, pc_matrix)
start = [best1par[0], best1par[0], best1par[1]]
if options.benchmark:
# start timing test for ALT model
bench_file.write("alt %s %s %d %d " % (snpinfo[0][0], snpinfo[0][1],
tot_as_counts, all_counts))
bench_file.flush()
t1 = time.time()
# maximize likelihood with alpha and beta as separate parameters
res = minimize(ll_two, start,
args=(test_snps, options.is_bnb_only, options.is_as_only,
bnb_sigmas, as_sigmas, options.read_error_rate,
pc_coefs, pc_matrix),
options={"maxiter" : 50000, "disp" : options.verbose},
method=OPTIMIZER)
best2par = res.x
time_taken = time.time() - t1
if options.verbose:
sys.stderr.write("alternative model optimization took %.3fs\n" % time_taken)
if options.benchmark:
bench_file.write("%.3f\n" % time_taken)
bench_file.flush()
loglike2par = ll_two(best2par, test_snps, options.is_bnb_only,
options.is_as_only, bnb_sigmas,
as_sigmas, options.read_error_rate,
pc_coefs, pc_matrix)
write_results(outfile, snpinfo, loglike1par, loglike2par, best2par,
tot_as_counts, ref_as_counts, alt_as_counts, all_counts)
except Exception as e:
write_empty_result(outfile, snpinfo)
# an error occured, write to output file, but put 0s for all params and
sys.stderr.write("An error occurred, writing line with 0s for SNP:\n%s\n" % str(e))
# continue
raise
# read next set of lines from input file
for i in range(len(infiles)):
line = infiles[i].readline().strip()
if line:
snpinfo[i] = line.split()
else:
# out of lines from at least one file, assume we are finished
finished = True
2
Example 33
def parse_section(self, module_name, section_name, file_name, lineno, lines):
self.sections_total += 1
# skip sections having whitespace in name
#if section_name.find(" ") >= 0 and section_name.find("::operator") < 0:
if (section_name.find(" ") >= 0 and not bool(re.match(r"(\w+::)*operator\s*(\w+|>>|<<|\(\)|->|\+\+|--|=|==|\+=|-=)", section_name)) ) or section_name.endswith(":"):
if show_errors:
print >> sys.stderr, "RST parser warning W%03d: SKIPPED: \"%s\" File: %s:%s" % (WARNING_002_HDRWHITESPACE, section_name, file_name, lineno)
self.sections_skipped += 1
return
func = {}
func["name"] = section_name
func["file"] = file_name
func["line"] = lineno
func["module"] = module_name
# parse section name
section_name = self.parse_namespace(func, section_name)
class_separator_idx = section_name.find("::")
if class_separator_idx > 0:
func["class"] = section_name[:class_separator_idx]
func["method"] = section_name[class_separator_idx+2:]
else:
func["method"] = section_name
capturing_seealso = False
skip_code_lines = False
expected_brief = True
was_code_line = False
fdecl = DeclarationParser()
pdecl = ParamParser()
ll = None
for l in lines:
# read tail of function/method declaration if needed
if not fdecl.isready():
fdecl.append(ll)
if fdecl.isready():
self.add_new_fdecl(func, fdecl)
continue
# continue capture seealso
if capturing_seealso:
if not l or l.startswith(" "):
seealso = func.get("seealso", [])
seealso.extend(l.split(","))
func["seealso"] = seealso
continue
else:
capturing_seealso = False
ll = l.strip()
if ll == "..":
expected_brief = False
skip_code_lines = False
continue
# skip lines if line-skipping mode is activated
if skip_code_lines:
if not l:
continue
if not l.startswith(" "):
skip_code_lines = False
if ll.startswith(".. code-block::") or ll.startswith(".. image::"):
skip_code_lines = True
continue
# todo: parse structure members; skip them for now
if ll.startswith(".. ocv:member::"):
#print ll
skip_code_lines = True
continue
#ignore references (todo: collect them)
if l.startswith(".. ["):
continue
if ll.startswith(".. "):
expected_brief = False
elif ll.endswith("::"):
# turn on line-skipping mode for code fragments
#print ll
skip_code_lines = True
ll = ll[:len(ll)-2]
# continue param parsing (process params after processing .. at the beginning of the line and :: at the end)
if pdecl.active:
pdecl.append(l)
if pdecl.active:
continue
else:
self.add_new_pdecl(func, pdecl)
# do not continue - current line can contain next parameter definition
# parse ".. seealso::" blocks
if ll.startswith(".. seealso::"):
if ll.endswith(".. seealso::"):
capturing_seealso = True
else:
seealso = func.get("seealso", [])
seealso.extend(ll[ll.find("::")+2:].split(","))
func["seealso"] = seealso
continue
# skip ".. index::"
if ll.startswith(".. index::"):
continue
# parse class & struct definitions
if ll.startswith(".. ocv:class::"):
func["class"] = ll[ll.find("::")+2:].strip()
if "method" in func:
del func["method"]
func["isclass"] = True
expected_brief = True
continue
if ll.startswith(".. ocv:struct::"):
func["class"] = ll[ll.find("::")+2:].strip()
if "method" in func:
del func["method"]
func["isstruct"] = True
expected_brief = True
continue
# parse function/method definitions
if fdecl.hasDeclaration(ll):
fdecl = DeclarationParser(ll)
if fdecl.isready():
self.add_new_fdecl(func, fdecl)
continue
# parse parameters
if pdecl.hasDeclaration(l):
pdecl = ParamParser(l)
continue
# record brief description
if expected_brief:
func["brief"] = func.get("brief", "") + "\n" + ll
if skip_code_lines:
expected_brief = False # force end brief if code block begins
continue
# record other lines as long description
if (skip_code_lines):
ll = ll.replace("/*", "/ *")
ll = ll.replace("*/", "* /")
if (was_code_line):
func["long"] = func.get("long", "") + "\n" + ll + "\n"
else:
was_code_line = True
func["long"] = func.get("long", "") + ll +"\n<code>\n\n // C++ code:\n\n"
else:
if (was_code_line):
func["long"] = func.get("long", "") + "\n" + ll + "\n</code>\n"
was_code_line = False
else:
func["long"] = func.get("long", "") + "\n" + ll
# endfor l in lines
if fdecl.balance != 0:
if show_critical_errors:
print >> sys.stderr, "RST parser error E%03d: invalid parentheses balance in \"%s\" at %s:%s" % (ERROR_003_PARENTHESES, section_name, file_name, lineno)
return
# save last parameter if needed
if pdecl.active:
self.add_new_pdecl(func, pdecl)
# add definition to list
func = self.normalize(func)
if self.validate(func):
self.definitions[func["name"]] = func
self.sections_parsed += 1
if verbose:
self.print_info(func)
elif func:
if func["name"] in known_text_sections_names:
if show_errors:
print >> sys.stderr, "RST parser warning W%03d: SKIPPED: \"%s\" File: %s:%s" % (WARNING_002_HDRWHITESPACE, section_name, file_name, lineno)
self.sections_skipped += 1
elif show_errors:
self.print_info(func, True, sys.stderr)
2
Example 34
Project: mutagen Source File: mid3v2.py
def write_files(edits, filenames, escape):
# unescape escape sequences and decode values
encoded_edits = []
for frame, value in edits:
if not value:
continue
try:
frame = frame_from_fsnative(frame)
except ValueError as err:
print_(text_type(err), file=sys.stderr)
assert isinstance(frame, str)
# strip "--"
frame = frame[2:]
try:
value = value_from_fsnative(value, escape)
except ValueError as err:
error(u"%s: %s" % (frame, text_type(err)))
assert isinstance(value, text_type)
encoded_edits.append((frame, value))
edits = encoded_edits
# preprocess:
# for all [frame,value] pairs in the edits list
# gather values for identical frames into a list
tmp = {}
for frame, value in edits:
if frame in tmp:
tmp[frame].append(value)
else:
tmp[frame] = [value]
# edits is now a dictionary of frame -> [list of values]
edits = tmp
# escape also enables escaping of the split separator
if escape:
string_split = split_escape
else:
string_split = lambda s, *args, **kwargs: s.split(*args, **kwargs)
for filename in filenames:
with _sig.block():
if verbose:
print_(u"Writing", filename, file=sys.stderr)
try:
id3 = mutagen.id3.ID3(filename)
except mutagen.id3.ID3NoHeaderError:
if verbose:
print_(u"No ID3 header found; creating a new tag",
file=sys.stderr)
id3 = mutagen.id3.ID3()
except Exception as err:
print_(str(err), file=sys.stderr)
continue
for (frame, vlist) in edits.items():
if frame == "POPM":
for value in vlist:
values = string_split(value, ":")
if len(values) == 1:
email, rating, count = values[0], 0, 0
elif len(values) == 2:
email, rating, count = values[0], values[1], 0
else:
email, rating, count = values
frame = mutagen.id3.POPM(
email=email, rating=int(rating), count=int(count))
id3.add(frame)
elif frame == "APIC":
for value in vlist:
values = string_split(value, ":")
# FIXME: doesn't support filenames with an invalid
# encoding since we have already decoded at that point
fn = values[0]
if len(values) >= 2:
desc = values[1]
else:
desc = u"cover"
if len(values) >= 3:
try:
picture_type = int(values[2])
except ValueError:
error(u"Invalid picture type: %r" % values[1])
else:
picture_type = PictureType.COVER_FRONT
if len(values) >= 4:
mime = values[3]
else:
mime = mimetypes.guess_type(fn)[0] or "image/jpeg"
if len(values) >= 5:
error("APIC: Invalid format")
encoding = get_frame_encoding(frame, desc)
try:
with open(fn, "rb") as h:
data = h.read()
except IOError as e:
error(text_type(e))
frame = mutagen.id3.APIC(encoding=encoding, mime=mime,
desc=desc, type=picture_type, data=data)
id3.add(frame)
elif frame == "COMM":
for value in vlist:
values = string_split(value, ":")
if len(values) == 1:
value, desc, lang = values[0], "", "eng"
elif len(values) == 2:
desc, value, lang = values[0], values[1], "eng"
else:
value = ":".join(values[1:-1])
desc, lang = values[0], values[-1]
frame = mutagen.id3.COMM(
encoding=3, text=value, lang=lang, desc=desc)
id3.add(frame)
elif frame == "UFID":
for value in vlist:
values = string_split(value, ":")
if len(values) != 2:
error(u"Invalid value: %r" % values)
owner = values[0]
data = values[1].encode("utf-8")
frame = mutagen.id3.UFID(owner=owner, data=data)
id3.add(frame)
elif frame == "TXXX":
for value in vlist:
values = string_split(value, ":", 1)
if len(values) == 1:
desc, value = "", values[0]
else:
desc, value = values[0], values[1]
frame = mutagen.id3.TXXX(
encoding=3, text=value, desc=desc)
id3.add(frame)
elif issubclass(mutagen.id3.Frames[frame],
mutagen.id3.UrlFrame):
frame = mutagen.id3.Frames[frame](encoding=3, url=vlist)
id3.add(frame)
else:
frame = mutagen.id3.Frames[frame](encoding=3, text=vlist)
id3.add(frame)
id3.save(filename)
2
Example 35
Project: cloud-scheduler Source File: config.py
def setup(path=None):
"""Setup cloudscheduler using config file.
setup will look for a configuration file specified on the command line,
or in ~/.cloudscheduler.conf or /etc/cloudscheduler.conf
"""
global condor_webservice_url
global condor_collector_url
global condor_retrieval_method
global condor_q_command
global condor_status_command
global condor_status_master_command
global condor_hold_command
global condor_release_command
global condor_off_command
global condor_on_command
global condor_advertise_command
global ssh_path
global openssl_path
global condor_context_file
global condor_host
global condor_host_on_vm
global vm_lifetime
global cert_file
global key_file
global cert_file_on_vm
global key_file_on_vm
global ca_root_certs
global ca_signing_policies
global cloudscheduler_ssh_key
global cloud_resource_config
global image_attach_device
global scratch_attach_device
global info_server_port
global admin_server_port
global workspace_path
global persistence_file
global user_limit_file
global target_cloud_alias_file
global job_ban_timeout
global ban_tracking
global ban_file
global ban_min_track
global ban_failrate_threshold
global polling_error_threshold
global condor_register_time_limit
global graceful_shutdown
global graceful_shutdown_method
global retire_before_lifetime
global retire_before_lifetime_factor
global cleanup_missing_vms
global clean_shutdown_idle
global getclouds
global scheduling_metric
global scheduling_algorithm
global job_distribution_type
global high_priority_job_support
global high_priority_job_weight
global cpu_distribution_weight
global memory_distribution_weight
global storage_distribution_weight
global cleanup_interval
global vm_poller_interval
global job_poller_interval
global machine_poller_interval
global scheduler_interval
global job_proxy_refresher_interval
global job_proxy_renewal_threshold
global vm_proxy_refresher_interval
global vm_proxy_renewal_threshold
global vm_proxy_shutdown_threshold
global vm_connection_fail_threshold
global vm_start_running_timeout
global vm_idle_threshold
global max_starting_vm
global max_keepalive
global proxy_cache_dir
global myproxy_logon_command
global override_vmtype
global vm_reqs_from_condor_reqs
global adjust_insufficient_resources
global use_cloud_init
global default_yaml
global validate_yaml
global retire_reallocate
global default_VMType
global default_VMNetwork
global default_VMCPUArch
global default_VMHypervisor
global default_VMName
global default_VMLoc
global default_VMAMI
global default_VMMem
global default_VMCPUCores
global default_VMStorage
global default_VMInstanceType
global default_VMInstanceTypeList
global default_VMMaximumPrice
global default_VMProxyNonBoot
global default_VMUserData
global default_TargetClouds
global default_VMAMIConfig
global default_VMInjectCA
global default_VMJobPerCore
global log_level
global log_location
global log_location_cloud_admin
global admin_log_comments
global log_stdout
global log_syslog
global log_max_size
global log_format
global use_pyopenssl
homedir = os.path.expanduser('~')
# Find config file
if not path:
if os.path.exists(homedir + "/.cloudscheduler/cloud_scheduler.conf"):
path = homedir + "/.cloudscheduler/cloud_scheduler.conf"
elif os.path.exists("/etc/cloudscheduler/cloud_scheduler.conf"):
path = "/etc/cloudscheduler/cloud_scheduler.conf"
elif os.path.exists("/usr/local/share/cloud-scheduler/cloud_scheduler.conf"):
path = "/usr/local/share/cloud-scheduler/cloud_scheduler.conf"
else:
print >> sys.stderr, "Configuration file problem: There doesn't " \
"seem to be a configuration file. " \
"You can specify one with the --config-file parameter, " \
"or put one in ~/.cloudscheduler/cloud_scheduler.conf or "\
"/etc/cloudscheduler/cloud_scheduler.conf"
sys.exit(1)
# Read config file
config_file = ConfigParser.ConfigParser()
try:
config_file.read(path)
except IOError:
print >> sys.stderr, "Configuration file problem: There was a " \
"problem reading %s. Check that it is readable," \
"and that it exists. " % path
raise
except ConfigParser.ParsingError:
print >> sys.stderr, "Configuration file problem: Couldn't " \
"parse your file. Check for spaces before or after variables."
raise
except:
print "Configuration file problem: There is something wrong with " \
"your config file."
raise
if config_file.has_option("global", "condor_retrieval_method"):
condor_retrieval_method = config_file.get("global",
"condor_retrieval_method")
if config_file.has_option("global", "condor_q_command"):
condor_q_command = config_file.get("global",
"condor_q_command")
if config_file.has_option("global", "condor_off_command"):
condor_off_command = config_file.get("global",
"condor_off_command")
if config_file.has_option("global", "condor_on_command"):
condor_on_command = config_file.get("global",
"condor_on_command")
if config_file.has_option("global", "ssh_path"):
ssh_path = config_file.get("global", "ssh_path")
if config_file.has_option("global", "openssl_path"):
openssl_path = config_file.get("global", "openssl_path")
if config_file.has_option("global", "condor_status_command"):
condor_status_command = config_file.get("global",
"condor_status_command")
if config_file.has_option("global", "condor_status_master_command"):
condor_status_master_command = config_file.get("global",
"condor_status_master_command")
if config_file.has_option("global", "condor_hold_command"):
condor_hold_command = config_file.get("global",
"condor_hold_command")
if config_file.has_option("global", "condor_release_command"):
condor_release_command = config_file.get("global",
"condor_release_command")
if config_file.has_option("global", "condor_advertise_command"):
condor_advertise_command = config_file.get("global",
"condor_advertise_command")
if config_file.has_option("global", "condor_webservice_url"):
condor_webservice_url = config_file.get("global",
"condor_webservice_url")
if config_file.has_option("global", "condor_collector_url"):
condor_collector_url = config_file.get("global",
"condor_collector_url")
if config_file.has_option("global", "condor_host_on_vm"):
condor_host_on_vm = config_file.get("global",
"condor_host_on_vm")
if config_file.has_option("global", "condor_context_file"):
condor_context_file = config_file.get("global",
"condor_context_file")
if config_file.has_option("global", "vm_lifetime"):
try:
vm_lifetime = config_file.getint("global", "vm_lifetime")
except ValueError:
print "Configuration file problem: vm_lifetime must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "cert_file"):
cert_file = config_file.get("global", "cert_file")
if config_file.has_option("global", "key_file"):
key_file = config_file.get("global", "key_file")
if config_file.has_option("global", "cert_file_on_vm"):
cert_file_on_vm = config_file.get("global", "cert_file_on_vm")
if config_file.has_option("global", "key_file_on_vm"):
key_file_on_vm = config_file.get("global", "key_file_on_vm")
if config_file.has_option("global", "ca_root_certs"):
ca_root_certs = config_file.get("global", "ca_root_certs").split(',')
if config_file.has_option("global", "ca_signing_policies"):
ca_signing_policies = config_file.get("global", "ca_signing_policies").split(',')
if config_file.has_option("global", "cloudscheduler_ssh_key"):
cloudscheduler_ssh_key = config_file.get("global", "cloudscheduler_ssh_key")
if config_file.has_option("global", "cloud_resource_config"):
cloud_resource_config = config_file.get("global",
"cloud_resource_config")
if config_file.has_option("global", "image_attach_device"):
image_attach_device = config_file.get("global",
"image_attach_device")
if config_file.has_option("global", "scratch_attach_device"):
scratch_attach_device = config_file.get("global",
"scratch_attach_device")
if config_file.has_option("global", "info_server_port"):
try:
info_server_port = config_file.getint("global", "info_server_port")
except ValueError:
print "Configuration file problem: info_server_port must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "admin_server_port"):
try:
info_server_port = config_file.getint("global", "admin_server_port")
except ValueError:
print "Configuration file problem: admin_server_port must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "workspace_path"):
workspace_path = config_file.get("global", "workspace_path")
if config_file.has_option("global", "persistence_file"):
persistence_file = config_file.get("global", "persistence_file")
if config_file.has_option("global", "user_limit_file"):
user_limit_file = config_file.get("global", "user_limit_file")
if config_file.has_option("global", "target_cloud_alias_file"):
target_cloud_alias_file = config_file.get("global", "target_cloud_alias_file")
if config_file.has_option("global", "job_ban_timeout"):
try:
job_ban_timeout = 60 * config_file.getint("global", "job_ban_timeout")
except ValueError:
print "Configuration file problem: job_ban_timeout must be an " \
"integer value in minutes."
sys.exit(1)
if config_file.has_option("global", "ban_file"):
ban_file = config_file.get("global", "ban_file")
if config_file.has_option("global", "polling_error_threshold"):
try:
polling_error_threshold = config_file.getint("global", "polling_error_threshold")
except ValueError:
print "Configuration file problem: polling_error_threshold must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "ban_failrate_threshold"):
try:
ban_failrate_threshold = config_file.getfloat("global", "ban_failrate_threshold")
if ban_failrate_threshold == 0:
print "Please use a float value (0, 1.0]"
sys.exit(1)
except ValueError:
print "Configuration file problem: ban_failrate_threshold must be an " \
"float value."
sys.exit(1)
if config_file.has_option("global", "ban_min_track"):
try:
ban_min_track = config_file.getint("global", "ban_min_track")
except ValueError:
print "Configuration file problem: ban_min_track must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "condor_register_time_limit"):
try:
condor_register_time_limit = 60*config_file.getint("global", "condor_register_time_limit")
except ValueError:
print "Configuration file problem: condor_register_time_limit must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "ban_tracking"):
try:
ban_tracking = config_file.getboolean("global", "ban_tracking")
except ValueError:
print "Configuration file problem: ban_tracking must be an " \
"boolean value."
sys.exit(1)
if config_file.has_option("global", "graceful_shutdown"):
try:
graceful_shutdown = config_file.getboolean("global", "graceful_shutdown")
except ValueError:
print "Configuration file problem: graceful_shutdown must be an " \
"boolean value."
sys.exit(1)
if config_file.has_option("global", "graceful_shutdown_method"):
graceful_shutdown_method = config_file.get("global", "graceful_shutdown_method")
if config_file.has_option("global", "retire_before_lifetime"):
try:
retire_before_lifetime = config_file.getboolean("global", "retire_before_lifetime")
except ValueError:
print "Configuration file problem: retire_before_lifetime must be an " \
"boolean value."
sys.exit(1)
if config_file.has_option("global", "retire_before_lifetime_factor"):
try:
retire_before_lifetime_factor = config_file.getfloat("global", "retire_before_lifetime_factor")
if retire_before_lifetime_factor < 1.0:
print "Please use a float value (1.0, X] for the retire_before_lifetime_factor"
sys.exit(1)
except ValueError:
print "Configuration file problem: retire_before_lifetime_factor must be a " \
"float value."
sys.exit(1)
if config_file.has_option("global", "cleanup_missing_vms"):
try:
cleanup_missing_vms = config_file.getboolean("global", "cleanup_missing_vms")
except ValueError:
print "Configuration file problem: cleanup_missing_vms must be an " \
"boolean value."
sys.exit(1)
if config_file.has_option("global", "clean_shutdown_idle"):
try:
clean_shutdown_idle = config_file.getboolean("global", "clean_shutdown_idle")
except ValueError:
print "Configuration file problem: clean_shutdown_idle must be an " \
"boolean value."
sys.exit(1)
if config_file.has_option("global", "getclouds"):
try:
getclouds = config_file.getboolean("global", "getclouds")
except ValueError:
print "Configuration file problem: getclouds must be an " \
"boolean value."
sys.exit(1)
if config_file.has_option("global", "scheduling_metric"):
scheduling_metric = config_file.get("global", "scheduling_metric")
if config_file.has_option("global", "job_distribution_type"):
job_distribution_type = config_file.get("global", "job_distribution_type")
if config_file.has_option("global", "memory_distribution_weight"):
try:
memory_distribution_weight = config_file.getfloat("global", "memory_distribution_weight")
if ban_failrate_threshold <= 0:
print "Please use a float value (0, x]"
sys.exit(1)
except ValueError:
print "Configuration file problem: memory_distribution_weight must be an " \
"float value."
sys.exit(1)
if config_file.has_option("global", "cpu_distribution_weight"):
try:
cpu_distribution_weight = config_file.getfloat("global", "cpu_distribution_weight")
if ban_failrate_threshold <= 0:
print "Please use a float value (0, x]"
sys.exit(1)
except ValueError:
print "Configuration file problem: cpu_distribution_weight must be an " \
"float value."
sys.exit(1)
if config_file.has_option("global", "storage_distribution_weight"):
try:
storage_distribution_weight = config_file.getfloat("global", "storage_distribution_weight")
if ban_failrate_threshold <= 0:
print "Please use a float value (0, x]"
sys.exit(1)
except ValueError:
print "Configuration file problem: storage_distribution_weight must be an " \
"float value."
sys.exit(1)
if config_file.has_option("global", "scheduling_algorithm"):
scheduling_algorithm = config_file.get("global", "scheduling_algorithm")
if config_file.has_option("global", "high_priority_job_support"):
try:
high_priority_job_support = config_file.getboolean("global", "high_priority_job_support")
except ValueError:
print "Configuration file problem: high_priority_job_support must be an " \
"boolean value."
sys.exit(1)
if config_file.has_option("global", "high_priority_job_weight"):
try:
high_priority_job_weight = config_file.getint("global", "high_priority_job_weight")
except ValueError:
print "Configuration file problem: high_priority_job_weight must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "scheduler_interval"):
try:
scheduler_interval = config_file.getint("global", "scheduler_interval")
except ValueError:
print "Configuration file problem: scheduler_interval must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "vm_poller_interval"):
try:
vm_poller_interval = config_file.getint("global", "vm_poller_interval")
except ValueError:
print "Configuration file problem: vm_poller_interval must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "job_poller_interval"):
try:
job_poller_interval = config_file.getint("global", "job_poller_interval")
except ValueError:
print "Configuration file problem: job_poller_interval must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "machine_poller_interval"):
try:
machine_poller_interval = config_file.getint("global", "machine_poller_interval")
except ValueError:
print "Configuration file problem: machine_poller_interval must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "cleanup_interval"):
try:
cleanup_interval = config_file.getint("global", "cleanup_interval")
except ValueError:
print "Configuration file problem: cleanup_interval must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "job_proxy_refresher_interval"):
try:
job_proxy_refresher_interval = config_file.getint("global", "job_proxy_refresher_interval")
except ValueError:
print "Configuration file problem: job_proxy_refresher_interval must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "job_proxy_renewal_threshold"):
try:
job_proxy_renewal_threshold = config_file.getint("global", "job_proxy_renewal_threshold")
except ValueError:
print "Configuration file problem: job_proxy_renewal_threshold must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "vm_proxy_refresher_interval"):
try:
vm_proxy_refresher_interval = config_file.getint("global", "vm_proxy_refresher_interval")
except ValueError:
print "Configuration file problem: vm_proxy_refresher_interval must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "vm_proxy_renewal_threshold"):
try:
vm_proxy_renewal_threshold = config_file.getint("global", "vm_proxy_renewal_threshold")
except ValueError:
print "Configuration file problem: vm_proxy_renewal_threshold must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "vm_proxy_shutdown_threshold"):
try:
vm_proxy_shutdown_threshold = config_file.getint("global", "vm_proxy_shutdown_threshold")
except ValueError:
print "Configuration file problem: vm_proxy_shutdown_threshold must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "vm_connection_fail_threshold"):
try:
vm_connection_fail_threshold = config_file.getint("global", "vm_connection_fail_threshold")
except ValueError:
print "Configuration file problem: vm_connection_fail_threshold must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "vm_idle_threshold"):
try:
vm_idle_threshold = config_file.getint("global", "vm_idle_threshold")
except ValueError:
print "Configuration file problem: vm_idle_threshold must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "vm_start_running_timeout"):
try:
vm_start_running_timeout = config_file.getint("global", "vm_start_running_timeout")
except ValueError:
print "Configuration file problem: vm_start_running_timeout must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "max_starting_vm"):
try:
max_starting_vm = config_file.getint("global", "max_starting_vm")
if max_starting_vm < -1:
max_starting_vm = -1
except ValueError:
print "Configuration file problem: max_starting_vm must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "max_keepalive"):
try:
max_keepalive = config_file.getint("global", "max_keepalive") * 60
if max_keepalive < 0:
max_keepalive = 0
except ValueError:
print "Configuration file problem: max_keepalive must be an " \
"integer value(# of minutes)."
sys.exit(1)
if config_file.has_option("global", "max_destroy_threads"):
try:
max_destroy_threads = config_file.getint("global", "max_destroy_threads")
if max_destroy_threads <= 0:
max_destroy_threads = 1
except ValueError:
print "Configuration file problem: max_destroy_threads must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "proxy_cache_dir"):
proxy_cache_dir = config_file.get("global", "proxy_cache_dir")
if config_file.has_option("global", "myproxy_logon_command"):
myproxy_logon_command = config_file.get("global", "myproxy_logon_command")
if config_file.has_option("global", "override_vmtype"):
try:
override_vmtype = config_file.getboolean("global", "override_vmtype")
except ValueError:
print "Configuration file problem: override_vmtype must be a" \
" Boolean value."
if config_file.has_option("global", "vm_reqs_from_condor_reqs"):
try:
vm_reqs_from_condor_reqs = config_file.getboolean("global", "vm_reqs_from_condor_reqs")
except ValueError:
print "Configuration file problem: vm_reqs_from_condor_reqs must be a" \
" Boolean value."
if config_file.has_option("global", "adjust_insufficient_resources"):
try:
adjust_insufficient_resources = config_file.getboolean("global", "adjust_insufficient_resources")
except ValueError:
print "Configuration file problem: adjust_insufficient_resources must be a" \
" Boolean value."
if config_file.has_option("global", "connection_fail_disable_time"):
try:
connection_fail_disable_time = config_file.getint("global", "connection_fail_disable_time")
except ValueError:
print "Configuration file problem: connection_fail_disable_time must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("global", "use_cloud_init"):
try:
use_cloud_init = config_file.getboolean("global", "use_cloud_init")
except ValueError:
print "Configuration file problem: use_cloud_init must be a" \
" Boolean value."
if config_file.has_option("global", "default_yaml"):
default_yaml = config_file.get("global", "default_yaml")
if config_file.has_option("global", "validate_yaml"):
try:
validate_yaml = config_file.getboolean("global", "validate_yaml")
except ValueError:
print "Configuration file problem: validate_yaml must be a" \
" Boolean value."
if config_file.has_option("global", "retire_reallocate"):
try:
retire_reallocate = config_file.getboolean("global", "retire_reallocate")
except ValueError:
print "Configuration file problem: retire_reallocate must be a" \
" Boolean value."
# Default Logging options
if config_file.has_option("logging", "log_level"):
log_level = config_file.get("logging", "log_level")
if config_file.has_option("logging", "log_location"):
log_location = os.path.expanduser(config_file.get("logging", "log_location"))
if config_file.has_option("logging", "log_location_cloud_admin"):
log_location_cloud_admin = os.path.expanduser(config_file.get("logging", "log_location_cloud_admin"))
if config_file.has_option("logging", "admin_log_comments"):
try:
admin_log_comments = config_file.getboolean("logging", "admin_log_comments")
except ValueError:
print "Configuration file problem: admin_log_comments must be a" \
" Boolean value."
if config_file.has_option("logging", "log_stdout"):
try:
log_stdout = config_file.getboolean("logging", "log_stdout")
except ValueError:
print "Configuration file problem: log_stdout must be a" \
" Boolean value."
if config_file.has_option("logging", "log_syslog"):
try:
log_syslog = config_file.getboolean("logging", "log_syslog")
except ValueError:
print "Configuration file problem: log_syslog must be a" \
" Boolean value."
if config_file.has_option("logging", "log_max_size"):
try:
log_max_size = config_file.getint("logging", "log_max_size")
except ValueError:
print "Configuration file problem: log_max_size must be an " \
"integer value in bytes."
sys.exit(1)
if config_file.has_option("logging", "log_format"):
log_format = config_file.get("logging", "log_format", raw=True)
# Default Job options
if config_file.has_option("job", "default_VMType"):
default_VMType = config_file.get("job", "default_VMType")
if config_file.has_option("job", "default_VMNetwork"):
default_VMNetwork = config_file.get("job", "default_VMNetwork")
if config_file.has_option("job", "default_VMCPUArch"):
default_VMCPUArch = config_file.get("job", "default_VMCPUArch")
if config_file.has_option("job", "default_VMHypervisor"):
default_VMHypervisor = config_file.get("job", "default_VMHypervisor")
if config_file.has_option("job", "default_VMName"):
default_VMName = config_file.get("job", "default_VMName")
if config_file.has_option("job", "default_VMLoc"):
default_VMLoc = config_file.get("job", "default_VMLoc")
if config_file.has_option("job", "default_VMAMI"):
default_VMAMI = config_file.get("job", "default_VMAMI")
if config_file.has_option("job", "default_VMMem"):
try:
default_VMMem = config_file.getint("job", "default_VMMem")
except ValueError:
print "Configuration file problem: default_VMMem must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("job", "default_VMCPUCores"):
try:
default_VMCPUCores = config_file.getint("job", "default_VMCPUCores")
except ValueError:
print "Configuration file problem: default_VMCPUCores must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("job", "default_VMStorage"):
try:
default_VMStorage = config_file.getint("job", "default_VMStorage")
except ValueError:
print "Configuration file problem: default_VMStorage must be an " \
"integer value."
sys.exit(1)
if config_file.has_option("job", "default_VMInstanceType"):
default_VMInstanceType = config_file.get("job", "default_VMInstanceType")
if config_file.has_option("job", "default_VMInstanceTypeList"):
default_VMInstanceTypeList = config_file.get("job", "default_VMInstanceTypeList")
if config_file.has_option("job", "default_VMMaximumPrice"):
try:
default_VMMaximumPrice = config_file.getfloat("job", "default_VMMaximumPrice")
except ValueError:
print "Configuration file problem: default_VMMaximumPrice must be an " \
"floating point value."
sys.exit(1)
if config_file.has_option("job", "default_VMProxyNonBoot"):
try:
default_VMProxyNonBoot = config_file.getboolean("global", "default_VMProxyNonBoot")
except ValueError:
print "Configuration file problem: default_VMProxyNonBoot must be a" \
" Boolean value."
if config_file.has_option("job", "default_VMUserData"):
default_VMUserData = config_file.get("job", "default_VMUserData").replace(' ', '').strip('"').split(',')
if config_file.has_option("job", "default_TargetClouds"):
default_TargetClouds = config_file.get("job", "default_TargetClouds")
if config_file.has_option("job", "default_VMAMIConfig"):
default_VMAMIConfig = config_file.get("job", "default_VMAMIConfig")
if config_file.has_option("job", "default_VMInjectCA"):
try:
default_VMInjectCA = config_file.getboolean("job", "default_VMInjectCA")
except ValueError:
print "Configuration file problem: default_VMInjectCA must be a" \
" Boolean value."
if config_file.has_option("job", "default_VMJobPerCore"):
try:
default_VMJobPerCore = config_file.getboolean("job", "default_VMJobPerCore")
except ValueError:
print "Configuration file problem: default_VMJobPerCore must be a" \
" Boolean value."
# Derived options
if condor_host_on_vm:
condor_host = condor_host_on_vm
else:
condor_host = utilities.get_hostname_from_url(condor_webservice_url)
if config_file.has_option("global", "use_pyopenssl"):
use_pyopenssl = config_file.getboolean("global", "use_pyopenssl")
2
Example 36
Project: camr Source File: Aligner.py
def align_single_concept(self,sent,tokens,cur_var,amr,alignment,unmatched_vars,triples,NEXT=False):
'''align single concept'''
if cur_var in amr.node_to_concepts:
cur_concept = amr.node_to_concepts[cur_var]
else:
cur_concept = cur_var
if cur_var in alignment and not NEXT and not isinstance(cur_var,(StrLiteral,Quantity,Polarity)) : # already aligned
return True, sent,tokens
match = self.concept_patterns.match(cur_concept)
if match:
rule_type = match.lastgroup
span = None
update = True
if rule_type == "NameEntity":
NE_items = [v[0] for k,v in amr[cur_var].items() if isinstance(v[0],StrLiteral)]
nep = r'%s|%s'%(r'\s'.join(NE_items),r'\s'.join(n[:4] if len(n) > 3 else n for n in NE_items))
NE_pattern = re.compile(nep,re.IGNORECASE)
start,end = self._search_sent(NE_pattern,sent,tokens)
assert end-start == len(NE_items)
span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],NE_items)
alignment[cur_var].append(span)
for k,v in amr[cur_var].items():
if isinstance(v[0],StrLiteral):
self.remove_aligned_concepts(cur_var,k,v[0],unmatched_vars,triples)
elif rule_type in ["DateEntity", "haveOrgRole91","RateEntity"]:
EN_items = []
EN_spans = []
for k,v in amr[cur_var].items():
vconcept = amr.node_to_concepts[v[0]] if v[0] in amr.node_to_concepts else v[0]
EN_items.append(vconcept)
success, sent, tokens = self.align_single_concept(sent,tokens,v[0],amr,alignment,unmatched_vars,triples)
sp = alignment[v[0]][-1]
sp.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
EN_spans.append(sp)
self.remove_aligned_concepts(cur_var,k,v[0],unmatched_vars,triples)
#print NE_spans,alignment
start = EN_spans[0].start
end = EN_spans[-1].end
span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],EN_items)
span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
alignment[cur_var].append(span)
elif rule_type == "QuantityEntity":
quantity = ''
unit = ''
unit_var = None
q_success = False
u_success = False
for k,v in amr[cur_var].items():
if k == 'quant':
quantity = v[0]
q_success, sent, tokens = self.align_single_concept(sent,tokens,quantity,amr,alignment,unmatched_vars,triples)
elif k == 'unit':
unit_var = v[0]
unit = amr.node_to_concepts[v[0]]
u_success, sent, tokens = self.align_single_concept(sent,tokens,unit_var,amr,alignment,unmatched_vars,triples)
else:
pass
if q_success and u_success:
#QTY_pattern = r'(%s|%s)\s+(%s)s?' % (quantity,english_number(int(quantity)),unit)
#QTY_items = [quantity,unit]
#start,end = self._search_sent(QTY_pattern,QTY_items,sent,tokens)
#assert end - start == len(QTY_items)
quantity_span = alignment[quantity][-1]
unit_span = alignment[unit_var][0]
start = quantity_span.start if quantity_span.start < unit_span.end else unit_span.start
end = unit_span.end if quantity_span.start < unit_span.end else quantity_span.end
while not (end - len(quantity_span.words)-len(unit_span.words) - start < 2): # wrong match more than one quantity to map in sentence
alignment[quantity].pop()
q_success, sent, tokens = self.align_single_concept(sent,tokens,quantity,amr,alignment,unmatched_vars,triples,NEXT=True) # redo it on updated sentence
quantity_span = alignment[quantity][-1]
start = quantity_span.start
#assert start == end - 2
span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],[quantity,unit])
self.remove_aligned_concepts(cur_var,'quant',quantity,unmatched_vars,triples)
alignment[cur_var].append(span)
elif q_success and not u_success: # does not have unit or unit cannot be aligned
quantity_span = alignment[quantity][0]
start = quantity_span.start
end = quantity_span.end
span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],[quantity])
self.remove_aligned_concepts(cur_var,'quant',quantity,unmatched_vars,triples)
alignment[cur_var].append(span)
#self.remove_aligned_concepts(unmatched_vars,amr[cur_var].items())
elif not q_success and u_success:
unit_span = alignment[unit_var][0]
span = Span(unit_span.start,unit_span.end,Aligner.ENTITY_TAG_TABLE[rule_type],[unit])
self.remove_aligned_concepts(cur_var,'unit',unit_var,unmatched_vars,triples)
alignment[cur_var].append(span)
else:
rule_type = 'SingleConcept'
elif rule_type == "Number":
'''
aligned = False
num = [cur_var]
num.extend(english_number(int(cur_var)).split('|'))
for i,token in tokens:
if token.lower() in num:
aligned = True
break
if aligned:
span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE[rule_type],[token])
alignment[cur_var].append(span)
else:
print >> sys.stderr, 'Variable/Concept %s/%s cannot be aligned'%(cur_var,cur_concept)
update = False
'''
if re.match('[0-9]+:[0-9]+',cur_concept):
num = [('time','(\\s|^)('+cur_concept+')(\\s|&)'),
('english','(\\s|^)('+to_time(cur_concept)+')(\\s|&)')]
else:
num = [('digit','(\\s|^)('+cur_concept+'|'+format_num(cur_concept)+')(\\s|&)'),
('string','(\\s|^)('+english_number(int(cur_concept))+')(\\s|&)'),
('order','(\\s|^)('+to_order(cur_concept)+')(\\s|&)'),
('round','(\\s|^)('+to_round(int(cur_concept))+')(\\s|&)')
]
NUM_pattern = self._compile_regex_rule(num)
#print NUM_pattern.pattern
try:
start,end = self._search_sent(NUM_pattern,sent,tokens)
span = Span(start,end,Aligner.ENTITY_TAG_TABLE[rule_type],[w for i,w in tokens if i in range(start,end)])
alignment[cur_var].append(span)
except Exception as e:
update = False
print >> sys.stderr,e
#raw_input('CONTINUE')
elif rule_type == 'multiple':
op1 = amr[cur_var]['op1'][0]
success, sent, tokens = self.align_single_concept(sent,tokens,op1,amr,alignment,unmatched_vars,triples)
if success:
span = alignment[op1][0]
alignment[cur_var].append(span)
self.remove_aligned_concepts(cur_var,'op1',op1,unmatched_vars,triples)
else:
update = False
elif rule_type in ["person","picture","country","state","city","desert","organization"]:
if 'name' in amr[cur_var]:
k_var = amr[cur_var]['name'][0]
success, sent, tokens = self.align_single_concept(sent,tokens,k_var,amr,alignment,unmatched_vars,triples)
span = alignment[k_var][0]
span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type+'-name'])
alignment[cur_var].append(span)
else:
ind,span = self.try_align_as_single_concept(cur_var,cur_concept,amr,alignment,tokens,unmatched_vars,triples)
if ind:
pass
elif 'ARG0-of' in amr[cur_var]:
k_var = amr[cur_var]['ARG0-of'][0]
success, sent, tokens = self.align_single_concept(sent,tokens,k_var,amr,alignment,unmatched_vars,triples)
if success:
span = alignment[k_var][0]
span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
alignment[cur_var].append(span)
else:
update = False
else:
update = False
elif rule_type == "NegPolarity":
aligned = False
for i,token in tokens:
if token.lower() in Aligner.neg_polarity:
aligned = True
break
if aligned:
span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE[rule_type],[token])
alignment[cur_var].append(span)
else:
print >> sys.stderr, 'Variable/Concept %s/%s cannot be aligned'%(cur_var,cur_concept)
update = False
elif rule_type == "thing":
if 'ARG1-of' in amr[cur_var]:
k_var = amr[cur_var]['ARG1-of'][0]
success, sent, tokens = self.align_single_concept(sent,tokens,k_var,amr,alignment,unmatched_vars,triples)
if success:
span = alignment[k_var][0]
span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
alignment[cur_var].append(span)
else:
update = False
else:
rule_type = 'SingleConcept'
elif rule_type == 'OrdinalEntity':
val = amr[cur_var]['value'][0]
success, sent, tokens = self.align_single_concept(sent,tokens,val,amr,alignment,unmatched_vars,triples)
self.remove_aligned_concepts(cur_var,'value',val,unmatched_vars,triples)
span = alignment[val][0]
span.set_entity_tag(Aligner.ENTITY_TAG_TABLE[rule_type])
alignment[cur_var].append(span)
elif rule_type == 'RelativePosition':
if 'direction' in amr[cur_var]:
dir_var = amr[cur_var]['direction'][0]
if amr.node_to_concepts[dir_var] == 'away':
aligned = False
for i,tok in tokens:
if tok.lower() == 'from':
aligned = True
break
if aligned:
span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE[rule_type],[tok])
alignment[cur_var].append(span)
alignment[dir_var].append(span)
else:
print >> sys.stderr, 'Variable/Concept %s/%s cannot be aligned'%(cur_var,cur_concept)
update = False
else:
rule_type = 'SingleConcept'
else:
rule_type = 'SingleConcept'
elif self.is_ago(cur_var,cur_concept,amr):
k_var = amr[cur_var]['op1'][0]
aligned = False
for i,tok in tokens:
if tok.lower() == 'ago':
aligned = True
break
if aligned:
span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE['ago'],[tok])
alignment[cur_var].append(span)
alignment[k_var].append(span)
else:
print >> sys.stderr, '(%s/%s) :op1 (%s/%s) cannot be aligned'%(cur_var,cur_concept,k_var,amr.node_to_concepts[k_var])
update = False
elif self.is_why_question(cur_var,amr):
arg0_var = amr[cur_var]['ARG0'][0]
aligned = False
for i,tok in tokens:
if tok.lower() == 'why':
aligned = True
break
if aligned:
span = Span(i,i+1,Aligner.ENTITY_TAG_TABLE['cause'],[tok])
alignment[cur_var].append(span)
alignment[arg0_var].append(span)
else:
print >> sys.stderr, '(%s/%s) :op1 (%s/%s) cannot be aligned'%(cur_var,cur_concept,arg0_var,amr.node_to_concepts[arg0_var])
update = False
else:
pass
if rule_type == "SingleConcept":
update,span = self.try_align_as_single_concept(cur_var,cur_concept,amr,alignment,tokens,unmatched_vars,triples)
elif cur_var in alignment:
pass
else:
print >> sys.stderr, 'Can not find type of concept %s / %s'%(cur_var,cur_concept)
# update
#print cur_concept,rule_type
if update:
tokens = [(i,tk) for i,tk in tokens if i not in range(span.start,span.end)]
sent = ' '.join(x for i,x in tokens)
if self.verbose > 2:
print >> sys.stderr, "Concept '%s' Matched to span '%s' "%(cur_concept,' '.join(w for i,w in enumerate(sentence.split()) if i+1 in range(span[0],span[1])))
print sent
print alignment
#raw_input('ENTER to continue')
return update, sent, tokens
2
Example 37
Project: internetarchive Source File: item.py
def upload_file(self, body,
key=None,
metadata=None,
headers=None,
access_key=None,
secret_key=None,
queue_derive=None,
verbose=None,
verify=None,
checksum=None,
delete=None,
retries=None,
retries_sleep=None,
debug=None,
request_kwargs=None):
"""Upload a single file to an item. The item will be created
if it does not exist.
:type body: Filepath or file-like object.
:param body: File or data to be uploaded.
:type key: str
:param key: (optional) Remote filename.
:type metadata: dict
:param metadata: (optional) Metadata used to create a new item.
:type headers: dict
:param headers: (optional) Add additional IA-S3 headers to request.
:type queue_derive: bool
:param queue_derive: (optional) Set to False to prevent an item from
being derived after upload.
:type verify: bool
:param verify: (optional) Verify local MD5 checksum matches the MD5
checksum of the file received by IAS3.
:type checksum: bool
:param checksum: (optional) Skip based on checksum.
:type delete: bool
:param delete: (optional) Delete local file after the upload has been
successfully verified.
:type retries: int
:param retries: (optional) Number of times to retry the given request
if S3 returns a 503 SlowDown error.
:type retries_sleep: int
:param retries_sleep: (optional) Amount of time to sleep between
``retries``.
:type verbose: bool
:param verbose: (optional) Print progress to stdout.
:type debug: bool
:param debug: (optional) Set to True to print headers to stdout, and
exit without sending the upload request.
Usage::
>>> import internetarchive
>>> item = internetarchive.Item('identifier')
>>> item.upload_file('/path/to/image.jpg',
... key='photos/image1.jpg')
True
"""
# Set defaults.
headers = {} if headers is None else headers
metadata = {} if metadata is None else metadata
access_key = self.session.access_key if access_key is None else access_key
secret_key = self.session.secret_key if secret_key is None else secret_key
queue_derive = True if queue_derive is None else queue_derive
verbose = False if verbose is None else verbose
verify = True if verify is None else verify
delete = False if delete is None else delete
# Set checksum after delete.
checksum = True if delete or checksum is None else checksum
retries = 0 if retries is None else retries
retries_sleep = 30 if retries_sleep is None else retries_sleep
debug = False if debug is None else debug
request_kwargs = {} if request_kwargs is None else request_kwargs
md5_sum = None
if not hasattr(body, 'read'):
body = open(body, 'rb')
size = get_file_size(body)
if not headers.get('x-archive-size-hint'):
headers['x-archive-size-hint'] = str(size)
# Build IA-S3 URL.
key = body.name.split('/')[-1] if key is None else key
base_url = '{0.session.protocol}//s3.us.archive.org/{0.identifier}'.format(self)
url = '{0}/{1}'.format(
base_url, urllib.parse.quote(key.lstrip('/').encode('utf-8')))
# Skip based on checksum.
if checksum:
md5_sum = get_md5(body)
ia_file = self.get_file(key)
if (not self.tasks) and (ia_file) and (ia_file.md5 == md5_sum):
log.info('{f} already exists: {u}'.format(f=key, u=url))
if verbose:
print(' {f} already exists, skipping.'.format(f=key))
if delete:
log.info(
'{f} successfully uploaded to '
'https://archive.org/download/{i}/{f} '
'and verified, deleting '
'local copy'.format(i=self.identifier,
f=key))
os.remove(body.name)
# Return an empty response object if checksums match.
# TODO: Is there a better way to handle this?
return Response()
# require the Content-MD5 header when delete is True.
if verify or delete:
if not md5_sum:
md5_sum = get_md5(body)
headers['Content-MD5'] = md5_sum
def _build_request():
body.seek(0, os.SEEK_SET)
if verbose:
try:
chunk_size = 1048576
expected_size = size / chunk_size + 1
chunks = chunk_generator(body, chunk_size)
progress_generator = progress.bar(
chunks,
expected_size=expected_size,
label=' uploading {f}: '.format(f=key))
data = IterableToFileAdapter(progress_generator, size)
except:
print(' uploading {f}'.format(f=key))
data = body
else:
data = body
request = S3Request(method='PUT',
url=url,
headers=headers,
data=data,
metadata=metadata,
access_key=access_key,
secret_key=secret_key,
queue_derive=queue_derive)
return request
if debug:
return _build_request()
else:
try:
error_msg = ('s3 is overloaded, sleeping for '
'{0} seconds and retrying. '
'{1} retries left.'.format(retries_sleep, retries))
while True:
if retries > 0:
if self.session.s3_is_overloaded(access_key):
sleep(retries_sleep)
log.info(error_msg)
if verbose:
print(' warning: {0}'.format(error_msg), file=sys.stderr)
retries -= 1
continue
request = _build_request()
prepared_request = request.prepare()
response = self.session.send(prepared_request,
stream=True,
**request_kwargs)
if (response.status_code == 503) and (retries > 0):
log.info(error_msg)
if verbose:
print(' warning: {0}'.format(error_msg), file=sys.stderr)
sleep(retries_sleep)
retries -= 1
continue
else:
if response.status_code == 503:
log.info('maximum retries exceeded, upload failed.')
break
response.raise_for_status()
log.info('uploaded {f} to {u}'.format(f=key, u=url))
if delete and response.status_code == 200:
log.info(
'{f} successfully uploaded to '
'https://archive.org/download/{i}/{f} and verified, deleting '
'local copy'.format(i=self.identifier,
f=key))
os.remove(body.name)
return response
except HTTPError as exc:
msg = get_s3_xml_text(exc.response.content)
error_msg = (' error uploading {0} to {1}, '
'{2}'.format(key, self.identifier, msg))
log.error(error_msg)
if verbose:
print(' error uploading {0}: {1}'.format(key, msg), file=sys.stderr)
# Raise HTTPError with error message.
raise type(exc)(error_msg, response=exc.response, request=exc.request)
2
Example 38
Project: ros_buildfarm Source File: generate_prerelease_script.py
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'prerelease' script")
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'source')
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_output_dir(parser, required=True)
group = parser.add_argument_group(
'Repositories in underlay workspace',
description='The repositories in the underlay workspace will be ' +
'built and installed as well as built and tested. ' +
'Dependencies will be provided by binary packages.')
group.add_argument(
'source_repos',
nargs='*',
default=[],
metavar='REPO_NAME',
help="A name of a 'repository' from the distribution file")
group.add_argument(
'--custom-branch',
nargs='*',
type=_repository_name_and_branch,
default=[],
metavar='REPO_NAME:BRANCH_OR_TAG_NAME',
help="A name of a 'repository' from the distribution file followed " +
'by a colon and a branch / tag name')
group.add_argument(
'--custom-repo',
nargs='*',
type=_repository_name_and_type_and_url_and_branch,
default=[],
metavar='REPO_NAME:REPO_TYPE:REPO_URL:BRANCH_OR_TAG_NAME',
help='The name, type, url and branch / tag name of a repository')
add_overlay_arguments(parser)
args = parser.parse_args(argv)
print('Fetching buildfarm configuration...')
config = get_config_index(args.config_url)
build_files = get_source_build_files(config, args.rosdistro_name)
build_file = build_files[args.source_build_name]
print('Fetching rosdistro cache...')
# Targets defined by source build file are subset of targets
# defined by release build files. To increase the number of supported
# pre-release targets, we combine all targets defined by all release
# build files and use that when configuring the devel job.
release_build_files = get_release_build_files(config, args.rosdistro_name)
release_targets_combined = {}
if release_build_files:
release_targets_combined[args.os_name] = {}
for build_name, rel_obj in release_build_files.items():
if args.os_name not in rel_obj.targets:
continue
for dist_name, targets in rel_obj.targets[args.os_name].items():
if dist_name not in release_targets_combined[args.os_name]:
release_targets_combined[args.os_name][dist_name] = {}
release_targets_combined[args.os_name][dist_name].update(targets)
index = get_index(config.rosdistro_index_url)
dist_cache = get_distribution_cache(index, args.rosdistro_name)
dist_file = dist_cache.distribution_file
# determine source repositories for underlay workspace
repositories = {}
for repo_name in args.source_repos:
if repo_name in repositories:
print("The repository '%s' appears multiple times" % repo_name,
file=sys.stderr)
return 1
try:
repositories[repo_name] = \
dist_file.repositories[repo_name].source_repository
except KeyError:
print(("The repository '%s' was not found in the distribution " +
"file") % repo_name, file=sys.stderr)
return 1
for repo_name, custom_version in args.custom_branch:
if repo_name in repositories:
print("The repository '%s' appears multiple times" % repo_name,
file=sys.stderr)
return 1
try:
source_repo = dist_file.repositories[repo_name].source_repository
except KeyError:
print(("The repository '%s' was not found in the distribution " +
"file") % repo_name, file=sys.stderr)
return 1
source_repo = deepcopy(source_repo)
source_repo.version = custom_version
repositories[repo_name] = source_repo
for repo_name, repo_type, repo_url, version in args.custom_repo:
if repo_name in repositories and repositories[repo_name]:
print("custom_repos option overriding '%s' to pull via '%s' "
"from '%s' with version '%s'. " %
(repo_name, repo_type, repo_url, version),
file=sys.stderr)
source_repo = RepositorySpecification(
repo_name, {
'type': repo_type,
'url': repo_url,
'version': version,
})
repositories[repo_name] = source_repo
scms = [(repositories[k], 'catkin_workspace/src/%s' % k)
for k in sorted(repositories.keys())]
# collect all template snippets of specific types
class IncludeHook(Hook):
def __init__(self):
Hook.__init__(self)
self.scripts = []
def beforeInclude(self, *args, **kwargs):
template_path = kwargs['file'].name
if template_path.endswith('/snippet/builder_shell.xml.em'):
self.scripts.append(kwargs['locals']['script'])
hook = IncludeHook()
from ros_buildfarm import templates
templates.template_hooks = [hook]
# use random source repo to pass to devel job template
source_repository = deepcopy(list(repositories.values())[0])
if not source_repository:
print(("The repository '%s' does not have a source entry in the distribution " +
'file. We cannot generate a prerelease without a source entry.') % repo_name,
file=sys.stderr)
return 1
source_repository.name = 'prerelease'
print('Evaluating job templates...')
configure_devel_job(
args.config_url, args.rosdistro_name, args.source_build_name,
None, args.os_name, args.os_code_name, args.arch,
config=config, build_file=build_file,
index=index, dist_file=dist_file, dist_cache=dist_cache,
jenkins=False, views=False,
source_repository=source_repository,
build_targets=release_targets_combined)
templates.template_hooks = None
# derive scripts for overlay workspace from underlay
overlay_scripts = []
for script in hook.scripts:
# skip cloning of ros_buildfarm repository
if 'git clone' in script and '.git ros_buildfarm' in script:
continue
# skip build-and-install step
if 'build and install' in script:
continue
# add prerelease overlay flag
run_devel_job = '/run_devel_job.py'
if run_devel_job in script:
script = script.replace(
run_devel_job, run_devel_job + ' --prerelease-overlay')
# replace mounted workspace volume with overlay and underlay
# used by:
# - create_devel_task_generator.py needs to find packages in both
# the underlay as well as the overlay workspace
# - catkin_make_isolated_and_test.py needs to source the environment of
# the underlay before building the overlay
mount_volume = '-v $WORKSPACE/catkin_workspace:/tmp/catkin_workspace'
if mount_volume in script:
script = script.replace(
mount_volume, mount_volume + ':ro ' + '-v $WORKSPACE/' +
'catkin_workspace_overlay:/tmp/catkin_workspace_overlay')
# relocate all docker files
docker_path = '$WORKSPACE/docker_'
if docker_path in script:
script = script.replace(
docker_path, docker_path + 'overlay_')
# rename all docker images
name_suffix = '_prerelease'
if name_suffix in script:
script = script.replace(
name_suffix, name_suffix + '_overlay')
overlay_scripts.append(script)
from ros_buildfarm import __file__ as ros_buildfarm_file
data = deepcopy(args.__dict__)
data.update({
'scms': scms,
'scripts': hook.scripts,
'overlay_scripts': overlay_scripts,
'ros_buildfarm_python_path': os.path.dirname(
os.path.dirname(os.path.abspath(ros_buildfarm_file))),
'python_executable': sys.executable,
'prerelease_script_path': os.path.dirname(os.path.abspath(__file__))})
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# generate multiple scripts
for script_name in [
'prerelease',
'prerelease_build_overlay',
'prerelease_build_underlay',
'prerelease_clone_overlay',
'prerelease_clone_underlay']:
content = expand_template(
'prerelease/%s_script.sh.em' % script_name, data,
options={BANGPATH_OPT: False})
script_file = os.path.join(args.output_dir, script_name + '.sh')
with open(script_file, 'w') as h:
h.write(content)
os.chmod(script_file, os.stat(script_file).st_mode | stat.S_IEXEC)
print('')
print('Generated prerelease script - to execute it run:')
if os.path.abspath(args.output_dir) != os.path.abspath(os.curdir):
print(' cd %s' % args.output_dir)
print(' ./prerelease.sh')
2
Example 39
Project: biocode Source File: batch_attributor.py
def main():
parser = argparse.ArgumentParser( description='Generates a shell script of Attributor runs given a range of pipeline IDs')
## output file to be written
parser.add_argument('-o', '--output_file', type=str, required=True, help='Path to an output shell file to be created' )
parser.add_argument('-od', '--output_directory', type=str, required=True, help='Directory where output files will be written' )
args = parser.parse_args()
# Either set this variable to a path, or use the pipeline_min/pipeline_max below
# If you dont' want to use this, set it to None
srs_id_list_file = '/path/to/some.id.list'
srs_ids_to_keep = list()
# inclusive. These are NOT read if the srs_id_list_file is defined above
#pipeline_min = 10927248802
#pipeline_max = 11214274766
pipeline_min = 11305294429
pipeline_max = 11373273817
# CONFIG
project_area = '/local/hmp/dacc/t3/'
owners = ['hhuot', 'cmccracken']
config_template = '/home/hhuot/git/Attributor/assign_functional_annotation.example_hhc.config'
# assumes all child names are like: SRS147134.rapsearch2.m8.gz
rapsearch2_base = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmgi/rapsearch'
attributor_path = '/home/hhuot/git/Attributor/assign_functional_annotation.py'
# names are like either SRS045739.metagenemark.gff3 or SRS045763.metagenemark3.gff3
gff_base_dir = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmorf/gff3'
# names are like either SRS018791.metagenemark.faa or SRS018794.metagenemark3.faa
fasta_base_dir = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmorf/faa'
# key = pipeline ID, value = SRS ID
pipelines = dict()
# We're either getting our SRS ID list to keep from a file or a range of pipeline IDs
if srs_id_list_file is not None:
for line in open(srs_id_list_file):
line = line.rstrip()
srs_ids_to_keep.append(line)
## make sure the config template exists
if not os.path.exists(config_template):
raise Exception("ERROR: config template file not found: {0}".format(config_template))
for pipeline_id in os.listdir("{0}/workflow/runtime/pipeline".format(project_area)):
m = re.match("^\d+$", pipeline_id)
if m:
pipeline_id = int(pipeline_id)
if srs_id_list_file is None:
if pipeline_id > pipeline_max or pipeline_id < pipeline_min:
continue
else:
continue
pipeline_path = "{0}/workflow/runtime/pipeline/{1}/pipeline.xml".format(project_area, pipeline_id)
owner = find_owner(pipeline_path)
if owner not in owners:
continue
pipeline_comment_path = "{0}/workflow/runtime/pipeline/{1}/pipeline.xml.comment".format(project_area, pipeline_id)
if os.path.exists(pipeline_comment_path):
comment = open(pipeline_comment_path).read()
m = re.search("(SRS\d+)", comment)
if m:
srs_id = m.group(1)
pipelines[pipeline_id] = srs_id
else:
m = re.search("(\d+\..{2,4})", comment)
if m:
srs_id = m.group(1)
pipelines[pipeline_id] = srs_id
else:
print("WARNING: Pipeline skipped without an SRS comment: {0}".format(pipeline_comment_path), file=sys.stderr)
print("INFO: found {0} pipelines".format(len(pipelines)))
batch_fh = open(args.output_file, 'wt')
for pipeline_id in pipelines:
# do we have a limit on the SRS IDs?
if srs_id_list_file is not None:
if pipelines[pipeline_id] not in srs_ids_to_keep:
continue
config_path = "{0}/{1}.config".format(args.output_directory, pipelines[pipeline_id])
tmp_config_path = "/tmp/{0}.config".format(pipeline_id)
## get the source FASTA path
if os.path.exists("{0}/{1}.metagenemark.faa".format(fasta_base_dir, pipelines[pipeline_id])):
fasta_path = "{0}/{1}.metagenemark.faa".format(fasta_base_dir, pipelines[pipeline_id])
elif os.path.exists("{0}/{1}.metagenemark3.faa".format(fasta_base_dir, pipelines[pipeline_id])):
fasta_path = "{0}/{1}.metagenemark3.faa".format(fasta_base_dir, pipelines[pipeline_id])
else:
print("WARNING: failed to find FASTA file for {0}".format(pipelines[pipeline_id]), file=sys.stderr)
continue
#raise Exception("ERROR: failed to find FASTA file for {0}".format(pipelines[pipeline_id]))
## get the source GFF path
if os.path.exists("{0}/{1}.metagenemark.gff3".format(gff_base_dir, pipelines[pipeline_id])):
gff3_path = "{0}/{1}.metagenemark.gff3".format(gff_base_dir, pipelines[pipeline_id])
elif os.path.exists("{0}/{1}.metagenemark3.gff3".format(gff_base_dir, pipelines[pipeline_id])):
gff3_path = "{0}/{1}.metagenemark3.gff3".format(gff_base_dir, pipelines[pipeline_id])
else:
print("WARNING: failed to find GFF3 file for {0}".format(pipelines[pipeline_id]), file=sys.stderr)
continue
#raise Exception("ERROR: failed to find GFF3 file for {0}".format(pipelines[pipeline_id]))
## copy the config file
shutil.copy(config_template, config_path)
## modify it with these paths
ofh = open(tmp_config_path, 'wt')
last_label = None
for line in open(config_path):
line = line.rstrip()
m = re.match(" - label: (.+)", line)
if m:
last_label = m.group(1)
m = re.match(" path: (.+)", line)
if m:
if last_label == 'coding_hmm_lib__equivalog':
ofh.write(" path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
continue
elif last_label == 'rapsearch2__uniref100__all_full':
ofh.write(" path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
continue
elif last_label == 'rapsearch2__uniref100__all_partial':
ofh.write(" path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
continue
elif last_label == 'rapsearch2__uniref100__trusted_full':
ofh.write(" path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
continue
elif last_label == 'rapsearch2__uniref100__trusted_partial':
ofh.write(" path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
continue
elif last_label == 'coding_hmm_lib__equivalog_domain':
ofh.write(" path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
continue
elif last_label == 'coding_hmm_lib__all_trusted':
ofh.write(" path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
continue
elif last_label == 'tmhmm':
ofh.write(" path: {0}/output_repository/tmhmm/{1}_default/tmhmm.raw.list\n".format(project_area, pipeline_id))
continue
elif last_label == 'lipoprotein_motif':
ofh.write(" path: {0}/output_repository/lipoprotein_motif/{1}_default/lipoprotein_motif.bsml.list\n".format(project_area, pipeline_id))
continue
m = re.match(" polypeptide_fasta: (\S+)", line)
if m:
ofh.write(" polypeptide_fasta: {0}\n".format(fasta_path))
continue
m = re.match(" gff3: (\S+)", line)
if m:
ofh.write(" gff3: {0}\n".format(gff3_path))
continue
ofh.write("{0}\n".format(line))
ofh.close()
shutil.move(tmp_config_path, config_path)
batch_fh.write("{0} -f gff3 -c {1} -o {2}/{3}\n".format(attributor_path, config_path, args.output_directory,
pipelines[pipeline_id]))
2
Example 40
Project: camr Source File: oracle.py
def give_ref_action_aux(self,currentIdx,currentChildIdx,currentGraph,ref_graph):
def isCorrectReplace(childIdx,node,rgraph):
for p in node.parents:
if p in rgraph.nodes and childIdx in rgraph.nodes[p].children:
return True
return False
def getABTParent(gn):
for p in gn.parents:
if not isinstance(p,int):
return p
return None
def need_infer(cidx,gn,cn,gnset,cg,rg):
for p in cn.parents:
#if p not in gnset and rg.isContained(p):
# return False
if p in gn.parents or gn.contains(cg.nodes[p]) or p in gn.children: # correct/merge/swap arc
return False
return True
def is_aligned(abtIdx,rg,cg):
return abtIdx in rg.abt_node_table and rg.abt_node_table[abtIdx] in cg.nodes
def need_swap(idx,idx_p,cg,gc,rg):
cur_head_p = cg.find_true_head(idx_p)
try:
cur_head = cg.abt_node_table[cur_head_p] if not isinstance(cur_head_p,int) else cur_head_p
except KeyError:
print >> sys.stderr,"inferred abt node cur_head_p not in ref graph!"
return None
if cur_head in gc.children:
return cur_head
cur_grand_head = getABTParent(rg.nodes[cur_head])
if (not isinstance(cur_head,int) and cur_grand_head in gc.children):
return cur_grand_head
return None
if currentIdx == START_ID:
return {'type':NEXT2},None
currentNode = currentGraph.nodes[currentIdx] #state.get_current_node()
currentChild = currentGraph.nodes[currentChildIdx] if currentChildIdx in currentGraph.nodes else None #state.get_current_child()
goldNodeSet = [ref_graph.abt_node_table[k] if k in ref_graph.abt_node_table else k for k in ref_graph.nodes]
result_act_type = None
result_act_label = None
if currentIdx in goldNodeSet:
currentIdx_p = currentIdx
if currentIdx not in ref_graph.nodes:
currentIdx = currentGraph.abt_node_table[currentIdx]
goldNode = ref_graph.nodes[currentIdx]
#for child in currentNode.children:
if currentChildIdx:
if currentChildIdx == START_ID:
abtParentIdx = getABTParent(goldNode)
#if state.action_history and state.action_history[-1]['type'] in [REPLACEHEAD,NEXT2,DELETENODE]
if need_infer(currentIdx,goldNode,currentNode,goldNodeSet,currentGraph,ref_graph) \
and abtParentIdx and not is_aligned(abtParentIdx,ref_graph,currentGraph) \
and (len(ref_graph.nodes[abtParentIdx].children) == 1 or not isinstance(currentIdx_p,int) or ((goldNode.words[0] in NOMLIST or currentGraph.sent[currentIdx_p]['lemma'].lower() in NOMLIST) and len(goldNode.words) == 1)):
gold_tag = ref_graph.nodes[abtParentIdx].tag
abt_node_index = ABT_PREFIX+str(currentGraph.abt_node_num)
ref_graph.add_abt_mapping(abtParentIdx,abt_node_index)
currentGraph.add_abt_mapping(abt_node_index,abtParentIdx)
return {'type':INFER},gold_tag
else:
return {'type':NEXT1},START_EDGE
if currentChildIdx in goldNodeSet:
currentChildIdx_p = currentChildIdx
if currentChildIdx not in ref_graph.nodes:
currentChildIdx = currentGraph.abt_node_table[currentChildIdx]
try:
goldChild = ref_graph.nodes[currentChildIdx]
except KeyError:
import pdb
pdb.set_trace()
if goldChild.contains(currentNode) or goldNode.contains(currentChild):
return {'type':MERGE}, None # merge
#result_act_type = {'type':MERGE}
#if currentIdx in goldChild.children and \
# currentChildIdx in goldNode.children:
'''
if currentChildIdx in goldNode.children and ref_graph.is_cycle(currentIdx):
print >> sys.stderr, "Circle detected in gold graph!"
gold_edge = ref_graph.get_edge_label(currentIdx,currentChildIdx)
return {'type':NEXT1}, gold_edge # next
'''
#result_act_type = {'type':NEXT1}
#result_act_label = gold_edge
if currentChildIdx in goldNode.children: # correct
#parents_to_add = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_reentrance_constrained(currentIdx_p,currentChildIdx_p)]
parents_to_add = [p for p in goldChild.parents if p not in [currentGraph.abt_node_table[cp] if cp in currentGraph.abt_node_table else cp for cp in currentChild.parents]]
if parents_to_add:
pta = parents_to_add[0]
if pta in ref_graph.abt_node_table: pta = ref_graph.abt_node_table[pta]
if pta in currentGraph.get_possible_parent_unconstrained(currentIdx_p,currentChildIdx_p):
gold_edge = ref_graph.get_edge_label(parents_to_add[0],currentChildIdx)
return {'type':REENTRANCE,'parent_to_add':pta},gold_edge
else:
gold_edge = ref_graph.get_edge_label(currentIdx,currentChildIdx)
return {'type':NEXT1}, gold_edge # next
else:
gold_edge = ref_graph.get_edge_label(currentIdx,currentChildIdx)
return {'type':NEXT1}, gold_edge # next
thead = need_swap(currentIdx,currentIdx_p,currentGraph,goldChild,ref_graph)
if thead is not None:#in goldChild.children:
gold_edge = ref_graph.get_edge_label(currentChildIdx,thead)
return {'type':SWAP}, gold_edge # swap
#result_act_type = {'type':SWAP}
parents_to_attach = [p for p in goldChild.parents if p not in [currentGraph.abt_node_table[cp] if cp in currentGraph.abt_node_table else cp for cp in currentChild.parents]]
if parents_to_attach:
pta = parents_to_attach[0]
if pta in ref_graph.abt_node_table: pta = ref_graph.abt_node_table[pta]
if pta in currentGraph.get_possible_parent_unconstrained(currentIdx_p,currentChildIdx_p):
gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
return {'type':REATTACH,'parent_to_attach':pta},gold_edge
#elif not isinstance(pta,int): # abstract (or unaligned) nodes
# abt_node_index = ABT_PREFIX+str(currentGraph.abt_node_num)
# if pta == parents_to_attach[0]:
# ref_graph.add_abt_mapping(pta,abt_node_index)
# currentGraph.add_abt_mapping(abt_node_index,pta)
# else:
# currentGraph.add_abt_mapping(abt_node_index,parents_to_attach[0])
# return {'type':INFER},None
else:
return {'type':NEXT1},None # violates the attachment constraints
else:
if self.verbose > 1: print >> sys.stderr, "Current child node %s doesn't have parents in gold span graph!"%(currentChildIdx)
return {'type':NEXT1},None
#return {'type':REATTACH,'parent_to_attach':None},None
else:
if goldNode.contains(currentChild):
return {'type':MERGE},None
#result_act_type = {'type':MERGE}
else:
#return {'type':DELETEEDGE} # delete edge
#result_act_type = {'type':DELETEEDGE}
#k = ref_graph.isContained(currentChildIdx)
#if k:
# return {'type':REATTACH,'parent_to_attach':k}
#else:
return {'type':NEXT1},None
#return {'type':REATTACH,'parent_to_attach':None},None
else:
#if len(currentNode.children) <= len(goldNode.children) and set(currentNode.children).issubset(set(goldNode.children)):
#children_to_add = [c for c in goldNode.children if c not in currentNode.children and c in currentGraph.get_possible_children_constrained(currentIdx)]
#if children_to_add:
# child_to_add = children_to_add[0]
# gold_edge = ref_graph.get_edge_label(currentIdx,child_to_add)
# return {'type':ADDCHILD,'child_to_add':child_to_add,'edge_label':gold_edge}
#else:
gold_tag = goldNode.tag
return {'type':NEXT2, 'tag':gold_tag},None # next: done with the current node move to next one
#else:
# if self.verbose > 2:
# print >> sys.stderr, "ERROR: Missing actions, current node's and gold node's children:%s %s"%(str(currentNode.children), str(goldNode.children))
# pass
elif ref_graph.isContained(currentIdx):
if currentChildIdx:
if currentChildIdx in goldNodeSet:
#goldChild = ref_graph.nodes[currentChildIdx] if currentChildIdx in ref_graph.nodes else ref_graph.nodes[currentGraph.abt_node_table[currentChildIdx]]
currentChildIdx_p = currentChildIdx
if currentChildIdx not in ref_graph.nodes:
currentChildIdx = currentGraph.abt_node_table[currentChildIdx]
goldChild = ref_graph.nodes[currentChildIdx]
if goldChild.contains(currentNode):
return {'type':MERGE},None
else:
parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx_p)]
#parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx)]
if parents_to_attach:
if ref_graph.nodes[parents_to_attach[0]].contains(currentNode):
return {'type':NEXT1},None # delay action for future merge
else:
gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
#if gold_edge == 'x': # not actually gold edge, skip this
# return {'type':NEXT1},None
#else:
return {'type':REATTACH,'parent_to_attach':parents_to_attach[0]},gold_edge
else:
return {'type':NEXT1},None
#return {'type':REATTACH,'parent_to_attach':None},None
else:
return {'type':NEXT1},None
#return {'type':REATTACH,'parent_to_attach':None},None
else:
return {'type':NEXT2},None
else:
'''
if currentChildIdx:
#assert len(currentNode.children) == 1
if currentChildIdx in goldNodeSet:
goldChild = ref_graph.nodes[currentChildIdx]
if (isCorrectReplace(currentChildIdx,currentNode,ref_graph,state.beta) or len(currentNode.children) == 1):
return {'type':REPLACEHEAD},None # replace head
#result_act_type = {'type':REPLACEHEAD}
else:
parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_constrained(currentIdx,currentChildIdx)]
#parents_to_attach = [p for p in goldChild.parents if p not in currentChild.parents and p in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx)]
if parents_to_attach:
if isCorrectReplace(parents_to_attach[0],currentNode,ref_graph,state.beta):
return {'type':NEXT1},None # delay action for future replace head
else:
gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
#if gold_edge == 'x': # not actually gold edge, skip this
# return {'type':NEXT1},None
#else:
return {'type':REATTACH,'parent_to_attach':parents_to_attach[0]},gold_edge
else:
return {'type':NEXT1},None
#return {'type':REATTACH,'parent_to_attach':None},None
else:
#return {'type':DELETEEDGE}
#result_act_type = {'type':DELETEEDGE}
return {'type':NEXT1},None
#return {'type':REATTACH,'parent_to_attach':None},None
'''
if currentChildIdx:
if currentChildIdx == START_ID:
return {'type':NEXT1},None
if currentChildIdx in goldNodeSet:
#goldChild = ref_graph.nodes[currentChildIdx] if currentChildIdx in ref_graph.nodes else ref_graph.nodes[currentGraph.abt_node_table[currentChildIdx]]
currentChildIdx_p = currentChildIdx
if currentChildIdx not in ref_graph.nodes:
currentChildIdx = currentGraph.abt_node_table[currentChildIdx]
goldChild = ref_graph.nodes[currentChildIdx]
if isCorrectReplace(currentChildIdx,currentNode,ref_graph) or len(currentNode.children) == 1: #current node's parents are already aligned
return {'type':REPLACEHEAD},None # replace head
else:
parents_to_attach = [p for p in goldChild.parents if p not in [currentGraph.abt_node_table[cp] if cp in currentGraph.abt_node_table else cp for cp in currentChild.parents]]
if parents_to_attach:
pta = parents_to_attach[0]
if pta in ref_graph.abt_node_table: pta = ref_graph.abt_node_table[pta]
if pta in currentGraph.get_possible_parent_unconstrained(currentIdx,currentChildIdx_p):
gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
return {'type':REATTACH,'parent_to_attach':pta},gold_edge
#elif not isinstance(pta,int) and pta not in currentGraph.nodes: # abstract (or unaligned) nodes
'''
#heuristic 1: just assign the abstract node index according to its first gold child
#assert pta != currentIdx
if pta == parents_to_attach[0] and state.is_possible_align(currentIdx,parents_to_attach[0],ref_graph):
ref_graph.add_abt_mapping(pta,currentIdx)
currentGraph.add_abt_mapping(currentIdx,pta)
gold_edge = ref_graph.get_edge_label(parents_to_attach[0],currentChildIdx)
return {'type':NEXT1},gold_edge
else:
return {'type':REPLACEHEAD},None
'''
else:
return {'type':NEXT1},None # violates the attachment constraints
else:
if self.verbose > 1: print >> sys.stderr, "Current child node %s doesn't have parents in gold span graph!"%(currentChildIdx)
return {'type':NEXT1},None
else:
if self.verbose > 1: print >> sys.stderr, "Current child node %s should have been deleted!"%(currentChildIdx)
return {'type':NEXT1},None
else:
# here currentNode.children must be empty
if currentNode.children and self.verbose > 1: print >> sys.stderr, "Unaligned node %s has children"%(currentNode.start)
# avoid delete name entity; will delete that in further reentrance action
#if currentGraph.sent[currentIdx]['ne'] == 'O' or currentGraph.sent[currentIdx]['pos'] in FUNCTION_TAG:
return {'type':DELETENODE},None
#else:
# return {'type':NEXT2},None
skip = NEXT1 if currentChildIdx else NEXT2
return {'type':skip},None
2
Example 41
Project: bloodhound Source File: standalone.py
def main():
from optparse import OptionParser, OptionValueError
parser = OptionParser(usage='usage: %prog [options] [projenv] ...',
version='%%prog %s' % VERSION)
auths = {}
def _auth_callback(option, opt_str, value, parser, cls):
info = value.split(',', 3)
if len(info) != 3:
raise OptionValueError("Incorrect number of parameters for %s"
% option)
env_name, filename, realm = info
if env_name in auths:
print >> sys.stderr, 'Ignoring duplicate authentication option ' \
'for project: %s' % env_name
else:
auths[env_name] = cls(os.path.abspath(filename), realm)
def _validate_callback(option, opt_str, value, parser, valid_values):
if value not in valid_values:
raise OptionValueError('%s must be one of: %s, not %s'
% (opt_str, '|'.join(valid_values), value))
setattr(parser.values, option.dest, value)
def _octal(option, opt_str, value, parser):
try:
setattr(parser.values, option.dest, int(value, 8))
except ValueError:
raise OptionValueError('Invalid octal umask value: %r' % value)
parser.add_option('-a', '--auth', action='callback', type='string',
metavar='DIGESTAUTH', callback=_auth_callback,
callback_args=(DigestAuthentication,),
help='[projectdir],[htdigest_file],[realm]')
parser.add_option('--basic-auth', action='callback', type='string',
metavar='BASICAUTH', callback=_auth_callback,
callback_args=(BasicAuthentication,),
help='[projectdir],[htpasswd_file],[realm]')
parser.add_option('-p', '--port', action='store', type='int', dest='port',
help='the port number to bind to')
parser.add_option('-b', '--hostname', action='store', dest='hostname',
help='the host name or IP address to bind to')
parser.add_option('--protocol', action='callback', type="string",
dest='protocol', callback=_validate_callback,
callback_args=(('http', 'scgi', 'ajp', 'fcgi'),),
help='http|scgi|ajp|fcgi')
parser.add_option('-q', '--unquote', action='store_true',
dest='unquote',
help='unquote PATH_INFO (may be needed when using ajp)')
parser.add_option('--http10', action='store_false', dest='http11',
help='use HTTP/1.0 protocol version instead of HTTP/1.1')
parser.add_option('--http11', action='store_true', dest='http11',
help='use HTTP/1.1 protocol version (default)')
parser.add_option('-e', '--env-parent-dir', action='store',
dest='env_parent_dir', metavar='PARENTDIR',
help='parent directory of the project environments')
parser.add_option('--base-path', action='store', type='string', # XXX call this url_base_path?
dest='base_path',
help='the initial portion of the request URL\'s "path"')
parser.add_option('-r', '--auto-reload', action='store_true',
dest='autoreload',
help='restart automatically when sources are modified')
parser.add_option('-s', '--single-env', action='store_true',
dest='single_env', help='only serve a single '
'project without the project list', default=False)
if os.name == 'posix':
parser.add_option('-d', '--daemonize', action='store_true',
dest='daemonize',
help='run in the background as a daemon')
parser.add_option('--pidfile', action='store',
dest='pidfile',
help='when daemonizing, file to which to write pid')
parser.add_option('--umask', action='callback', type='string',
dest='umask', metavar='MASK', callback=_octal,
help='when daemonizing, file mode creation mask '
'to use, in octal notation (default 022)')
try:
import grp, pwd
def _group(option, opt_str, value, parser):
try:
value = int(value)
except ValueError:
try:
value = grp.getgrnam(value)[2]
except KeyError:
raise OptionValueError('group not found: %r' % value)
setattr(parser.values, option.dest, value)
def _user(option, opt_str, value, parser):
try:
value = int(value)
except ValueError:
try:
value = pwd.getpwnam(value)[2]
except KeyError:
raise OptionValueError('user not found: %r' % value)
setattr(parser.values, option.dest, value)
parser.add_option('--group', action='callback', type='string',
dest='group', metavar='GROUP', callback=_group,
help='the group to run as')
parser.add_option('--user', action='callback', type='string',
dest='user', metavar='USER', callback=_user,
help='the user to run as')
except ImportError:
pass
parser.set_defaults(port=None, hostname='', base_path='', daemonize=False,
protocol='http', http11=True, umask=022, user=None,
group=None)
options, args = parser.parse_args()
if not args and not options.env_parent_dir:
parser.error('either the --env-parent-dir option or at least one '
'environment must be specified')
if options.single_env:
if options.env_parent_dir:
parser.error('the --single-env option cannot be used with '
'--env-parent-dir')
elif len(args) > 1:
parser.error('the --single-env option cannot be used with '
'more than one enviroment')
if options.daemonize and options.autoreload:
parser.error('the --auto-reload option cannot be used with '
'--daemonize')
if options.port is None:
options.port = {
'http': 80,
'scgi': 4000,
'ajp': 8009,
'fcgi': 8000,
}[options.protocol]
server_address = (options.hostname, options.port)
# relative paths don't work when daemonized
args = [os.path.abspath(a) for a in args]
if options.env_parent_dir:
options.env_parent_dir = os.path.abspath(options.env_parent_dir)
if parser.has_option('pidfile') and options.pidfile:
options.pidfile = os.path.abspath(options.pidfile)
wsgi_app = TracEnvironMiddleware(dispatch_request,
options.env_parent_dir, args,
options.single_env)
if auths:
if options.single_env:
project_name = os.path.basename(args[0])
wsgi_app = AuthenticationMiddleware(wsgi_app, auths, project_name)
else:
wsgi_app = AuthenticationMiddleware(wsgi_app, auths)
base_path = options.base_path.strip('/')
if base_path:
wsgi_app = BasePathMiddleware(wsgi_app, base_path)
if options.protocol == 'http':
def serve():
addr, port = server_address
if not addr or addr == '0.0.0.0':
loc = '0.0.0.0:%s view at http://127.0.0.1:%s/%s' \
% (port, port, base_path)
else:
loc = 'http://%s:%s/%s' % (addr, port, base_path)
try:
httpd = TracHTTPServer(server_address, wsgi_app,
options.env_parent_dir, args,
use_http_11=options.http11)
except socket.error, e:
print 'Error starting Trac server on %s' % loc
print e.strerror
sys.exit(1)
print 'Server starting in PID %i.' % os.getpid()
print 'Serving on %s' % loc
if options.http11:
print 'Using HTTP/1.1 protocol version'
httpd.serve_forever()
elif options.protocol in ('scgi', 'ajp', 'fcgi'):
def serve():
server_cls = __import__('flup.server.%s' % options.protocol,
None, None, ['']).WSGIServer
flup_app = wsgi_app
if options.unquote:
from trac.web.fcgi_frontend import FlupMiddleware
flup_app = FlupMiddleware(flup_app)
ret = server_cls(flup_app, bindAddress=server_address).run()
sys.exit(42 if ret else 0) # if SIGHUP exit with status 42
try:
if options.daemonize:
daemon.daemonize(pidfile=options.pidfile, progname='tracd',
umask=options.umask)
if options.group is not None:
os.setgid(options.group)
if options.user is not None:
os.setuid(options.user)
if options.autoreload:
def modification_callback(file):
print >> sys.stderr, 'Detected modification of %s, ' \
'restarting.' % file
autoreload.main(serve, modification_callback)
else:
serve()
except OSError, e:
print >> sys.stderr, '%s: %s' % (e.__class__.__name__, e)
sys.exit(1)
except KeyboardInterrupt:
pass
2
Example 42
Project: python-future Source File: main.py
def main(args=None):
"""Main program.
Args:
fixer_pkg: the name of a package where the fixers are located.
args: optional; a list of command line arguments. If omitted,
sys.argv[1:] is used.
Returns a suggested exit status (0, 1, 2).
"""
# Set up option parser
parser = optparse.OptionParser(usage="futurize [options] file|dir ...")
parser.add_option("-V", "--version", action="store_true",
help="Report the version number of futurize")
parser.add_option("-a", "--all-imports", action="store_true",
help="Add all __future__ and future imports to each module")
parser.add_option("-1", "--stage1", action="store_true",
help="Modernize Python 2 code only; no compatibility with Python 3 (or dependency on ``future``)")
parser.add_option("-2", "--stage2", action="store_true",
help="Take modernized (stage1) code and add a dependency on ``future`` to provide Py3 compatibility.")
parser.add_option("-0", "--both-stages", action="store_true",
help="Apply both stages 1 and 2")
parser.add_option("-u", "--unicode-literals", action="store_true",
help="Add ``from __future__ import unicode_literals`` to implicitly convert all unadorned string literals '' into unicode strings")
parser.add_option("-f", "--fix", action="append", default=[],
help="Each FIX specifies a transformation; default: all.\nEither use '-f division -f metaclass' etc. or use the fully-qualified module name: '-f lib2to3.fixes.fix_types -f libfuturize.fixes.fix_unicode_keep_u'")
parser.add_option("-j", "--processes", action="store", default=1,
type="int", help="Run 2to3 concurrently")
parser.add_option("-x", "--nofix", action="append", default=[],
help="Prevent a fixer from being run.")
parser.add_option("-l", "--list-fixes", action="store_true",
help="List available transformations")
parser.add_option("-p", "--print-function", action="store_true",
help="Modify the grammar so that print() is a function")
parser.add_option("-v", "--verbose", action="store_true",
help="More verbose logging")
parser.add_option("--no-diffs", action="store_true",
help="Don't show diffs of the refactoring")
parser.add_option("-w", "--write", action="store_true",
help="Write back modified files")
parser.add_option("-n", "--nobackups", action="store_true", default=False,
help="Don't write backups for modified files.")
parser.add_option("-o", "--output-dir", action="store", type="str",
default="", help="Put output files in this directory "
"instead of overwriting the input files. Requires -n. "
"For Python >= 2.7 only.")
parser.add_option("-W", "--write-unchanged-files", action="store_true",
help="Also write files even if no changes were required"
" (useful with --output-dir); implies -w.")
parser.add_option("--add-suffix", action="store", type="str", default="",
help="Append this string to all output filenames."
" Requires -n if non-empty. For Python >= 2.7 only."
"ex: --add-suffix='3' will generate .py3 files.")
# Parse command line arguments
flags = {}
refactor_stdin = False
options, args = parser.parse_args(args)
if options.write_unchanged_files:
flags["write_unchanged_files"] = True
if not options.write:
warn("--write-unchanged-files/-W implies -w.")
options.write = True
# If we allowed these, the original files would be renamed to backup names
# but not replaced.
if options.output_dir and not options.nobackups:
parser.error("Can't use --output-dir/-o without -n.")
if options.add_suffix and not options.nobackups:
parser.error("Can't use --add-suffix without -n.")
if not options.write and options.no_diffs:
warn("not writing files and not printing diffs; that's not very useful")
if not options.write and options.nobackups:
parser.error("Can't use -n without -w")
if "-" in args:
refactor_stdin = True
if options.write:
print("Can't write to stdin.", file=sys.stderr)
return 2
# Is this ever necessary?
if options.print_function:
flags["print_function"] = True
# Set up logging handler
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(format='%(name)s: %(message)s', level=level)
logger = logging.getLogger('libfuturize.main')
if options.stage1 or options.stage2:
assert options.both_stages is None
options.both_stages = False
else:
options.both_stages = True
avail_fixes = set()
if options.stage1 or options.both_stages:
avail_fixes.update(lib2to3_fix_names_stage1)
avail_fixes.update(libfuturize_fix_names_stage1)
if options.stage2 or options.both_stages:
avail_fixes.update(lib2to3_fix_names_stage2)
avail_fixes.update(libfuturize_fix_names_stage2)
if options.unicode_literals:
avail_fixes.add('libfuturize.fixes.fix_unicode_literals_import')
if options.version:
print(__version__)
return 0
if options.list_fixes:
print("Available transformations for the -f/--fix option:")
# for fixname in sorted(refactor.get_all_fix_names(fixer_pkg)):
for fixname in sorted(avail_fixes):
print(fixname)
if not args:
return 0
if not args:
print("At least one file or directory argument required.",
file=sys.stderr)
print("Use --help to show usage.", file=sys.stderr)
return 2
unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix)
extra_fixes = set()
if options.all_imports:
if options.stage1:
prefix = 'libfuturize.fixes.'
extra_fixes.add(prefix +
'fix_add__future__imports_except_unicode_literals')
else:
# In case the user hasn't run stage1 for some reason:
prefix = 'libpasteurize.fixes.'
extra_fixes.add(prefix + 'fix_add_all__future__imports')
extra_fixes.add(prefix + 'fix_add_future_standard_library_import')
extra_fixes.add(prefix + 'fix_add_all_future_builtins')
explicit = set()
if options.fix:
all_present = False
for fix in options.fix:
if fix == 'all':
all_present = True
else:
if ".fix_" in fix:
explicit.add(fix)
else:
# Infer the full module name for the fixer.
# First ensure that no names clash (e.g.
# lib2to3.fixes.fix_blah and libfuturize.fixes.fix_blah):
found = [f for f in avail_fixes
if f.endswith('fix_{0}'.format(fix))]
if len(found) > 1:
print("Ambiguous fixer name. Choose a fully qualified "
"module name instead from these:\n" +
"\n".join(" " + myf for myf in found),
file=sys.stderr)
return 2
elif len(found) == 0:
print("Unknown fixer. Use --list-fixes or -l for a list.",
file=sys.stderr)
return 2
explicit.add(found[0])
if len(explicit & unwanted_fixes) > 0:
print("Conflicting usage: the following fixers have been "
"simultaneously requested and disallowed:\n" +
"\n".join(" " + myf for myf in (explicit & unwanted_fixes)),
file=sys.stderr)
return 2
requested = avail_fixes.union(explicit) if all_present else explicit
else:
requested = avail_fixes.union(explicit)
fixer_names = (requested | extra_fixes) - unwanted_fixes
input_base_dir = os.path.commonprefix(args)
if (input_base_dir and not input_base_dir.endswith(os.sep)
and not os.path.isdir(input_base_dir)):
# One or more similar names were passed, their directory is the base.
# os.path.commonprefix() is ignorant of path elements, this corrects
# for that weird API.
input_base_dir = os.path.dirname(input_base_dir)
if options.output_dir:
input_base_dir = input_base_dir.rstrip(os.sep)
logger.info('Output in %r will mirror the input directory %r layout.',
options.output_dir, input_base_dir)
# Initialize the refactoring tool
if future.utils.PY26:
extra_kwargs = {}
else:
extra_kwargs = {
'append_suffix': options.add_suffix,
'output_dir': options.output_dir,
'input_base_dir': input_base_dir,
}
rt = StdoutRefactoringTool(
sorted(fixer_names), flags, sorted(explicit),
options.nobackups, not options.no_diffs,
**extra_kwargs)
# Refactor all files and directories passed as arguments
if not rt.errors:
if refactor_stdin:
rt.refactor_stdin()
else:
try:
rt.refactor(args, options.write, None,
options.processes)
except refactor.MultiprocessingUnsupported:
assert options.processes > 1
print("Sorry, -j isn't " \
"supported on this platform.", file=sys.stderr)
return 1
rt.summarize()
# Return error status (0 if rt.errors is zero)
return int(bool(rt.errors))
2
Example 43
Project: mypaint Source File: meta.py
def _get_versions(gitprefix="gitexport"):
"""Gets all version strings for use in release/build scripting.
:param str gitprefix: how to denote git-derived build metainfo
:rtype: tuple
:returns: all 3 version strings: (base, formal, ceremonial)
This function must only be called by Python build scripts like
SConscript, or by release.sh (which invokes the interpreter).
It assumes that the current working directory is either the
one-level-down directory in an unpacked export generated by
release.sh (when a `release_info` file exists), or a working git
repository (when a `.git` directory exists).
The `gitprefix` is only used when examining the local git reporitory
to derive the additional information.
"""
import re
import os
import sys
import subprocess
# Establish some fallbacks for use when there's no .git present,
# or no release_info.
base_version = MYPAINT_VERSION
formal_version = base_version
ceremonial_version = formal_version + "+unknown"
if os.path.isfile("release_info"):
# If release information from release.sh exists, use that
relinfo = {}
with open("release_info", "rb") as relinfo_fp:
exec(relinfo_fp, relinfo)
base_version = relinfo.get(
"MYPAINT_VERSION_BASE",
base_version,
)
formal_version = relinfo.get(
"MYPAINT_VERSION_FORMAL",
formal_version,
)
ceremonial_version = relinfo.get(
"MYPAINT_VERSION_CEREMONIAL",
ceremonial_version,
)
elif base_version.endswith("-alpha"):
# There will be no matching git tag for initial alpha (active
# development) phases.
if os.path.isdir(".git"):
cmd = ["git", "rev-parse", "--short", "HEAD"]
try:
objsha = str(subprocess.check_output(cmd)).strip()
except:
print("ERROR: Failed to invoke %r. Build will be marked as "
"unsupported." % (" ".join(cmd), ),
file=sys.stderr)
else:
build_ids = [gitprefix, objsha]
build_metadata = ".".join(build_ids)
ceremonial_version = "{}+{}".format(
formal_version,
build_metadata,
)
elif os.path.isdir(".git"):
# Pull the additional info from git.
cmd = ["git", "describe", "--tags", "--long", "--dirty", "--always"]
try:
git_desc = str(subprocess.check_output(cmd)).strip()
except:
print("ERROR: Failed to invoke %r. Build will be marked as "
"unsupported." % (" ".join(cmd), ),
file=sys.stderr)
else:
# If MYPAINT_VERSION matches the most recent tag in git,
# then use the extra information from `git describe`.
parse_pattern = r'''
^ v{base_version} # Expected base version.
(?:-(\d+))? #1 Number of commits since the tag.
(?:-g([0-9a-f]+))? #2 Abbr'd SHA of the git tree exported.
(?:-(dirty))? #3 Highlight uncommitted changes.
$
'''.rstrip().format(base_version = re.escape(base_version))
parse_re = re.compile(parse_pattern, re.VERBOSE|re.IGNORECASE)
match = parse_re.match(git_desc)
objsha = None
nrevs = 0
dirty = None
if match:
(nrevs, objsha, dirty) = match.groups()
else:
print("WARNING: Failed to parse output of \"{cmd}\". "
"The base MYPAINT_VERSION ({ver}) from the code "
"should be present in the output of this command "
"({git_desc}).".format(cmd=" ".join(cmd),
git_desc=repr(git_desc),
ver=base_version),
file=sys.stderr)
print("HINT: make sure you have the most recent tags: "
"git fetch --tags",
file=sys.stderr)
cmd = ["git", "rev-parse", "--short", "HEAD"]
print(
"WARNING: falling back to using just \"{cmd}\".".format(
cmd=" ".join(cmd)),
file=sys.stderr)
try:
cmdout = str(subprocess.check_output(cmd)).strip()
except:
print("ERROR: Failed to invoke %r. Build will be marked "
"as unsupported." % (" ".join(cmd), ),
file=sys.stderr)
if re.match(r'^([0-9a-f]{7,})$', cmdout, re.I):
objsha = cmdout
else:
print("WARNING: Output of {cmd} ({output}) does not look "
"like a git revision SHA.".format(cmd=" ".join(cmd),
output=cmdout),
file=sys.stderr)
# nrevs is None or zero if this commit is the matched tag.
# If not, then incorporate the numbers somehow.
if nrevs and int(nrevs)>0:
if "-" not in base_version:
raise ValueError(
"The code's MYPAINT_VERSION ({ver}) "
"denotes a final release but there are commits "
"after the tag v{ver} in this git branch. "
"A new 'vX.Y.Z-alpha' phase tag needs to be "
"created for the next version now, "
"and lib.meta.MYPAINT_VERSION needs to be "
"updated to match it."
.format (
ver = base_version,
)
)
# Can't just fake it with a hyphen: that would
# have lower priority than the final release.
else:
# It's already something like "1.2.0-alpha",
# so we can use a dot-suffix: "1.2.0-alpha.42".
formal_version = "%s.%s" % (base_version, nrevs)
# The super-long version may be overridden later too,
# but for now it must incorporate the normal long
# version string.
ceremonial_version = formal_version
# Collect details about the build after a plus sign.
# objsha is None if this commit is the matched tag.
# The dirty flag is only present if there are uncommitted
# changes (which shouldn't happen).
build_ids = []
if objsha:
build_ids.append(gitprefix + "." + objsha)
if dirty:
build_ids.append("dirty")
if build_ids:
build_metadata = ".".join(build_ids)
ceremonial_version = "{}+{}".format(
formal_version,
build_metadata,
)
else:
pass
return (base_version, formal_version, ceremonial_version)
2
Example 44
def Start(parser=None,
argv=sys.argv,
quiet=False,
no_parsing=False,
add_csv_options=False,
add_database_options=False,
add_pipe_options=True,
add_cluster_options=False,
add_output_options=False,
return_parser=False):
"""set up an experiment.
The :py:func:`Start` method will set up a file logger and add some
default and some optional options to the command line parser. It
will then parse the command line and set up input/output
redirection and start a timer for benchmarking purposes.
The default options added by this method are:
``-v/--verbose``
the :term:`loglevel`
``timeit``
turn on benchmarking information and save to file
``timeit-name``
name to use for timing information,
``timeit-header``
output header for timing information.
``seed``
the random seed. If given, the python random
number generator will be initialized with this
seed.
Optional options added are:
add_csv_options
``dialect``
csv_dialect. the default is ``excel-tab``, defaulting to
:term:`tsv` formatted files.
add_database_options
``-C/--connection``
psql connection string
``-U/--user``
psql user name
add_cluster_options
``--use-cluster``
use cluster
``--cluster-priority``
cluster priority to request
``--cluster-queue-manager``
cluster queue manager to use
``--cluster-queue``
cluster queue to use
``--cluster-num-jobs``
number of jobs to submit to the cluster at the same time
``--cluster-memory-resource``
name of the cluster memory resource (SGE specific option)
``--cluster-memory-default``
default amount of memory allocated per job.
``--cluster-options``
additional options to the cluster for each job.
add_output_options
``-P/--output-filename-pattern``
Pattern to use for output filenames.
Arguments
---------
param parser : :py:class:`E.OptionParser`
instance with command line options.
argv : list
command line options to parse. Defaults to
:py:data:`sys.argv`
quiet : bool
set :term:`loglevel` to 0 - no logging
no_parsing : bool
do not parse command line options
return_parser : bool
return the parser object, no parsing. Useful for inspecting
the command line options of a script without running it.
add_csv_options : bool
add common options for parsing :term:`tsv` separated files.
add_database_options : bool
add common options for connecting to various databases.
add_pipe_options : bool
add common options for redirecting input/output
add_cluster_options : bool
add common options for scripts submitting jobs to the cluster
add_output_options : bool
add commond options for working with multiple output files
Returns
-------
tuple
(:py:class:`E.OptionParser` object, list of positional
arguments)
"""
if not parser:
parser = OptionParser(
version="%prog version: $Id$")
global global_options, global_args, global_starting_time
# save default values given by user
user_defaults = copy.copy(parser.defaults)
global_starting_time = time.time()
group = OptionGroup(parser, "Script timing options")
group.add_option("--timeit", dest='timeit_file', type="string",
help="store timeing information in file [%default].")
group.add_option("--timeit-name", dest='timeit_name', type="string",
help="name in timing file for this class of jobs "
"[%default].")
group.add_option("--timeit-header", dest='timeit_header',
action="store_true",
help="add header for timing information [%default].")
parser.add_option_group(group)
group = OptionGroup(parser, "Common options")
group.add_option("--random-seed", dest='random_seed', type="int",
help="random seed to initialize number generator "
"with [%default].")
group.add_option("-v", "--verbose", dest="loglevel", type="int",
help="loglevel [%default]. The higher, the more output.")
group.add_option("-?", dest="short_help", action="callback",
callback=callbackShortHelp,
help="output short help (command line options only.")
parser.add_option_group(group)
if quiet:
parser.set_defaults(loglevel=0)
else:
parser.set_defaults(loglevel=1)
parser.set_defaults(
timeit_file=None,
timeit_name='all',
timeit_header=None,
random_seed=None,
)
if add_csv_options:
parser.add_option("--csv-dialect", dest="csv_dialect", type="string",
help="csv dialect to use [%default].")
parser.set_defaults(
csv_dialect="excel-tab",
csv_lineterminator="\n",
)
if add_cluster_options:
group = OptionGroup(parser, "cluster options")
group.add_option("--no-cluster", "--local", dest="without_cluster",
action="store_true",
help="do no use cluster - run locally [%default].")
group.add_option("--cluster-priority", dest="cluster_priority",
type="int",
help="set job priority on cluster [%default].")
group.add_option("--cluster-queue-manager", dest="cluster_queue_manager",
type="string",
help="set cluster queue manager [%default].")
group.add_option("--cluster-queue", dest="cluster_queue",
type="string",
help="set cluster queue [%default].")
group.add_option("--cluster-memory-resource", dest="cluster_memory_resource",
type="string",
help="set cluster memory resource [%default].")
group.add_option("--cluster-memory-default", dest="cluster_memory_default",
type="string",
help="set cluster memory default [%default].")
group.add_option("--cluster-num-jobs", dest="cluster_num_jobs",
type="int",
help="number of jobs to submit to the queue execute "
"in parallel [%default].")
group.add_option("--cluster-parallel",
dest="cluster_parallel_environment",
type="string",
help="name of the parallel environment to use "
"[%default].")
group.add_option("--cluster-options", dest="cluster_options",
type="string",
help="additional options for cluster jobs, passed "
"on to queuing system [%default].")
parser.set_defaults(without_cluster=False,
cluster_queue_manager=None,
cluster_queue=None,
cluster_memory_resource=None,
cluster_memory_default=None,
cluster_priority=None,
cluster_num_jobs=None,
cluster_parallel_environment=None,
cluster_options=None)
parser.add_option_group(group)
if add_output_options or add_pipe_options:
group = OptionGroup(parser, "Input/output options")
if add_output_options:
group.add_option(
"-P", "--output-filename-pattern",
dest="output_filename_pattern", type="string",
help="OUTPUT filename pattern for various methods "
"[%default].")
group.add_option("-F", "--force-output", dest="output_force",
action="store_true",
help="force over-writing of existing files.")
parser.set_defaults(output_filename_pattern="%s",
output_force=False)
if add_pipe_options:
group.add_option("-I", "--stdin", dest="stdin", type="string",
help="file to read stdin from [default = stdin].",
metavar="FILE")
group.add_option("-L", "--log", dest="stdlog", type="string",
help="file with logging information "
"[default = stdout].",
metavar="FILE")
group.add_option("-E", "--error", dest="stderr", type="string",
help="file with error information "
"[default = stderr].",
metavar="FILE")
group.add_option("-S", "--stdout", dest="stdout", type="string",
help="file where output is to go "
"[default = stdout].",
metavar="FILE")
parser.set_defaults(stderr=sys.stderr)
parser.set_defaults(stdout=sys.stdout)
parser.set_defaults(stdlog=sys.stdout)
parser.set_defaults(stdin=sys.stdin)
parser.add_option_group(group)
if add_database_options:
group = OptionGroup(parser, "Database connection options")
group.add_option(
"--database-backend", dest="database_backend", type="choice",
choices=("sqlite", "mysql", "postgres"),
help="database backend [%default].")
group.add_option(
"--database-host", dest="database_host", type="string",
help="database host [%default].")
group.add_option(
"--database-name", dest="database_name", type="string",
help="name of the database [%default].")
group.add_option(
"--database-username", dest="database_username", type="string",
help="database username [%default].")
group.add_option(
"--database-password", dest="database_password", type="string",
help="database password [%default].")
group.add_option(
"--database-port", dest="database_port", type="int",
help="database port [%default].")
parser.set_defaults(
database_backend="sqlite",
database_name="csvdb",
database_host="",
database_port=3306,
database_username="",
database_password="")
parser.add_option_group(group)
# restore user defaults
parser.defaults.update(user_defaults)
if return_parser:
return parser
if not no_parsing:
(global_options, global_args) = parser.parse_args(argv[1:])
if global_options.random_seed is not None:
random.seed(global_options.random_seed)
if add_pipe_options:
if global_options.stdout != sys.stdout:
global_options.stdout = openFile(global_options.stdout, "w")
if global_options.stderr != sys.stderr:
if global_options.stderr == "stderr":
global_options.stderr = global_options.stderr
else:
global_options.stderr = openFile(global_options.stderr, "w")
if global_options.stdlog != sys.stdout:
global_options.stdlog = openFile(global_options.stdlog, "a")
if global_options.stdin != sys.stdin:
global_options.stdin = openFile(global_options.stdin, "r")
else:
global_options.stderr = sys.stderr
global_options.stdout = sys.stdout
global_options.stdlog = sys.stdout
global_options.stdin = sys.stdin
if global_options.loglevel >= 1:
global_options.stdlog.write(getHeader() + "\n")
global_options.stdlog.write(getParams(global_options) + "\n")
global_options.stdlog.flush()
# configure logging
# map from 0-10 to logging scale
# 0: quiet
# 1: little verbositiy
# >1: increased verbosity
if global_options.loglevel == 0:
lvl = logging.ERROR
elif global_options.loglevel == 1:
lvl = logging.INFO
else:
lvl = logging.DEBUG
if global_options.stdout == global_options.stdlog:
format = '# %(asctime)s %(levelname)s %(message)s'
else:
format = '%(asctime)s %(levelname)s %(message)s'
logging.basicConfig(
level=lvl,
format=format,
stream=global_options.stdlog)
# set up multi-line logging
# Note that .handlers is not part of the API, might change
# Solution is to configure handlers explicitely.
for handler in logging.getLogger().handlers:
handler.setFormatter(MultiLineFormatter(format))
return global_options, global_args
2
Example 45
Project: Live-Blog Source File: cmdline.py
def main(argv):
if not color_terminal():
# Windows' poor cmd box doesn't understand ANSI sequences
nocolor()
try:
opts, args = getopt.getopt(argv[1:], 'ab:t:d:c:CD:A:ng:NEqQWw:P')
allopts = set(opt[0] for opt in opts)
srcdir = confdir = path.abspath(args[0])
if not path.isdir(srcdir):
print('Error: Cannot find source directory.', file=sys.stderr)
return 1
if not path.isfile(path.join(srcdir, 'conf.py')) and \
'-c' not in allopts and '-C' not in allopts:
print(('Error: Source directory doesn\'t '
'contain conf.py file.'), file=sys.stderr)
return 1
outdir = path.abspath(args[1])
if not path.isdir(outdir):
print('Making output directory...', file=sys.stderr)
os.makedirs(outdir)
except (IndexError, getopt.error):
usage(argv)
return 1
filenames = args[2:]
err = 0
for filename in filenames:
if not path.isfile(filename):
print('Cannot find file %r.' % filename, file=sys.stderr)
err = 1
if err:
return 1
# likely encoding used for command-line arguments
try:
locale = __import__('locale') # due to submodule of the same name
likely_encoding = locale.getpreferredencoding()
except Exception:
likely_encoding = None
buildername = None
force_all = freshenv = warningiserror = use_pdb = False
status = sys.stdout
warning = sys.stderr
error = sys.stderr
warnfile = None
confoverrides = {}
tags = []
doctreedir = path.join(outdir, '.doctrees')
for opt, val in opts:
if opt == '-b':
buildername = val
elif opt == '-a':
if filenames:
usage(argv, 'Cannot combine -a option and filenames.')
return 1
force_all = True
elif opt == '-t':
tags.append(val)
elif opt == '-d':
doctreedir = path.abspath(val)
elif opt == '-c':
confdir = path.abspath(val)
if not path.isfile(path.join(confdir, 'conf.py')):
print(('Error: Configuration directory '
'doesn\'t contain conf.py file.'), file=sys.stderr)
return 1
elif opt == '-C':
confdir = None
elif opt == '-D':
try:
key, val = val.split('=')
except ValueError:
print(('Error: -D option argument must be '
'in the form name=value.'), file=sys.stderr)
return 1
try:
val = int(val)
except ValueError:
if likely_encoding and isinstance(val, bytes):
try:
val = val.decode(likely_encoding)
except UnicodeError:
pass
confoverrides[key] = val
elif opt == '-A':
try:
key, val = val.split('=')
except ValueError:
print(('Error: -A option argument must be '
'in the form name=value.'), file=sys.stderr)
return 1
try:
val = int(val)
except ValueError:
if likely_encoding and isinstance(val, bytes):
try:
val = val.decode(likely_encoding)
except UnicodeError:
pass
confoverrides['html_context.%s' % key] = val
elif opt == '-n':
confoverrides['nitpicky'] = True
elif opt == '-N':
nocolor()
elif opt == '-E':
freshenv = True
elif opt == '-q':
status = None
elif opt == '-Q':
status = None
warning = None
elif opt == '-W':
warningiserror = True
elif opt == '-w':
warnfile = val
elif opt == '-P':
use_pdb = True
if warning and warnfile:
warnfp = open(warnfile, 'w')
warning = Tee(warning, warnfp)
error = warning
try:
app = Sphinx(srcdir, confdir, outdir, doctreedir, buildername,
confoverrides, status, warning, freshenv,
warningiserror, tags)
app.build(force_all, filenames)
return app.statuscode
except KeyboardInterrupt:
if use_pdb:
import pdb
print(red('Interrupted while building, starting debugger:'), file=error)
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[2])
return 1
except Exception as err:
if use_pdb:
import pdb
print(red('Exception occurred while building, '
'starting debugger:'), file=error)
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[2])
else:
print(file=error)
if isinstance(err, SystemMessage):
print(red('reST markup error:'), file=error)
print(terminal_safe(err.args[0]), file=error)
elif isinstance(err, SphinxError):
print(red('%s:' % err.category), file=error)
print(terminal_safe(str(err)), file=error)
else:
print(red('Exception occurred:'), file=error)
print(format_exception_cut_frames().rstrip(), file=error)
tbpath = save_traceback()
print(red('The full traceback has been saved '
'in %s, if you want to report the '
'issue to the developers.' % tbpath), file=error)
print(('Please also report this if it was a user '
'error, so that a better error message '
'can be provided next time.'), file=error)
print((
'Either send bugs to the mailing list at '
'<http://groups.google.com/group/sphinx-dev/>,\n'
'or report them in the tracker at '
'<http://bitbucket.org/birkenfeld/sphinx/issues/>. Thanks!'), file=error)
return 1
2
Example 46
Project: p2pool-n Source File: main.py
@defer.inlineCallbacks
def main(args, net, datadir_path, merged_urls, worker_endpoint):
try:
print 'p2pool (version %s)' % (p2pool.__version__,)
print
@defer.inlineCallbacks
def connect_p2p():
# connect to bitcoind over bitcoin-p2p
print '''Testing bitcoind P2P connection to '%s:%s'...''' % (args.bitcoind_address, args.bitcoind_p2p_port)
factory = bitcoin_p2p.ClientFactory(net.PARENT)
reactor.connectTCP(args.bitcoind_address, args.bitcoind_p2p_port, factory)
def long():
print ''' ...taking a while. Common reasons for this include all of bitcoind's connection slots being used...'''
long_dc = reactor.callLater(5, long)
yield factory.getProtocol() # waits until handshake is successful
if not long_dc.called: long_dc.cancel()
print ' ...success!'
print
defer.returnValue(factory)
if args.testnet: # establish p2p connection first if testnet so bitcoind can work without connections
factory = yield connect_p2p()
# connect to bitcoind over JSON-RPC and do initial getmemorypool
url = '%s://%s:%i/' % ('https' if args.bitcoind_rpc_ssl else 'http', args.bitcoind_address, args.bitcoind_rpc_port)
print '''Testing bitcoind RPC connection to '%s' with username '%s'...''' % (url, args.bitcoind_rpc_username)
bitcoind = jsonrpc.HTTPProxy(url, dict(Authorization='Basic ' + base64.b64encode(args.bitcoind_rpc_username + ':' + args.bitcoind_rpc_password)), timeout=30)
yield helper.check(bitcoind, net)
temp_work = yield helper.getwork(bitcoind)
bitcoind_getinfo_var = variable.Variable(None)
@defer.inlineCallbacks
def poll_warnings():
bitcoind_getinfo_var.set((yield deferral.retry('Error while calling getinfo:')(bitcoind.rpc_getinfo)()))
yield poll_warnings()
deferral.RobustLoopingCall(poll_warnings).start(20*60)
print ' ...success!'
print ' Current block hash: %x' % (temp_work['previous_block'],)
print ' Current block height: %i' % (temp_work['height'] - 1,)
print
if not args.testnet:
factory = yield connect_p2p()
print 'Determining payout address...'
if args.pubkey_hash is None:
address_path = os.path.join(datadir_path, 'cached_payout_address')
if os.path.exists(address_path):
with open(address_path, 'rb') as f:
address = f.read().strip('\r\n')
print ' Loaded cached address: %s...' % (address,)
else:
address = None
if address is not None:
res = yield deferral.retry('Error validating cached address:', 5)(lambda: bitcoind.rpc_validateaddress(address))()
if not res['isvalid'] or not res['ismine']:
print ' Cached address is either invalid or not controlled by local bitcoind!'
address = None
if address is None:
print ' Getting payout address from bitcoind...'
address = yield deferral.retry('Error getting payout address from bitcoind:', 5)(lambda: bitcoind.rpc_getaccountaddress('p2pool'))()
with open(address_path, 'wb') as f:
f.write(address)
my_pubkey_hash = bitcoin_data.address_to_pubkey_hash(address, net.PARENT)
else:
my_pubkey_hash = args.pubkey_hash
print ' ...success! Payout address:', bitcoin_data.pubkey_hash_to_address(my_pubkey_hash, net.PARENT)
print
print "Loading shares..."
shares = {}
known_verified = set()
def share_cb(share):
share.time_seen = 0 # XXX
shares[share.hash] = share
if len(shares) % 1000 == 0 and shares:
print " %i" % (len(shares),)
ss = p2pool_data.ShareStore(os.path.join(datadir_path, 'shares.'), net, share_cb, known_verified.add)
print " ...done loading %i shares (%i verified)!" % (len(shares), len(known_verified))
print
print 'Initializing work...'
node = p2pool_node.Node(factory, bitcoind, shares.values(), known_verified, net)
yield node.start()
for share_hash in shares:
if share_hash not in node.tracker.items:
ss.forget_share(share_hash)
for share_hash in known_verified:
if share_hash not in node.tracker.verified.items:
ss.forget_verified_share(share_hash)
node.tracker.removed.watch(lambda share: ss.forget_share(share.hash))
node.tracker.verified.removed.watch(lambda share: ss.forget_verified_share(share.hash))
def save_shares():
for share in node.tracker.get_chain(node.best_share_var.value, min(node.tracker.get_height(node.best_share_var.value), 2*net.CHAIN_LENGTH)):
ss.add_share(share)
if share.hash in node.tracker.verified.items:
ss.add_verified_hash(share.hash)
deferral.RobustLoopingCall(save_shares).start(60)
print ' ...success!'
print
print 'Joining p2pool network using port %i...' % (args.p2pool_port,)
@defer.inlineCallbacks
def parse(host):
port = net.P2P_PORT
if ':' in host:
host, port_str = host.split(':')
port = int(port_str)
defer.returnValue(((yield reactor.resolve(host)), port))
addrs = {}
if os.path.exists(os.path.join(datadir_path, 'addrs')):
try:
with open(os.path.join(datadir_path, 'addrs'), 'rb') as f:
addrs.update(dict((tuple(k), v) for k, v in json.loads(f.read())))
except:
print >>sys.stderr, 'error parsing addrs'
for addr_df in map(parse, net.BOOTSTRAP_ADDRS):
try:
addr = yield addr_df
if addr not in addrs:
addrs[addr] = (0, time.time(), time.time())
except:
log.err()
connect_addrs = set()
for addr_df in map(parse, args.p2pool_nodes):
try:
connect_addrs.add((yield addr_df))
except:
log.err()
node.p2p_node = p2pool_node.P2PNode(node,
port=args.p2pool_port,
max_incoming_conns=args.p2pool_conns,
addr_store=addrs,
connect_addrs=connect_addrs,
desired_outgoing_conns=args.p2pool_outgoing_conns,
advertise_ip=args.advertise_ip,
)
node.p2p_node.start()
def save_addrs():
with open(os.path.join(datadir_path, 'addrs'), 'wb') as f:
f.write(json.dumps(node.p2p_node.addr_store.items()))
deferral.RobustLoopingCall(save_addrs).start(60)
print ' ...success!'
print
if args.upnp:
@defer.inlineCallbacks
def upnp_thread():
while True:
try:
is_lan, lan_ip = yield ipdiscover.get_local_ip()
if is_lan:
pm = yield portmapper.get_port_mapper()
yield pm._upnp.add_port_mapping(lan_ip, args.p2pool_port, args.p2pool_port, 'p2pool', 'TCP')
except defer.TimeoutError:
pass
except:
if p2pool.DEBUG:
log.err(None, 'UPnP error:')
yield deferral.sleep(random.expovariate(1/120))
upnp_thread()
# start listening for workers with a JSON-RPC server
print 'Listening for workers on %r port %i...' % (worker_endpoint[0], worker_endpoint[1])
wb = work.WorkerBridge(node, my_pubkey_hash, args.donation_percentage, merged_urls, args.worker_fee)
web_root = web.get_web_root(wb, datadir_path, bitcoind_getinfo_var)
caching_wb = worker_interface.CachingWorkerBridge(wb)
worker_interface.WorkerInterface(caching_wb).attach_to(web_root, get_handler=lambda request: request.redirect('static/'))
web_serverfactory = server.Site(web_root)
serverfactory = switchprotocol.FirstByteSwitchFactory({'{': stratum.StratumServerFactory(caching_wb)}, web_serverfactory)
deferral.retry('Error binding to worker port:', traceback=False)(reactor.listenTCP)(worker_endpoint[1], serverfactory, interface=worker_endpoint[0])
with open(os.path.join(os.path.join(datadir_path, 'ready_flag')), 'wb') as f:
pass
print ' ...success!'
print
# done!
print 'Started successfully!'
print 'Go to http://127.0.0.1:%i/ to view graphs and statistics!' % (worker_endpoint[1],)
if args.donation_percentage > 1.1:
print '''Donating %.1f%% of work towards P2Pool's development. Thanks for the tip!''' % (args.donation_percentage,)
elif args.donation_percentage < .9:
print '''Donating %.1f%% of work towards P2Pool's development. Please donate to encourage further development of P2Pool!''' % (args.donation_percentage,)
else:
print '''Donating %.1f%% of work towards P2Pool's development. Thank you!''' % (args.donation_percentage,)
print 'You can increase this amount with --give-author argument! (or decrease it, if you must)'
print
if hasattr(signal, 'SIGALRM'):
signal.signal(signal.SIGALRM, lambda signum, frame: reactor.callFromThread(
sys.stderr.write, 'Watchdog timer went off at:\n' + ''.join(traceback.format_stack())
))
signal.siginterrupt(signal.SIGALRM, False)
deferral.RobustLoopingCall(signal.alarm, 30).start(1)
if args.irc_announce:
from twisted.words.protocols import irc
class IRCClient(irc.IRCClient):
nickname = 'p2pool%02i' % (random.randrange(100),)
channel = net.ANNOUNCE_CHANNEL
def lineReceived(self, line):
if p2pool.DEBUG:
print repr(line)
irc.IRCClient.lineReceived(self, line)
def signedOn(self):
self.in_channel = False
irc.IRCClient.signedOn(self)
self.factory.resetDelay()
self.join(self.channel)
@defer.inlineCallbacks
def new_share(share):
if not self.in_channel:
return
if share.pow_hash <= share.header['bits'].target and abs(share.timestamp - time.time()) < 10*60:
yield deferral.sleep(random.expovariate(1/60))
message = '\x02%s BLOCK FOUND by %s! %s%064x' % (net.NAME.upper(), bitcoin_data.script2_to_address(share.new_script, net.PARENT), net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header_hash)
if all('%x' % (share.header_hash,) not in old_message for old_message in self.recent_messages):
self.say(self.channel, message)
self._remember_message(message)
self.watch_id = node.tracker.verified.added.watch(new_share)
self.recent_messages = []
def joined(self, channel):
self.in_channel = True
def left(self, channel):
self.in_channel = False
def _remember_message(self, message):
self.recent_messages.append(message)
while len(self.recent_messages) > 100:
self.recent_messages.pop(0)
def privmsg(self, user, channel, message):
if channel == self.channel:
self._remember_message(message)
def connectionLost(self, reason):
node.tracker.verified.added.unwatch(self.watch_id)
print 'IRC connection lost:', reason.getErrorMessage()
class IRCClientFactory(protocol.ReconnectingClientFactory):
protocol = IRCClient
reactor.connectTCP("irc.freenode.net", 6667, IRCClientFactory(), bindAddress=(worker_endpoint[0], 0))
@defer.inlineCallbacks
def status_thread():
last_str = None
last_time = 0
while True:
yield deferral.sleep(3)
try:
height = node.tracker.get_height(node.best_share_var.value)
this_str = 'P2Pool: %i shares in chain (%i verified/%i total) Peers: %i (%i incoming)' % (
height,
len(node.tracker.verified.items),
len(node.tracker.items),
len(node.p2p_node.peers),
sum(1 for peer in node.p2p_node.peers.itervalues() if peer.incoming),
) + (' FDs: %i R/%i W' % (len(reactor.getReaders()), len(reactor.getWriters())) if p2pool.DEBUG else '')
datums, dt = wb.local_rate_monitor.get_datums_in_last()
my_att_s = sum(datum['work']/dt for datum in datums)
my_shares_per_s = sum(datum['work']/dt/bitcoin_data.target_to_average_attempts(datum['share_target']) for datum in datums)
this_str += '\n Local: %sH/s in last %s Local dead on arrival: %s Expected time to share: %s' % (
math.format(int(my_att_s)),
math.format_dt(dt),
math.format_binomial_conf(sum(1 for datum in datums if datum['dead']), len(datums), 0.95),
math.format_dt(1/my_shares_per_s) if my_shares_per_s else '???',
)
if height > 2:
(stale_orphan_shares, stale_doa_shares), shares, _ = wb.get_stale_counts()
stale_prop = p2pool_data.get_average_stale_prop(node.tracker, node.best_share_var.value, min(60*60//net.SHARE_PERIOD, height))
real_att_s = p2pool_data.get_pool_attempts_per_second(node.tracker, node.best_share_var.value, min(height - 1, 60*60//net.SHARE_PERIOD)) / (1 - stale_prop)
this_str += '\n Shares: %i (%i orphan, %i dead) Stale rate: %s Efficiency: %s Current payout: %.4f %s' % (
shares, stale_orphan_shares, stale_doa_shares,
math.format_binomial_conf(stale_orphan_shares + stale_doa_shares, shares, 0.95),
math.format_binomial_conf(stale_orphan_shares + stale_doa_shares, shares, 0.95, lambda x: (1 - x)/(1 - stale_prop)),
node.get_current_txouts().get(bitcoin_data.pubkey_hash_to_script2(my_pubkey_hash), 0)*1e-8, net.PARENT.SYMBOL,
)
this_str += '\n Pool: %sH/s Stale rate: %.1f%% Expected time to block: %s' % (
math.format(int(real_att_s)),
100*stale_prop,
math.format_dt(2**256 / node.bitcoind_work.value['bits'].target / real_att_s),
)
for warning in p2pool_data.get_warnings(node.tracker, node.best_share_var.value, net, bitcoind_getinfo_var.value, node.bitcoind_work.value):
print >>sys.stderr, '#'*40
print >>sys.stderr, '>>> Warning: ' + warning
print >>sys.stderr, '#'*40
if gc.garbage:
print '%i pieces of uncollectable cyclic garbage! Types: %r' % (len(gc.garbage), map(type, gc.garbage))
if this_str != last_str or time.time() > last_time + 15:
print this_str
last_str = this_str
last_time = time.time()
except:
log.err()
status_thread()
except:
reactor.stop()
log.err(None, 'Fatal error:')
2
Example 47
Project: lowrank-gru Source File: RNN_adding.py
def main(n_iter, n_batch, n_hidden, time_steps, learning_rate, savefile, scale_penalty, use_scale,
model, n_hidden_lstm, loss_function, n_gru_lr_proj, initial_b_u):
#import pdb; pdb.set_trace()
# --- Set optimization params --------
gradient_clipping = np.float32(50000)
# --- Set data params ----------------
n_input = 2
n_output = 1
# --- Manage data --------------------
n_train = 1e5
n_test = 1e4
num_batches = n_train / n_batch
train_x = np.asarray(np.zeros((time_steps, n_train, 2)),
dtype=theano.config.floatX)
train_x[:,:,0] = np.asarray(np.random.uniform(low=0.,
high=1.,
size=(time_steps, n_train)),
dtype=theano.config.floatX)
# inds = np.asarray([np.random.choice(time_steps, 2, replace=False) for i in xrange(train_x.shape[1])])
inds = np.asarray(np.random.randint(time_steps/2, size=(train_x.shape[1],2)))
inds[:, 1] += time_steps/2
for i in range(train_x.shape[1]):
train_x[inds[i, 0], i, 1] = 1.0
train_x[inds[i, 1], i, 1] = 1.0
train_y = (train_x[:,:,0] * train_x[:,:,1]).sum(axis=0)
train_y = np.reshape(train_y, (n_train, 1))
test_x = np.asarray(np.zeros((time_steps, n_test, 2)),
dtype=theano.config.floatX)
test_x[:,:,0] = np.asarray(np.random.uniform(low=0.,
high=1.,
size=(time_steps, n_test)),
dtype=theano.config.floatX)
inds = np.asarray([np.random.choice(time_steps, 2, replace=False) for i in xrange(test_x.shape[1])])
for i in range(test_x.shape[1]):
test_x[inds[i, 0], i, 1] = 1.0
test_x[inds[i, 1], i, 1] = 1.0
test_y = (test_x[:,:,0] * test_x[:,:,1]).sum(axis=0)
test_y = np.reshape(test_y, (n_test, 1))
#######################################################################
gradient_clipping = np.float32(1)
if (model == 'LSTM'):
#inputs, parameters, costs = LSTM(n_input, n_hidden_lstm, n_output, loss_function=loss_function)
inputs, parameters, costs = LSTM(n_input, n_hidden_lstm, n_output, loss_function=loss_function, initial_b_f=initial_b_u)
gradients = T.grad(costs[0], parameters)
gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]
#by AnvaMiba
elif (model == 'GRU'):
inputs, parameters, costs = GRU(n_input, n_hidden_lstm, n_output, loss_function=loss_function, initial_b_u=initial_b_u)
gradients = T.grad(costs[0], parameters)
gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]
#by AnvaMiba
elif (model == 'GRU_LR'):
inputs, parameters, costs = GRU_LR(n_input, n_hidden_lstm, n_output, n_gru_lr_proj, loss_function=loss_function, initial_b_u=initial_b_u)
gradients = T.grad(costs[0], parameters)
gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]
elif (model == 'complex_RNN'):
inputs, parameters, costs = complex_RNN(n_input, n_hidden, n_output, scale_penalty, loss_function=loss_function)
if use_scale is False:
parameters.pop()
gradients = T.grad(costs[0], parameters)
elif (model == 'complex_RNN_LSTM'):
inputs, parameters, costs = complex_RNN_LSTM(n_input, n_hidden, n_hidden_lstm, n_output, scale_penalty, loss_function=loss_function)
elif (model == 'IRNN'):
inputs, parameters, costs = IRNN(n_input, n_hidden, n_output, loss_function=loss_function)
gradients = T.grad(costs[0], parameters)
gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]
elif (model == 'RNN'):
inputs, parameters, costs = tanhRNN(n_input, n_hidden, n_output, loss_function=loss_function)
gradients = T.grad(costs[0], parameters)
gradients = [T.clip(g, -gradient_clipping, gradient_clipping) for g in gradients]
else:
print >> sys.stderr, "Unsuported model:", model
return
s_train_x = theano.shared(train_x)
s_train_y = theano.shared(train_y)
s_test_x = theano.shared(test_x)
s_test_y = theano.shared(test_y)
# --- Compile theano functions --------------------------------------------------
index = T.iscalar('i')
updates, rmsprop = rms_prop(learning_rate, parameters, gradients)
givens = {inputs[0] : s_train_x[:, n_batch * index : n_batch * (index + 1), :],
inputs[1] : s_train_y[n_batch * index : n_batch * (index + 1), :]}
givens_test = {inputs[0] : s_test_x,
inputs[1] : s_test_y}
train = theano.function([index], costs[0], givens=givens, updates=updates)
test = theano.function([], costs[1], givens=givens_test)
# --- Training Loop ---------------------------------------------------------------
# f1 = file('/data/lisatmp3/shahamar/adding/complexRNN_400.pkl', 'rb')
# data1 = cPickle.load(f1)
# f1.close()
# train_loss = data1['train_loss']
# test_loss = data1['test_loss']
# best_params = data1['best_params']
# best_test_loss = data1['best_test_loss']
# for i in xrange(len(parameters)):
# parameters[i].set_value(data1['parameters'][i])
# for i in xrange(len(parameters)):
# rmsprop[i].set_value(data1['rmsprop'][i])
# import pdb; pdb.set_trace()
train_loss = []
test_loss = []
best_params = [p.get_value() for p in parameters]
best_test_loss = 1e6
for i in xrange(n_iter):
# start_time = timeit.default_timer()
if (n_iter % int(num_batches) == 0):
#import pdb; pdb.set_trace()
inds = np.random.permutation(int(n_train))
data_x = s_train_x.get_value()
s_train_x.set_value(data_x[:,inds,:])
data_y = s_train_y.get_value()
s_train_y.set_value(data_y[inds,:])
mse = train(i % int(num_batches))
train_loss.append(mse)
print >> sys.stderr, "Iteration:", i
print >> sys.stderr, "mse:", mse
print >> sys.stderr, ''
#if (i % 50==0):
if (i % 500==0):
mse = test()
print >> sys.stderr, ''
print >> sys.stderr, "TEST"
print >> sys.stderr, "mse:", mse
print >> sys.stderr, ''
test_loss.append(mse)
if mse < best_test_loss:
best_params = [p.get_value() for p in parameters]
best_test_loss = mse
save_vals = {'parameters': [p.get_value() for p in parameters],
'rmsprop': [r.get_value() for r in rmsprop],
'train_loss': train_loss,
'test_loss': test_loss,
'best_params': best_params,
'best_test_loss': best_test_loss,
'model': model,
'time_steps': time_steps}
cPickle.dump(save_vals,
file(savefile, 'wb'),
cPickle.HIGHEST_PROTOCOL)
2
Example 48
def main():
"""Main function called upon script execution."""
if OPTIONS.get('--no-colors'):
disable_all_colors()
elif OPTIONS.get('--colors'):
enable_all_colors()
if is_enabled() and os.name == 'nt':
Windows.enable(auto_colors=True, reset_atexit=True)
elif OPTIONS.get('--light-bg'):
set_light_background()
elif OPTIONS.get('--dark-bg'):
set_dark_background()
# Light or dark colors.
print('Autocolors for all backgrounds:')
print(Color(' {autoblack}Black{/black} {autored}Red{/red} {autogreen}Green{/green} '), end='')
print(Color('{autoyellow}Yellow{/yellow} {autoblue}Blue{/blue} {automagenta}Magenta{/magenta} '), end='')
print(Color('{autocyan}Cyan{/cyan} {autowhite}White{/white}'))
print(Color(' {autobgblack}{autoblack}Black{/black}{/bgblack} '), end='')
print(Color('{autobgblack}{autored}Red{/red}{/bgblack} {autobgblack}{autogreen}Green{/green}{/bgblack} '), end='')
print(Color('{autobgblack}{autoyellow}Yellow{/yellow}{/bgblack} '), end='')
print(Color('{autobgblack}{autoblue}Blue{/blue}{/bgblack} '), end='')
print(Color('{autobgblack}{automagenta}Magenta{/magenta}{/bgblack} '), end='')
print(Color('{autobgblack}{autocyan}Cyan{/cyan}{/bgblack} {autobgblack}{autowhite}White{/white}{/bgblack}'))
print(Color(' {autobgred}{autoblack}Black{/black}{/bgred} {autobgred}{autored}Red{/red}{/bgred} '), end='')
print(Color('{autobgred}{autogreen}Green{/green}{/bgred} {autobgred}{autoyellow}Yellow{/yellow}{/bgred} '), end='')
print(Color('{autobgred}{autoblue}Blue{/blue}{/bgred} {autobgred}{automagenta}Magenta{/magenta}{/bgred} '), end='')
print(Color('{autobgred}{autocyan}Cyan{/cyan}{/bgred} {autobgred}{autowhite}White{/white}{/bgred}'))
print(Color(' {autobggreen}{autoblack}Black{/black}{/bggreen} '), end='')
print(Color('{autobggreen}{autored}Red{/red}{/bggreen} {autobggreen}{autogreen}Green{/green}{/bggreen} '), end='')
print(Color('{autobggreen}{autoyellow}Yellow{/yellow}{/bggreen} '), end='')
print(Color('{autobggreen}{autoblue}Blue{/blue}{/bggreen} '), end='')
print(Color('{autobggreen}{automagenta}Magenta{/magenta}{/bggreen} '), end='')
print(Color('{autobggreen}{autocyan}Cyan{/cyan}{/bggreen} {autobggreen}{autowhite}White{/white}{/bggreen}'))
print(Color(' {autobgyellow}{autoblack}Black{/black}{/bgyellow} '), end='')
print(Color('{autobgyellow}{autored}Red{/red}{/bgyellow} '), end='')
print(Color('{autobgyellow}{autogreen}Green{/green}{/bgyellow} '), end='')
print(Color('{autobgyellow}{autoyellow}Yellow{/yellow}{/bgyellow} '), end='')
print(Color('{autobgyellow}{autoblue}Blue{/blue}{/bgyellow} '), end='')
print(Color('{autobgyellow}{automagenta}Magenta{/magenta}{/bgyellow} '), end='')
print(Color('{autobgyellow}{autocyan}Cyan{/cyan}{/bgyellow} {autobgyellow}{autowhite}White{/white}{/bgyellow}'))
print(Color(' {autobgblue}{autoblack}Black{/black}{/bgblue} {autobgblue}{autored}Red{/red}{/bgblue} '), end='')
print(Color('{autobgblue}{autogreen}Green{/green}{/bgblue} '), end='')
print(Color('{autobgblue}{autoyellow}Yellow{/yellow}{/bgblue} {autobgblue}{autoblue}Blue{/blue}{/bgblue} '), end='')
print(Color('{autobgblue}{automagenta}Magenta{/magenta}{/bgblue} '), end='')
print(Color('{autobgblue}{autocyan}Cyan{/cyan}{/bgblue} {autobgblue}{autowhite}White{/white}{/bgblue}'))
print(Color(' {autobgmagenta}{autoblack}Black{/black}{/bgmagenta} '), end='')
print(Color('{autobgmagenta}{autored}Red{/red}{/bgmagenta} '), end='')
print(Color('{autobgmagenta}{autogreen}Green{/green}{/bgmagenta} '), end='')
print(Color('{autobgmagenta}{autoyellow}Yellow{/yellow}{/bgmagenta} '), end='')
print(Color('{autobgmagenta}{autoblue}Blue{/blue}{/bgmagenta} '), end='')
print(Color('{autobgmagenta}{automagenta}Magenta{/magenta}{/bgmagenta} '), end='')
print(Color('{autobgmagenta}{autocyan}Cyan{/cyan}{/bgmagenta} '), end='')
print(Color('{autobgmagenta}{autowhite}White{/white}{/bgmagenta}'))
print(Color(' {autobgcyan}{autoblack}Black{/black}{/bgcyan} {autobgcyan}{autored}Red{/red}{/bgcyan} '), end='')
print(Color('{autobgcyan}{autogreen}Green{/green}{/bgcyan} '), end='')
print(Color('{autobgcyan}{autoyellow}Yellow{/yellow}{/bgcyan} {autobgcyan}{autoblue}Blue{/blue}{/bgcyan} '), end='')
print(Color('{autobgcyan}{automagenta}Magenta{/magenta}{/bgcyan} '), end='')
print(Color('{autobgcyan}{autocyan}Cyan{/cyan}{/bgcyan} {autobgcyan}{autowhite}White{/white}{/bgcyan}'))
print(Color(' {autobgwhite}{autoblack}Black{/black}{/bgwhite} '), end='')
print(Color('{autobgwhite}{autored}Red{/red}{/bgwhite} {autobgwhite}{autogreen}Green{/green}{/bgwhite} '), end='')
print(Color('{autobgwhite}{autoyellow}Yellow{/yellow}{/bgwhite} '), end='')
print(Color('{autobgwhite}{autoblue}Blue{/blue}{/bgwhite} '), end='')
print(Color('{autobgwhite}{automagenta}Magenta{/magenta}{/bgwhite} '), end='')
print(Color('{autobgwhite}{autocyan}Cyan{/cyan}{/bgwhite} {autobgwhite}{autowhite}White{/white}{/bgwhite}'))
print()
# Light colors.
print('Light colors for dark backgrounds:')
print(Color(' {hiblack}Black{/black} {hired}Red{/red} {higreen}Green{/green} '), end='')
print(Color('{hiyellow}Yellow{/yellow} {hiblue}Blue{/blue} {himagenta}Magenta{/magenta} '), end='')
print(Color('{hicyan}Cyan{/cyan} {hiwhite}White{/white}'))
print(Color(' {hibgblack}{hiblack}Black{/black}{/bgblack} '), end='')
print(Color('{hibgblack}{hired}Red{/red}{/bgblack} {hibgblack}{higreen}Green{/green}{/bgblack} '), end='')
print(Color('{hibgblack}{hiyellow}Yellow{/yellow}{/bgblack} '), end='')
print(Color('{hibgblack}{hiblue}Blue{/blue}{/bgblack} '), end='')
print(Color('{hibgblack}{himagenta}Magenta{/magenta}{/bgblack} '), end='')
print(Color('{hibgblack}{hicyan}Cyan{/cyan}{/bgblack} {hibgblack}{hiwhite}White{/white}{/bgblack}'))
print(Color(' {hibgred}{hiblack}Black{/black}{/bgred} {hibgred}{hired}Red{/red}{/bgred} '), end='')
print(Color('{hibgred}{higreen}Green{/green}{/bgred} {hibgred}{hiyellow}Yellow{/yellow}{/bgred} '), end='')
print(Color('{hibgred}{hiblue}Blue{/blue}{/bgred} {hibgred}{himagenta}Magenta{/magenta}{/bgred} '), end='')
print(Color('{hibgred}{hicyan}Cyan{/cyan}{/bgred} {hibgred}{hiwhite}White{/white}{/bgred}'))
print(Color(' {hibggreen}{hiblack}Black{/black}{/bggreen} '), end='')
print(Color('{hibggreen}{hired}Red{/red}{/bggreen} {hibggreen}{higreen}Green{/green}{/bggreen} '), end='')
print(Color('{hibggreen}{hiyellow}Yellow{/yellow}{/bggreen} '), end='')
print(Color('{hibggreen}{hiblue}Blue{/blue}{/bggreen} '), end='')
print(Color('{hibggreen}{himagenta}Magenta{/magenta}{/bggreen} '), end='')
print(Color('{hibggreen}{hicyan}Cyan{/cyan}{/bggreen} {hibggreen}{hiwhite}White{/white}{/bggreen}'))
print(Color(' {hibgyellow}{hiblack}Black{/black}{/bgyellow} '), end='')
print(Color('{hibgyellow}{hired}Red{/red}{/bgyellow} '), end='')
print(Color('{hibgyellow}{higreen}Green{/green}{/bgyellow} '), end='')
print(Color('{hibgyellow}{hiyellow}Yellow{/yellow}{/bgyellow} '), end='')
print(Color('{hibgyellow}{hiblue}Blue{/blue}{/bgyellow} '), end='')
print(Color('{hibgyellow}{himagenta}Magenta{/magenta}{/bgyellow} '), end='')
print(Color('{hibgyellow}{hicyan}Cyan{/cyan}{/bgyellow} {hibgyellow}{hiwhite}White{/white}{/bgyellow}'))
print(Color(' {hibgblue}{hiblack}Black{/black}{/bgblue} {hibgblue}{hired}Red{/red}{/bgblue} '), end='')
print(Color('{hibgblue}{higreen}Green{/green}{/bgblue} '), end='')
print(Color('{hibgblue}{hiyellow}Yellow{/yellow}{/bgblue} {hibgblue}{hiblue}Blue{/blue}{/bgblue} '), end='')
print(Color('{hibgblue}{himagenta}Magenta{/magenta}{/bgblue} '), end='')
print(Color('{hibgblue}{hicyan}Cyan{/cyan}{/bgblue} {hibgblue}{hiwhite}White{/white}{/bgblue}'))
print(Color(' {hibgmagenta}{hiblack}Black{/black}{/bgmagenta} '), end='')
print(Color('{hibgmagenta}{hired}Red{/red}{/bgmagenta} '), end='')
print(Color('{hibgmagenta}{higreen}Green{/green}{/bgmagenta} '), end='')
print(Color('{hibgmagenta}{hiyellow}Yellow{/yellow}{/bgmagenta} '), end='')
print(Color('{hibgmagenta}{hiblue}Blue{/blue}{/bgmagenta} '), end='')
print(Color('{hibgmagenta}{himagenta}Magenta{/magenta}{/bgmagenta} '), end='')
print(Color('{hibgmagenta}{hicyan}Cyan{/cyan}{/bgmagenta} '), end='')
print(Color('{hibgmagenta}{hiwhite}White{/white}{/bgmagenta}'))
print(Color(' {hibgcyan}{hiblack}Black{/black}{/bgcyan} {hibgcyan}{hired}Red{/red}{/bgcyan} '), end='')
print(Color('{hibgcyan}{higreen}Green{/green}{/bgcyan} '), end='')
print(Color('{hibgcyan}{hiyellow}Yellow{/yellow}{/bgcyan} {hibgcyan}{hiblue}Blue{/blue}{/bgcyan} '), end='')
print(Color('{hibgcyan}{himagenta}Magenta{/magenta}{/bgcyan} '), end='')
print(Color('{hibgcyan}{hicyan}Cyan{/cyan}{/bgcyan} {hibgcyan}{hiwhite}White{/white}{/bgcyan}'))
print(Color(' {hibgwhite}{hiblack}Black{/black}{/bgwhite} '), end='')
print(Color('{hibgwhite}{hired}Red{/red}{/bgwhite} {hibgwhite}{higreen}Green{/green}{/bgwhite} '), end='')
print(Color('{hibgwhite}{hiyellow}Yellow{/yellow}{/bgwhite} '), end='')
print(Color('{hibgwhite}{hiblue}Blue{/blue}{/bgwhite} '), end='')
print(Color('{hibgwhite}{himagenta}Magenta{/magenta}{/bgwhite} '), end='')
print(Color('{hibgwhite}{hicyan}Cyan{/cyan}{/bgwhite} {hibgwhite}{hiwhite}White{/white}{/bgwhite}'))
print()
# Dark colors.
print('Dark colors for light backgrounds:')
print(Color(' {black}Black{/black} {red}Red{/red} {green}Green{/green} {yellow}Yellow{/yellow} '), end='')
print(Color('{blue}Blue{/blue} {magenta}Magenta{/magenta} {cyan}Cyan{/cyan} {white}White{/white}'))
print(Color(' {bgblack}{black}Black{/black}{/bgblack} {bgblack}{red}Red{/red}{/bgblack} '), end='')
print(Color('{bgblack}{green}Green{/green}{/bgblack} {bgblack}{yellow}Yellow{/yellow}{/bgblack} '), end='')
print(Color('{bgblack}{blue}Blue{/blue}{/bgblack} {bgblack}{magenta}Magenta{/magenta}{/bgblack} '), end='')
print(Color('{bgblack}{cyan}Cyan{/cyan}{/bgblack} {bgblack}{white}White{/white}{/bgblack}'))
print(Color(' {bgred}{black}Black{/black}{/bgred} {bgred}{red}Red{/red}{/bgred} '), end='')
print(Color('{bgred}{green}Green{/green}{/bgred} {bgred}{yellow}Yellow{/yellow}{/bgred} '), end='')
print(Color('{bgred}{blue}Blue{/blue}{/bgred} {bgred}{magenta}Magenta{/magenta}{/bgred} '), end='')
print(Color('{bgred}{cyan}Cyan{/cyan}{/bgred} {bgred}{white}White{/white}{/bgred}'))
print(Color(' {bggreen}{black}Black{/black}{/bggreen} {bggreen}{red}Red{/red}{/bggreen} '), end='')
print(Color('{bggreen}{green}Green{/green}{/bggreen} {bggreen}{yellow}Yellow{/yellow}{/bggreen} '), end='')
print(Color('{bggreen}{blue}Blue{/blue}{/bggreen} {bggreen}{magenta}Magenta{/magenta}{/bggreen} '), end='')
print(Color('{bggreen}{cyan}Cyan{/cyan}{/bggreen} {bggreen}{white}White{/white}{/bggreen}'))
print(Color(' {bgyellow}{black}Black{/black}{/bgyellow} {bgyellow}{red}Red{/red}{/bgyellow} '), end='')
print(Color('{bgyellow}{green}Green{/green}{/bgyellow} {bgyellow}{yellow}Yellow{/yellow}{/bgyellow} '), end='')
print(Color('{bgyellow}{blue}Blue{/blue}{/bgyellow} {bgyellow}{magenta}Magenta{/magenta}{/bgyellow} '), end='')
print(Color('{bgyellow}{cyan}Cyan{/cyan}{/bgyellow} {bgyellow}{white}White{/white}{/bgyellow}'))
print(Color(' {bgblue}{black}Black{/black}{/bgblue} {bgblue}{red}Red{/red}{/bgblue} '), end='')
print(Color('{bgblue}{green}Green{/green}{/bgblue} {bgblue}{yellow}Yellow{/yellow}{/bgblue} '), end='')
print(Color('{bgblue}{blue}Blue{/blue}{/bgblue} {bgblue}{magenta}Magenta{/magenta}{/bgblue} '), end='')
print(Color('{bgblue}{cyan}Cyan{/cyan}{/bgblue} {bgblue}{white}White{/white}{/bgblue}'))
print(Color(' {bgmagenta}{black}Black{/black}{/bgmagenta} {bgmagenta}{red}Red{/red}{/bgmagenta} '), end='')
print(Color('{bgmagenta}{green}Green{/green}{/bgmagenta} {bgmagenta}{yellow}Yellow{/yellow}{/bgmagenta} '), end='')
print(Color('{bgmagenta}{blue}Blue{/blue}{/bgmagenta} {bgmagenta}{magenta}Magenta{/magenta}{/bgmagenta} '), end='')
print(Color('{bgmagenta}{cyan}Cyan{/cyan}{/bgmagenta} {bgmagenta}{white}White{/white}{/bgmagenta}'))
print(Color(' {bgcyan}{black}Black{/black}{/bgcyan} {bgcyan}{red}Red{/red}{/bgcyan} '), end='')
print(Color('{bgcyan}{green}Green{/green}{/bgcyan} {bgcyan}{yellow}Yellow{/yellow}{/bgcyan} '), end='')
print(Color('{bgcyan}{blue}Blue{/blue}{/bgcyan} {bgcyan}{magenta}Magenta{/magenta}{/bgcyan} '), end='')
print(Color('{bgcyan}{cyan}Cyan{/cyan}{/bgcyan} {bgcyan}{white}White{/white}{/bgcyan}'))
print(Color(' {bgwhite}{black}Black{/black}{/bgwhite} {bgwhite}{red}Red{/red}{/bgwhite} '), end='')
print(Color('{bgwhite}{green}Green{/green}{/bgwhite} {bgwhite}{yellow}Yellow{/yellow}{/bgwhite} '), end='')
print(Color('{bgwhite}{blue}Blue{/blue}{/bgwhite} {bgwhite}{magenta}Magenta{/magenta}{/bgwhite} '), end='')
print(Color('{bgwhite}{cyan}Cyan{/cyan}{/bgwhite} {bgwhite}{white}White{/white}{/bgwhite}'))
if OPTIONS['--wait']:
print('Waiting for {0} to exist within 10 seconds...'.format(OPTIONS['--wait']), file=sys.stderr, end='')
stop_after = time.time() + 20
while not os.path.exists(OPTIONS['--wait']) and time.time() < stop_after:
print('.', file=sys.stderr, end='')
sys.stderr.flush()
time.sleep(0.5)
print(' done')
2
Example 49
Project: nesoni Source File: fill_scaffolds.py
def fill_scaffolds(args):
max_filler_length, args = grace.get_option_value(args, '--max-filler', int, 4000)
if len(args) < 2:
print USAGE
return 1
(output_dir, graph_dir), args = args[:2], args[2:]
scaffolds = [ ]
def scaffold(args):
circular, args = grace.get_option_value(args, '--circular', grace.as_bool, False)
scaffold = [ ]
for item in args:
scaffold.append( ('contig', int(item)) )
scaffold.append( ('gap', None) )
if not circular: scaffold = scaffold[:-1]
name = 'custom_scaffold_%d' % (len(scaffolds)+1)
scaffolds.append( (name, scaffold) )
grace.execute(args, [scaffold])
custom_scaffolds = (len(scaffolds) != 0)
sequences = dict(
(a.split()[0], b.upper())
for a,b in
io.read_sequences(os.path.join(
graph_dir, '454AllContigs.fna')))
sequence_names = sorted(sequences)
sequence_ids = dict(zip(sequence_names, xrange(1,len(sequence_names)+1)))
contexts = { }
context_names = { }
context_depths = { }
for i in xrange(1,len(sequence_names)+1):
seq = sequences[sequence_names[i-1]]
contexts[ i ] = seq
context_names[ i ] = sequence_names[i-1]+'-fwd'
contexts[ -i ] = bio.reverse_complement(seq)
context_names[ -i ] = sequence_names[i-1]+'-rev'
links = collections.defaultdict(list)
for line in open(
os.path.join(graph_dir, '454ContigGraph.txt'),
'rU'):
parts = line.rstrip('\n').split('\t')
if parts[0].isdigit():
seq = sequence_ids[parts[1]]
context_depths[ seq] = float(parts[3])
context_depths[-seq] = float(parts[3])
if parts[0] == 'C':
name1 = 'contig%05d' % int(parts[1])
dir1 = {"3'" : 1, "5'" : -1 }[parts[2]]
name2 = 'contig%05d' % int(parts[3])
dir2 = {"5'" : 1, "3'" : -1 }[parts[4]]
depth = int(parts[5])
#print name1, dir1, name2, dir2, depth
links[ sequence_ids[name1] * dir1 ].append( (depth, sequence_ids[name2] * dir2) )
links[ sequence_ids[name2] * -dir2 ].append( (depth, sequence_ids[name1] * -dir1) )
if parts[0] == 'S' and not custom_scaffolds:
name = 'scaffold%05d' % int(parts[2])
components = parts[3].split(';')
scaffold = [ ]
for component in components:
a,b = component.split(':')
if a == 'gap':
scaffold.append( ('gap',int(b)) )
else:
strand = { '+': +1, '-': -1 }[ b ]
scaffold.append( ('contig', sequence_ids['contig%05d'%int(a)] * strand) )
scaffolds.append( (name, scaffold) )
#paths = { }
#
#todo = [ ]
#for i in contexts:
# for depth_left, neg_left in links[-i]:
# left = -neg_left
# for depth_right, right in links[i]:
# todo.append( ( max(-depth_left,-depth_right,-context_depths[i]), left, right, (i,)) )
#
#heapq.heapify(todo)
#while todo:
# score, source, dest, path = heapq.heappop(todo)
# if (source,dest) in paths: continue
#
# paths[(source,dest)] = path
#
# if len(contexts[dest]) > max_filler_length: continue
#
# for depth, next in links[dest]:
# heapq.heappush(todo,
# ( max(score,-depth,-context_depths[dest]), source, next, path+(dest,))
# )
path_source_dest = collections.defaultdict(dict) # source -> dest -> next
path_dest_source = collections.defaultdict(dict) # dest -> source -> next
# Use links, in order to depth of coverage, to construct paths between contigs
# Thus: paths have maximum minimum depth
# subsections of paths also have this property
todo = [ ]
for i in contexts:
for depth_link, right in links[i]:
todo.append( ( depth_link, i, right) )
todo.sort(reverse=True)
for score, left, right in todo:
if right in path_source_dest[left]: continue
sources = [(left,right)]
if len(contexts[left]) <= max_filler_length:
sources += path_dest_source[left].items()
destinations = [right]
if len(contexts[right]) <= max_filler_length:
destinations += path_source_dest[right].keys()
for source, next in sources:
for dest in destinations:
if dest in path_source_dest[source]: continue
path_source_dest[source][dest] = next
path_dest_source[dest][source] = next
workspace = io.Workspace(output_dir)
scaffold_f = workspace.open('scaffolds.fa','wb')
#comments = [ ]
features = [ ]
used = set()
previous_total = 0
for i, (name, scaffold) in enumerate(scaffolds):
result = '' # Inefficient. Meh.
n_filled = 0
n_failed = 0
for j, item in enumerate(scaffold):
if item[0] == 'contig':
result += contexts[item[1]]
used.add(abs(item[1]))
else:
left = scaffold[j-1]
right = scaffold[ (j+1) % len(scaffold) ] #If gap at end, assume circular
assert left[0] == 'contig'
assert right[0] == 'contig'
gap_start = len(result)
can_fill = right[1] in path_source_dest[left[1]]
if can_fill:
n = 0
k = path_source_dest[left[1]][right[1]]
while k != right[1]:
n += len(contexts[k])
result += contexts[k].lower()
used.add(abs(k))
k = path_source_dest[k][right[1]]
n_filled += 1
if item[1] is not None and max(n,item[1]) > min(n,item[1])*4:
print >> sys.stderr, 'Warning: gap size changed from %d to %d in scaffold %d' % (item[1],n,i+1)
else:
n_failed += 1
#print >> sys.stderr, 'Warning: No path to fill a gap in scaffold %d' % (i+1)
result += 'n' * (9 if item[1] is None else item[1])
gap_end = len(result)
#features.append( '%s\t%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s' % (
# 'all-scaffolds',
# 'fill-scaffolds',
# 'gap',
# previous_total + gap_start+1,
# previous_total + max(gap_end, gap_start+1), #Allow for zeroed out gaps. Hmm.
# '.', #score
# '+', #strand
# '.', #frame
# '' #properties
#))
features.append( '%s\t%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s' % (
name,
'fill-scaffolds',
'gap',
gap_start+1,
max(gap_end, gap_start+1), #Allow for zeroed out gaps. Hmm.
'.', #score
'+', #strand
'.', #frame
'' #properties
))
io.write_fasta(scaffold_f, name, result)
previous_total += len(result)
#comments.append('##sequence-region %s %d %d' % (name, 1, len(result)))
print >> sys.stderr, 'Scaffold%05d: %d gaps filled, %d could not be filled' % (i+1, n_filled, n_failed)
scaffold_f.close()
gff_f = workspace.open('scaffolds.gff', 'wb')
#print >>gff_f, '##gff-version 3'
#for comment in comments:
# print >>gff_f, comment
for feature in features:
print >>gff_f, feature
gff_f.close()
leftovers_f = workspace.open('leftovers.fa', 'wb')
for name in sequence_names:
if sequence_ids[name] not in used:
io.write_fasta(leftovers_f, name, sequences[name])
leftovers_f.close()
ends = { }
for i, (name, scaffold) in enumerate(scaffolds):
if scaffold[-1][0] == 'gap': continue
ends[ '%s start' % name ] = scaffold[-1][1]
ends[ '%s end ' % name ] = -scaffold[0][1]
for end1 in sorted(ends):
options = [ end2 for end2 in ends if -ends[end2] in path_source_dest[ends[end1]] ]
if len(options) == 1:
print >> sys.stderr, 'Note: from', end1, 'only', options[0], 'is reachable'
2
Example 50
Project: Mobile-Security-Framework-MobSF Source File: systrace-legacy.py
def main():
parser = optparse.OptionParser()
parser.add_option('-o', dest='output_file', help='write HTML to FILE',
default='trace.html', metavar='FILE')
parser.add_option('-t', '--time', dest='trace_time', type='int',
help='trace for N seconds', metavar='N')
parser.add_option('-b', '--buf-size', dest='trace_buf_size', type='int',
help='use a trace buffer size of N KB', metavar='N')
parser.add_option('-d', '--disk', dest='trace_disk', default=False,
action='store_true', help='trace disk I/O (requires root)')
parser.add_option('-f', '--cpu-freq', dest='trace_cpu_freq', default=False,
action='store_true', help='trace CPU frequency changes')
parser.add_option('-i', '--cpu-idle', dest='trace_cpu_idle', default=False,
action='store_true', help='trace CPU idle events')
parser.add_option('-l', '--cpu-load', dest='trace_cpu_load', default=False,
action='store_true', help='trace CPU load')
parser.add_option('-s', '--no-cpu-sched', dest='trace_cpu_sched', default=True,
action='store_false', help='inhibit tracing CPU ' +
'scheduler (allows longer trace times by reducing data ' +
'rate into buffer)')
parser.add_option('-u', '--bus-utilization', dest='trace_bus_utilization',
default=False, action='store_true',
help='trace bus utilization (requires root)')
parser.add_option('-w', '--workqueue', dest='trace_workqueue', default=False,
action='store_true', help='trace the kernel workqueues ' +
'(requires root)')
parser.add_option('--set-tags', dest='set_tags', action='store',
help='set the enabled trace tags and exit; set to a ' +
'comma separated list of: ' +
', '.join(trace_tag_bits.iterkeys()))
parser.add_option('--link-assets', dest='link_assets', default=False,
action='store_true', help='link to original CSS or JS resources '
'instead of embedding them')
parser.add_option('--from-file', dest='from_file', action='store',
help='read the trace from a file rather than running a live trace')
parser.add_option('--asset-dir', dest='asset_dir', default='trace-viewer',
type='string', help='')
parser.add_option('-e', '--serial', dest='device_serial', type='string',
help='adb device serial number')
options, args = parser.parse_args()
if options.set_tags:
flags = 0
tags = options.set_tags.split(',')
for tag in tags:
try:
flags |= trace_tag_bits[tag]
except KeyError:
parser.error('unrecognized tag: %s\nknown tags are: %s' %
(tag, ', '.join(trace_tag_bits.iterkeys())))
atrace_args = ['adb', 'shell', 'setprop', 'debug.atrace.tags.enableflags', hex(flags)]
add_adb_serial(atrace_args, options.device_serial)
try:
subprocess.check_call(atrace_args)
except subprocess.CalledProcessError, e:
print >> sys.stderr, 'unable to set tags: %s' % e
print '\nSet enabled tags to: %s\n' % ', '.join(tags)
print ('You will likely need to restart the Android framework for this to ' +
'take effect:\n\n adb shell stop\n adb shell ' +
'start\n')
return
atrace_args = ['adb', 'shell', 'atrace', '-z']
add_adb_serial(atrace_args, options.device_serial)
if options.trace_disk:
atrace_args.append('-d')
if options.trace_cpu_freq:
atrace_args.append('-f')
if options.trace_cpu_idle:
atrace_args.append('-i')
if options.trace_cpu_load:
atrace_args.append('-l')
if options.trace_cpu_sched:
atrace_args.append('-s')
if options.trace_bus_utilization:
atrace_args.append('-u')
if options.trace_workqueue:
atrace_args.append('-w')
if options.trace_time is not None:
if options.trace_time > 0:
atrace_args.extend(['-t', str(options.trace_time)])
else:
parser.error('the trace time must be a positive number')
if options.trace_buf_size is not None:
if options.trace_buf_size > 0:
atrace_args.extend(['-b', str(options.trace_buf_size)])
else:
parser.error('the trace buffer size must be a positive number')
if options.from_file is not None:
atrace_args = ['cat', options.from_file]
script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
if options.link_assets:
src_dir = os.path.join(script_dir, options.asset_dir, 'src')
build_dir = os.path.join(script_dir, options.asset_dir, 'build')
js_files, js_flattenizer, css_files, templates = get_assets(src_dir, build_dir)
css = '\n'.join(linked_css_tag % (os.path.join(src_dir, f)) for f in css_files)
js = '<script language="javascript">\n%s</script>\n' % js_flattenizer
js += '\n'.join(linked_js_tag % (os.path.join(src_dir, f)) for f in js_files)
else:
css_filename = os.path.join(script_dir, flattened_css_file)
js_filename = os.path.join(script_dir, flattened_js_file)
css = compiled_css_tag % (open(css_filename).read())
js = compiled_js_tag % (open(js_filename).read())
templates = ''
html_filename = options.output_file
trace_started = False
leftovers = ''
adb = subprocess.Popen(atrace_args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
dec = zlib.decompressobj()
while True:
ready = select.select([adb.stdout, adb.stderr], [], [adb.stdout, adb.stderr])
if adb.stderr in ready[0]:
err = os.read(adb.stderr.fileno(), 4096)
sys.stderr.write(err)
sys.stderr.flush()
if adb.stdout in ready[0]:
out = leftovers + os.read(adb.stdout.fileno(), 4096)
if options.from_file is None:
out = out.replace('\r\n', '\n')
if out.endswith('\r'):
out = out[:-1]
leftovers = '\r'
else:
leftovers = ''
if not trace_started:
lines = out.splitlines(True)
out = ''
for i, line in enumerate(lines):
if line == 'TRACE:\n':
sys.stdout.write("downloading trace...")
sys.stdout.flush()
out = ''.join(lines[i+1:])
html_prefix = read_asset(script_dir, 'prefix.html')
html_file = open(html_filename, 'w')
html_file.write(html_prefix % (css, js, templates))
trace_started = True
break
elif 'TRACE:'.startswith(line) and i == len(lines) - 1:
leftovers = line + leftovers
else:
sys.stdout.write(line)
sys.stdout.flush()
if len(out) > 0:
out = dec.decompress(out)
html_out = out.replace('\n', '\\n\\\n')
if len(html_out) > 0:
html_file.write(html_out)
result = adb.poll()
if result is not None:
break
if result != 0:
print >> sys.stderr, 'adb returned error code %d' % result
elif trace_started:
html_out = dec.flush().replace('\n', '\\n\\\n').replace('\r', '')
if len(html_out) > 0:
html_file.write(html_out)
html_suffix = read_asset(script_dir, 'suffix.html')
html_file.write(html_suffix)
html_file.close()
print " done\n\n wrote file://%s\n" % (os.path.abspath(options.output_file))
else:
print >> sys.stderr, ('An error occured while capturing the trace. Output ' +
'file was not written.')