Here are the examples of the python api sys.executable taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
135 Examples
0
Example 51
def submit(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
inw = job.getInputWorkspace()
outw = job.getOutputWorkspace()
#scriptpath = self.preparejob(jobconfig,inw,outw)
scriptpath = self.preparejob(jobconfig, master_input_sandbox)
# FIX from Angelo Carbone
# stderr_option = '-e '+str(outw.getPath())+'stderr'
# stdout_option = '-o '+str(outw.getPath())+'stdout'
# FIX from Alex Richards - see Savannah #87477
stdout_option = self.config['stdoutConfig'] % str(outw.getPath())
stderr_option = self.config['stderrConfig'] % str(outw.getPath())
queue_option = ''
if self.queue:
queue_option = '-q ' + str(self.queue)
try:
jobnameopt = "-" + self.config['jobnameopt']
except Exception, err:
logger.debug("Unknown error: %s" % str(err))
jobnameopt = False
if self.extraopts:
import re
for opt in re.compile(r'(-\w+)').findall(self.extraopts):
if opt in ('-o', '-e', '-oo', '-eo'):
logger.warning("option %s is forbidden", opt)
return False
if self.queue and opt == '-q':
logger.warning("option %s is forbidden if queue is defined ( queue = '%s')", opt, self.queue)
return False
if jobnameopt and opt == jobnameopt:
jobnameopt = False
queue_option = queue_option + " " + self.extraopts
if jobnameopt and job.name != '':
# PBS doesn't like names with spaces
tmp_name = job.name
if isType(self, PBS):
tmp_name = tmp_name.replace(" ", "_")
queue_option = queue_option + " " + \
jobnameopt + " " + "'%s'" % (tmp_name)
# bugfix #16646
if self.config['shared_python_executable']:
import sys
script_cmd = "%s %s" % (sys.executable, scriptpath)
else:
script_cmd = scriptpath
command_str = self.config['submit_str'] % (inw.getPath(), queue_option, stderr_option, stdout_option, script_cmd)
self.command_string = command_str
rc, soutfile = self.command(command_str)
with open(soutfile) as sout_file:
sout = sout_file.read()
import re
m = re.compile(self.config['submit_res_pattern'], re.M).search(sout)
if m is None:
logger.warning('could not match the output and extract the Batch job identifier!')
logger.warning('command output \n %s ', sout)
else:
self.id = m.group('id')
try:
queue = m.group('queue')
if self.queue != queue:
if self.queue:
logger.warning('you requested queue "%s" but the job was submitted to queue "%s"', self.queue, queue)
logger.warning('command output \n %s ', sout)
else:
logger.info('using default queue "%s"', queue)
self.actualqueue = queue
except IndexError:
logger.info('could not match the output and extract the Batch queue name')
# clean up the tmp file
if os.path.exists(soutfile):
os.remove(soutfile)
return rc == 0
0
Example 52
Project: monaco Source File: monaco.py
def monaco():
'''
The management daemon for Monaco
'''
# Set up top-level console logger
consolehandler = logging.StreamHandler()
consolehandler.setLevel(logging.ERROR)
consolehandler.setFormatter(logging.Formatter('%(name)s - %(levelname)s - %(message)s'))
logger = logging.getLogger('monaco')
logger.setLevel(logging.DEBUG)
logger.addHandler(consolehandler)
# Load config
config.initLoggers()
# Install reload handler
def USR1_handler(signum, frame):
''' SIGUSR1 handler '''
logger.info('Received SIGUSR1, reloading Monaco...')
# this kills the current process tree and re-executes the
# original call arguments in the same PID (good for upstart)
os.execl(sys.executable, *([sys.executable]+sys.argv))
signal.signal(signal.SIGUSR1, USR1_handler)
# Setup scheduling
sched = Scheduler(daemon=True)
atexit.register(lambda: sched.shutdown(wait=False))
sched.start()
# Config and start slave thread to maintain app state
slave = Slave()
slave.update_subs()
#slave.start() starting in apsched inherits daemon=True
@sched.interval_schedule(seconds=1)
def maintain_slave():
'''
This periodically ensures that the slave is subscribed to all the
channels it should be. Again, this should be redundant
'''
if not hasattr(maintain_slave, 'started'):
maintain_slave.started = True
slave.start()
logger.info('Slave thread started')
if not slave.is_alive():
logger.info('Slave thread died')
slave.start()
# lifeRaft - Use Raft algorithm to maintain management leader
liferaft = LifeRaft()
# Start master/monitor thread
master = Master(liferaft)
master.start()
# maintain mgmt redis role from LifeRaft
r = redis.StrictRedis(port=config.config['mgmt_port'])
mastercli = None
while True:
time.sleep(1)
try:
master_tup = liferaft.value
if not master_tup:
logger.warn("Couldn't learn master value!")
else:
logger.debug('MGMT DB maintaining: %s', repr(master_tup))
if master_tup[0] == config.config['IP']:
if r.info()['role'] == 'master':
# We're the master, and properly configured
continue
# Else set to master, rewrite config to persist
logger.debug('Promoting self to master')
r.slaveof()
r.config_rewrite()
else:
if r.info()['role'] == 'slave' and r.info()['master_host'] == master_tup[0]:
# We're a slave and properly configured to the master
continue
for idx in xrange(3):
if idx != 0:
# Decongest on retry
time.sleep(0.5)
try:
# Try to get a connection to the mgmt master and verify it thinks it's master
if not mastercli:
mastercli = redis.StrictRedis(host=master_tup[0], port=config.config['mgmt_port'])
if mastercli.info()['role'] != 'master':
# If it doesn't think it's master, delete client and continue to retry
del mastercli
continue
break
except Exception:
continue
else:
# We didn't break
logger.debug('Assigned master (%s) has not yet assumed role', master_tup[0])
continue
# Set slave to proper master
logger.debug('Assigning self as slaveof %s:6379', master_tup[0])
r.slaveof(host=master_tup[0], port=config.config['mgmt_port'])
except Exception:
try:
r = redis.StrictRedis(port=config.config['mgmt_port'])
except Exception:
logger.exception("Can't connect to mgmt db!")
0
Example 53
Project: tvnamer Source File: functional_runner.py
def run_tvnamer(with_files, with_flags = None, with_input = "", with_config = None, run_on_directory = False):
"""Runs tvnamer on list of file-names in with_files.
with_files is a list of strings.
with_flags is a list of command line arguments to pass to tvnamer.
with_input is the sent to tvnamer's stdin
with_config is a string containing the tvnamer to run tvnamer with.
Returns a dict with stdout, stderr and a list of files created
"""
# Create dummy files (config and episodes)
tvnpath = get_tvnamer_path()
episodes_location = make_temp_dir()
dummy_files = make_dummy_files(with_files, episodes_location)
if with_config is not None:
configfname = make_temp_config(with_config)
conf_args = ['-c', configfname]
else:
conf_args = []
if with_flags is None:
with_flags = []
if run_on_directory:
files = [episodes_location]
else:
files = dummy_files
# Construct command
cmd = [sys.executable, tvnpath] + conf_args + with_flags + files
p("Running command:")
p(" ".join(cmd))
# Copy sys.path to PYTHONPATH so same modules are available as in
# test environmen
env = os.environ.copy()
env['PYTHONPATH'] = ":".join(sys.path)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT, # All stderr to stdout
stdin = subprocess.PIPE,
env=env)
proc.stdin.write(with_input.encode("utf-8"))
output, _ = proc.communicate()
if PY2:
def unicodify(obj, encoding = "utf-8"):
if isinstance(obj, basestring):
if not isinstance(obj, unicode):
obj = unicode(obj, encoding)
return obj
output = unicodify(output)
created_files = []
for walkroot, walkdirs, walkfiles in os.walk(string_type(episodes_location)):
curlist = [os.path.join(walkroot, name) for name in walkfiles]
# Remove episodes_location from start of path
curlist = [relpath(x, episodes_location) for x in curlist]
created_files.extend(curlist)
# Clean up dummy files and config
clear_temp_dir(episodes_location)
if with_config is not None:
os.unlink(configfname)
return {
'output': output,
'files': created_files,
'returncode': proc.returncode}
0
Example 54
Project: EventGhost Source File: CheckDependencies.py
def CheckDependencies(buildSetup):
failedDeps = []
for dep in DEPENDENCIES:
dep.buildSetup = buildSetup
try:
dep.Check()
except (WrongVersion, MissingDependency):
failedDeps.append(dep)
if failedDeps and buildSetup.args.make_env and not os.environ.get("_REST"):
if not IsAdmin():
print WrapText(
"ERROR: Can't create virtual environment from a command "
"prompt without administrative privileges."
)
return False
if not VirtualEnv.Running():
if not VirtualEnv.Exists():
print "Creating our virtual environment..."
CreateVirtualEnv()
print ""
VirtualEnv.Activate()
for dep in failedDeps[:]:
print "Installing %s..." % dep.name
try:
if InstallDependency(dep): #and dep.Check():
failedDeps.remove(dep)
else:
print "ERROR: Installation of %s failed!" % dep.name
except MissingChocolatey:
print WrapText(
"ERROR: To complete installation of this package, I need "
"package manager Chocolatey, which wasn't found and "
"couldn't be installed automatically. Please install it "
"by hand and try again."
)
except MissingPip:
print WrapText(
"ERROR: To complete installation of this package, I need "
"package manager pip, which wasn't found. Note that "
"all versions of Python capable of building EventGhost "
"come bundled with pip, so please install a supported "
"version of Python and try again."
)
except MissingPowerShell:
print WrapText(
"ERROR: To complete installation of this package, I need "
"package manager Chocolatey, which can't be installed "
"without PowerShell. Please install PowerShell by hand "
"and try again."
)
print ""
VirtualEnv.Restart()
if failedDeps:
print WrapText(
"Before we can continue, the following dependencies must "
"be installed:"
)
print ""
for dep in failedDeps:
print " *", dep.name, dep.version
if dep.url:
print " Link:", dep.url
print ""
print WrapText(
"Dependencies without an associated URL can be installed via "
"`pip install [package-name]`. Dependencies in .whl format "
"can be installed via `pip install [url]`. All other dependencies "
"will need to be installed manually or via Chocolatey "
"<https://chocolatey.org/>."
)
if not buildSetup.args.make_env:
print ""
print WrapText(
"Alternately, from a command prompt with administrative "
"privileges, I can try to create a virtual environment for "
"you that satisfies all dependencies via `%s %s --make-env`."
% (basename(sys.executable).split(".")[0], sys.argv[0])
)
return not failedDeps
0
Example 55
Project: tfmesos Source File: scheduler.py
def to_task_info(self, offer, master_addr, gpu_uuids=[],
gpu_resource_type=None):
ti = Dict()
ti.task_id.value = str(self.mesos_task_id)
ti.agent_id.value = offer.agent_id.value
ti.name = '/job:%s/task:%s' % (self.job_name, self.task_index)
ti.resources = resources = []
cpus = Dict()
resources.append(cpus)
cpus.name = 'cpus'
cpus.type = 'SCALAR'
cpus.scalar.value = self.cpus
mem = Dict()
resources.append(mem)
mem.name = 'mem'
mem.type = 'SCALAR'
mem.scalar.value = self.mem
image = os.environ.get('DOCKER_IMAGE')
if image is not None:
ti.container.type = 'DOCKER'
ti.container.docker.image = image
ti.container.volumes = volumes = []
for path in ['/etc/passwd', '/etc/group']:
v = Dict()
volumes.append(v)
v.host_path = v.container_path = path
v.mode = 'RO'
for src, dst in iteritems(self.volumes):
v = Dict()
volumes.append(v)
v.container_path = dst
v.host_path = src
v.mode = 'RW'
if self.gpus and gpu_uuids and gpu_resource_type is not None:
if gpu_resource_type == 'SET':
hostname = offer.hostname
url = 'http://%s:3476/docker/cli?dev=%s' % (
hostname, urllib.parse.quote(
' '.join(gpu_uuids)
)
)
try:
ti.container.docker.parameters = parameters = []
docker_args = urllib.request.urlopen(url).read()
for arg in docker_args.split():
k, v = arg.split('=')
assert k.startswith('--')
k = k[2:]
p = Dict()
parameters.append(p)
p.key = k
p.value = v
gpus = Dict()
resources.append(gpus)
gpus.name = 'gpus'
gpus.type = 'SET'
gpus.set.item = gpu_uuids
except Exception:
logger.exception(
'fail to determine remote device parameter,'
' disable gpu resources'
)
else:
gpus = Dict()
resources.append(gpus)
gpus.name = 'gpus'
gpus.type = 'SCALAR'
gpus.scalar.value = len(gpu_uuids)
else:
if self.gpus and gpu_uuids and gpu_resource_type is not None:
if gpu_resource_type == 'SET':
gpus = Dict()
resources.append(gpus)
gpus.name = 'gpus'
gpus.type = 'SET'
gpus.set.item = gpu_uuids
else:
gpus = Dict()
resources.append(gpus)
gpus.name = 'gpus'
gpus.type = 'SCALAR'
gpus.scalar.value = len(gpu_uuids)
ti.command.shell = True
cmd = [
sys.executable, '-m', '%s.server' % __package__,
str(self.mesos_task_id), master_addr
]
ti.command.value = ' '.join(cmd)
ti.command.environment.variables = variables = []
env = Dict()
variables.append(env)
env.name = 'PYTHONPATH'
env.value = ':'.join(sys.path)
return ti
0
Example 56
Project: pygame_cffi Source File: run_tests.py
def run(*args, **kwds):
"""Run the Pygame unit test suite and return (total tests run, fails dict)
Positional arguments (optional):
The names of tests to include. If omitted then all tests are run. Test
names need not include the trailing '_test'.
Keyword arguments:
incomplete - fail incomplete tests (default False)
nosubprocess - run all test suites in the current process
(default False, use separate subprocesses)
dump - dump failures/errors as dict ready to eval (default False)
file - if provided, the name of a file into which to dump failures/errors
timings - if provided, the number of times to run each individual test to
get an average run time (default is run each test once)
exclude - A list of TAG names to exclude from the run. The items may be
comma or space separated.
show_output - show silenced stderr/stdout on errors (default False)
all - dump all results, not just errors (default False)
randomize - randomize order of tests (default False)
seed - if provided, a seed randomizer integer
multi_thread - if provided, the number of THREADS in which to run
subprocessed tests
time_out - if subprocess is True then the time limit in seconds before
killing a test (default 30)
fake - if provided, the name of the fake tests package in the
run_tests__tests subpackage to run instead of the normal
Pygame tests
python - the path to a python executable to run subprocessed tests
(default sys.executable)
interative - allow tests tagged 'interative'.
Return value:
A tuple of total number of tests run, dictionary of error information. The
dictionary is empty if no errors were recorded.
By default individual test modules are run in separate subprocesses. This
recreates normal Pygame usage where pygame.init() and pygame.quit() are
called only once per program execution, and avoids unfortunate
interactions between test modules. Also, a time limit is placed on test
execution, so frozen tests are killed when there time allotment expired.
Use the single process option if threading is not working properly or if
tests are taking too long. It is not guaranteed that all tests will pass
in single process mode.
Tests are run in a randomized order if the randomize argument is True or a
seed argument is provided. If no seed integer is provided then the system
time is used.
Individual test modules may have a corresponding *_tags.py module,
defining a __tags__ attribute, a list of tag strings used to selectively
omit modules from a run. By default only the 'interactive', 'ignore', and
'subprocess_ignore' tags are ignored. 'interactive' is for modules that
take user input, like cdrom_test.py. 'ignore' and 'subprocess_ignore' for
for disabling modules for foreground and subprocess modes respectively.
These are for disabling tests on optional modules or for experimental
modules with known problems. These modules can be run from the console as
a Python program.
This function can only be called once per Python session. It is not
reentrant.
"""
global was_run
if was_run:
raise RuntimeError("run() was already called this session")
was_run = True
options = kwds.copy()
option_nosubprocess = options.get('nosubprocess', False)
option_dump = options.pop('dump', False)
option_file = options.pop('file', None)
option_all = options.pop('all', False)
option_randomize = options.get('randomize', False)
option_seed = options.get('seed', None)
option_multi_thread = options.pop('multi_thread', 1)
option_time_out = options.pop('time_out', 120)
option_fake = options.pop('fake', None)
option_python = options.pop('python', sys.executable)
option_exclude = options.pop('exclude', ())
option_interactive = options.pop('interactive', False)
if not option_interactive and 'interactive' not in option_exclude:
option_exclude += ('interactive',)
if not option_nosubprocess and 'subprocess_ignore' not in option_exclude:
option_exclude += ('subprocess_ignore',)
elif 'ignore' not in option_exclude:
option_exclude += ('ignore',)
if sys.version_info < (3, 0, 0):
option_exclude += ('python2_ignore',)
else:
option_exclude += ('python3_ignore',)
main_dir, test_subdir, fake_test_subdir = prepare_test_env()
test_runner_py = os.path.join(test_subdir, "test_utils", "test_runner.py")
cur_working_dir = os.path.abspath(os.getcwd())
###########################################################################
# Compile a list of test modules. If fake, then compile list of fake
# xxxx_test.py from run_tests__tests
TEST_MODULE_RE = re.compile('^(.+_test)\.py$')
test_mods_pkg_name = test_pkg_name
if option_fake is not None:
test_mods_pkg_name = '.'.join([test_mods_pkg_name,
'run_tests__tests',
option_fake])
test_subdir = os.path.join(fake_test_subdir, option_fake)
working_dir = test_subdir
else:
working_dir = main_dir
# Added in because some machines will need os.environ else there will be
# false failures in subprocess mode. Same issue as python2.6. Needs some
# env vars.
test_env = os.environ
fmt1 = '%s.%%s' % test_mods_pkg_name
fmt2 = '%s.%%s_test' % test_mods_pkg_name
if args:
test_modules = [
m.endswith('_test') and (fmt1 % m) or (fmt2 % m) for m in args
]
else:
test_modules = []
for f in sorted(os.listdir(test_subdir)):
for match in TEST_MODULE_RE.findall(f):
test_modules.append(fmt1 % (match,))
###########################################################################
# Remove modules to be excluded.
tmp = test_modules
test_modules = []
for name in tmp:
tag_module_name = "%s_tags" % (name[0:-5],)
try:
tag_module = import_submodule(tag_module_name)
except ImportError:
test_modules.append(name)
else:
try:
tags = tag_module.__tags__
except AttributeError:
print ("%s has no tags: ignoring" % (tag_module_name,))
test_module.append(name)
else:
for tag in tags:
if tag in option_exclude:
print ("skipping %s (tag '%s')" % (name, tag))
break
else:
test_modules.append(name)
del tmp, tag_module_name, name
###########################################################################
# Meta results
results = {}
meta_results = {'__meta__' : {}}
meta = meta_results['__meta__']
###########################################################################
# Randomization
if option_randomize or option_seed is not None:
if option_seed is None:
option_seed = time.time()
meta['random_seed'] = option_seed
print ("\nRANDOM SEED USED: %s\n" % option_seed)
random.seed(option_seed)
random.shuffle(test_modules)
###########################################################################
# Single process mode
if option_nosubprocess:
unittest_patch.patch(**options)
options['exclude'] = option_exclude
t = time.time()
for module in test_modules:
results.update(run_test(module, **options))
t = time.time() - t
###########################################################################
# Subprocess mode
#
if not option_nosubprocess:
if is_pygame_pkg:
from pygame.tests.test_utils.async_sub import proc_in_time_or_kill
else:
from test.test_utils.async_sub import proc_in_time_or_kill
pass_on_args = ['--exclude', ','.join(option_exclude)]
for option in ['timings', 'seed']:
value = options.pop(option, None)
if value is not None:
pass_on_args.append('--%s' % option)
pass_on_args.append(str(value))
for option, value in options.items():
option = option.replace('_', '-')
if value:
pass_on_args.append('--%s' % option)
def sub_test(module):
print ('loading %s' % module)
cmd = [option_python, test_runner_py, module ] + pass_on_args
return (module,
(cmd, test_env, working_dir),
proc_in_time_or_kill(cmd, option_time_out, env=test_env,
wd=working_dir))
if option_multi_thread > 1:
def tmap(f, args):
return pygame.threads.tmap (
f, args, stop_on_error = False,
num_workers = option_multi_thread
)
else:
tmap = map
t = time.time()
for module, cmd, (return_code, raw_return) in tmap(sub_test,
test_modules):
test_file = '%s.py' % os.path.join(test_subdir, module)
cmd, test_env, working_dir = cmd
test_results = get_test_results(raw_return)
if test_results:
results.update(test_results)
else:
results[module] = {}
add_to_results = [
'return_code', 'raw_return', 'cmd', 'test_file',
'test_env', 'working_dir', 'module',
]
results[module].update(from_namespace(locals(), add_to_results))
t = time.time() - t
###########################################################################
# Output Results
#
untrusty_total, combined = combine_results(results, t)
total, fails = test_failures(results)
meta['total_tests'] = total
meta['combined'] = combined
results.update(meta_results)
if option_nosubprocess:
assert total == untrusty_total
if not option_dump:
print (combined)
else:
results = option_all and results or fails
print (TEST_RESULTS_START)
print (pformat(results))
if option_file is not None:
results_file = open(option_file, 'w')
try:
results_file.write(pformat(results))
finally:
results_file.close()
return total, fails
0
Example 57
Project: gonewilder Source File: serve.py
def run_cgi(self):
"""Execute a CGI script."""
path = self.path
dir, rest = self.cgi_info
i = path.find('/', len(dir) + 1)
while i >= 0:
nextdir = path[:i]
nextrest = path[i+1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = path.find('/', len(dir) + 1)
else:
break
# find an explicit query string, if present.
i = rest.rfind('?')
if i >= 0:
rest, query = rest[:i], rest[i+1:]
else:
query = ''
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
print rest
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(404, "No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(403, "CGI script is not a plain file (%r)" %
scriptname)
return
ispy = self.is_python(scriptname)
# print str(ispy) + "am i python?"
if not ispy:
if not (self.have_fork or self.have_popen2 or self.have_popen3):
self.send_error(403, "CGI script is not a Python script (%r)" %
scriptname)
return
if not self.is_executable(scriptfile):
self.send_error(403, "CGI script is not executable (%r)" %
scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
host = self.address_string()
if host != self.client_address[0]:
env['REMOTE_HOST'] = host
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.getheader("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = base64.decodestring(authorization[1])
except binascii.Error:
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.typeheader is None:
env['CONTENT_TYPE'] = self.headers.type
else:
env['CONTENT_TYPE'] = self.headers.typeheader
length = self.headers.getheader('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.getheader('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.getheader('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.getheaders('cookie'))
if co:
env['HTTP_COOKIE'] = ', '.join(co)
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(200, "Script output follows")
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except os.error:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non Unix - use subprocess
import subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin = subprocess.PIPE,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
env = env
)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
0
Example 58
Project: PythonScript Source File: CGIHTTPServer.py
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
i = rest.find('/')
while i >= 0:
nextdir = rest[:i]
nextrest = rest[i+1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = rest.find('/')
else:
break
# find an explicit query string, if present.
i = rest.rfind('?')
if i >= 0:
rest, query = rest[:i], rest[i+1:]
else:
query = ''
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(404, "No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(403, "CGI script is not a plain file (%r)" %
scriptname)
return
ispy = self.is_python(scriptname)
if not ispy:
if not (self.have_fork or self.have_popen2 or self.have_popen3):
self.send_error(403, "CGI script is not a Python script (%r)" %
scriptname)
return
if not self.is_executable(scriptfile):
self.send_error(403, "CGI script is not executable (%r)" %
scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
host = self.address_string()
if host != self.client_address[0]:
env['REMOTE_HOST'] = host
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.getheader("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = base64.decodestring(authorization[1])
except binascii.Error:
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.typeheader is None:
env['CONTENT_TYPE'] = self.headers.type
else:
env['CONTENT_TYPE'] = self.headers.typeheader
length = self.headers.getheader('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.getheader('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.getheader('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.getheaders('cookie'))
if co:
env['HTTP_COOKIE'] = ', '.join(co)
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(200, "Script output follows")
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except os.error:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non Unix - use subprocess
import subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin = subprocess.PIPE,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
env = env
)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
0
Example 59
def platform(aliased=0, terse=0):
""" Returns a single string identifying the underlying platform
with as much useful information as possible (but no more :).
The output is intended to be human readable rather than
machine parseable. It may look different on different
platforms and this is intended.
If "aliased" is true, the function will use aliases for
various platforms that report system names which differ from
their common names, e.g. SunOS will be reported as
Solaris. The system_alias() function is used to implement
this.
Setting terse to true causes the function to return only the
absolute minimum information needed to identify the platform.
"""
result = _platform_cache.get((aliased, terse), None)
if result is not None:
return result
# Get uname information and then apply platform specific cosmetics
# to it...
system, node, release, version, machine, processor = uname()
if machine == processor:
processor = ''
if aliased:
system, release, version = system_alias(system, release, version)
if system == 'Windows':
# MS platforms
rel, vers, csd, ptype = win32_ver(version)
if terse:
platform = _platform(system, release)
else:
platform = _platform(system, release, version, csd)
elif system in ('Linux',):
# Linux based systems
distname, distversion, distid = dist('')
if distname and not terse:
platform = _platform(system, release, machine, processor,
'with',
distname, distversion, distid)
else:
# If the distribution name is unknown check for libc vs. glibc
libcname, libcversion = libc_ver(sys.executable)
platform = _platform(system, release, machine, processor,
'with',
libcname+libcversion)
elif system == 'Java':
# Java platforms
r, v, vminfo, (os_name, os_version, os_arch) = java_ver()
if terse or not os_name:
platform = _platform(system, release, version)
else:
platform = _platform(system, release, version,
'on',
os_name, os_version, os_arch)
elif system == 'MacOS':
# MacOS platforms
if terse:
platform = _platform(system, release)
else:
platform = _platform(system, release, machine)
else:
# Generic handler
if terse:
platform = _platform(system, release)
else:
bits, linkage = architecture(sys.executable)
platform = _platform(system, release, machine,
processor, bits, linkage)
_platform_cache[(aliased, terse)] = platform
return platform
0
Example 60
def test_encoded_file(self):
# Test that tracebacks are correctly printed for encoded source files:
# - correct line number (Issue2384)
# - respect file encoding (Issue3975)
import tempfile, sys, subprocess, os
# The spawned subprocess has its stdout redirected to a PIPE, and its
# encoding may be different from the current interpreter, on Windows
# at least.
process = subprocess.Popen([sys.executable, "-c",
"import sys; print(sys.stdout.encoding)"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, stderr = process.communicate()
output_encoding = text_type(stdout, 'ascii').splitlines()[0]
def do_test(firstlines, message, charset, lineno, output_encoding):
# Raise the message in a subprocess, and catch the output
with fixtures.TempDir() as d:
TESTFN = d.path + '/fname'
output = io.open(TESTFN, "w", encoding=charset)
output.write(u("""{0}if 1:
import traceback;
raise RuntimeError('{1}')
""").format(firstlines, message))
output.close()
process = subprocess.Popen([sys.executable, TESTFN],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = process.communicate()
if output_encoding == 'None':
output_encoding = charset
stdout = stdout.decode(output_encoding).splitlines()
# The source lines are encoded with the 'backslashreplace' handler
encoded_message = message.encode(output_encoding,
'backslashreplace')
# and we just decoded them with the output_encoding.
message_ascii = encoded_message.decode(output_encoding)
err_line = u("raise RuntimeError('{0}')").format(message_ascii)
err_msg = u("RuntimeError: {0}").format(message_ascii)
if platform.python_implementation() == 'PyPy':
# PyPy includes its own top level app_main.py in the traceback.
del stdout[1]
self.assertIn(("line %s" % lineno), stdout[1],
"Invalid line number: {0!r} instead of {1}".format(
stdout[1], lineno))
self.assertTrue(stdout[2].endswith(err_line),
"Invalid traceback line: {0!r} instead of {1!r}".format(
stdout[2], err_line))
self.assertTrue(stdout[3] == err_msg,
"Invalid error message: {0!r} instead of {1!r}".format(
stdout[3], err_msg))
do_test("", "foo", "ascii", 3, output_encoding)
for charset in ("ascii", "iso-8859-1", "utf-8", "GBK"):
if charset == "ascii":
text = u("foo")
elif charset == "GBK":
text = u("\u4E02\u5100")
else:
text = u("h\xe9 ho")
do_test("# coding: {0}\n".format(charset),
text, charset, 4, output_encoding)
do_test("#!shebang\n# coding: {0}\n".format(charset),
text, charset, 5, output_encoding)
do_test(" \t\f\n# coding: {0}\n".format(charset),
text, charset, 5, output_encoding)
# Issue #18960: coding spec should has no effect
# (Fixed in 3.4)
if sys.version_info[:2] > (3, 3):
do_test(
"0\n# coding: GBK\n", u("h\xe9 ho"), 'utf-8', 5,
output_encoding)
0
Example 61
Project: raster-functions Source File: setup.py
def main():
pipURL = "http://bootstrap.pypa.io/get-pip.py"
vcURL = "http://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi"
pipExePath = path.join(path.dirname(sys.executable), r"Scripts\pip.exe")
setupHome = path.join(path.abspath(path.dirname(__file__)), "scripts")
distHome = path.join(path.abspath(path.dirname(__file__)), "dist")
try:
log("Installing PIP")
pipPyPath = path.join(setupHome, "get-pip.py")
locateFile(pipURL, pipPyPath)
call([sys.executable, pipPyPath])
if path.isfile(pipExePath):
log("PIP installed successfully")
else:
raise Exception("PIP failed")
call([pipExePath, "install", "--upgrade", "--no-index", "--find-links={0}".format(distHome), "pip"])
call([pipExePath, "install", "--upgrade", "--no-index", "--find-links={0}".format(distHome), "wheel"])
except:
die("PIP installation failed!", 1)
try:
if sys.version_info[0] == 2:
log("Installing Microsoft Visual C++ Compiler")
vcSetupPath = path.join(distHome, "VCForPython27.msi")
locateFile(vcURL, vcSetupPath)
c = ["msiexec", "/i", vcSetupPath, "/qb-"]
log("Executing: {0}".format(" ".join(c)))
call(c)
log("C++ Compiler for Python installed successfully")
except:
die("VC++ Compiler for Python installation failed!.", 4)
try:
log("Installing Python dependencies")
reqFilePath = path.join(setupHome, "requirements.txt")
if not path.isfile(reqFilePath):
die("Dependency listing file not found: {0}".format(reqFilePath), 5)
c = [pipExePath, "install", "--upgrade", "--no-index", "--find-links={0}".format(distHome), "-r", reqFilePath]
log("Executing: {0}".format(" ".join(c)))
call(c)
except:
die("Dependency installation failed!", 6)
try:
arcpy = __import__('arcpy')
info = arcpy.GetInstallInfo()
bVersionOK = True
minArcGISVersion = '10.3.1'
if info['Version'].split(".")[0] == 10 and (tuple(map(int, (info['Version'].split(".")))) < tuple(map(int, (minArcGISVersion.split("."))))):
bVersionOK = False
minProVersion = '1.0'
if info['Version'].split(".")[0] == 1 and (tuple(map(int, (info['Version'].split(".")))) < tuple(map(int, (minProVersion.split("."))))):
bVersionOK = False
if not bVersionOK:
raise Exception("No ArcGIS")
print("\n\n")
if info['Version'][0] == 10:
log("Python extensions for raster functions in ArcGIS {} {} build {} successfully installed.".format(
info['ProductName'], info['Version'], info['BuildNumber']))
else:
log("Python extensions for raster functions in {} {} build {} successfully installed.".format(
info['ProductName'], info['Version'], info['BuildNumber']))
except:
logging.warn("Unable to find ArcGIS 10.3.1/ArcGIS Pro 1.0 or above.")
log("Done.")
sleep(2)
exit(0)
0
Example 62
Project: pip-update-requirements Source File: req_install.py
def install(self, install_options, global_options=[], root=None,
prefix=None):
if self.editable:
self.install_editable(
install_options, global_options, prefix=prefix)
return
if self.is_wheel:
version = pip.wheel.wheel_version(self.source_dir)
pip.wheel.check_compatibility(version, self.name)
self.move_wheel_files(self.source_dir, root=root, prefix=prefix)
self.install_succeeded = True
return
# Extend the list of global and install options passed on to
# the setup.py call with the ones from the requirements file.
# Options specified in requirements file override those
# specified on the command line, since the last option given
# to setup.py is the one that is used.
global_options += self.options.get('global_options', [])
install_options += self.options.get('install_options', [])
if self.isolated:
global_options = list(global_options) + ["--no-user-cfg"]
temp_location = tempfile.mkdtemp('-record', 'pip-')
record_filename = os.path.join(temp_location, 'install-record.txt')
try:
install_args = [sys.executable, "-u"]
install_args.append('-c')
install_args.append(SETUPTOOLS_SHIM % self.setup_py)
install_args += list(global_options) + \
['install', '--record', record_filename]
if not self.as_egg:
install_args += ['--single-version-externally-managed']
if root is not None:
install_args += ['--root', root]
if prefix is not None:
install_args += ['--prefix', prefix]
if self.pycompile:
install_args += ["--compile"]
else:
install_args += ["--no-compile"]
if running_under_virtualenv():
py_ver_str = 'python' + sysconfig.get_python_version()
install_args += ['--install-headers',
os.path.join(sys.prefix, 'include', 'site',
py_ver_str, self.name)]
msg = 'Running setup.py install for %s' % (self.name,)
with open_spinner(msg) as spinner:
with indent_log():
call_subprocess(
install_args + install_options,
cwd=self.setup_py_dir,
show_stdout=False,
spinner=spinner,
)
if not os.path.exists(record_filename):
logger.debug('Record file %s not found', record_filename)
return
self.install_succeeded = True
if self.as_egg:
# there's no --always-unzip option we can pass to install
# command so we unable to save the installed-files.txt
return
def prepend_root(path):
if root is None or not os.path.isabs(path):
return path
else:
return change_root(root, path)
with open(record_filename) as f:
for line in f:
directory = os.path.dirname(line)
if directory.endswith('.egg-info'):
egg_info_dir = prepend_root(directory)
break
else:
logger.warning(
'Could not find .egg-info directory in install record'
' for %s',
self,
)
# FIXME: put the record somewhere
# FIXME: should this be an error?
return
new_lines = []
with open(record_filename) as f:
for line in f:
filename = line.strip()
if os.path.isdir(filename):
filename += os.path.sep
new_lines.append(
os.path.relpath(
prepend_root(filename), egg_info_dir)
)
inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
with open(inst_files_path, 'w') as f:
f.write('\n'.join(new_lines) + '\n')
finally:
if os.path.exists(record_filename):
os.remove(record_filename)
rmtree(temp_location)
0
Example 63
def main():
p = OptionParser(usage=__doc__.strip())
p.add_option("--clean", "-c", action="store_true",
help="clean source directory")
options, args = p.parse_args()
if not args:
p.error('no submodules given')
else:
dirs = ['numpy/%s' % x for x in map(os.path.basename, args)]
# Prepare
if not os.path.isdir(TEMP):
os.makedirs(TEMP)
# Set up dummy files (for building only submodules)
dummy_files = {
'__init__.py': 'from numpy.version import version as __version__',
'version.py': 'version = "1.4.0.dev"'
}
for fn, content in dummy_files.items():
fn = os.path.join(TEMP, 'numpy', fn)
if not os.path.isfile(fn):
try:
os.makedirs(os.path.dirname(fn))
except OSError:
pass
f = open(fn, 'wb+')
f.write(content.encode('ascii'))
f.close()
# Environment
pp = [os.path.abspath(TEMP)]
def getenv():
env = dict(os.environ)
env.update({'PYTHONPATH': ':'.join(pp)})
return env
# Copy
for d in dirs:
src = os.path.join(BASE, d)
dst = os.path.join(TEMP, d)
# Run 2to3
sync_2to3(dst=dst,
src=src,
patchfile=os.path.join(TEMP, os.path.basename(d) + '.patch'),
clean=options.clean)
# Run setup.py, falling back to Pdb post-mortem on exceptions
setup_py = os.path.join(dst, 'setup.py')
if os.path.isfile(setup_py):
code = """\
import pdb, sys, traceback
p = pdb.Pdb()
try:
import __main__
__main__.__dict__.update({
"__name__": "__main__", "__file__": "setup.py",
"__builtins__": __builtins__})
fp = open("setup.py", "rb")
try:
exec(compile(fp.read(), "setup.py", 'exec'))
finally:
fp.close()
except SystemExit:
raise
except:
traceback.print_exc()
t = sys.exc_info()[2]
p.interaction(None, t)
"""
ret = subprocess.call([sys.executable, '-c', code,
'build_ext', '-i'],
cwd=dst,
env=getenv())
if ret != 0:
raise RuntimeError("Build failed.")
# Run nosetests
subprocess.call(['nosetests3', '-v', d], cwd=TEMP)
0
Example 64
Project: fail2ban Source File: fail2banclienttestcase.py
@with_tmpdir
@with_kill_srv
def testClientStartBackgroundCall(self, tmp):
global INTERACT
startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
# if fast, start server process from client started direct here:
if unittest.F2B.fast: # pragma: no cover
self.execSuccess(startparams + ("start",))
else:
# start (in new process, using the same python version):
cmd = (sys.executable, pjoin(BIN, CLIENT))
logSys.debug('Start %s ...', cmd)
cmd = cmd + startparams + ("--async", "start",)
ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
self.assertTrue(len(ret) and ret[0])
# wait for server (socket and ready):
self._wait_for_srv(tmp, True, startparams=cmd)
self.assertLogged("Server ready")
self.pruneLog()
try:
# echo from client (inside):
self.execSuccess(startparams, "echo", "TEST-ECHO")
self.assertLogged("TEST-ECHO")
self.assertLogged("Exit with code 0")
self.pruneLog()
# test ping timeout:
self.execSuccess(startparams, "ping", "0.1")
self.assertLogged("Server replied: pong")
self.pruneLog()
# python 3 seems to bypass such short timeouts also,
# so suspend/resume server process and test between it...
pid = _get_pid_from_file(pjoin(tmp, "f2b.pid"))
try:
# suspend:
os.kill(pid, signal.SIGSTOP); # or SIGTSTP?
time.sleep(Utils.DEFAULT_SHORT_INTERVAL)
# test ping with short timeout:
self.execFailed(startparams, "ping", "1e-10")
finally:
# resume:
os.kill(pid, signal.SIGCONT)
self.assertLogged("timed out")
self.pruneLog()
# interactive client chat with started server:
INTERACT += [
"echo INTERACT-ECHO",
"status",
"exit"
]
self.execSuccess(startparams, "-i")
self.assertLogged("INTERACT-ECHO")
self.assertLogged("Status", "Number of jail:")
self.assertLogged("Exit with code 0")
self.pruneLog()
# test reload and restart over interactive client:
INTERACT += [
"reload",
"restart",
"exit"
]
self.execSuccess(startparams, "-i")
self.assertLogged("Reading config files:")
self.assertLogged("Shutdown successful")
self.assertLogged("Server ready")
self.assertLogged("Exit with code 0")
self.pruneLog()
# test reload missing jail (interactive):
INTERACT += [
"reload ~~unknown~jail~fail~~",
"exit"
]
self.execSuccess(startparams, "-i")
self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
self.pruneLog()
# test reload missing jail (direct):
self.execFailed(startparams, "reload", "~~unknown~jail~fail~~")
self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
self.assertLogged("Exit with code -1")
self.pruneLog()
finally:
self.pruneLog()
# stop:
self.execSuccess(startparams, "stop")
self.assertLogged("Shutdown successful")
self.assertLogged("Exit with code 0")
0
Example 65
Project: bde-tools Source File: configurehelper.py
def _configure_external_libs(self):
if len(self.build_config.external_dep) == 0:
return
self._configure_distribution_refroot()
try:
self.ctx.find_program('pkg-config', var='PKGCONFIG')
except self.ctx.errors.ConfigurationError:
Logs.warn('Could not find pkg-config on the PATH. Using the'
'built-in python based pkg-config (pykg-config) '
'instead.')
self.ctx.env['PKGCONFIG'] = [sys.executable, os.path.join(
sysutil.repo_root_path(), 'bin', 'tools', 'pykg-config',
'pykg-config.py')]
self.ctx.find_program('pkg-config', var='PKGCONFIG')
pkgconfig_args = ['--libs', '--cflags']
if 'shr' not in self.ufid.flags:
pkgconfig_args.append('--static')
# If the static build is chosen (the default), waf assumes that all
# libraries queried from pkg-config are to be built statically, which
# is not true for some libraries. We work around this issue by manually
# changing the affected libraries to be linked dynamically instead.
dl_overrides = ['pthread', 'rt', 'nsl', 'socket']
# If lib_suffix is set, we expect the pkgconfig files being depended on
# to have the same suffix as well. Since the .dep files will not have
# the suffix, we will remove the suffix from the names of the options
# loaded into the waf environment.
rename_keys = ['defines', 'includes', 'lib', 'libpath', 'stlib',
'stlibpath']
lib_suffix = self.ctx.options.lib_suffix
for lib in sorted(self.build_config.external_dep):
actual_lib = lib + str(lib_suffix or '')
help_str = """failed to find the library using pkg-config
Maybe "%s.pc" is missing from "PKG_CONFIG_PATH"? Inspect config.log in the
build output directory for details.""" % \
actual_lib
self.ctx.check_cfg(
package=actual_lib,
args=pkgconfig_args,
errmsg=help_str)
if lib_suffix:
for k in rename_keys:
key_old = (k + '_' + actual_lib).upper()
key_new = (k + '_' + lib).upper()
self.ctx.env[key_new] = self.ctx.env[key_old]
del self.ctx.env[key_old]
sl_key = ('stlib_' + lib).upper()
dl_key = ('lib_' + lib).upper()
# preserve the order of libraries
for l in dl_overrides:
if l in self.ctx.env[sl_key]:
if dl_key not in self.ctx.env:
self.ctx.env[dl_key] = []
self.ctx.env[sl_key].remove(l)
self.ctx.env[dl_key].append(l)
if lib_suffix:
defines_old = self.ctx.env['DEFINES']
defines_new = []
for d in defines_old:
index = d.find('%s=1' % lib_suffix.upper())
if index >= 0:
defines_new.append('%s=1' % d[0:index])
else:
defines_new.append(d)
self.ctx.env['DEFINES'] = defines_new
0
Example 66
Project: nikola Source File: plugin.py
def do_install(self, url, name, show_install_notes=True):
"""Download and install a plugin."""
data = self.get_json(url)
if name in data:
utils.makedirs(self.output_dir)
url = data[name]
LOGGER.info("Downloading '{0}'".format(url))
try:
zip_data = requests.get(url).content
except requests.exceptions.SSLError:
LOGGER.warning("SSL error, using http instead of https (press ^C to abort)")
time.sleep(1)
url = url.replace('https', 'http', 1)
zip_data = requests.get(url).content
zip_file = io.BytesIO()
zip_file.write(zip_data)
LOGGER.info('Extracting: {0} into {1}/'.format(name, self.output_dir))
utils.extract_all(zip_file, self.output_dir)
dest_path = os.path.join(self.output_dir, name)
else:
try:
plugin_path = utils.get_plugin_path(name)
except:
LOGGER.error("Can't find plugin " + name)
return 1
utils.makedirs(self.output_dir)
dest_path = os.path.join(self.output_dir, name)
if os.path.exists(dest_path):
LOGGER.error("{0} is already installed".format(name))
return 1
LOGGER.info('Copying {0} into plugins'.format(plugin_path))
shutil.copytree(plugin_path, dest_path)
reqpath = os.path.join(dest_path, 'requirements.txt')
if os.path.exists(reqpath):
LOGGER.notice('This plugin has Python dependencies.')
LOGGER.info('Installing dependencies with pip...')
try:
subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath))
except subprocess.CalledProcessError:
LOGGER.error('Could not install the dependencies.')
print('Contents of the requirements.txt file:\n')
with io.open(reqpath, 'r', encoding='utf-8') as fh:
print(utils.indent(fh.read(), 4 * ' '))
print('You have to install those yourself or through a '
'package manager.')
else:
LOGGER.info('Dependency installation succeeded.')
reqnpypath = os.path.join(dest_path, 'requirements-nonpy.txt')
if os.path.exists(reqnpypath):
LOGGER.notice('This plugin has third-party '
'dependencies you need to install '
'manually.')
print('Contents of the requirements-nonpy.txt file:\n')
with io.open(reqnpypath, 'r', encoding='utf-8') as fh:
for l in fh.readlines():
i, j = l.split('::')
print(utils.indent(i.strip(), 4 * ' '))
print(utils.indent(j.strip(), 8 * ' '))
print()
print('You have to install those yourself or through a package '
'manager.')
req_plug_path = os.path.join(dest_path, 'requirements-plugins.txt')
if os.path.exists(req_plug_path):
LOGGER.notice('This plugin requires other Nikola plugins.')
LOGGER.info('Installing plugins...')
try:
with io.open(req_plug_path, 'r', encoding='utf-8') as inf:
for plugname in inf.readlines():
self.do_install(url, plugname, show_install_notes)
except subprocess.CalledProcessError:
LOGGER.error('Could not install a plugin.')
print('Contents of the requirements-plugins.txt file:\n')
with io.open(req_plug_path, 'r', encoding='utf-8') as fh:
print(utils.indent(fh.read(), 4 * ' '))
print('You have to install those yourself manually.')
else:
LOGGER.info('Dependency installation succeeded.')
confpypath = os.path.join(dest_path, 'conf.py.sample')
if os.path.exists(confpypath) and show_install_notes:
LOGGER.notice('This plugin has a sample config file. Integrate it with yours in order to make this plugin work!')
print('Contents of the conf.py.sample file:\n')
with io.open(confpypath, 'r', encoding='utf-8') as fh:
if self.site.colorful:
print(utils.indent(pygments.highlight(
fh.read(), PythonLexer(), TerminalFormatter()),
4 * ' '))
else:
print(utils.indent(fh.read(), 4 * ' '))
return 0
0
Example 67
Project: pybagit Source File: bagit.py
def update(self, full=True):
""" Scans the data directory, adds any new files it finds to the
manifest, and removes any files from the manifest that it does not
find in the directory.
Args:
full (bool): Only add new files and remove missing files and skip
reindexing of previously indexed files
"""
filelist = os.listdir(self.bag_directory)
# Read old manifest so we can detect new, existing and deleted files
md5_hashes = dict()
sha1_hashes = dict()
datamanifests = [f for f in filelist
if re.match(r"^manifest-(sha1|md5)\.txt", f)]
for man in datamanifests:
man = os.path.join(self.bag_directory, man)
if 'manifest-md5.txt' in man:
for line in codecs.open(man, 'rb', self.tag_file_encoding):
hash_, file_ = line.split(' ', 1)
md5_hashes[file_] = hash_
elif 'manifest-sha1.txt' in man:
for line in codecs.open(man, 'rb', self.tag_file_encoding):
hash_, file_ = line.split(' ', 1)
sha1_hashes[file_] = hash_
if full:
os.unlink(man)
# clean up any old manifest files. We'll be regenerating them later.
tagmanifests = [f for f in filelist
if re.match(r"^tagmanifest-(sha1|md5)\.txt", f)]
for man in tagmanifests:
man = os.path.join(self.bag_directory, man)
os.unlink(man)
self.new_files = set()
self.existing_files = set()
self.removed_files = set()
for path, dirs, files in os.walk(self.data_directory):
# add an empty .keep file in empty directories.
if not dirs and not files:
open(os.path.join(path, '.keep'), 'w').close()
for name in files:
self.new_filesfile = self._sanitize_filename(name)
full_file = os.path.join(path, self.new_filesfile)
if self.new_filesfile != name:
os.rename(os.path.join(path, name), full_file)
relative_file = os.path.relpath(full_file, self.data_directory)
if relative_file in md5_hashes:
self.existing_files.add(relative_file)
elif relative_file in sha1_hashes:
self.existing_files.add(relative_file)
else:
self.new_files.add(relative_file)
known_files = set(md5_hashes) | set(sha1_hashes)
self.removed_files = known_files - self.existing_files
self.existing_files -= self.removed_files
# checksum the data directory
cmd = [
sys.executable,
self._path_to_multichecksum,
"-a", self.hash_encoding,
"-c", self.tag_file_encoding,
self.data_directory,
]
if not full:
cmd.append('-u')
p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
while p.returncode is None:
time.sleep(0.1)
p.poll()
# read in the manifest to an instance variable
self._read_manifest_to_dict()
# clean out any previous tag manifest contents.
self.tag_manifest_contents = {}
for f in ['bagit.txt', 'bag-info.txt', 'fetch.txt', self.manifest_file]:
csum = self._calculate_checksum(os.path.join(self.bag_directory, f))
relp = os.path.relpath(os.path.join(self.bag_directory, f), self.bag_directory)
self.tag_manifest_contents[relp] = csum
# write out the tag manifest
self._write_dict_to_manifest(mode="t")
# read it back in
self._read_manifest_to_dict(mode="t")
0
Example 68
Project: easybuild-easyblocks Source File: pythonpackage.py
def pick_python_cmd(req_maj_ver=None, req_min_ver=None):
"""
Pick 'python' command to use, based on specified version requirements.
If the major version is specified, it must be an exact match (==).
If the minor version is specified, it is considered a minimal minor version (>=).
List of considered 'python' commands (in order)
* 'python' available through $PATH
* 'python<major_ver>' available through $PATH
* 'python<major_ver>.<minor_ver>' available through $PATH
* Python executable used in current session (sys.executable)
"""
log = fancylogger.getLogger('pick_python_cmd', fname=False)
def check_python_cmd(python_cmd):
"""Check whether specified Python command satisfies requirements."""
# check whether specified Python command is available
if os.path.isabs(python_cmd):
if not os.path.isfile(python_cmd):
log.debug("Python command '%s' does not exist", python_cmd)
return False
else:
python_cmd_path = which(python_cmd)
if python_cmd_path is None:
log.debug("Python command '%s' not available through $PATH", python_cmd)
return False
if req_maj_ver is not None:
if req_min_ver is None:
req_majmin_ver = '%s.0' % req_maj_ver
else:
req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver)
pycode = 'import sys; print("%s.%s" % sys.version_info[:2])'
out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False)
out = out.strip()
# (strict) check for major version
maj_ver = out.split('.')[0]
if maj_ver != str(req_maj_ver):
log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver)
return False
# check for minimal minor version
if LooseVersion(out) < LooseVersion(req_majmin_ver):
log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", out, req_majmin_ver)
return False
# all check passed
log.debug("All check passed for Python command '%s'!", python_cmd)
return True
# compose list of 'python' commands to consider
python_cmds = ['python']
if req_maj_ver:
python_cmds.append('python%s' % req_maj_ver)
if req_min_ver:
python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver))
python_cmds.append(sys.executable)
log.debug("Considering Python commands: %s", ', '.join(python_cmds))
# try and find a 'python' command that satisfies the requirements
res = None
for python_cmd in python_cmds:
if check_python_cmd(python_cmd):
log.debug("Python command '%s' satisfies version requirements!", python_cmd)
if os.path.isabs(python_cmd):
res = python_cmd
else:
res = which(python_cmd)
log.debug("Absolute path to retained Python command: %s", res)
break
else:
log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on",
req_maj_ver, req_min_ver, python_cmd)
return res
0
Example 69
Project: git-lockup Source File: test_git_lockup.py
def test_setup(self):
self.basedir = self.make_basedir("Create.setup")
upstream = self.subpath("upstream")
os.makedirs(upstream)
one = self.subpath("one")
self.git("init", "--bare", subdir="upstream")
self.git("clone", os.path.abspath(upstream), os.path.abspath(one),
workdir=upstream)
self.add_change(message="initial-unsigned")
# first push needs to be explicit, since we haven't added a
# remote.origin.push refspec in .git/config yet
self.git("push", "origin", "master", subdir="one")
out = self.run_command(["git-lockup", "setup-publish"], one)
self.assertIn("the post-commit hook will now sign changes on branch 'master'", out)
self.assertIn("verifykey: vk0-", out)
vk_s = re.search(r"(vk0-\w+)", out).group(1)
#self.assertIn("you should now commit the generated 'setup-lockup'", out)
# setup-publish automatically adds setup-lockup and lockup.config
# pyflakes one/.git/lockup-tool
# pyflakes one/setup-lockup
# now that the publishing repo is configured to sign commits, adding
# a change should get a note with a signature
self.add_change(message="first-signed")
head = self.git("rev-parse", "HEAD", subdir="one")
notes = self.git("notes", "list", head, subdir="one").split("\n")
self.assertEqual(len(notes), 1, notes)
# the updated refspec should push the notes along with the commits
self.git("push", subdir="one")
# so they should be present in the upstream (bare) repo
notes = self.git("notes", "list", head, subdir="upstream").split("\n")
self.assertEqual(len(notes), 1, notes)
# cloning the repo doesn't get the notes by default
two = self.subpath("two")
self.git("clone", os.path.abspath(upstream), os.path.abspath(two),
workdir=upstream)
notes = self.git("notes", subdir="two")
self.assertEqual(notes, "")
# run the downstream setup script
out = self.run_command([sys.executable, "./setup-lockup"], two)
self.assertIn("remote 'origin' configured to use verification proxy", out)
self.assertIn("branch 'master' configured to verify with key %s" % vk_s, out)
# now downstream pulls should work, fetch notes, and check signatures
out = self.git("pull", subdir="two")
#self.assertNotIn("Could not find local refs/notes/commits", out)
#print "FIRST PULL", out
self.add_change(message="second-signed")
self.git("push", subdir="one")
out = self.git("pull", subdir="two")
#print "SECOND PULL", out
one_head = self.git("rev-parse", "HEAD", subdir="one")
two_head = self.git("rev-parse", "HEAD", subdir="two")
self.assertEqual(one_head, two_head)
# unsigned commits should be rejected by the downstream
unsigned = self.subpath("unsigned")
self.git("clone", os.path.abspath(upstream), os.path.abspath(unsigned),
workdir=upstream)
self.add_change(subdir="unsigned", message="unsigned")
unsigned_head = self.git("rev-parse", "HEAD", subdir="unsigned")
self.assertNotEqual(unsigned, one_head)
self.git("push", subdir="unsigned")
rc,out,err = self.run_failing_command(["git", "pull"],
self.subpath("two"))
self.assertEqual(rc, 1)
self.assertEqual(out, "")
self.assertIn("\nno valid signature found for branch refs/heads/master (rev %s)\n" % unsigned_head, err)
self.assertIn("\nfatal: Could not read from remote repository.\n", err)
# should fail
#print "THIRD PULL (unsigned)", out
two_head = self.git("rev-parse", "HEAD", subdir="two")
self.assertNotEqual(two_head, unsigned_head)
0
Example 70
Project: ironpython3 Source File: server.py
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
i = rest.find('/')
while i >= 0:
nextdir = rest[:i]
nextrest = rest[i+1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = rest.find('/')
else:
break
# find an explicit query string, if present.
i = rest.rfind('?')
if i >= 0:
rest, query = rest[:i], rest[i+1:]
else:
query = ''
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(404, "No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(403, "CGI script is not a plain file (%r)" %
scriptname)
return
ispy = self.is_python(scriptname)
if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
self.send_error(403, "CGI script is not executable (%r)" %
scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.parse.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.get("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = authorization[1].encode('ascii')
authorization = base64.decodebytes(authorization).\
decode('ascii')
except (binascii.Error, UnicodeError):
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.get('content-type') is None:
env['CONTENT_TYPE'] = self.headers.get_content_type()
else:
env['CONTENT_TYPE'] = self.headers['content-type']
length = self.headers.get('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.get('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.get('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.get_all('cookie', []))
cookie_str = ', '.join(co)
if cookie_str:
env['HTTP_COOKIE'] = cookie_str
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(200, "Script output follows")
self.flush_headers()
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except OSError:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non-Unix -- use subprocess
import subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env = env
)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
0
Example 71
def create_plugin():
if not options.url:
options.url = ''
if not os.path.exists(options.basepath):
print('basepath does not exist')
return
if not options.configdir:
options.configdir = deluge.common.get_default_config_dir()
options.configdir = os.path.realpath(options.configdir)
real_name = options.name
name = real_name.replace(' ', '_')
safe_name = name.lower()
if options.module_name:
safe_name = options.module_name.lower()
plugin_base = os.path.realpath(os.path.join(options.basepath, name))
deluge_namespace = os.path.join(plugin_base, 'deluge')
plugins_namespace = os.path.join(deluge_namespace, 'plugins')
src = os.path.join(plugins_namespace, safe_name)
data_dir = os.path.join(src, 'data')
python_path = sys.executable
if os.path.exists(plugin_base):
print('the directory %s already exists, delete it first' % plugin_base)
return
def write_file(path, filename, template, include_gpl=True):
plugin_args = {
'author_name': options.author_name,
'author_email': options.author_email,
'name': name,
'safe_name': safe_name,
'filename': filename,
'plugin_base': plugin_base,
'python_path': python_path,
'url': options.url,
'configdir': options.configdir,
'current_year': datetime.utcnow().year
}
filename = os.path.join(path, filename)
with open(filename, 'w') as _file:
if filename.endswith('.py') and include_gpl:
_file.write(GPL % plugin_args)
_file.write(template % plugin_args)
print('creating folders..')
os.mkdir(plugin_base)
os.mkdir(deluge_namespace)
os.mkdir(plugins_namespace)
os.mkdir(src)
os.mkdir(data_dir)
print('creating files..')
write_file(plugin_base, 'setup.py', SETUP)
write_file(deluge_namespace, '__init__.py', NAMESPACE_INIT, False)
write_file(plugins_namespace, '__init__.py', NAMESPACE_INIT, False)
write_file(src, '__init__.py', INIT)
write_file(src, 'gtkui.py', GTKUI)
write_file(src, 'webui.py', WEBUI)
write_file(src, 'core.py', CORE)
write_file(src, 'common.py', COMMON)
write_file(data_dir, 'config.glade', GLADE)
write_file(data_dir, '%s.js' % safe_name, DEFAULT_JS)
# add an input parameter for this?
print('building dev-link..')
write_file(plugin_base, 'create_dev_link.sh', CREATE_DEV_LINK)
dev_link_path = os.path.join(plugin_base, 'create_dev_link.sh')
os.system('chmod +x %s' % dev_link_path) # lazy..
os.system(dev_link_path)
0
Example 72
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--control',
metavar='BRANCH',
default='django-control',
help="Django version to use as control. If --vcs git refers to a git"
"branch name or commit-id. If --vcs none refers to a path."
)
parser.add_argument(
'--experiment',
metavar='BRANCH',
default='django-experiment',
help="Django version to use as experiment. If --vcs git refers to a git"
"branch name or commit-id. If --vcs none refers to a path."
)
parser.add_argument(
'--control-python',
metavar='PATH',
default=sys.executable,
help="Python executable to use as control. Can be used to test Python "
"2 vs 3 performance on the benchmarks."
)
parser.add_argument(
'--experiment-python',
metavar='PATH',
default=sys.executable,
help="Python executable to use as experiment. Can be used to test "
"Python 2 vs 3 performance on the benchmarks."
)
parser.add_argument(
'--vcs',
choices=['git', 'hg', 'none'],
default='git',
help='Specify which VCS to use for control/experiment. Set to none to '
'specify paths, not branch or commit-id\'s.'
)
parser.add_argument(
'-t', '--trials',
type=int,
default=50,
help='Number of times to run each benchmark.'
)
parser.add_argument(
'-r', '--record',
default=None,
metavar='PATH',
help='Directory to record detailed output as a series of JSON files.',
)
parser.add_argument(
'--benchmark-dir',
dest='benchmark_dir',
metavar='PATH',
default=DEFAULT_BENCHMARK_DIR,
help='Directory to inspect for benchmarks. Defaults to the benchmarks '
'included with djangobench.',
)
parser.add_argument(
'benchmarks',
metavar='name',
default=None,
help="Benchmarks to be run. Defaults to all.",
nargs='*'
)
parser.add_argument(
'-p',
'--profile-dir',
dest='profile_dir',
default=None,
metavar='PATH',
help='Directory to record profiling statistics for the control and '
'experimental run of each benchmark'
)
parser.add_argument(
'--continue-on-error',
dest='continue_on_error',
action='store_true',
help='Continue with the remaining benchmarks if any fail',
)
parser.add_argument(
'-l',
'--list',
dest='list_benchmarks',
action='store_true',
help='List all available benchmarks and exit.',
)
parser.add_argument(
'--log',
dest='loglevel',
default='WARNING',
help='Define log level, set to INFO to show executed commands. Useful '
'for debugging benchmarks.'
)
args = parser.parse_args()
numeric_level = getattr(logging, args.loglevel.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % args.loglevel)
logging.basicConfig(level=numeric_level)
if args.list_benchmarks:
print_benchmarks(args.benchmark_dir)
else:
run_benchmarks(
control=args.control,
experiment=args.experiment,
benchmark_dir=args.benchmark_dir,
benchmarks=args.benchmarks,
trials=args.trials,
vcs=None if args.vcs == 'none' else args.vcs,
record_dir=args.record,
profile_dir=args.profile_dir,
continue_on_error=args.continue_on_error,
control_python=args.control_python,
experiment_python=args.experiment_python,
)
0
Example 73
Project: stash Source File: subprocess.py
def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=None):
# vars
self._fds = []
self.returncode = None
self._worker = None
self._cwd = cwd
self._environ = (env if env is not None else {})
if isinstance(args, (str, unicode)):
self.cmd = args
else:
if args[0] == sys.executable:
# common use case
args = ["python"] + list(args)
self.cmd = l2c.list2cmdline(args)
# === setup std* ===
rfm = "rU" if universal_newlines else "rb"
# setup stdout
if stdout is None:
# use own stdout
self.stdout = None
self._sp_stdout = None
elif stdout == PIPE:
# create new pipe
rfd, wfd = os.pipe()
self._fds += [rfd, wfd]
self.stdout = os.fdopen(rfd, rfm, bufsize)
self._sp_stdout = os.fdopen(wfd, "wb")
elif isinstance(stdout, (int, long)):
# use fd
self.stdout = None
self._fds.append(stdout)
self._sp_stdout = os.fdopen(stdout, "wb")
else:
self.stdout = None
self._sp_stdout = stdout
# setup stderr
if stderr is None:
# use own stdout
self.stderr = None
self._sp_stderr = None
elif stderr == PIPE:
# create new pipe
rfd, wfd = os.pipe()
self._fds += [rfd, wfd]
self.stderr = os.fdopen(rfd, rfm, bufsize)
self._sp_stderr = os.fdopen(wfd, "wb")
elif stderr == STDOUT:
self.stderr = self.stdout
self._sp_stderr = self._sp_stdout
elif isinstance(stderr, (int, long)):
# use fd
self.stderr = None
self._fds.append(stderr)
self._sp_stderr = os.fdopen(stderr, "wb")
else:
self.stderr = None
self._sp_stderr = stderr
# setup stdin
if stdin is None:
# use own stdout
self.stdin = None
self._sp_stdin = None
elif stdin == PIPE:
# create new pipe
rfd, wfd = os.pipe()
self._fds += [rfd, wfd]
self.stdin = os.fdopen(rfd, "wb")
self._sp_stdin = os.fdopen(wfd, "rb")
elif isinstance(stdin, (int, long)):
# use fd
self.stdin = None
self._fds.append(stdin)
self._sp_stdin = os.fdopen(stdin)
else:
self.stdin = None
self._sp_stdin = stdin
# run
self._run()
0
Example 74
Project: HealthStarter Source File: req_install.py
def install(self, install_options, global_options=[], root=None):
if self.editable:
self.install_editable(install_options, global_options)
return
if self.is_wheel:
version = pip.wheel.wheel_version(self.source_dir)
pip.wheel.check_compatibility(version, self.name)
self.move_wheel_files(self.source_dir, root=root)
self.install_succeeded = True
return
# Extend the list of global and install options passed on to
# the setup.py call with the ones from the requirements file.
# Options specified in requirements file override those
# specified on the command line, since the last option given
# to setup.py is the one that is used.
global_options += self.options.get('global_options', [])
install_options += self.options.get('install_options', [])
if self.isolated:
global_options = list(global_options) + ["--no-user-cfg"]
temp_location = tempfile.mkdtemp('-record', 'pip-')
record_filename = os.path.join(temp_location, 'install-record.txt')
try:
install_args = [sys.executable]
install_args.append('-c')
install_args.append(
"import setuptools, tokenize;__file__=%r;"
"exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
".replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py
)
install_args += list(global_options) + \
['install', '--record', record_filename]
if not self.as_egg:
install_args += ['--single-version-externally-managed']
if root is not None:
install_args += ['--root', root]
if self.pycompile:
install_args += ["--compile"]
else:
install_args += ["--no-compile"]
if running_under_virtualenv():
py_ver_str = 'python' + sysconfig.get_python_version()
install_args += ['--install-headers',
os.path.join(sys.prefix, 'include', 'site',
py_ver_str, self.name)]
logger.info('Running setup.py install for %s', self.name)
with indent_log():
call_subprocess(
install_args + install_options,
cwd=self.source_dir,
show_stdout=False,
)
if not os.path.exists(record_filename):
logger.debug('Record file %s not found', record_filename)
return
self.install_succeeded = True
if self.as_egg:
# there's no --always-unzip option we can pass to install
# command so we unable to save the installed-files.txt
return
def prepend_root(path):
if root is None or not os.path.isabs(path):
return path
else:
return change_root(root, path)
with open(record_filename) as f:
for line in f:
directory = os.path.dirname(line)
if directory.endswith('.egg-info'):
egg_info_dir = prepend_root(directory)
break
else:
logger.warning(
'Could not find .egg-info directory in install record'
' for %s',
self,
)
# FIXME: put the record somewhere
# FIXME: should this be an error?
return
new_lines = []
with open(record_filename) as f:
for line in f:
filename = line.strip()
if os.path.isdir(filename):
filename += os.path.sep
new_lines.append(
make_path_relative(
prepend_root(filename), egg_info_dir)
)
inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
with open(inst_files_path, 'w') as f:
f.write('\n'.join(new_lines) + '\n')
finally:
if os.path.exists(record_filename):
os.remove(record_filename)
rmtree(temp_location)
0
Example 75
Project: bcbio-nextgen Source File: tophat.py
def tophat_align(fastq_file, pair_file, ref_file, out_base, align_dir, data,
names=None):
"""
run alignment using Tophat v2
"""
config = data["config"]
options = get_in(config, ("resources", "tophat", "options"), {})
options = _set_fusion_mode(options, config)
options = _set_quality_flag(options, data)
options = _set_transcriptome_option(options, data, ref_file)
options = _set_cores(options, config)
options = _set_rg_options(options, names)
options = _set_stranded_flag(options, config)
ref_file, runner = _determine_aligner_and_reference(ref_file, config)
# fusion search does not work properly with Bowtie2
if options.get("fusion-search", False):
ref_file = ref_file.replace("/bowtie2", "/bowtie")
if _tophat_major_version(config) == 1:
raise NotImplementedError("Tophat versions < 2.0 are not supported, please "
"download the newest version of Tophat here: "
"http://tophat.cbcb.umd.edu")
if _ref_version(ref_file) == 1 or options.get("fusion-search", False):
options["bowtie1"] = True
out_dir = os.path.join(align_dir, "%s_tophat" % out_base)
final_out = os.path.join(out_dir, "{0}.bam".format(names["sample"]))
if file_exists(final_out):
return final_out
out_file = os.path.join(out_dir, "accepted_hits.bam")
unmapped = os.path.join(out_dir, "unmapped.bam")
files = [ref_file, fastq_file]
if not file_exists(out_file):
with file_transaction(config, out_dir) as tx_out_dir:
safe_makedir(tx_out_dir)
if pair_file and not options.get("mate-inner-dist", None):
d, d_stdev = _estimate_paired_innerdist(fastq_file, pair_file,
ref_file, out_base,
tx_out_dir, data)
options["mate-inner-dist"] = d
options["mate-std-dev"] = d_stdev
files.append(pair_file)
options["output-dir"] = tx_out_dir
options["no-coverage-search"] = True
options["no-mixed"] = True
tophat_runner = sh.Command(config_utils.get_program("tophat",
config))
ready_options = {}
for k, v in options.iteritems():
ready_options[k.replace("-", "_")] = v
# tophat requires options before arguments,
# otherwise it silently ignores them
tophat_ready = tophat_runner.bake(**ready_options)
cmd = "%s %s" % (sys.executable, str(tophat_ready.bake(*files)))
do.run(cmd, "Running Tophat on %s and %s." % (fastq_file, pair_file), None)
if pair_file and _has_alignments(out_file):
fixed = _fix_mates(out_file, os.path.join(out_dir, "%s-align.bam" % out_base),
ref_file, config)
else:
fixed = out_file
fixed_unmapped = _fix_unmapped(fixed, unmapped, data)
fixed = merge_unmapped(fixed, fixed_unmapped, config)
fixed = _add_rg(fixed, config, names)
fixed = bam.sort(fixed, config)
picard = broad.runner_from_path("picard", config)
# set the contig order to match the reference file so GATK works
fixed = picard.run_fn("picard_reorder", fixed, data["sam_ref"],
os.path.splitext(fixed)[0] + ".picard.bam")
fixed = fix_insert_size(fixed, config)
if not file_exists(final_out):
symlink_plus(fixed, final_out)
return final_out
0
Example 76
Project: qibuild Source File: venv.py
def configure_virtualenv(config, python_worktree, build_worktree=None,
remote_packages=None, site_packages=True,
python_executable=None, env=None):
""" Main entry point. Called by ``qipy bootstrap``
:param: remote_packages List of third-party packages to add in the virtualenv
:param: site_packages Allow access to global site packages
"""
ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
if not remote_packages:
remote_packages = list()
# create a new virtualenv
python_worktree.config = config
venv_path = python_worktree.venv_path
pip = python_worktree.pip
if not python_executable:
python_executable = sys.executable
virtualenv_py = virtualenv.__file__
if virtualenv_py.endswith(".pyc"):
virtualenv_py = virtualenv_py[:-1]
cmd = [python_executable, virtualenv_py]
cmd.append(venv_path)
if site_packages:
cmd.append("--system-site-packages")
if not env:
env = os.environ.copy()
# Make sure PYTHONHOME is never set when we call bootstrap
env.pop("PYTHONHOME", None)
try:
qisys.command.call(cmd, env=env)
except qisys.command.CommandFailedException:
ui.error("Failed to create virtualenv")
return False
ui.info(ui.blue, "::", ui.reset, "Adding python projects")
# Write a qi.pth file containing path to C/C++ extensions and
# path to pure python modules or packages
pure_python_ok = handle_pure_python(venv_path, python_worktree, env=env)
if build_worktree:
handle_extensions(venv_path, python_worktree, build_worktree)
handle_modules(venv_path, python_worktree)
if not pure_python_ok:
ui.info(ui.red, "Failed to add some python projects")
ui.info()
binaries_path = virtualenv.path_locations(venv_path)[-1]
pip_binary = os.path.join(binaries_path, "pip")
remote_ok = True
if remote_packages:
ui.info(ui.blue, "::", ui.reset,
"Adding other requirements: " + ", ".join(remote_packages))
cmd = [pip_binary, "install"]
if not ui.CONFIG["verbose"]:
cmd.append("--quiet")
cmd.extend(remote_packages)
rc = qisys.command.call(cmd, ignore_ret_code=True, env=env)
remote_ok = (rc == 0)
if not remote_ok:
ui.info(ui.red, "Failed to add some third party requirements")
ui.info()
projects_with_requirements = list()
for project in python_worktree.python_projects:
path = os.path.join(project.path, "requirements.txt")
if os.path.isfile(path):
projects_with_requirements.append(project)
ui.info(ui.blue, "::", ui.reset,
"Installing deps from requirements.txt files")
requirements_ok = True
for i, project in enumerate(projects_with_requirements):
ui.info_count(i, len(projects_with_requirements),
ui.blue, project.name)
cmd = [pip_binary, "install"]
if not ui.CONFIG["verbose"]:
cmd.append("--quiet")
path = os.path.join(project.path, "requirements.txt")
cmd.extend(["--requirement", path])
rc = qisys.command.call(cmd, ignore_ret_code=True, env=env)
requirements_ok = (rc == 0)
ui.info()
res = (pure_python_ok and remote_ok and requirements_ok)
if res:
ui.info(ui.green, "Done")
else:
ui.error("Bootstrap failed")
return res
0
Example 77
Project: comtypes Source File: register.py
def _registry_entries(self, cls):
"""Return a sequence of tuples containing registry entries.
The tuples must be (key, subkey, name, value).
Required entries:
=================
_reg_clsid_ - a string or GUID instance
_reg_clsctx_ - server type(s) to register
Optional entries:
=================
_reg_desc_ - a string
_reg_progid_ - a string naming the progid, typically 'MyServer.MyObject.1'
_reg_novers_progid_ - version independend progid, typically 'MyServer.MyObject'
_reg_typelib_ - an tuple (libid, majorversion, minorversion) specifying a typelib.
_reg_threading_ - a string specifying the threading model
Note that the first part of the progid string is typically the
IDL library name of the type library containing the coclass.
"""
HKCR = _winreg.HKEY_CLASSES_ROOT
# table format: rootkey, subkey, valuename, value
table = []
append = lambda *args: table.append(args)
# basic entry - names the comobject
reg_clsid = str(cls._reg_clsid_) # that's the only required attribute for registration
reg_desc = getattr(cls, "_reg_desc_", "")
if not reg_desc:
# Simple minded algorithm to construct a description from
# the progid:
reg_desc = getattr(cls, "_reg_novers_progid_", "") or \
getattr(cls, "_reg_progid_", "")
if reg_desc:
reg_desc = reg_desc.replace(".", " ")
append(HKCR, "CLSID\\%s" % reg_clsid, "", reg_desc)
reg_progid = getattr(cls, "_reg_progid_", None)
if reg_progid:
# for ProgIDFromCLSID:
append(HKCR, "CLSID\\%s\\ProgID" % reg_clsid, "", reg_progid) # 1
# for CLSIDFromProgID
if reg_desc:
append(HKCR, reg_progid, "", reg_desc) # 2
append(HKCR, "%s\\CLSID" % reg_progid, "", reg_clsid) # 3
reg_novers_progid = getattr(cls, "_reg_novers_progid_", None)
if reg_novers_progid:
append(HKCR, "CLSID\\%s\\VersionIndependentProgID" % reg_clsid, # 1a
"", reg_novers_progid)
if reg_desc:
append(HKCR, reg_novers_progid, "", reg_desc) # 2a
append(HKCR, "%s\\CurVer" % reg_novers_progid, "", reg_progid) #
append(HKCR, "%s\\CLSID" % reg_novers_progid, "", reg_clsid) # 3a
clsctx = getattr(cls, "_reg_clsctx_", 0)
if clsctx & comtypes.CLSCTX_LOCAL_SERVER \
and not hasattr(sys, "frozendllhandle"):
exe = sys.executable
if " " in exe:
exe = '"%s"' % exe
if not hasattr(sys, "frozen"):
if not __debug__:
exe = "%s -O" % exe
script = os.path.abspath(sys.modules[cls.__module__].__file__)
if " " in script:
script = '"%s"' % script
append(HKCR, "CLSID\\%s\\LocalServer32" % reg_clsid, "", "%s %s" % (exe, script))
else:
append(HKCR, "CLSID\\%s\\LocalServer32" % reg_clsid, "", "%s" % exe)
# Register InprocServer32 only when run from script or from
# py2exe dll server, not from py2exe exe server.
if clsctx & comtypes.CLSCTX_INPROC_SERVER \
and getattr(sys, "frozen", None) in (None, "dll"):
append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
"", self._get_serverdll())
# only for non-frozen inproc servers the PythonPath/PythonClass is needed.
if not hasattr(sys, "frozendllhandle") \
or not comtypes.server.inprocserver._clsid_to_class:
append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
"PythonClass", self._get_full_classname(cls))
append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
"PythonPath", self._get_pythonpath(cls))
reg_threading = getattr(cls, "_reg_threading_", None)
if reg_threading is not None:
append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
"ThreadingModel", reg_threading)
reg_tlib = getattr(cls, "_reg_typelib_", None)
if reg_tlib is not None:
append(HKCR, "CLSID\\%s\\Typelib" % reg_clsid, "", reg_tlib[0])
return table
0
Example 78
Project: coala Source File: LinterTest.py
def test_process_output_regex(self):
# Also test the case when an unknown severity is matched.
test_output = ("12:4-14:0-Serious issue (error) -> ORIGIN=X -> D\n"
"0:0-0:1-This is a warning (warning) -> ORIGIN=Y -> A\n"
"813:77-1024:32-Just a note (info) -> ORIGIN=Z -> C\n"
"0:0-0:0-Some unknown sev (???) -> ORIGIN=W -> B\n")
regex = (r"(?P<line>\d+):(?P<column>\d+)-"
r"(?P<end_line>\d+):(?P<end_column>\d+)-"
r"(?P<message>.*) \((?P<severity>.*)\) -> "
r"ORIGIN=(?P<origin>.*) -> (?P<additional_info>.*)")
uut = (linter(sys.executable,
output_format="regex",
output_regex=regex)
(self.EmptyTestLinter)
(self.section, None))
uut.warn = Mock()
sample_file = "some-file.xtx"
results = list(uut.process_output(test_output, sample_file, [""]))
expected = [Result.from_values("EmptyTestLinter (X)",
"Serious issue",
sample_file,
12, 4, 14, 0,
RESULT_SEVERITY.MAJOR,
additional_info="D"),
Result.from_values("EmptyTestLinter (Y)",
"This is a warning",
sample_file,
0, 0, 0, 1,
RESULT_SEVERITY.NORMAL,
additional_info="A"),
Result.from_values("EmptyTestLinter (Z)",
"Just a note",
sample_file,
813, 77, 1024, 32,
RESULT_SEVERITY.INFO,
additional_info="C"),
Result.from_values("EmptyTestLinter (W)",
"Some unknown sev",
sample_file,
0, 0, 0, 0,
RESULT_SEVERITY.NORMAL,
additional_info="B")]
self.assertEqual(results, expected)
uut.warn.assert_called_once_with(
"'???' not found in severity-map. Assuming "
"`RESULT_SEVERITY.NORMAL`.")
# Test when providing a sequence as output.
test_output = ["",
"12:4-14:0-Serious issue (error) -> ORIGIN=X -> XYZ\n"]
results = list(uut.process_output(test_output, sample_file, [""]))
expected = [Result.from_values("EmptyTestLinter (X)",
"Serious issue",
sample_file,
12, 4, 14, 0,
RESULT_SEVERITY.MAJOR,
additional_info="XYZ")]
self.assertEqual(results, expected)
# Test with using `result_message` parameter.
uut = (linter(sys.executable,
output_format="regex",
output_regex=regex,
result_message="Hello world")
(self.EmptyTestLinter)
(self.section, None))
results = list(uut.process_output(test_output, sample_file, [""]))
expected = [Result.from_values("EmptyTestLinter (X)",
"Hello world",
sample_file,
12, 4, 14, 0,
RESULT_SEVERITY.MAJOR,
additional_info="XYZ")]
self.assertEqual(results, expected)
0
Example 79
Project: evennia Source File: evennia_launcher.py
def server_operation(mode, service, interactive, profiler, logserver=False):
"""
Handle argument options given on the command line.
Args:
mode (str): Start/stop/restart and so on.
service (str): "server", "portal" or "all".
interactive (bool). Use interactive mode or daemon.
profiler (bool): Run the service under the profiler.
logserver (bool, optional): Log Server data to logfile
specified by settings.SERVER_LOG_FILE.
"""
cmdstr = [sys.executable, EVENNIA_RUNNER]
errmsg = "The %s does not seem to be running."
if mode == 'start':
# launch the error checker. Best to catch the errors already here.
error_check_python_modules()
# starting one or many services
if service == 'server':
if profiler:
cmdstr.append('--pserver')
if interactive:
cmdstr.append('--iserver')
if logserver:
cmdstr.append('--logserver')
cmdstr.append('--noportal')
elif service == 'portal':
if profiler:
cmdstr.append('--pportal')
if interactive:
cmdstr.append('--iportal')
cmdstr.append('--noserver')
django.core.management.call_command(
'collectstatic', verbosity=1, interactive=False)
else:
# all
# for convenience we don't start logging of
# portal, only of server with this command.
if profiler:
# this is the common case
cmdstr.append('--pserver')
if interactive:
cmdstr.append('--iserver')
if logserver:
cmdstr.append('--logserver')
django.core.management.call_command(
'collectstatic', verbosity=1, interactive=False)
cmdstr.extend([
GAMEDIR, TWISTED_BINARY, SERVER_LOGFILE,
PORTAL_LOGFILE, HTTP_LOGFILE])
# start the server
process = Popen(cmdstr, env=getenv())
if interactive:
try:
process.wait()
except KeyboardInterrupt:
server_operation("stop", "portal", False, False)
return
finally:
print(NOTE_KEYBOARDINTERRUPT)
elif mode == 'reload':
# restarting services
if os.name == 'nt':
print(
"Restarting from command line is not supported under Windows. "
"Log into the game to restart.")
return
if service == 'server':
kill(SERVER_PIDFILE, SIG, "Server reloaded.",
errmsg % 'Server', SERVER_RESTART, restart=True)
elif service == 'portal':
print(
"Note: Portal usually doesnt't need to be reloaded unless you "
"are debugging in interactive mode. If Portal was running in "
"default Daemon mode, it cannot be restarted. In that case "
"you have to restart it manually with 'evennia.py "
"start portal'")
kill(PORTAL_PIDFILE, SIG,
"Portal reloaded (or stopped, if it was in daemon mode).",
errmsg % 'Portal', PORTAL_RESTART, restart=True)
else:
# all
# default mode, only restart server
kill(SERVER_PIDFILE, SIG,
"Server reload.",
errmsg % 'Server', SERVER_RESTART, restart=True)
elif mode == 'stop':
# stop processes, avoiding reload
if service == 'server':
kill(SERVER_PIDFILE, SIG,
"Server stopped.", errmsg % 'Server', SERVER_RESTART)
elif service == 'portal':
kill(PORTAL_PIDFILE, SIG,
"Portal stopped.", errmsg % 'Portal', PORTAL_RESTART)
else:
kill(PORTAL_PIDFILE, SIG,
"Portal stopped.", errmsg % 'Portal', PORTAL_RESTART)
kill(SERVER_PIDFILE, SIG,
"Server stopped.", errmsg % 'Server', SERVER_RESTART)
0
Example 80
def run(self):
os.chdir(ROOT)
log.info("getting revision number")
call('{} ./burpui -m manage compile_translation'.format(sys.executable).split())
rev = 'stable'
try:
branch = check_output('sed s@^.*/@@g .git/HEAD'.split()).rstrip()
ver = open(os.path.join('burpui', 'VERSION')).read().rstrip()
if branch and 'dev' in ver:
rev = branch
try:
with open('burpui/RELEASE', 'w') as f:
f.write(rev)
except:
pass
except:
pass
# Not sure bower was a great idea...
keep = [
'burpui/static/vendor/bootswatch/slate/bootstrap.min.css',
'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.eot',
'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.svg',
'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.ttf',
'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.woff',
'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.woff2',
'burpui/static/vendor/nvd3/build/nv.d3.min.css',
'burpui/static/vendor/datatables.net-bs/css/dataTables.bootstrap.min.css',
'burpui/static/vendor/datatables.net-responsive-bs/css/responsive.bootstrap.min.css',
'burpui/static/vendor/jquery.fancytree/dist/skin-bootstrap/ui.fancytree.min.css',
'burpui/static/vendor/bootstrap-switch/dist/css/bootstrap3/bootstrap-switch.min.css',
'burpui/static/vendor/ui-select/dist/select.min.css',
'burpui/static/vendor/jquery/dist/jquery.min.js',
'burpui/static/vendor/jquery-ui/jquery-ui.min.js',
'burpui/static/vendor/bootstrap/dist/js/bootstrap.min.js',
'burpui/static/vendor/typeahead.js/dist/typeahead.bundle.min.js',
'burpui/static/vendor/d3/d3.min.js',
'burpui/static/vendor/nvd3/build/nv.d3.min.js',
'burpui/static/vendor/datatables.net/js/jquery.dataTables.min.js',
'burpui/static/vendor/datatables.net-bs/js/dataTables.bootstrap.min.js',
'burpui/static/vendor/datatables.net-responsive/js/dataTables.responsive.js',
'burpui/static/vendor/datatables.net-responsive-bs/js/responsive.bootstrap.js',
'burpui/static/vendor/jquery.fancytree/dist/jquery.fancytree-all.min.js',
'burpui/static/vendor/jquery-file-download/src/Scripts/jquery.fileDownload.js',
'burpui/static/vendor/lodash/dist/lodash.min.js',
'burpui/static/vendor/angular/angular.min.js',
'burpui/static/vendor/angular-route/angular-route.min.js',
'burpui/static/vendor/angular-sanitize/angular-sanitize.min.js',
'burpui/static/vendor/angular-resource/angular-resource.min.js',
'burpui/static/vendor/angular-animate/angular-animate.min.js',
'burpui/static/vendor/bootstrap-switch/dist/js/bootstrap-switch.min.js',
'burpui/static/vendor/angular-bootstrap-switch/dist/angular-bootstrap-switch.min.js',
'burpui/static/vendor/ui-select/dist/select.min.js',
'burpui/static/vendor/angular-strap/dist/angular-strap.min.js',
'burpui/static/vendor/angular-strap/dist/angular-strap.tpl.min.js',
'burpui/static/vendor/angular-onbeforeunload/build/angular-onbeforeunload.js',
'burpui/static/vendor/moment/min/moment.min.js',
'burpui/static/vendor/moment/locale/fr.js',
'burpui/static/vendor/angular-ui-calendar/src/calendar.js',
'burpui/static/vendor/fullcalendar/dist/fullcalendar.min.css',
'burpui/static/vendor/fullcalendar/dist/fullcalendar.print.css',
'burpui/static/vendor/fullcalendar/dist/fullcalendar.min.js',
'burpui/static/vendor/fullcalendar/dist/gcal.js',
'burpui/static/vendor/fullcalendar/dist/lang/fr.js',
'burpui/static/vendor/angular-bootstrap/ui-bootstrap.min.js',
'burpui/static/vendor/angular-bootstrap/ui-bootstrap-tpls.min.js',
'burpui/static/vendor/components-font-awesome/css/font-awesome.min.css',
'burpui/static/vendor/components-font-awesome/fonts/FontAwesome.otf',
'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.eot',
'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.svg',
'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.ttf',
'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.woff',
'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.woff2',
]
dirlist = []
for dirname, subdirs, files in os.walk('burpui/static/vendor'):
for filename in files:
path = os.path.join(dirname, filename)
_, ext = os.path.splitext(path)
if os.path.isfile(path) and path not in keep and filename not in ['bower.json', 'package.json']:
if (rev == 'stable' and ext == '.map') or ext != '.map':
os.unlink(path)
elif os.path.isdir(path):
dirlist.append(path)
dirlist.sort(reverse=True)
for d in dirlist:
if os.path.isdir(d) and not os.listdir(d):
os.rmdir(d)
0
Example 81
Project: eventlet Source File: server.py
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
path = dir + '/' + rest
i = path.find('/', len(dir)+1)
while i >= 0:
nextdir = path[:i]
nextrest = path[i+1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = path.find('/', len(dir)+1)
else:
break
# find an explicit query string, if present.
rest, _, query = rest.partition('?')
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(
HTTPStatus.NOT_FOUND,
"No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not a plain file (%r)" % scriptname)
return
ispy = self.is_python(scriptname)
if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not executable (%r)" % scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.parse.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.get("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = authorization[1].encode('ascii')
authorization = base64.decodebytes(authorization).\
decode('ascii')
except (binascii.Error, UnicodeError):
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.get('content-type') is None:
env['CONTENT_TYPE'] = self.headers.get_content_type()
else:
env['CONTENT_TYPE'] = self.headers['content-type']
length = self.headers.get('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.get('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.get('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.get_all('cookie', []))
cookie_str = ', '.join(co)
if cookie_str:
env['HTTP_COOKIE'] = cookie_str
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(HTTPStatus.OK, "Script output follows")
self.flush_headers()
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except OSError:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non-Unix -- use subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env = env
)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
0
Example 82
Project: crossbar Source File: process.py
def _start_native_worker(self, wtype, id, options=None, details=None):
assert(wtype in ['router', 'container', 'websocket-testee'])
# prohibit starting a worker twice
#
if id in self._workers:
emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(id)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.worker_already_running', emsg)
# check worker options
#
options = options or {}
try:
if wtype == 'router':
checkconfig.check_router_options(options)
elif wtype == 'container':
checkconfig.check_container_options(options)
elif wtype == 'websocket-testee':
checkconfig.check_websocket_testee_options(options)
else:
raise Exception("logic error")
except Exception as e:
emsg = "Could not start native worker: invalid configuration ({})".format(e)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
# allow override Python executable from options
#
if 'python' in options:
exe = options['python']
# the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy
#
if not os.path.isabs(exe):
emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format(exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
# of course the path must exist and actually be executable
#
if not (os.path.isfile(exe) and os.access(exe, os.X_OK)):
emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format(exe)
self.log.error(emsg)
raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
else:
exe = sys.executable
# assemble command line for forking the worker
#
# all native workers (routers and containers for now) start
# from the same script in crossbar/worker/process.py -- we're
# invoking via "-m" so that .pyc files, __pycache__ etc work
# properly.
args = [exe, "-u", "-m", "crossbar.worker.process"]
args.extend(["--cbdir", self._node._cbdir])
args.extend(["--worker", str(id)])
args.extend(["--realm", self._realm])
args.extend(["--type", wtype])
args.extend(["--loglevel", get_global_log_level()])
# allow override worker process title from options
#
if options.get('title', None):
args.extend(['--title', options['title']])
# forward explicit reactor selection
#
if 'reactor' in options and sys.platform in options['reactor']:
args.extend(['--reactor', options['reactor'][sys.platform]])
# FIXME
# elif self._node.options.reactor:
# args.extend(['--reactor', self._node.options.reactor])
# create worker process environment
#
worker_env = create_process_env(options)
# We need to use the same PYTHONPATH we were started with, so we can
# find the Crossbar we're working with -- it may not be the same as the
# one on the default path
worker_env["PYTHONPATH"] = os.pathsep.join(sys.path)
# log name of worker
#
worker_logname = {
'router': 'Router',
'container': 'Container',
'websocket-testee': 'WebSocketTestee'
}.get(wtype, 'Worker')
# topic URIs used (later)
#
if wtype == 'router':
starting_topic = 'crossbar.node.on_router_starting'
started_topic = 'crossbar.node.on_router_started'
elif wtype == 'container':
starting_topic = 'crossbar.node.on_container_starting'
started_topic = 'crossbar.node.on_container_started'
elif wtype == 'websocket-testee':
starting_topic = 'crossbar.node.on_websocket_testee_starting'
started_topic = 'crossbar.node.on_websocket_testee_started'
else:
raise Exception("logic error")
# add worker tracking instance to the worker map ..
#
if wtype == 'router':
worker = RouterWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))
elif wtype == 'container':
worker = ContainerWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))
elif wtype == 'websocket-testee':
worker = WebSocketTesteeWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))
else:
raise Exception("logic error")
self._workers[id] = worker
# create a (custom) process endpoint.
#
if platform.isWindows():
childFDs = None # Use the default Twisted ones
else:
# The communication between controller and container workers is
# using WAMP running over 2 pipes.
# For controller->container traffic this runs over FD 0 (`stdin`)
# and for the container->controller traffic, this runs over FD 3.
#
# Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for
# container->controller traffic, so that components running in the
# container which happen to write to `stdout` or `stderr` do not
# interfere with the container-controller communication.
childFDs = {0: "w", 1: "r", 2: "r", 3: "r"}
ep = WorkerProcessEndpoint(
self._node._reactor, exe, args, env=worker_env, worker=worker,
childFDs=childFDs)
# ready handling
#
def on_ready_success(id):
self.log.info("{worker} with ID '{id}' and PID {pid} started",
worker=worker_logname, id=worker.id, pid=worker.pid)
self._node._reactor.addSystemEventTrigger(
'before', 'shutdown',
self._cleanup_worker, self._node._reactor, worker,
)
worker.status = 'started'
worker.started = datetime.utcnow()
started_info = {
u'id': worker.id,
u'status': worker.status,
u'started': utcstr(worker.started),
u'who': worker.who,
}
# FIXME: make start of stats printer dependent on log level ..
worker.log_stats(5.)
self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))
return started_info
def on_ready_error(err):
del self._workers[worker.id]
emsg = 'Failed to start native worker: {}'.format(err.value)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.cannot_start", emsg, worker.getlog())
worker.ready.addCallbacks(on_ready_success, on_ready_error)
def on_exit_success(_):
self.log.info("Node worker {worker.id} ended successfully", worker=worker)
worker.log_stats(0)
del self._workers[worker.id]
return True
def on_exit_error(err):
self.log.info("Node worker {worker.id} ended with error ({err})", worker=worker, err=err)
worker.log_stats(0)
del self._workers[worker.id]
return False
def check_for_shutdown(was_successful):
shutdown = False
# automatically shutdown node whenever a worker ended (successfully, or with error)
#
if checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info("Node worker ended, and trigger '{trigger}' active", trigger=checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT)
shutdown = True
# automatically shutdown node when worker ended with error
#
if not was_successful and checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers:
self.log.info("Node worker ended with error, and trigger '{trigger}' active", trigger=checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR)
shutdown = True
# automatically shutdown node when no more workers are left
#
if len(self._workers) == 0 and checkconfig.NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers:
self.log.info("No more node workers running, and trigger '{trigger}' active", trigger=checkconfig.NODE_SHUTDOWN_ON_LAST_WORKER_EXIT)
shutdown = True
# initiate shutdown (but only if we are not already shutting down)
#
if shutdown:
if not self._shutdown_requested:
self.log.info("Node shutting down ..")
self.shutdown()
else:
# ignore: shutdown already initiated ..
self.log.info("Node is already shutting down.")
else:
self.log.info(
"Node will continue to run (node shutdown triggers active: {triggers})",
triggers=self._node._node_shutdown_triggers,
)
d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error)
d_on_exit.addBoth(check_for_shutdown)
# create a transport factory for talking WAMP to the native worker
#
transport_factory = create_native_worker_client_factory(self._node._router_session_factory, worker.ready, worker.exit)
transport_factory.noisy = False
self._workers[id].factory = transport_factory
# now (immediately before actually forking) signal the starting of the worker
#
starting_info = {
u'id': id,
u'status': worker.status,
u'created': utcstr(worker.created),
u'who': worker.who,
}
# the caller gets a progressive result ..
if details.progress:
details.progress(starting_info)
# .. while all others get an event
self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))
# now actually fork the worker ..
#
self.log.info("Starting {worker} with ID '{id}'...",
worker=worker_logname, id=id)
self.log.debug("{worker} '{id}' command line is '{cmdline}'",
worker=worker_logname, id=id, cmdline=' '.join(args))
d = ep.connect(transport_factory)
def on_connect_success(proto):
# this seems to be called immediately when the child process
# has been forked. even if it then immediately fails because
# e.g. the executable doesn't even exist. in other words,
# I'm not sure under what conditions the deferred will errback ..
pid = proto.transport.pid
self.log.debug("Native worker process connected with PID {pid}",
pid=pid)
# note the PID of the worker
worker.pid = pid
# proto is an instance of NativeWorkerClientProtocol
worker.proto = proto
worker.status = 'connected'
worker.connected = datetime.utcnow()
def on_connect_error(err):
# not sure when this errback is triggered at all ..
self.log.error("Interal error: connection to forked native worker failed ({err})", err=err)
# in any case, forward the error ..
worker.ready.errback(err)
d.addCallbacks(on_connect_success, on_connect_error)
return worker.ready
0
Example 83
def get_stack_trace(self, source=None, script=None,
breakpoint='PyObject_Print',
cmds_after_breakpoint=None,
import_site=False):
'''
Run 'python -c SOURCE' under gdb with a breakpoint.
Support injecting commands after the breakpoint is reached
Returns the stdout from gdb
cmds_after_breakpoint: if provided, a list of strings: gdb commands
'''
# We use "set breakpoint pending yes" to avoid blocking with a:
# Function "foo" not defined.
# Make breakpoint pending on future shared library load? (y or [n])
# error, which typically happens python is dynamically linked (the
# breakpoints of interest are to be found in the shared library)
# When this happens, we still get:
# Function "PyObject_Print" not defined.
# emitted to stderr each time, alas.
# Initially I had "--eval-command=continue" here, but removed it to
# avoid repeated print breakpoints when traversing hierarchical data
# structures
# Generate a list of commands in gdb's language:
commands = ['set breakpoint pending yes',
'break %s' % breakpoint,
# The tests assume that the first frame of printed
# backtrace will not contain program counter,
# that is however not guaranteed by gdb
# therefore we need to use 'set print address off' to
# make sure the counter is not there. For example:
# #0 in PyObject_Print ...
# is assumed, but sometimes this can be e.g.
# #0 0x00003fffb7dd1798 in PyObject_Print ...
'set print address off',
'run']
# GDB as of 7.4 onwards can distinguish between the
# value of a variable at entry vs current value:
# http://sourceware.org/gdb/onlinedocs/gdb/Variables.html
# which leads to the selftests failing with errors like this:
# AssertionError: 'v@entry=()' != '()'
# Disable this:
if (gdb_major_version, gdb_minor_version) >= (7, 4):
commands += ['set print entry-values no']
if cmds_after_breakpoint:
commands += cmds_after_breakpoint
else:
commands += ['backtrace']
# print commands
# Use "commands" to generate the arguments with which to invoke "gdb":
args = ["gdb", "--batch", "-nx"]
args += ['--eval-command=%s' % cmd for cmd in commands]
args += ["--args",
sys.executable]
if not import_site:
# -S suppresses the default 'import site'
args += ["-S"]
if source:
args += ["-c", source]
elif script:
args += [script]
# print args
# print ' '.join(args)
# Use "args" to invoke gdb, capturing stdout, stderr:
out, err = run_gdb(*args, PYTHONHASHSEED='0')
errlines = err.splitlines()
unexpected_errlines = []
# Ignore some benign messages on stderr.
ignore_patterns = (
'Function "%s" not defined.' % breakpoint,
"warning: no loadable sections found in added symbol-file"
" system-supplied DSO",
"warning: Unable to find libthread_db matching"
" inferior's thread library, thread debugging will"
" not be available.",
"warning: Cannot initialize thread debugging"
" library: Debugger service failed",
'warning: Could not load shared library symbols for '
'linux-vdso.so',
'warning: Could not load shared library symbols for '
'linux-gate.so',
'warning: Could not load shared library symbols for '
'linux-vdso64.so',
'Do you need "set solib-search-path" or '
'"set sysroot"?',
'warning: Source file is more recent than executable.',
# Issue #19753: missing symbols on System Z
'Missing separate debuginfo for ',
'Try: zypper install -C ',
)
for line in errlines:
if not line.startswith(ignore_patterns):
unexpected_errlines.append(line)
# Ensure no unexpected error messages:
self.assertEqual(unexpected_errlines, [])
return out
0
Example 84
Project: VisTrails Source File: engine_manager.py
def start_engines(self, nb=None, prompt="Number of engines to start"):
"""Start some engines locally
"""
c = self.ensure_controller()
if c is None:
if qt_available:
QtGui.QMessageBox.warning(
None,
"No controller",
"Can't start engines: couldn't connect to a "
"controller")
print "parallelflow: no controller, not starting engines"
else:
if not nb and qt_available:
nb, res = QtGui.QInputDialog.getInt(
None,
"Start engines",
prompt,
1, # value
1, # min
16) # max
if not res:
return
elif nb is None:
nb = 1
print "parallelflow: about to start %d engines" % nb
if qt_available:
bar = QtGui.QProgressDialog(
"Starting engines...",
None,
0, nb)
def progress(n):
bar.setValue(n)
bar.show()
else:
def progress(n): pass
progress(0)
init_engines = set(c.ids)
# Start the processes
starting = set()
for i in xrange(nb):
proc, res = self.start_process(
None,
sys.executable,
'-m',
'IPython.parallel.apps.ipengineapp',
'--profile=%s' % self.profile)
starting.add(proc)
# Wait for each one to either fail or connect
failed = []
connected = 0
while connected < len(starting):
connected = len(set(c.ids) - init_engines)
progress(len(failed) + connected)
time.sleep(0.5)
for p in list(starting):
res = p.poll()
if res is not None:
failed.append(res)
starting.remove(p)
if failed:
nb_failed = len(failed)
if nb_failed > 3:
failed = "%s, ..." % (', '.join('%d' % f for f in failed))
else:
failed = ', '.join('%d' % f for f in failed)
if qt_available:
QtGui.QMessageBox.critical(
None,
"Error",
"%d engine(s) exited with codes: %s" % (
nb_failed, failed))
print "parallelflow: %d engine(s) exited with codes: %s" % (
nb_failed, failed)
self.started_engines.update(starting)
if qt_available:
bar.hide()
bar.deleteLater()
print "parallelflow: %d engines started" % nb
0
Example 85
def resubmit(self):
job = self.getJobObject()
inw = job.getInputWorkspace()
outw = job.getOutputWorkspace()
statusfilename = outw.getPath('__jobstatus__')
try:
os.remove(statusfilename)
except OSError as x:
if x.errno != 2:
logger.warning("OSError:" + str(x))
scriptpath = inw.getPath('__jobscript__')
#stderr_option = '-e '+str(outw.getPath())+'stderr'
#stdout_option = '-o '+str(outw.getPath())+'stdout'
# FIX from Alex Richards - see Savannah #87477
stdout_option = self.config['stdoutConfig'] % str(outw.getPath())
stderr_option = self.config['stderrConfig'] % str(outw.getPath())
queue_option = ''
if self.queue:
queue_option = '-q ' + str(self.queue)
try:
jobnameopt = "-" + self.config['jobnameopt']
except Exception as err:
logger.debug("Err: %s" % str(err))
jobnameopt = False
if self.extraopts:
import re
for opt in re.compile(r'(-\w+)').findall(self.extraopts):
if opt in ('-o', '-e', '-oo', '-eo'):
logger.warning("option %s is forbidden", opt)
return False
if self.queue and opt == '-q':
logger.warning("option %s is forbidden if queue is defined ( queue = '%s')", opt, self.queue)
return False
if jobnameopt and opt == jobnameopt:
jobnameopt = False
queue_option = queue_option + " " + self.extraopts
if jobnameopt and job.name != '':
# PBS doesn't like names with spaces
tmp_name = job.name
if isType(self, PBS):
tmp_name = tmp_name.replace(" ", "_")
queue_option = queue_option + " " + \
jobnameopt + " " + "'%s'" % (tmp_name)
# bugfix #16646
if self.config['shared_python_executable']:
import sys
script_cmd = "%s %s" % (sys.executable, scriptpath)
else:
script_cmd = scriptpath
command_str = self.config['submit_str'] % (
inw.getPath(), queue_option, stderr_option, stdout_option, script_cmd)
self.command_string = command_str
rc, soutfile = self.command(command_str)
logger.debug('from command get rc: "%d"', rc)
if rc == 0:
with open(soutfile) as sout_file:
sout = sout_file.read()
import re
m = re.compile(
self.config['submit_res_pattern'], re.M).search(sout)
if m is None:
logger.warning('could not match the output and extract the Batch job identifier!')
logger.warning('command output \n %s ', sout)
else:
self.id = m.group('id')
try:
queue = m.group('queue')
if self.queue != queue:
if self.queue:
logger.warning('you requested queue "%s" but the job was submitted to queue "%s"', self.queue, queue)
logger.warning('command output \n %s ', sout)
else:
logger.info('using default queue "%s"', queue)
self.actualqueue = queue
except IndexError:
logger.info('could not match the output and extract the Batch queue name')
else:
with open(soutfile) as sout_file:
logger.warning(sout_file.read())
return rc == 0
0
Example 86
Project: vent Source File: menu_launcher.py
def processmenu(path_dirs, menu, parent=None):
""" processes the execution of the interaction sent to the menu """
optioncount = len(menu['options'])
exitmenu = False
while not exitmenu:
getin = runmenu(menu, parent)
if getin == optioncount:
exitmenu = True
elif menu['options'][getin]['type'] in [PROMPT]:
curses.def_prog_mode()
os.system('reset')
screen.clear()
if prompt():
os.system(menu['options'][getin]['command'])
screen.clear()
curses.reset_prog_mode()
try:
curses.curs_set(1)
curses.curs_set(0)
except Exception as e:
pass
elif menu['options'][getin]['type'] in [COMMAND, CONFIRM]:
curses.def_prog_mode()
os.system('reset')
screen.clear()
os.system(menu['options'][getin]['command'])
if menu['options'][getin]['type'] == CONFIRM:
if menu['title'] == "Remove Plugins":
exitmenu = True
confirm()
screen.clear()
curses.reset_prog_mode()
try:
curses.curs_set(1)
curses.curs_set(0)
except Exception as e:
pass
elif menu['options'][getin]['type'] in [INFO, DISPLAY]:
pass
elif menu['options'][getin]['type'] == INPUT:
if menu['options'][getin]['title'] == "Add Plugins":
plugin_url = get_param("Enter the HTTPS Git URL that contains the new plugins, e.g. https://github.com/CyberReboot/vent-plugins.git:")
curses.def_prog_mode()
os.system('reset')
screen.clear()
if not "https://" in plugin_url:
os.system("echo No plugins added, url is not formatted correctly.")
os.system("echo Please use a git url, e.g. https://github.com/CyberReboot/vent-plugins.git")
else:
os.system("python2.7 "+path_dirs.data_dir+"plugin_parser.py add_plugins "+plugin_url+" "+path_dirs.base_dir+" "+path_dirs.data_dir)
confirm()
screen.clear()
os.execl(sys.executable, sys.executable, *sys.argv)
elif menu['options'][getin]['title'] == "Files":
filename = get_param("Enter the name of the processed file to lookup logs for:")
curses.def_prog_mode()
os.system('reset')
os.system("clear")
screen.clear()
try:
# ensure directory exists
os.system("if [ ! -d /tmp/vent_logs ]; then mkdir /tmp/vent_logs; fi;")
# check logs exist for that file
found = call("python2.7 "+path_dirs.info_dir+"get_logs.py -f "+filename+" | grep "+filename, shell=True)
if found == 0:
# print logs
os.system("python2.7 "+path_dirs.info_dir+"get_logs.py -f "+filename+" | tee /tmp/vent_logs/vent_file_"+filename+" | less")
else:
os.system("echo \"No logs found for that file.\" | less")
except Exception as e:
os.system("echo \"Error retrieving logs for that file.\" | less")
screen.clear()
curses.reset_prog_mode()
try:
curses.curs_set(1)
curses.curs_set(0)
except Exception as e:
pass
elif menu['options'][getin]['type'] == MENU:
if menu['options'][getin]['title'] == "Remove Plugins":
screen.clear()
installed_plugins = get_installed_plugin_repos(path_dirs, CONFIRM, "remove")
processmenu(path_dirs, installed_plugins, menu)
screen.clear()
elif menu['options'][getin]['title'] == "Show Installed Plugins":
screen.clear()
installed_plugins = get_installed_plugin_repos(path_dirs, DISPLAY, "")
processmenu(path_dirs, installed_plugins, menu)
screen.clear()
elif menu['options'][getin]['title'] == "Update Plugins":
screen.clear()
installed_plugins = get_installed_plugin_repos(path_dirs, CONFIRM, "update")
processmenu(path_dirs, installed_plugins, menu)
screen.clear()
elif menu['options'][getin]['title'] == "Status":
screen.clear()
plugins = get_plugin_status(path_dirs)
processmenu(path_dirs, plugins, menu)
screen.clear()
elif menu['options'][getin]['title'] == "Services":
screen.clear()
containers = get_container_menu(path_dirs)
processmenu(path_dirs, containers, menu)
screen.clear()
elif menu['options'][getin]['title'] == "Namespaces":
screen.clear()
namespaces = get_namespace_menu(path_dirs)
processmenu(path_dirs, namespaces, menu)
screen.clear()
else:
screen.clear()
processmenu(path_dirs, menu['options'][getin], menu)
screen.clear()
elif menu['options'][getin]['type'] == EXITMENU:
exitmenu = True
0
Example 87
Project: gsutil Source File: test_update.py
@unittest.skipUnless(CERTIFICATE_VALIDATION_ENABLED,
'Test requires https certificate validation enabled.')
def test_update(self):
"""Tests that the update command works or raises proper exceptions."""
if os.environ.get('CLOUDSDK_WRAPPER') == '1':
stderr = self.RunGsUtil(['update'], stdin='n',
return_stderr=True, expected_status=1)
self.assertIn('update command is disabled for Cloud SDK', stderr)
return
if gslib.IS_PACKAGE_INSTALL:
# The update command is not present when installed via package manager.
stderr = self.RunGsUtil(['update'], return_stderr=True, expected_status=1)
self.assertIn('Invalid command', stderr)
return
# Create two temp directories, one of which we will run 'gsutil update' in
# to pull the changes from the other.
tmpdir_src = self.CreateTempDir()
tmpdir_dst = self.CreateTempDir()
# Copy gsutil to both source and destination directories.
gsutil_src = os.path.join(tmpdir_src, 'gsutil')
gsutil_dst = os.path.join(tmpdir_dst, 'gsutil')
# Path when executing from tmpdir (Windows doesn't support in-place rename)
gsutil_relative_dst = os.path.join('gsutil', 'gsutil')
shutil.copytree(GSUTIL_DIR, gsutil_src)
# Copy specific files rather than all of GSUTIL_DIR so we don't pick up temp
# working files left in top-level directory by gsutil developers (like tags,
# .git*, etc.)
os.makedirs(gsutil_dst)
for comp in ('CHANGES.md', 'CHECKSUM', 'COPYING', 'gslib', 'gsutil',
'gsutil.py', 'MANIFEST.in', 'README.md', 'setup.py', 'test',
'third_party', 'VERSION'):
if os.path.isdir(os.path.join(GSUTIL_DIR, comp)):
func = shutil.copytree
else:
func = shutil.copyfile
func(os.path.join(GSUTIL_DIR, comp), os.path.join(gsutil_dst, comp))
# Create a fake version number in the source so we can verify it in the
# destination.
expected_version = '17.25'
src_version_file = os.path.join(gsutil_src, 'VERSION')
self.assertTrue(os.path.exists(src_version_file))
with open(src_version_file, 'w') as f:
f.write(expected_version)
# Create a tarball out of the source directory and copy it to a bucket.
src_tarball = os.path.join(tmpdir_src, 'gsutil.test.tar.gz')
normpath = os.path.normpath
try:
# We monkey patch os.path.normpath here because the tarfile module
# normalizes the ./gsutil path, but the update command expects the tar
# file to be prefixed with . This preserves the ./gsutil path.
os.path.normpath = lambda fname: fname
tar = tarfile.open(src_tarball, 'w:gz')
tar.add(gsutil_src, arcname='./gsutil')
tar.close()
finally:
os.path.normpath = normpath
prefix = [sys.executable] if sys.executable else []
# Run with an invalid gs:// URI.
p = subprocess.Popen(prefix + ['gsutil', 'update', 'gs://pub'],
cwd=gsutil_dst, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
self.assertEqual(p.returncode, 1)
self.assertIn('update command only works with tar.gz', stderr)
# Run with non-existent gs:// URI.
p = subprocess.Popen(
prefix + ['gsutil', 'update', 'gs://pub/Jdjh38)(;.tar.gz'],
cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
self.assertEqual(p.returncode, 1)
self.assertIn('NotFoundException', stderr)
# Run with file:// URI wihout -f option.
p = subprocess.Popen(prefix + ['gsutil', 'update', suri(src_tarball)],
cwd=gsutil_dst, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
self.assertEqual(p.returncode, 1)
self.assertIn('command does not support', stderr)
# Run with a file present that was not distributed with gsutil.
with open(os.path.join(gsutil_dst, 'userdata.txt'), 'w') as fp:
fp.write('important data\n')
p = subprocess.Popen(prefix + ['gsutil', 'update', '-f', suri(src_tarball)],
cwd=gsutil_dst, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stdin=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
# Clean up before next test, and before assertions so failure doesn't leave
# this file around.
os.unlink(os.path.join(gsutil_dst, 'userdata.txt'))
self.assertEqual(p.returncode, 1)
self.assertIn(
'The update command cannot run with user data in the gsutil directory',
stderr.replace(os.linesep, ' '))
# Determine whether we'll need to decline the analytics prompt.
analytics_prompt = not (
os.path.exists(_UUID_FILE_PATH) or
boto.config.get_value('GSUtil', 'disable_analytics_prompt'))
update_input = 'n\r\ny\r\n' if analytics_prompt else 'y\r\n'
# Now do the real update, which should succeed.
p = subprocess.Popen(prefix + [gsutil_relative_dst, 'update', '-f',
suri(src_tarball)],
cwd=tmpdir_dst, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stdin=subprocess.PIPE)
(_, stderr) = p.communicate(input=update_input)
p.stdout.close()
p.stderr.close()
self.assertEqual(p.returncode, 0, msg=(
'Non-zero return code (%d) from gsutil update. stderr = \n%s' %
(p.returncode, stderr)))
# Verify that version file was updated.
dst_version_file = os.path.join(tmpdir_dst, 'gsutil', 'VERSION')
with open(dst_version_file, 'r') as f:
self.assertEqual(f.read(), expected_version)
# If the analytics prompt was given, that means we disabled analytics. We
# should reset to the default by deleting the UUID file.
if analytics_prompt:
os.unlink(_UUID_FILE_PATH)
0
Example 88
Project: ClusterRunner Source File: base_config_loader.py
def configure_defaults(self, conf):
"""
This is the base configuration. All default configuration values belong here. These values may be overridden by
other configurations.
:type conf: Configuration
"""
if getattr(sys, 'frozen', False):
root_directory = dirname(sys.executable) # frozen
else:
root_directory = dirname(dirname(dirname(dirname(realpath(__file__))))) # unfrozen
conf.set('secret', None) # This must be overridden by conf or the service will not start for security reasons
conf.set('root_directory', root_directory) # the root directory of the application
conf.set('main_executable_path', sys.argv[0])
conf.set('version', autoversioning.get_version())
# If the user installed ClusterRunner manually, the directories used by the manual install should take
# precedence over the default install location (the home directory).
static_configured_base_directory = join('/var', 'lib', 'clusterrunner')
if isdir(static_configured_base_directory):
base_directory = static_configured_base_directory
else:
base_directory = join(expanduser('~/'), '.clusterrunner')
# where all of the clusterrunner specific files will be stored (other than source code)
conf.set('base_directory', base_directory)
# the path to the clusterrunner config file. We have to specify this in defaults since it cannot depend on
# values in the file it refers to.
conf.set('config_file', join(base_directory, 'clusterrunner.conf'))
# Where the clusterrunner service will save the process id to. These settings are set in base_config_loader
# and not in master_config_loader and slave_config_loader because CLI tools such as "clusterrunner stop"
# needs to read in these settings.
conf.set('master_pid_file', join(base_directory, '.clusterrunner_master.pid'))
conf.set('slave_pid_file', join(base_directory, '.clusterrunner_slave.pid'))
# contains symlinks to build-specific repos
conf.set('build_symlink_directory', join('/tmp', 'clusterrunner_build_symlinks'))
# where the repos are cloned to
conf.set('repo_directory', None)
conf.set('project_yaml_filename', 'clusterrunner.yaml')
conf.set('log_file', None)
conf.set('log_level', 'DEBUG')
conf.set('max_log_file_size', 1024 * 1024 * 50) # 50mb
conf.set('max_log_file_backups', 5)
# set eventlog file conf values to None to disable eventlogs by default
conf.set('log_filename', 'clusterrunner_default.log')
conf.set('eventlog_filename', 'eventlog_default.log')
conf.set('eventlog_file', None)
conf.set('max_eventlog_file_size', 1024 * 1024 * 50) # 50mb
conf.set('max_eventlog_file_backups', 5)
conf.set('hostname', platform.node())
conf.set('master_hostname', 'localhost')
conf.set('master_port', '43000')
conf.set('slaves', ['localhost'])
# Strict host key checking on git remote operations, disabled by default
conf.set('git_strict_host_key_checking', False)
# CORS support - a regex to match against allowed API request origins, or None to disable CORS
conf.set('cors_allowed_origins_regex', None)
# Helper executables
bin_dir = join(root_directory, 'bin')
conf.set('git_askpass_exe', join(bin_dir, 'git_askpass.sh'))
conf.set('git_ssh_exe', join(bin_dir, 'git_ssh.sh'))
# How slaves get the project
# Slaves would get the project from master if set to True. Otherwise it would just get the project in
# the same way how the master gets the project.
conf.set('get_project_from_master', True)
# Should we have shallow or full clones of the repository?
# The master must have full clones, as slaves fetch from the master, and one cannot fetch from a shallow clone.
conf.set('shallow_clones', False)
0
Example 89
def crondance(apppath, ctype='soft',startup=False):
cron_path = os.path.join(apppath,'admin','cron')
token = Token(cron_path)
cronmaster = token.acquire(startup=startup)
if not cronmaster:
return
now_s = time.localtime()
checks=(('min',now_s.tm_min),
('hr',now_s.tm_hour),
('mon',now_s.tm_mon),
('dom',now_s.tm_mday),
('dow',now_s.tm_wday))
apps = [x for x in os.listdir(apppath)
if os.path.isdir(os.path.join(apppath, x))]
for app in apps:
apath = os.path.join(apppath,app)
cronpath = os.path.join(apath, 'cron')
crontab = os.path.join(cronpath, 'crontab')
if not os.path.exists(crontab):
continue
try:
f = open(crontab, 'rt')
cronlines = f.readlines()
lines = [x for x in cronlines if x.strip() and x[0]!='#']
tasks = [parsecronline(cline) for cline in lines]
except Exception, e:
logging.error('WEB2PY CRON: crontab read error %s' % e)
continue
for task in tasks:
commands = [sys.executable]
if os.path.exists('web2py.py'):
commands.append('web2py.py')
citems = [(k in task and not v in task[k]) for k,v in checks]
task_min= task.get('min',[])
if not task:
continue
elif not startup and task_min == [-1]:
continue
elif task_min != [-1] and reduce(lambda a,b: a or b, citems):
continue
logging.info('WEB2PY CRON (%s): %s executing %s in %s at %s' \
% (ctype, app, task.get('cmd'),
os.getcwd(), datetime.datetime.now()))
action, command, models = False, task['cmd'], ''
if command.startswith('**'):
(action,models,command) = (True,'',command[2:])
elif command.startswith('*'):
(action,models,command) = (True,'-M',command[1:])
else:
action=False
if action and command.endswith('.py'):
commands.extend(('-P',
'-N',models,
'-S',app,
'-a','"<recycle>"',
'-R',command))
shell = True
elif action:
commands.extend(('-P',
'-N',models,
'-S',app+'/'+command,
'-a','"<recycle>"'))
shell = True
else:
commands = command
shell = False
try:
print time.ctime()+' '+ctype+' CRON RUNNING %s' % commands
cronlauncher(commands, shell=shell).start()
except Exception, e:
logging.warning(
'WEB2PY CRON: Execution error for %s: %s' \
% (task.get('cmd'), e))
token.release()
0
Example 90
Project: gcc-python-plugin Source File: run-test-suite.py
def run_test(testdir):
# Compile each 'input.c', using 'script.py'
# Assume success and empty stdout; compare against expected stderr, or empty if file not present
inputfiles = get_source_files(testdir)
outfile = os.path.join(testdir, 'output.o')
script_py = os.path.join(testdir, 'script.py')
out = TestStream(os.path.join(testdir, 'stdout.txt'))
err = TestStream(os.path.join(testdir, 'stderr.txt'))
cp = configparser.SafeConfigParser()
metadatapath = os.path.join(testdir, 'metadata.ini')
cp.read([metadatapath])
if cp.has_section('WhenToRun'):
if cp.has_option('WhenToRun', 'required_features'):
required_features = cp.get('WhenToRun', 'required_features').split()
for feature in required_features:
if feature not in features:
raise ValueError('%s in %s not found in %s'
% (feature, metadatapath, config_h))
if not features[feature]:
raise SkipTest('required feature %s not available in %s'
% (feature, config_h))
env = dict(os.environ)
env['LC_ALL'] = 'C'
# Generate the command-line for invoking gcc:
args = [CC]
if len(inputfiles) == 1:
args += ['-c'] # (don't run the linker)
else:
args += ['-fPIC', '-shared']
# Force LTO when there's more than one source file:
args += ['-flto', '-flto-partition=none']
if GCC_VERSION >= 4008:
# GCC 4.8 started showing the source line where the problem is,
# followed by another line showing a caret indicating column.
# This is a great usability feature, but totally breaks our "gold"
# output, so turn it off for running tests:
args += ['-fno-diagnostics-show-caret']
# Similarly, the macro expansion tracking is great for usability,
# but breaks the "gold" output, so we disable it during tests:
args += ['-ftrack-macro-expansion=0']
args += ['-o', outfile]
args += ['-fplugin=%s' % os.path.abspath('%s.so' % PLUGIN_NAME),
'-fplugin-arg-%s-script=%s' % (PLUGIN_NAME, script_py)]
# Force the signedness of char so that the tests have consistent
# behavior across all archs:
args += ['-fsigned-char']
# Special-case: add the python include dir (for this runtime) if the C code
# uses Python.h:
def uses_python_headers():
for inputfile in inputfiles:
with open(inputfile, 'r') as f:
code = f.read()
if '#include <Python.h>' in code:
return True
if uses_python_headers():
args += ['-I' + get_python_inc()]
# If there's a getopts.py, run it to get additional test-specific
# command-line options:
getopts_py = os.path.join(testdir, 'getopts.py')
if os.path.exists(getopts_py):
p = Popen([sys.executable, getopts_py], stdout=PIPE, stderr=PIPE)
opts_out, opts_err = p.communicate()
if six.PY3:
opts_out = opts_out.decode()
opts_err = opts_err.decode()
c = p.wait()
if c != 0:
raise CommandError()
args += opts_out.split()
# and the source files go at the end:
args += inputfiles
if options.show:
# Show the gcc invocation:
print(' '.join(args))
# Invoke the compiler:
p = Popen(args, env=env, stdout=PIPE, stderr=PIPE)
out.actual, err.actual = p.communicate()
if six.PY3:
out.actual = out.actual.decode()
err.actual = err.actual.decode()
#print 'out: %r' % out.actual
#print 'err: %r' % err.actual
exitcode_actual = p.wait()
if options.show:
# then the user wants to see the gcc invocation directly
sys.stdout.write(out.actual)
sys.stderr.write(err.actual)
# Expected exit code
# By default, we expect success if the expected stderr is empty, and
# and failure if it's non-empty.
# This can be overridden if the test has a metadata.ini, by setting
# exitcode within the [ExpectedBehavior] section:
if err.expdata == '':
exitcode_expected = 0
else:
exitcode_expected = 1
if cp.has_section('ExpectedBehavior'):
if cp.has_option('ExpectedBehavior', 'exitcode'):
exitcode_expected = cp.getint('ExpectedBehavior', 'exitcode')
# Check exit code:
if exitcode_actual != exitcode_expected:
sys.stderr.write(out.diff('stdout'))
sys.stderr.write(err.diff('stderr'))
raise CompilationError(out.actual, err.actual, p, args)
if exitcode_expected == 0:
assert os.path.exists(outfile)
out.check_for_diff(out.actual, err.actual, p, args, 'stdout', WRITEBACK)
err.check_for_diff(out.actual, err.actual, p, args, 'stderr', WRITEBACK)
0
Example 91
Project: python-3parclient Source File: HPE3ParClient_base.py
def setUp(self, withSSH=False, withFilePersona=False):
self.withSSH = withSSH
self.withFilePersona = withFilePersona
cwd = os.path.dirname(os.path.abspath(
inspect.getfile(inspect.currentframe())))
if self.unitTest:
self.printHeader('Using flask ' + self.flask_url)
parsed_url = urlparse(self.flask_url)
userArg = '-user=%s' % self.user
passwordArg = '-password=%s' % self.password
portArg = '-port=%s' % parsed_url.port
script = 'HPE3ParMockServer_flask.py'
path = "%s/%s" % (cwd, script)
try:
self.mockServer = subprocess.Popen([sys.executable,
path,
userArg,
passwordArg,
portArg],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE
)
except Exception:
pass
time.sleep(1)
if self.withFilePersona:
self.cl = file_client.HPE3ParFilePersonaClient(self.flask_url)
else:
self.cl = client.HPE3ParClient(self.flask_url)
if self.withSSH:
self.printHeader('Using paramiko SSH server on port %s' %
self.ssh_port)
ssh_script = 'HPE3ParMockServer_ssh.py'
ssh_path = "%s/%s" % (cwd, ssh_script)
self.mockSshServer = subprocess.Popen([sys.executable,
ssh_path,
str(self.ssh_port)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
time.sleep(1)
else:
if withFilePersona:
self.printHeader('Using 3PAR %s with File Persona' %
self.url_3par)
self.cl = file_client.HPE3ParFilePersonaClient(self.url_3par)
else:
self.printHeader('Using 3PAR ' + self.url_3par)
self.cl = client.HPE3ParClient(self.url_3par)
if self.withSSH:
# This seems to slow down the test cases, so only use this when
# requested
if self.unitTest:
# The mock SSH server can be accessed at 0.0.0.0.
ip = '0.0.0.0'
else:
parsed_3par_url = urlparse(self.url_3par)
ip = parsed_3par_url.hostname.split(':').pop()
try:
# Now that we don't do keep-alive, the conn_timeout needs to
# be set high enough to avoid sometimes slow response in
# the File Persona tests.
self.cl.setSSHOptions(
ip,
self.user,
self.password,
port=self.ssh_port,
conn_timeout=500,
known_hosts_file=self.known_hosts_file,
missing_key_policy=self.missing_key_policy)
except Exception as ex:
print(ex)
self.fail("failed to start ssh client")
# Setup remote copy target
if self.run_remote_copy:
parsed_3par_url = urlparse(self.secondary_url_3par)
ip = parsed_3par_url.hostname.split(':').pop()
self.secondary_cl = client.HPE3ParClient(self.secondary_url_3par)
try:
self.secondary_cl.setSSHOptions(
ip,
self.secondary_user,
self.secondary_password,
port=self.ssh_port,
conn_timeout=500,
known_hosts_file=self.known_hosts_file,
missing_key_policy=self.missing_key_policy)
except Exception as ex:
print(ex)
self.fail("failed to start ssh client")
self.secondary_cl.login(self.secondary_user,
self.secondary_password)
if self.debug:
self.cl.debug_rest(True)
self.cl.login(self.user, self.password)
if not self.port:
ports = self.cl.getPorts()
ports = [p for p in ports['members']
if p['linkState'] == 4 and # Ready
('device' not in p or not p['device']) and
p['mode'] == self.cl.PORT_MODE_TARGET]
self.port = ports[0]['portPos']
0
Example 92
Project: visionegg Source File: Configuration.py
def __init__(self):
"""Load global Vision Egg configuration information."""
cfg = ConfigParser.ConfigParser()
if hasattr(sys,'argv') and sys.executable == sys.argv[0]: # Windows binary
self.VISIONEGG_SYSTEM_DIR = os.curdir
self.VISIONEGG_USER_DIR = os.curdir
else:
# non-standard VisionEgg installations
try:
self.VISIONEGG_SYSTEM_DIR = os.environ['VISIONEGG_SYSTEM_DIR']
except KeyError:
self.VISIONEGG_SYSTEM_DIR = os.path.split(__file__)[0]
user_dir = os.path.expanduser("~")
self.VISIONEGG_USER_DIR = os.path.join(user_dir,"VisionEgg")
# See if there's an environment variable for the config file
if 'VISIONEGG_CONFIG_FILE' in os.environ.keys():
configFile = os.environ['VISIONEGG_CONFIG_FILE']
else:
# Is there one in VISIONEGG_USER_DIR?
configFile = os.path.join(self.VISIONEGG_USER_DIR,"VisionEgg.cfg")
if not os.path.isfile(configFile):
configFile = os.path.join(self.VISIONEGG_SYSTEM_DIR,"VisionEgg.cfg")
if not os.path.isfile(configFile):
configFile = None # No file, use defaults specified in environment variables then here
if configFile:
cfg.read(configFile)
else:
# pretend we have a config file
cfg.add_section('General')
for key in defaults.keys():
cfg.set('General',key,str(defaults[key]))
if sys.platform == 'darwin':
cfg.add_section('darwin')
for key in extra_darwin_defaults.keys():
cfg.set('darwin',key,str(extra_darwin_defaults[key]))
# Do the general stuff first
# Set the default values
for name in defaults.keys():
if name in os.environ.keys():
value = os.environ[name]
else:
value = defaults[name]
if isinstance(defaults[name], int):
if value == 'False':
value = 0
elif value == 'True':
value = 1
setattr(self,name,int(value))
elif isinstance(defaults[name], float):
setattr(self,name,float(value))
else:
setattr(self,name,value)
# Get the values from the configFile
general_options = cfg.options('General')
self._delayed_configuration_log_warnings = [] # chick and egg problem
# set defaults from config file
for option in general_options:
name = option.upper()
if name not in defaults.keys():
self._delayed_configuration_log_warnings.append(
"While reading %s: The variable \"%s\" is not (anymore) a Vision Egg variable."%(os.path.abspath(configFile),option))
continue
value = cfg.get('General',option)
if name in os.environ.keys():
value = os.environ[name]
if isinstance(defaults[name], int):
if value == 'False':
value = 0
elif value == 'True':
value = 1
setattr(self,name,int(value))
elif isinstance(defaults[name], float):
setattr(self,name,float(value))
else:
setattr(self,name,value)
# Do platform specific stuff
# Set the default values
platform_name = sys.platform
extra_name = "extra_%s_defaults"%(platform_name,)
if extra_name in globals().keys():
extra_defaults = globals()[extra_name]
for name in extra_defaults.keys():
setattr(self,name,extra_defaults[name])
# Get the values from the configFile
platform_options = cfg.options(platform_name)
for option in platform_options:
name = option.upper()
if name not in extra_defaults.keys():
raise KeyError("No Vision Egg configuration variable \"%s\""%option)
value = cfg.get(platform_name,option)
if name in os.environ.keys():
value = os.environ[name]
if isinstance(extra_defaults[name], int):
if value == 'False':
value = 0
elif value == 'True':
value = 1
setattr(self,name,int(value))
elif isinstance(extra_defaults[name], float):
setattr(self,name,float(value))
else:
setattr(self,name,value)
if(configFile):
self.VISIONEGG_CONFIG_FILE = os.path.abspath(configFile)
else:
self.VISIONEGG_CONFIG_FILE = None
0
Example 93
Project: python-hunter Source File: test_hunter.py
def test_pth_sample2(LineMatcher):
env = dict(os.environ, PYTHONHUNTER="module='__main__'")
env.pop('COVERAGE_PROCESS_START', None)
env.pop('COV_CORE_SOURCE', None)
output = subprocess.check_output(
[sys.executable, os.path.join(os.path.dirname(__file__), 'sample2.py')],
env=env,
stderr=subprocess.STDOUT,
)
lm = LineMatcher(output.decode('utf-8').splitlines())
lm.fnmatch_lines([
'*tests*sample2.py:* call if __name__ == "__main__": #*',
'*tests*sample2.py:* line if __name__ == "__main__": #*',
'*tests*sample2.py:* line import functools',
'*tests*sample2.py:* line def deco(opt):',
'*tests*sample2.py:* line @deco(1)',
'*tests*sample2.py:* call def deco(opt):',
'*tests*sample2.py:* line def decorator(func):',
'*tests*sample2.py:* line return decorator',
'*tests*sample2.py:* return return decorator',
'* * ... return value: <function deco*',
'*tests*sample2.py:* line @deco(2)',
'*tests*sample2.py:* call def deco(opt):',
'*tests*sample2.py:* line def decorator(func):',
'*tests*sample2.py:* line return decorator',
'*tests*sample2.py:* return return decorator',
'* * ... return value: <function deco*',
'*tests*sample2.py:* line @deco(3)',
'*tests*sample2.py:* call def deco(opt):',
'*tests*sample2.py:* line def decorator(func):',
'*tests*sample2.py:* line return decorator',
'*tests*sample2.py:* return return decorator',
'* * ... return value: <function deco*',
'*tests*sample2.py:* call def decorator(func):',
'*tests*sample2.py:* line @functools.wraps(func)',
'*tests*sample2.py:* line return wrapper',
'*tests*sample2.py:* return return wrapper',
'* * ... return value: <function foo *',
'*tests*sample2.py:* call def decorator(func):',
'*tests*sample2.py:* line @functools.wraps(func)',
'*tests*sample2.py:* line return wrapper',
'*tests*sample2.py:* return return wrapper',
'* * ... return value: <function foo *',
'*tests*sample2.py:* call def decorator(func):',
'*tests*sample2.py:* line @functools.wraps(func)',
'*tests*sample2.py:* line return wrapper',
'*tests*sample2.py:* return return wrapper',
'* * ... return value: <function foo *',
'*tests*sample2.py:* line foo(',
"*tests*sample2.py:* line 'a*',",
"*tests*sample2.py:* line 'b'",
'*tests*sample2.py:* call @functools.wraps(func)',
'* * | def wrapper(*args):',
'*tests*sample2.py:* line return func(*args)',
'*tests*sample2.py:* call @functools.wraps(func)',
'* * | def wrapper(*args):',
'*tests*sample2.py:* line return func(*args)',
'*tests*sample2.py:* call @functools.wraps(func)',
'* * | def wrapper(*args):',
'*tests*sample2.py:* line return func(*args)',
'*tests*sample2.py:* call @deco(1)',
'* * | @deco(2)',
'* * | @deco(3)',
'* * | def foo(*args):',
'*tests*sample2.py:* line return args',
'*tests*sample2.py:* return return args',
"* * ... return value: ('a*', 'b')",
"*tests*sample2.py:* return return func(*args)",
"* * ... return value: ('a*', 'b')",
"*tests*sample2.py:* return return func(*args)",
"* * ... return value: ('a*', 'b')",
"*tests*sample2.py:* return return func(*args)",
"* * ... return value: ('a*', 'b')",
"*tests*sample2.py:* line try:",
"*tests*sample2.py:* line None(",
"*tests*sample2.py:* line 'a',",
"*tests*sample2.py:* line 'b'",
"*tests*sample2.py:* exception 'b'",
"* * ... exception value: *",
"*tests*sample2.py:* line except:",
"*tests*sample2.py:* line pass",
"*tests*sample2.py:* return pass",
"* ... return value: None",
])
0
Example 94
Project: databus Source File: CGIHTTPServer.py
def run_cgi(self):
"""Execute a CGI script."""
path = self.path
dir, rest = self.cgi_info
i = path.find('/', len(dir) + 1)
while i >= 0:
nextdir = path[:i]
nextrest = path[i+1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = path.find('/', len(dir) + 1)
else:
break
# find an explicit query string, if present.
i = rest.rfind('?')
if i >= 0:
rest, query = rest[:i], rest[i+1:]
else:
query = ''
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(404, "No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(403, "CGI script is not a plain file (%r)" %
scriptname)
return
ispy = self.is_python(scriptname)
if not ispy:
if not (self.have_fork or self.have_popen2 or self.have_popen3):
self.send_error(403, "CGI script is not a Python script (%r)" %
scriptname)
return
if not self.is_executable(scriptfile):
self.send_error(403, "CGI script is not executable (%r)" %
scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = {}
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
host = self.address_string()
if host != self.client_address[0]:
env['REMOTE_HOST'] = host
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.getheader("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = base64.decodestring(authorization[1])
except binascii.Error:
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.typeheader is None:
env['CONTENT_TYPE'] = self.headers.type
else:
env['CONTENT_TYPE'] = self.headers.typeheader
length = self.headers.getheader('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.getheader('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.getheader('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.getheaders('cookie'))
if co:
env['HTTP_COOKIE'] = ', '.join(co)
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
os.environ.update(env)
self.send_response(200, "Script output follows")
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except os.error:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, os.environ)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
elif self.have_popen2 or self.have_popen3:
# Windows -- use popen2 or popen3 to create a subprocess
import shutil
if self.have_popen3:
popenx = os.popen3
else:
popenx = os.popen2
cmdline = scriptfile
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = "%s -u %s" % (interp, cmdline)
if '=' not in query and '"' not in query:
cmdline = '%s "%s"' % (cmdline, query)
self.log_message("command: %s", cmdline)
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
files = popenx(cmdline, 'b')
fi = files[0]
fo = files[1]
if self.have_popen3:
fe = files[2]
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
fi.write(data)
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
fi.close()
shutil.copyfileobj(fo, self.wfile)
if self.have_popen3:
errors = fe.read()
fe.close()
if errors:
self.log_error('%s', errors)
sts = fo.close()
if sts:
self.log_error("CGI script exit status %#x", sts)
else:
self.log_message("CGI script exited OK")
else:
# Other O.S. -- execute script in this process
save_argv = sys.argv
save_stdin = sys.stdin
save_stdout = sys.stdout
save_stderr = sys.stderr
try:
save_cwd = os.getcwd()
try:
sys.argv = [scriptfile]
if '=' not in decoded_query:
sys.argv.append(decoded_query)
sys.stdout = self.wfile
sys.stdin = self.rfile
execfile(scriptfile, {"__name__": "__main__"})
finally:
sys.argv = save_argv
sys.stdin = save_stdin
sys.stdout = save_stdout
sys.stderr = save_stderr
os.chdir(save_cwd)
except SystemExit, sts:
self.log_error("CGI script exit status %s", str(sts))
else:
self.log_message("CGI script exited OK")
0
Example 95
def get_stack_trace(self, source=None, script=None,
breakpoint=BREAKPOINT_FN,
cmds_after_breakpoint=None,
import_site=False):
'''
Run 'python -c SOURCE' under gdb with a breakpoint.
Support injecting commands after the breakpoint is reached
Returns the stdout from gdb
cmds_after_breakpoint: if provided, a list of strings: gdb commands
'''
# We use "set breakpoint pending yes" to avoid blocking with a:
# Function "foo" not defined.
# Make breakpoint pending on future shared library load? (y or [n])
# error, which typically happens python is dynamically linked (the
# breakpoints of interest are to be found in the shared library)
# When this happens, we still get:
# Function "textiowrapper_write" not defined.
# emitted to stderr each time, alas.
# Initially I had "--eval-command=continue" here, but removed it to
# avoid repeated print breakpoints when traversing hierarchical data
# structures
# Generate a list of commands in gdb's language:
commands = ['set breakpoint pending yes',
'break %s' % breakpoint,
'run']
if cmds_after_breakpoint:
commands += cmds_after_breakpoint
else:
commands += ['backtrace']
# print commands
# Use "commands" to generate the arguments with which to invoke "gdb":
args = ["gdb", "--batch"]
args += ['--eval-command=%s' % cmd for cmd in commands]
args += ["--args",
sys.executable]
if not import_site:
# -S suppresses the default 'import site'
args += ["-S"]
if source:
args += ["-c", source]
elif script:
args += [script]
# print args
# print ' '.join(args)
# Use "args" to invoke gdb, capturing stdout, stderr:
out, err = run_gdb(*args, PYTHONHASHSEED='0')
errlines = err.splitlines()
unexpected_errlines = []
# Ignore some benign messages on stderr.
ignore_patterns = (
'Function "%s" not defined.' % breakpoint,
"warning: no loadable sections found in added symbol-file"
" system-supplied DSO",
"warning: Unable to find libthread_db matching"
" inferior's thread library, thread debugging will"
" not be available.",
"warning: Cannot initialize thread debugging"
" library: Debugger service failed",
'warning: Could not load shared library symbols for '
'linux-vdso.so',
'warning: Could not load shared library symbols for '
'linux-gate.so',
'Do you need "set solib-search-path" or '
'"set sysroot"?',
)
for line in errlines:
if not line.startswith(ignore_patterns):
unexpected_errlines.append(line)
# Ensure no unexpected error messages:
self.assertEqual(unexpected_errlines, [])
return out
0
Example 96
Project: reprozip Source File: functional.py
@in_temp_dir
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
rpz_python = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz_python = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz_python.extend(['-W', 'error:signal'])
rpuz_python.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz_python.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz_python.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz = rpz_python + [reprozip_main.absolute().path] + verbose
rpuz = rpuz_python + [reprounzip_main.absolute().path] + verbose
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l
for l in output.splitlines())
output = check_output(rpz + ['testrun', '-a', '/fake/path/echo',
'/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + ['testrun', 'bash', '-c',
'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'])
check_call(rpz + ['trace', '--overwrite',
'bash', '-c', 'cat /etc/passwd;echo'])
check_call(rpz + ['trace', '--continue',
'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
def check_simple(args, stream, infile=1):
output = check_output(args, stream).splitlines()
try:
first = output.index(b"Read 6 bytes")
except ValueError:
stderr.write("output = %r\n" % output)
raise
if infile == 1:
assert output[first + 1] == b"a = 29, b = 13"
assert output[first + 2] == b"result = 42"
else: # infile == 2
assert output[first + 1] == b"a = 25, b = 11"
assert output[first + 2] == b"result = 36"
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + ['trace', '--overwrite', '-d', 'rpz-simple',
'./simple',
(tests / 'simple_input.txt').path,
'simple_output.txt'])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
inputs_outputs = conf['inputs_outputs']
# Exactly one input: "arg1", "...simple_input.txt"
# Output: 'arg2', "...simple_output.txt"
# There might be more output files: the C coverage files
found = 0
for fdict in inputs_outputs:
if Path(fdict['path']).name == b'simple_input.txt':
assert fdict['name'] == 'arg1'
assert fdict['read_by_runs'] == [0]
assert not fdict.get('written_by_runs')
found |= 0x01
elif Path(fdict['path']).name == b'simple_output.txt':
assert fdict['name'] == 'arg2'
assert not fdict.get('read_by_runs')
assert fdict['written_by_runs'] == [0]
found |= 0x02
else:
# No other inputs
assert not fdict.get('read_by_runs')
assert found == 0x03
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').rename('simple.orig')
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot'])
try:
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
# Run chroot
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
'arg2:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
'%s:binc.bin' % (Path.cwd() / 'simple')])
assert same_files('simple.orig', 'binc.bin')
# Replace input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:%s' % (tests / 'simple_input2.txt',
tests / 'simple_input.txt')])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
# Use reprounzip-vistrails with chroot
check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot_vt'])
try:
output_in_chroot = join_root(Path('simplechroot_vt/root'),
orig_output_location)
# Run using reprounzip-vistrails
check_simple(
sudo + rpuz_python +
['-m', 'reprounzip.plugins.vistrails', '1',
'chroot', 'simplechroot_vt', '0',
'--input-file', 'arg1:%s' % (tests / 'simple_input2.txt'),
'--output-file', 'arg2:output_vt.txt'],
'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot_vt'])
if not (tests / 'vagrant').exists():
check_call(['sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'}])
# Unpack Vagrant-chroot
check_call(rpuz + ['vagrant', 'setup/create', '--memory', '512',
'--use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrantchroot').path])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'%s:binvc.bin' % (Path.cwd() / 'simple')])
assert same_files('simple.orig', 'binvc.bin')
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput2.txt'])
with Path('voutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
'%s:%s' % (tests / 'simple_input2.txt',
tests / 'simple_input.txt')])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out', 2)
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrantchroot').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
'simple.rpz',
(tests / 'vagrant/simplevagrant').path])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput1.txt'])
with Path('woutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'%s:binvs.bin' % (Path.cwd() / 'simple')])
assert same_files('simple.orig', 'binvs.bin')
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput2.txt'])
with Path('woutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'%s:bind.bin' % (Path.cwd() / 'simple')])
assert same_files('simple.orig', 'bind.bin')
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:%s' % (tests / 'simple_input2.txt',
tests / 'simple_input.txt')])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads'], 'err')
assert any(b'successfully exec\'d /bin/./echo' in l
for l in output.splitlines())
# ########################################
# 'threads2' program: testrun
#
# Build
build('threads2', ['threads2.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads2'], 'err')
assert any(b'successfully exec\'d /bin/echo' in l
for l in output.splitlines())
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', '--overwrite',
'./exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup',
'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot',
'--cmdline', './exec_echo', 'changedexecechooutput'])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2 ** 32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
err = check_output(rpz + ['testrun', './connect'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
assert any(re.search(br'process connected to [0-9.]+:80', l)
for l in err)
# ########################################
# 'vfork' program: testrun
#
# Build
build('vfork', ['vfork.c'])
# Trace
err = check_output(rpz + ['testrun', './vfork'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
# ########################################
# 'rename' program: trace
#
# Build
build('rename', ['rename.c'])
# Trace
check_call(rpz + ['trace', '--overwrite', '-d', 'rename-trace',
'./rename'])
with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
# Check that written files were logged
database = Path.cwd() / 'rename-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
files = set(Path(r[0]) for r in rows)
for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
if (Path.cwd() / n) not in files:
raise AssertionError("Missing file: %s" % (Path.cwd() / n))
conn.close()
# Check that created files won't be packed
for f in config.get('other_files'):
if 'dir2' in Path(f).parent.components:
raise AssertionError("Created file shouldn't be packed: %s" %
Path(f))
# ########################################
# Test shebang corner-cases
#
Path('a').symlink('b')
with Path('b').open('w') as fp:
fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c'))
Path('b').chmod(0o744)
Path('c').symlink('d')
with Path('d').open('w') as fp:
fp.write('#!e')
Path('d').chmod(0o744)
with Path('e').open('w') as fp:
fp.write('#!/bin/echo')
Path('e').chmod(0o744)
# Trace
out = check_output(rpz + ['trace', '--overwrite', '-d', 'shebang-trace',
'--dont-identify-packages', './a', '1', '2'])
out = out.splitlines()[0]
assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii')
# Check config
with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
other_files = set(Path(f) for f in config['other_files']
if f.startswith('%s/' % Path.cwd()))
# Check database
database = Path.cwd() / 'shebang-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
opened = [Path(r[0]) for r in rows
if r[0].startswith('%s/' % Path.cwd())]
rows = conn.execute(
'''
SELECT name, argv FROM executed_files
''')
executed = [(Path(r[0]), r[1]) for r in rows
if Path(r[0]).lies_under(Path.cwd())]
print("other_files: %r" % sorted(other_files))
print("opened: %r" % opened)
print("executed: %r" % executed)
assert other_files == set(Path.cwd() / p
for p in ('a', 'b', 'c', 'd', 'e'))
assert opened == [Path.cwd() / 'c', Path.cwd() / 'e']
assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')]
# ########################################
# Test old packages
#
old_packages = [
('simple-0.4.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V'
'eDhXNTQ'),
('simple-0.6.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr'
'cUdtbGs'),
('simple-0.7.1.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V'
'QVpWOGs'),
]
for name, url in old_packages:
print("Testing old package %s" % name)
f = Path(name)
if not f.exists():
download_file(url, f)
# Info
check_call(rpuz + ['info', name])
# Show files
check_call(rpuz + ['showfiles', name])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', name])
# Unpack directory
check_call(rpuz + ['directory', 'setup', name, 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = Path('simpledir/root/tmp')
output_in_dir = output_in_dir.listdir('reprozip_*')[0]
output_in_dir = output_in_dir / 'simple_output.txt'
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
0
Example 97
Project: vispy Source File: _runners.py
def _unit(mode, extra_arg_string, coverage=False):
"""Run unit tests using a particular mode"""
import_dir = _get_import_dir()[0]
cwd = op.abspath(op.join(import_dir, '..'))
extra_args = [''] + extra_arg_string.split(' ')
del extra_arg_string
use_pytest = False
try:
import pytest # noqa, analysis:ignore
use_pytest = True
except ImportError:
try:
import nose # noqa, analysis:ignore
except ImportError:
raise SkipTest('Skipping unit tests, neither pytest nor nose '
'installed')
if mode == 'nobackend':
msg = 'Running tests with no backend'
if use_pytest:
extra_args += ['-m', '"not vispy_app_test"']
else:
extra_args += ['-a', '"!vispy_app_test"']
else:
# check to make sure we actually have the backend of interest
invalid = run_subprocess([sys.executable, '-c',
'import vispy.app; '
'vispy.app.use_app("%s"); exit(0)' % mode],
return_code=True)[2]
if invalid:
print('%s\n%s\n%s' % (_line_sep, 'Skipping backend %s, not '
'installed or working properly' % mode,
_line_sep))
raise SkipTest()
msg = 'Running tests with %s backend' % mode
if use_pytest:
extra_args += ['-m', 'vispy_app_test']
else:
extra_args += ['-a', 'vispy_app_test']
if coverage and use_pytest:
extra_args += ['--cov', 'vispy', '--no-cov-on-fail']
# make a call to "python" so that it inherits whatever the system
# thinks is "python" (e.g., virtualenvs)
extra_arg_string = ' '.join(extra_args)
insert = extra_arg_string if use_pytest else extra_args
extra_args += [import_dir] # positional argument
cmd = [sys.executable, '-c', _unit_script % insert]
env = deepcopy(os.environ)
# We want to set this for all app backends plus "nobackend" to
# help ensure that app tests are appropriately decorated
env.update(dict(_VISPY_TESTING_APP=mode, VISPY_IGNORE_OLD_VERSION='true'))
env_str = '_VISPY_TESTING_APP=%s ' % mode
if len(msg) > 0:
msg = ('%s\n%s:\n%s%s'
% (_line_sep, msg, env_str, extra_arg_string))
print(msg)
sys.stdout.flush()
return_code = run_subprocess(cmd, return_code=True, cwd=cwd,
env=env, stdout=None, stderr=None)[2]
if return_code:
raise RuntimeError('unit failure (%s)' % return_code)
if coverage:
# Running a py.test with coverage will wipe out any files that
# exist as .coverage or .coverage.*. It should work to pass
# COVERAGE_FILE env var when doing run_subprocess above, but
# it does not. Therefore we instead use our own naming scheme,
# and in Travis when we combine them, use COVERAGE_FILE with the
# `coverage combine` command.
out_name = op.join(cwd, '.vispy-coverage.%s' % mode)
if op.isfile(out_name):
os.remove(out_name)
os.rename(op.join(cwd, '.coverage'), out_name)
0
Example 98
Project: zest.releaser Source File: utils.py
def _execute_command(command, input_value=''):
"""commands.getoutput() replacement that also works on windows"""
logger.debug("Running command: %r", command)
if command.startswith(sys.executable):
env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
if ' upload ' in command or ' register ' in command:
# We really do want to see the stderr here, otherwise a
# failed upload does not even show up in the output.
show_stderr = True
else:
show_stderr = False
else:
env = None
show_stderr = True
p = subprocess.Popen(command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=MUST_CLOSE_FDS,
env=env)
i, o, e = (p.stdin, p.stdout, p.stderr)
if input_value:
i.write(input_value.encode(INPUT_ENCODING))
i.close()
stdout_output = o.read()
stderr_output = e.read()
# We assume that the output from commands we're running is text.
if not isinstance(stdout_output, six.text_type):
stdout_output = stdout_output.decode(OUTPUT_ENCODING)
if not isinstance(stderr_output, six.text_type):
stderr_output = stderr_output.decode(OUTPUT_ENCODING)
# TODO. Note that the returncode is always None, also after
# running p.kill(). The shell=True may be tripping us up. For
# some ideas, see http://stackoverflow.com/questions/4789837
if p.returncode or show_stderr or 'Traceback' in stderr_output:
# Some error occured
# print(Fore.RED + stderr_output)
stderr_output = stderr_output.strip()
if stderr_output:
# Make sure every error line is marked red. The stderr
# output also catches some warnings though. It would be
# really irritating if we start treating a line like this
# as an error: warning: no previously-included files
# matching '*.pyc' found anywhere in distribution. Same
# for empty lines. So try to be smart about it.
errors = []
for line in stderr_output.split('\n'):
line = line.strip()
if not line:
# Keep it in the errors, but do not mark it with a color.
errors.append(line)
continue
for known in KNOWN_WARNINGS:
if line.lower().startswith(known):
# Not a real error, so mark it as a warning.
errors.append(Fore.MAGENTA + line)
break
else:
# Not found in known warnings, so mark it as an error.
errors.append(Fore.RED + line)
errors = '\n'.join(errors)
else:
errors = ''
result = stdout_output + errors
else:
# Only return the stdout. Stderr only contains possible
# weird/confusing warnings that might trip up extraction of version
# numbers and so.
result = stdout_output
if stderr_output:
logger.debug("Stderr of running command '%s':\n%s",
command, stderr_output)
o.close()
e.close()
return result
0
Example 99
Project: django-nonrel Source File: debug.py
def get_traceback_html(self):
"Return HTML code for traceback."
if self.exc_type and issubclass(self.exc_type, TemplateDoesNotExist):
from django.template.loader import template_source_loaders
self.template_does_not_exist = True
self.loader_debug_info = []
for loader in template_source_loaders:
try:
module = import_module(loader.__module__)
if hasattr(loader, '__class__'):
source_list_func = loader.get_template_sources
else: # NOTE: Remember to remove this branch when we deprecate old template loaders in 1.4
source_list_func = module.get_template_sources
# NOTE: This assumes exc_value is the name of the template that
# the loader attempted to load.
template_list = [{'name': t, 'exists': os.path.exists(t)} \
for t in source_list_func(str(self.exc_value))]
except (ImportError, AttributeError):
template_list = []
if hasattr(loader, '__class__'):
loader_name = loader.__module__ + '.' + loader.__class__.__name__
else: # NOTE: Remember to remove this branch when we deprecate old template loaders in 1.4
loader_name = loader.__module__ + '.' + loader.__name__
self.loader_debug_info.append({
'loader': loader_name,
'templates': template_list,
})
if (settings.TEMPLATE_DEBUG and hasattr(self.exc_value, 'source') and
isinstance(self.exc_value, TemplateSyntaxError)):
self.get_template_exception_info()
frames = self.get_traceback_frames()
for i, frame in enumerate(frames):
if 'vars' in frame:
frame['vars'] = [(k, force_escape(pprint(v))) for k, v in frame['vars']]
frames[i] = frame
unicode_hint = ''
if self.exc_type and issubclass(self.exc_type, UnicodeError):
start = getattr(self.exc_value, 'start', None)
end = getattr(self.exc_value, 'end', None)
if start is not None and end is not None:
unicode_str = self.exc_value.args[1]
unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
from django import get_version
t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
c = Context({
'is_email': self.is_email,
'unicode_hint': unicode_hint,
'frames': frames,
'request': self.request,
'settings': get_safe_settings(),
'sys_executable': sys.executable,
'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
'server_time': datetime.datetime.now(),
'django_version_info': get_version(),
'sys_path' : sys.path,
'template_info': self.template_info,
'template_does_not_exist': self.template_does_not_exist,
'loader_debug_info': self.loader_debug_info,
})
# Check whether exception info is available
if self.exc_type:
c['exception_type'] = self.exc_type.__name__
if self.exc_value:
c['exception_value'] = smart_unicode(self.exc_value, errors='replace')
if frames:
c['lastframe'] = frames[-1]
return t.render(c)
0
Example 100
def run(self):
# Get the server port, spawning it if necessary
server_port = get_server_port()
if server_port == -1 or not Server.ping(server_port, timeout=5):
if len(self.client.server_prefs):
# Spawn the server
logger.info("Client spawning Server...")
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.bind(('localhost', 0))
sock.listen(1)
port = sock.getsockname()[1]
args = self.client.server_prefs + ( port, )
code = "from envisage.plugins.remote_editor.communication." \
"server import main; main(r'%s', '%s', %i)" % args
spawn_independent([sys.executable, '-c', code])
# Await a reponse from the server
try:
server, address = accept_no_intr(sock)
try:
command, arguments = receive(server)
if command == "__port__":
self.client._server_port = int(arguments)
else:
raise socket.error
finally:
# Use try...except to handle timeouts
try:
server.shutdown(socket.SHUT_RD)
except:
pass
except socket.error as e:
logger.error(repr(e))
logger.error("Client spawned a non-responsive Server! " \
"Unregistering...")
self.client.error = True
self.client.unregister()
return
finally:
sock.close()
else:
logger.error("Client could not contact the Server and no " \
"spawn command is defined. Unregistering...")
self.client.error = True
self.client.unregister()
return
else:
self.client._server_port = server_port
# Create the socket that will receive commands from the server
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('localhost', 0))
sock.listen(1)
self.client._port = sock.getsockname()[1]
# Register with the server
port = str(self.client._port)
arguments = MESSAGE_SEP.join((port, self.client.self_type,
self.client.other_type))
self.client.error = not send_port(self.client._server_port, 'register',
arguments)
self.client.registered = True
# Send queued commands (these can only exist if we spawned the server)
for command, args in self.client._queue:
arguments = MESSAGE_SEP.join((port, command, args))
self.client.error = not send_port(self.client._server_port, 'send',
arguments)
self.client._queue = []
# Start the loop to listen for commands from the Server
logger.info("Client listening on port %i..." % self.client._port)
try:
while not self._finished:
server, address = accept_no_intr(sock)
# Reject non-local connections
if address[0] != '127.0.0.1':
msg = "Client on port %s received connection from a " \
"non-local party (port %s). Ignoring..."
logger.warning(msg % (port, address[0]))
continue
# Receive the command from the server
self.client.error = False
command, arguments = receive(server)
msg = r"Client on port %s received: %s %s"
logger.debug(msg, port, command, arguments)
# Handle special commands from the server
if command == "__orphaned__":
self.client.orphaned = bool(int(arguments))
elif command == "__error__":
error_status = arguments[0]
error_message = ''
if len(arguments) > 0:
error_message = arguments[1:]
logger.warning("Error status received from the server: " \
"%s\n%s" % (error_status, error_message))
self.client.error = bool(int(error_status))
# Handle other commands through Client interface
else:
if self.client.ui_dispatch == 'off':
self.client.handle_command(command, arguments)
else:
if self.client.ui_dispatch == 'auto':
from pyface.gui import GUI
else:
exec('from pyface.ui.%s.gui import GUI' %
self.client.ui_dispatch)
GUI.invoke_later(self.client.handle_command,
command, arguments)
finally:
self.client.unregister()