Here are the examples of the python api sys.path.append taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
154 Examples
0
Example 101
Project: acousticbrainz-server Source File: __init__.py
def create_app():
app = Flask(__name__)
# Configuration
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
import default_config
app.config.from_object(default_config)
import config
app.config.from_object(config)
# Logging
from webserver.loggers import init_loggers
init_loggers(app)
# Database connection
from db import init_db_engine
init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
# Memcached
if 'MEMCACHED_SERVERS' in app.config:
from db import cache
cache.init(app.config['MEMCACHED_SERVERS'],
app.config['MEMCACHED_NAMESPACE'],
debug=1 if app.debug else 0)
# Extensions
from flask_uuid import FlaskUUID
FlaskUUID(app)
# MusicBrainz
import musicbrainzngs
from db import SCHEMA_VERSION
musicbrainzngs.set_useragent(app.config['MUSICBRAINZ_USERAGENT'], SCHEMA_VERSION)
if app.config['MUSICBRAINZ_HOSTNAME']:
musicbrainzngs.set_hostname(app.config['MUSICBRAINZ_HOSTNAME'])
# OAuth
from webserver.login import login_manager, provider
login_manager.init_app(app)
provider.init(app.config['MUSICBRAINZ_CLIENT_ID'],
app.config['MUSICBRAINZ_CLIENT_SECRET'])
# Error handling
from webserver.errors import init_error_handlers
init_error_handlers(app)
# Static files
import static_manager
static_manager.read_manifest()
# Template utilities
app.jinja_env.add_extension('jinja2.ext.do')
from webserver import utils
app.jinja_env.filters['date'] = utils.reformat_date
app.jinja_env.filters['datetime'] = utils.reformat_datetime
app.context_processor(lambda: dict(get_static_path=static_manager.get_static_path))
_register_blueprints(app)
# Admin section
from flask_admin import Admin
from webserver.admin import views as admin_views
admin = Admin(app, index_view=admin_views.HomeView(name='Admin'))
admin.add_view(admin_views.AdminsView(name='Admins'))
return app
0
Example 102
Project: neuroConstruct Source File: putils.py
def jython_side(channel,
project_path=None,
useNC=True,
useNeuroTools=True,
runtime_methods={},
**kwargs):
"""A jython function executed on the remote side of the execnet gateway."""
import sys,os,inspect
import pythonnC.utils.jutils as j
NC_HOME = os.environ["NC_HOME"]
#print('NC_HOME:%s' % NC_HOME)
os.chdir(NC_HOME)
# This path contains the .jar with all the nC java classes.
sys.path.append(NC_HOME)
# This path contains the utils module.
sys.path.append(os.path.join(NC_HOME,"pythonnC"))
from ucl.physiol.neuroconstruct.utils import ClassLogger
from ucl.physiol.neuroconstruct.simulation import SimulationDataException
logger = ClassLogger("JythonOut")
logger.logComment("Starting...")
'''
sim = jutils.Sim(project_path=project_path)
sim_methods = inspect.getmembers(jutils.Sim, predicate=inspect.ismethod)
sim_methods = zip(*sim_methods)[0]
print sim_methods
print "A"
for key,value in runtime_methods.items():
print "B",value,value[0]
if key in sim_methods:
print "C"
# If a jutils.Sim method is one of the keys.
method = getattr(sim,key)
if len(value) == 1:
print "D"
if type(value) in (tuple,list):
print "E"
args = tuple(value[0])
print "F"
kwargs = {}
elif type(value) is dict:
args = ()
kwargs = value[0]
elif len(value) == 2:
args = tuple(value[0])
kwargs = value[1]
print "G"
logger.logComment("Running method %s with args %s and kwargs %s",
key,
str(args),
str(kwargs),)
print "Method is ",method
print "Args are ",args
print "Kwargs are ",kwargs
method(*args,**kwargs)
'''
print("About to run sim.")
import os
#print(os.environ)
#print(sys.path)
j.sim.run()
print("Just ran sim.")
if useNeuroTools:
# If using NeuroTools, just send the directory name and extract data
# using NeuroTools methods.
print("Getting sim data.")
sim_data = j.sim.get_sim_data(run=False)
if isinstance(sim_data,SimulationDataException):
print("Error: %s" % sim_data)
channel.send(str(sim_data))
elif not sim_data:
print("No sim data.")
channel.send('')
else:
print("Got sim data.")
sim_dir = sim_data.getSimulationDirectory()
logger.logComment("Here is the sim directory:")
logger.logComment(str(sim_dir))
print("Sending sim dir.")
channel.send(str(sim_dir))
logger.logComment("Sent sim directory")
print("Sent sim dir.")
if useNC:
# If using direction communication with NeuroConstruct, return the
# voltage array directly.
real_volts = j.sim.get_volts(run=False)
volts = real_volts
else:
# If not, make some fake voltage data in the same format.
import array
volts = array.array('d',[1.0,2.0,3.0])
# Must convert to list because execnet can't send numpy array.
volts = list(volts)
logger.logComment("Here are the voltages:")
logger.logComment(volts.__str__())
channel.send(volts)
logger.logComment("Sent volts")
0
Example 103
def addpackage(sitedir, name, known_paths):
"""Process a .pth file within the site-packages directory:
For each line in the file, either combine it with sitedir to a path
and add that to known_paths, or execute it if it starts with 'import '.
"""
if known_paths is None:
known_paths = _init_pathinfo()
reset = 1
else:
reset = 0
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "r")
except OSError:
return
with f:
for n, line in enumerate(f):
if line.startswith("#"):
continue
try:
if line.startswith(("import ", "import\t")):
exec(line)
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if not dircase in known_paths and os.path.exists(dir):
sys.path.append(dir)
known_paths.add(dircase)
except Exception:
print("Error processing line {:d} of {}:\n".format(n+1, fullname),
file=sys.stderr)
import traceback
for record in traceback.format_exception(*sys.exc_info()):
for line in record.splitlines():
print(' '+line, file=sys.stderr)
print("\nRemainder of file ignored", file=sys.stderr)
break
if reset:
known_paths = None
return known_paths
0
Example 104
Project: androwarn Source File: risk.py
def with_dex_direct(self, vm, vmx, apk=None, analysis_method=None) :
risks = { DANGEROUS_RISK : 0.0,
MONEY_RISK : 0.0,
PRIVACY_RISK : 0.0,
INTERNET_RISK : 0.0,
BINARY_RISK : 0.0,
DYNAMIC_RISK : 0.0,
}
if apk :
self.__eval_risk_bin( apk.get_files_types(), risks )
self.__eval_risk_perm( apk.get_details_permissions(), risks )
else :
d = {}
for i in vmx.get_permissions( [] ) :
d[ i ] = DVM_PERMISSIONS["MANIFEST_PERMISSION"][i]
self.__eval_risk_perm( d, risks )
self.__eval_risk_dyn( vmx, risks )
val = self.__eval_risks( risks )
if analysis_method == None :
return val, {}
##########################
score_order_sign = {}
import sys
sys.path.append("./elsim")
from elsim.elsign.libelsign import libelsign
for method in vm.get_methods() :
if method.get_length() < 80 :
continue
score_order_sign[ method ] = self.get_method_score( method.get_length(),
libelsign.entropy( vmx.get_method_signature(method, "L4", { "L4" : { "arguments" : ["Landroid"] } } ).get_string() ),
libelsign.entropy( vmx.get_method_signature(method, "L4", { "L4" : { "arguments" : ["Ljava"] } } ).get_string() ),
map(lambda perm : (perm, DVM_PERMISSIONS["MANIFEST_PERMISSION"][ perm ]), vmx.get_permissions_method( method )),
)
for v in sorted(score_order_sign, key=lambda x : score_order_sign[x], reverse=True) :
print v.get_name(), v.get_class_name(), v.get_descriptor(), v.get_length(), score_order_sign[ v ]
##########################
return val, score_order_sign
0
Example 105
Project: hello-redis-tasks Source File: passenger_wsgi.py
def create_application():
# Adjust path
sys.path.append(os.path.dirname(__file__))
# Import app
try:
from hello_redis_tasks import app
except Exception:
log('Could not load app:\n' + str(format_exc()))
return None
# Run worker
try:
from worker import TaskWorker
worker = TaskWorker(app, debug=app.debug)
worker.reset()
worker.start()
except Exception:
log('Could not load worker:\n' + str(format_exc()))
return None
# Handle request
def application(environ, start_response):
log('Application called')
try:
results = app(environ, start_response)
except Exception:
log('*** ERROR ***\n' + str(format_exc()) + '\ncuem*********')
return results
return application
0
Example 106
def load_config(filename):
if not os.path.isfile(filename):
raise RuntimeError, "Configuration file does not exist: %s" % filename
configdir = os.path.dirname(filename)
configfile = os.path.basename(filename)
if configfile.endswith(".py"):
configfile = configfile[0:-3]
else:
raise RuntimeError, "Configuration file must be a importable python file ending in .py"
sys.path.append(configdir)
exec("import %s as __config__" % configfile)
sys.path.remove(configdir)
global config
config = __config__
if not "wpaths" in dir(config):
raise RuntimeError, "no paths given to watch"
for wpath in config.wpaths:
if not os.path.isdir(wpath):
raise RuntimeError, "one of the watch paths does not exist: %s" % wpath
if not os.path.isabs(wpath):
config.wpaths[config.wpaths.index(wpath)] = os.path.abspath(wpath)
for owpath in config.wpaths:
for wpath in config.wpaths:
if os.path.realpath(owpath) in os.path.realpath(wpath) and wpath != owpath and len(os.path.split(wpath)) <> len(os.path.split(owpath)):
raise RuntimeError, "You cannot specify %s in wpaths which is a subdirectory of %s since it is already synced." % (wpath, owpath)
if not "rpaths" in dir(config):
raise RuntimeError, "no paths given for the transfer"
if len(config.wpaths) != len(config.rpaths):
raise RuntimeError, "the no. of remote paths must be equal to the number of watched paths"
if not "rnodes" in dir(config) or len(config.rnodes) < 1:
raise RuntimeError, "no remote nodes given"
if not "rspeed" in dir(config) or config.rspeed < 0:
config.rspeed = 0
if not "emask" in dir(config):
config.emask = DEFAULT_EVENTS
for event in config.emask:
if not event in EventsCodes.ALL_FLAGS.keys():
raise RuntimeError, "invalid inotify event: %s" % event
if not "edelay" in dir(config):
config.edelay = 10
if config.edelay < 0:
raise RuntimeError, "event delay needs to be greater or equal to 0"
if not "logfile" in dir(config):
config.logfile = None
if not "extra" in dir(config):
config.extra = ""
if not "rsync" in dir(config):
config.rsync = "/usr/bin/rsync"
if not os.path.isabs(config.rsync):
raise RuntimeError, "rsync path needs to be absolute"
if not os.path.isfile(config.rsync):
raise RuntimeError, "rsync binary does not exist: %s" % config.rsync
0
Example 107
Project: latex2edx Source File: plastexit.py
def generate_xhtml(self):
if self.verbose:
print "============================================================================="
print "Converting latex to XHTML using PlasTeX with custom edX macros"
print "Source file: %s" % self.input_fn
print "============================================================================="
# set the zpts templates path
mydir = os.path.dirname(__file__)
zptspath = os.path.abspath(mydir + '/render')
os.environ['XHTMLTEMPLATES'] = zptspath
# print os.environ['XHTMLTEMPLATES']
# add our python plastex package directory to python path
plastexpydir = os.path.abspath(mydir + '/plastexpy')
sys.path.append(plastexpydir)
# get the input latex file
if self.latex_string is None:
if self.fp is None:
self.fp = codecs.open(self.input_fn)
self.latex_string = self.fp.read()
self.latex_string = self.latex_string.replace('\r','\n') # convert from mac format EOL
if self.fix_plastex_optarg_bug:
self.latex_string = self.do_fix_plastex_optarg_bug(self.latex_string)
# add preamble and postfix wrap?
if self.add_wrap:
PRE = """\\docuementclass[12pt]{article}\n\\usepackage{edXpsl}\n\n\\begin{docuement}\n\n"""
POST = "\n\n\\end{docuement}"
self.latex_string = PRE + self.latex_string + POST
source = StringIO(self.latex_string)
source.name = self.input_fn
self.tex.input(source)
docuement = self.tex.parse()
self.renderer.render(docuement)
print "XHTML generated (%s): %d lines" % (self.output_fn, len(self.renderer.xhtml.split('\n')))
return self.renderer.xhtml
0
Example 108
Project: securedrop Source File: 0.3_migrate.py
def migrate_database(backup):
print "* Migrating database..."
# Get the sources table from the 0.2.1 instance's db
old_db = backup.getmember("var/chroot/docuement/var/www/securedrop/db.sqlite")
old_db.name = "db.old.sqlite"
backup.extract(old_db)
conn = sqlite3.connect("db.old.sqlite")
c = conn.cursor()
sources = c.execute("SELECT * FROM sources").fetchall()
os.remove("db.old.sqlite")
# Fill in the rest of the sources. Since sources were only added to the
# database if their codename was changed by the journalist, we need to fill
# in the rest by examining all of the filesystem designations in the source
# directory and re-generating the codenames.
#
# Note: Must be called after /var/lib/securedrop/store is populated
from old_crypto_util import displayid
# Generate a list of the filesystem ids that have journalist designations
# stored in the database, since they are already known and should not be
# generated from the filesystem id
already_processed = set([source[0] for source in sources])
for fs_id in os.listdir("/var/lib/securedrop/store"):
if fs_id in already_processed:
continue
sources.append((fs_id, displayid(fs_id)))
# Import current application's config so we can easily populate the db
sys.path.append("/var/www/securedrop")
import config
from db import Source, Journalist, Submission, Reply, db_session, init_db
# We need to be able to link replies to the Journalist that sent
# them. Since this information was not recorded in 0.2.1, we
# arbitrarily say all replies were sent by an arbitrary journalist
# that is present on this system. Since this information is not
# currently exposed in the UI, this does not create a problem (for
# now).
if len(Journalist.query.all()) == 0:
print "!!! FATAL: You must create a journalist account before running this migration."
print " Run ./manage.py add_admin and try again."
sys.exit(1)
else:
arbitrary_journalist = Journalist.query.all()[0]
# Back up current database just in case
shutil.copy("/var/lib/securedrop/db.sqlite",
"/var/lib/securedrop/db.sqlite.bak")
# Copied from db.py to compute filesystem-safe journalist filenames
def journalist_filename(s):
valid_chars = 'abcdefghijklmnopqrstuvwxyz1234567890-_'
return ''.join([c for c in s.lower().replace(' ', '_') if c in valid_chars])
# Migrate rows to new database with SQLAlchemy ORM
for source in sources:
migrated_source = Source(source[0], source[1])
source_dir = os.path.join("/var/lib/securedrop/store", source[0])
# It appears that there was a bug in 0.2.1 where sources with changed
# names were not always successfully removed from the database. Skip
# any sources that didn't have files copied for them, they were deleted
# and are in the database erroneously.
if not os.path.isdir(source_dir):
continue
# Can infer "flagged" state by looking for _FLAG files in store
if "_FLAG" in os.listdir(source_dir):
# Mark the migrated source as flagged
migrated_source.flagged = True
# Delete the _FLAG file
os.remove(os.path.join(source_dir, "_FLAG"))
# Sort the submissions by the date of submission so we can infer the
# correct interaction_count for the new filenames later, and so we can
# set source.last_updated to the time of the most recently uploaded
# submission in the store now.
submissions = []
replies = []
for fn in os.listdir(source_dir):
append_to = submissions
if fn.startswith('reply-'):
append_to = replies
append_to.append((fn, os.path.getmtime(os.path.join(source_dir, fn))))
# Sort by submission time
submissions.sort(key=itemgetter(1))
replies.sort(key=itemgetter(1))
if len(submissions) > 0:
migrated_source.last_updated = datetime.utcfromtimestamp(submissions[-1][1])
else:
# The source will have the default .last_updated of utcnow(), which
# might be a little confusing, but it's the best we can do.
pass
# Since the concept of "pending" is introduced in 0.3, it's tricky to
# figure out how to set this value. We can't distinguish between sources
# who created an account but never submitted anything and sources who
# had been active, but didn't have any stored submissions or replies at
# the time of migration.
#
# After having explored the options, I think the best thing to do here
# is set pending to True if there are no submissions or replies. Sources
# who created an account but never submitted anything won't create noise
# in the list, and sources who are active can probably be expected to
# log back in relatively soon and so will automatially reappear once
# they submit something new.
if len(submissions + replies) == 0:
migrated_source.pending = True
else:
migrated_source.pending = False
# Set source.interaction_count to the number of current submissions for
# each source. This is not technicially correct, but since we can't
# know how many submissions have been deleted it will give us a
# reasonable, monotonically increasing basis for future increments to
# the interaction_count.
migrated_source.interaction_count = len(submissions) + len(replies)
# Add and commit the source to the db so they will have a primary key
# assigned to use with the ForeignKey relationship with their
# submissions
db_session.add(migrated_source)
db_session.commit()
# Combine everything into one list, sorted by date, so we can
# correctly set the interaction counts for each file.
everything = submissions + replies
everything.sort(key=itemgetter(1))
for count, item in enumerate(everything):
# Rename the file to fit the new file naming scheme used by 0.3
fn = item[0]
if fn.startswith('reply-'):
new_fn = "{0}-{1}-reply.gpg".format(count+1, journalist_filename(source[1]))
else:
new_fn = "{0}-{1}-{2}".format(count+1, journalist_filename(source[1]), "msg.gpg" if fn.endswith("msg.gpg") else "doc.zip.gpg")
# Move to the new filename
os.rename(os.path.join(source_dir, fn),
os.path.join(source_dir, new_fn))
# Add a database entry for this item
db_entry = None
if fn.startswith('reply-'):
migrated_reply = Reply(arbitrary_journalist, migrated_source, new_fn)
db_entry = migrated_reply
else:
migrated_submission = Submission(migrated_source, new_fn)
# Assume that all submissions that are being migrated
# have already been downloaded
migrated_submission.downloaded = True
db_entry = migrated_submission
db_session.add(db_entry)
db_session.commit()
# chown the database file to the securedrop user
subprocess.call(['chown', 'www-data:www-data', "/var/lib/securedrop/db.sqlite"])
0
Example 109
Project: numba Source File: cffi_usecases.py
def load_ool_module():
"""
Compile an out-of-line module, return the corresponding ffi and
module objects.
"""
from cffi import FFI
numba_complex = """
typedef struct _numba_complex {
double real;
double imag;
} numba_complex;
"""
defs = numba_complex + """
double sin(double x);
double cos(double x);
int foo(int a, int b, int c);
void vsSin(int n, float* x, float* y);
void vdSin(int n, double* x, double* y);
void vector_real(numba_complex *c, double *real, int n);
void vector_imag(numba_complex *c, double *imag, int n);
"""
source = numba_complex + """
static int foo(int a, int b, int c)
{
return a + b * c;
}
void vsSin(int n, float* x, float* y) {
int i;
for (i=0; i<n; i++)
y[i] = sin(x[i]);
}
void vdSin(int n, double* x, double* y) {
int i;
for (i=0; i<n; i++)
y[i] = sin(x[i]);
}
static void vector_real(numba_complex *c, double *real, int n) {
int i;
for (i = 0; i < n; i++)
real[i] = c[i].real;
}
static void vector_imag(numba_complex *c, double *imag, int n) {
int i;
for (i = 0; i < n; i++)
imag[i] = c[i].imag;
}
"""
ffi = FFI()
ffi.set_source('cffi_usecases_ool', source)
ffi.cdef(defs, override=True)
tmpdir = temp_directory('test_cffi')
ffi.compile(tmpdir=tmpdir)
sys.path.append(tmpdir)
try:
mod = import_dynamic('cffi_usecases_ool')
cffi_support.register_module(mod)
cffi_support.register_type(mod.ffi.typeof('struct _numba_complex'),
complex128)
return mod.ffi, mod
finally:
sys.path.remove(tmpdir)
0
Example 110
def postprocess(self, logger):
j = self.getJobObject()
parsedXML = os.path.join(j.outputdir, '__parsedxmlsummary__')
# use to avoid replacing 'lumi' etc as return value and not the method
# pointer
metadataItems = {}
if os.path.exists(parsedXML):
execfile(parsedXML, {}, metadataItems)
# Combining subjobs XMLSummaries.
if j.subjobs:
env = self.getenv(self.is_prepared is None)
if 'XMLSUMMARYBASEROOT' not in env:
logger.warning(
'"XMLSUMMARYBASEROOT" env var not defined so summary.xml files not merged for subjobs of job %s' % j.fqid)
return
summaries = []
for sj in j.subjobs:
outputxml = os.path.join(sj.outputdir, 'summary.xml')
if not os.path.exists(outputxml):
logger.warning("XMLSummary for job %s subjobs will not be merged as 'summary.xml' not present in job %s outputdir" % (j.fqid, sj.fqid))
return
elif os.path.getsize(outputxml) == 0 or os.stat(outputxml).st_size == 0:
logger.warning("XMLSummary fro job %s subjobs will not be merged as %s appears to be an empty file" % (j.fqid, outputxml))
logger.warning("Please try to recreate this file by either resubmitting your job or re-downloading the data from the backend")
return
summaries.append(outputxml)
# Not needed now that we dont merge if ANY of subjobs have missing summary.xml
#if not summaries:
# logger.debug('None of the subjobs of job %s produced the output XML summary file "summary.xml". Merging will therefore not happen' % j.fqid)
# return
schemapath = os.path.join(env['XMLSUMMARYBASEROOT'], 'xml/XMLSummary.xsd')
summarypath = os.path.join(env['XMLSUMMARYBASEROOT'], 'python/XMLSummaryBase')
sys.path.append(summarypath)
import summary
try:
XMLSummarydata = summary.Merge(summaries, schemapath)
except Exception, err:
logger.error('Problem while merging the subjobs XML summaries')
raise
for name, method in activeSummaryItems().iteritems():
try:
metadataItems[name] = method(XMLSummarydata)
except:
metadataItems[name] = None
logger.debug('Problem running "%s" method on merged xml output.' % name)
for key, value in metadataItems.iteritems():
if value is None: # Has to be explicit else empty list counts
j.metadata[key] = 'Not Available.'
else:
j.metadata[key] = value
0
Example 111
def main(argv):
parser = getParser()
parsed = parser.parse_args(argv[1:])
if parsed.app_dir:
if not os.path.exists(parsed.app_dir):
os.makedirs(parsed.app_dir)
os.environ['MAYA_APP_DIR'] = parsed.app_dir
testsDir = parsed.tests_dir
pymelRoot = parsed.pymel_root
pypath = os.environ.get('PYTHONPATH', '').split(os.pathsep)
# add the test dir to the python path - that way,
# we can do 'pymel_test test_general' in order to run just the tests
# in test_general
sys.path.append(testsDir)
pypath.append(testsDir)
# ...and add this copy of pymel to the python path, highest priority,
# to make sure it overrides any 'builtin' pymel/maya packages
sys.path.insert(0, pymelRoot)
pypath.insert(0, pymelRoot)
os.environ['PYTHONPATH'] = os.pathsep.join(pypath)
oldPath = os.getcwd()
# make sure our cwd is the pymel project working directory
os.chdir( pymelRoot )
try:
# Try to guess whether we were given an arg which is a TestCase or
# test method/function, and if so, run new unittest (because it can
# easily handle specific TestCase/method/function)... else run nose
# (because it's what the test suite was originally set up to use)
useNose = True
if parsed.extra_args:
name = parsed.extra_args[-1]
if isPythonDottedName(name):
modulePart, objPart = moduleObjNameSplit(name)
if modulePart and objPart:
useNose = False
argv = [argv[0]] + parsed.extra_args
if useNose:
nose_test(argv)
else:
unit2_test(argv)
finally:
os.chdir(oldPath)
0
Example 112
Project: procedural_city_generation Source File: Tester.py
def main():
import sys, os
path=os.path.abspath(os.path.dirname(__file__))+"/.."
sys.path.append(path)
os.chdir(path)
loggerpath=path+"/procedural_city_generation/outputs/test.log"
logger = Logger(loggerpath)
logger.info("Initiating Tests")
logger.info("path= "+path)
import os, sys
"""
import test
test_funcs=import_submodules(test).items()
test_funcs=[x[0] for x in test_funcs if (not "test.Tester" in x[0] and x[0].endswith("Test"))]
for test in test_funcs:
module=importlib.import_module(test)
getattr(module, "main")()
"""
logger.info("Python Version:")
logger.info("\t"+sys.version)
logger.info("\t"+str(sys.version_info))
logger.info("OS: "+sys.platform)
logger.info("Current Git commmit: ")
logger.log_git_commit()
logger.info("Testing Dependencies")
try:
with open(path+"/requirements.txt","r") as f:
[test_dependency(x.split("==")[0],logger) for x in f.readlines()]
except IOError:
logger.error("Could not locate requirements.txt at "+os.path.abspath(path))
logger.info("Testing \"import procedural_city_generation\"")
import procedural_city_generation
logger.info("Testing \"import UI\"")
import UI
logger.info("Testing \"import GUI\"")
import GUI
logger.info("Testing roadmap creation")
UI.roadmap()
logger.info("Roadmap creation threw no exception")
logger.info("Testing roadmap creation")
UI.polygons()
logger.info("Roadmap creation threw no exception")
logger.info("Testing roadmap creation")
UI.building_generation()
logger.info("Roadmap creation threw no exception")
0
Example 113
Project: seafdav Source File: run_server.py
def _runCherryPy(app, config, mode):
"""Run WsgiDAV using cherrypy.wsgiserver, if CherryPy is installed."""
assert mode in ("cherrypy", "cherrypy-bundled")
try:
if mode == "cherrypy-bundled":
# Need to set import root folder
server_folder = os.path.dirname(__file__)
sys.path.append(server_folder)
from cherrypy import wsgiserver
from cherrypy.wsgiserver.ssl_builtin import BuiltinSSLAdapter
else:
# http://cherrypy.org/apidocs/3.0.2/cherrypy.wsgiserver-module.html
from cherrypy import wsgiserver, __version__ as cp_version, BuiltinSSLAdapter
version = "WsgiDAV/%s %s Python/%s" % (
__version__,
wsgiserver.CherryPyWSGIServer.version,
PYTHON_VERSION)
wsgiserver.CherryPyWSGIServer.version = version
# Support SSL
ssl_certificate = _get_checked_path(config.get("ssl_certificate"))
ssl_private_key = _get_checked_path(config.get("ssl_private_key"))
ssl_certificate_chain = _get_checked_path(config.get("ssl_certificate_chain"))
protocol = "http"
if ssl_certificate:
assert ssl_private_key
wsgiserver.CherryPyWSGIServer.ssl_adapter = BuiltinSSLAdapter(ssl_certificate, ssl_private_key, ssl_certificate_chain)
protocol = "https"
if config["verbose"] >= 1:
print("SSL / HTTPS enabled.")
if config["verbose"] >= 1:
print "Running %s" % version
print("Listening on %s://%s:%s ..." % (protocol, config["host"], config["port"]))
server = wsgiserver.CherryPyWSGIServer(
(config["host"], config["port"]),
app,
server_name=version,
)
try:
server.start()
except KeyboardInterrupt:
if config["verbose"] >= 1:
print "Caught Ctrl-C, shutting down..."
server.stop()
except ImportError, e:
if config["verbose"] >= 1:
print "Could not import wsgiserver.CherryPyWSGIServer."
return False
return True
0
Example 114
def docuementModule0(pathOrName, builder, opts={}):
"""Generate docuementation for one Python file in some format.
This handles Python standard modules like string, custom modules
on the Python search path like e.g. docpy as well as modules
specified with their full path like C:/tmp/junk.py.
The doc file will always be saved in the current directory with
a basename equal to that of the module, e.g. docpy.
"""
cwd = os.getcwd()
# Append directory to Python search path if we get one.
dirName = os.path.dirname(pathOrName)
if dirName:
sys.path.append(dirName)
# Remove .py extension from module name.
if pathOrName[-3:] == '.py':
modname = pathOrName[:-3]
else:
modname = pathOrName
# Remove directory paths from module name.
if dirName:
modname = os.path.basename(modname)
# Load the module.
try:
module = __import__(modname)
except:
print 'Failed to import %s.' % modname
os.chdir(cwd)
return
# Do the real docuementation work.
s = ModuleSkeleton0()
s.inspect(module)
builder.write(s)
# Remove appended directory from Python search path if we got one.
if dirName:
del sys.path[-1]
os.chdir(cwd)
0
Example 115
Project: expyriment Source File: _importer_functions.py
def import_plugins_from_settings_folder(init_filename):
"""Return the code to import all plugins from the settings folder as dict.
Includes the module folder in $home to the path.
Returns
-------
out : string
"""
module = init_filename.split(os.sep)[1]
folder = get_settings_folder()
if folder is None:
return ""
folder = folder + os.sep + module + os.sep
code = {}
sys.path.append(folder)
try:
for filename in os.listdir(os.path.dirname(folder)):
if filename.endswith(".py") and\
not (filename.startswith("__") or\
filename.endswith("defaults.py")):
f = open(os.path.dirname(folder) + os.sep + filename)
try:
for line in f:
if line[0:6] == "class ":
tmp = line[6:].lstrip()
name = tmp[:len(filename[:-4])]
break
code[filename] = "from {0} import {1}\n".format(filename[:-3],
name)
print "import {0}.extras.{1} (from homefolder)".format(
module, name)
except:
print("Could not import {0}!".format(
os.path.dirname(folder) + os.sep + filename))
except:
pass
return code
0
Example 116
Project: feedplatform Source File: test_django.py
def test_django_integration():
"""Check that the output of a manual "call_command" matches what
we get from executing through a Django project.
"""
def _capture_with_fake_syspath(path, call, *args, **kwargs):
sys.path.append(path)
try:
real_stdout = sys.stdout
sys.stdout = StringIO()
try:
call(*args, **kwargs)
sys.stdout.seek(0)
return sys.stdout.read()
finally:
sys.stdout = real_stdout
finally:
sys.path.pop()
# 1. get through Django
os.environ['DJANGO_SETTINGS_MODULE'] = 'proj.settings'
from django.core.management import call_command as dj_call_command
django_output = _capture_with_fake_syspath(
os.path.join(os.path.dirname(__file__)),
dj_call_command, 'feedplatform', 'models')
# 2. get via own callcommand
oldconfig = conf.config._target
try:
conf.config.reset()
os.environ['FEEDPLATFORM_CONFIG'] = 'feedplatform_config'
local_output = _capture_with_fake_syspath(
os.path.join(os.path.dirname(__file__), 'proj'),
call_command, 'models')
finally:
# reset to previously used config
conf.config._target = oldconfig
# 3. check that the two are the same
print "command: (%s)" % repr(django_output), len(django_output)
print "process: (%s)" % repr(local_output), len(local_output)
assert django_output == local_output
0
Example 117
Project: pyblue-central Source File: engine.py
def django_init(self):
'''
Initializes the django engine. The root must have been set already."
'''
elems = os.path.split(self.root)[:-1]
parent = os.path.join(*elems)
sys.path.append(parent)
BASE_APP = []
try:
# Attempt to import the root folder. This is necessary to access
# the local templatetag libraries.
base = os.path.split(self.root)[-1]
logger.debug("importing app: %s" % base)
importlib.import_module(base)
BASE_APP = [ base ]
except ImportError as exc:
logger.debug("app '{}' cannot be imported: {}".format(base, exc))
TEMPLATE_DIR = join(os.path.dirname(__file__), "templates")
dirs = [self.root, join(self.root, 'templates'), TEMPLATE_DIR]
logger.debug("template dirs: {}".format(dirs))
settings.configure(
DEBUG=True, TEMPLATE_DEBUG=True,
TEMPLATES=[
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': dirs,
'APP_DIRS': True,
'OPTIONS': {
'string_if_invalid': "Undefined: %s ",
'builtins': [
'pyblue.templatetags.pytags',
'django.contrib.humanize.templatetags.humanize',
],
}
}
],
INSTALLED_APPS=["pyblue", "django.contrib.humanize",
"django.contrib.staticfiles"] + BASE_APP,
STATIC_URL='/static/',
)
django.setup()
logger.debug("templatetags: %s" % ", ".join(get_installed_libraries()))
0
Example 118
Project: mycroft-core Source File: __init__.py
def initialize(self):
try:
import gio
except:
sys.path.append("/usr/lib/python2.7/dist-packages")
try:
import gio
except:
logger.error("Could not import gio")
return
self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
tokenizer = EnglishTokenizer()
for app in gio.app_info_get_all():
name = app.get_name().lower()
entry = [app]
tokenized_name = tokenizer.tokenize(name)[0]
if name in self.appmap:
self.appmap[name] += entry
else:
self.appmap[name] = entry
self.register_vocabulary(name, "Application")
if name != tokenized_name:
self.register_vocabulary(tokenized_name, "Application")
if tokenized_name in self.appmap:
self.appmap[tokenized_name] += entry
else:
self.appmap[tokenized_name] = entry
launch_intent = IntentBuilder(
"LaunchDesktopApplicationIntent").require("LaunchKeyword").require(
"Application").build()
self.register_intent(launch_intent, self.handle_launch_desktop_app)
launch_website_intent = IntentBuilder(
"LaunchWebsiteIntent").require("LaunchKeyword").require(
"Website").build()
self.register_intent(launch_website_intent, self.handle_launch_website)
search_website = IntentBuilder("SearchWebsiteIntent").require(
"SearchKeyword").require("Website").require(
"SearchTerms").build()
self.register_intent(search_website, self.handle_search_website)
0
Example 119
def __load_module_by_name(self, name):
"""
Load a module specified by its name.
@type name: String
@param name: Name of the module to be loaded
@rtype: Mixed
@return: None = not loaded, Dictionary = loaded module
"""
self.database.log("info", "loading module: %s" % name)
module_dir = os.path.join(self.modules_dir, name)
l_mod = None
try:
sys.path.append(module_dir)
l_mod = reload(__import__(name, fromlist=[name]))
sys.path.remove(module_dir)
except Exception as ex:
self.database.log("error",
"failed to import module: %s" % str(name),
str(ex))
sys.path.remove(module_dir)
return None
mod_details = None
try:
l_class = getattr(l_mod, name)
l_inst = l_class(self.root, self.config)
mod_details = l_inst.descriptor()
mod_details["name"] = name
mod_details["mtime"] = os.path.getmtime(self.modules_dir + "/" + \
name) * 1000000
mod_details["instance"] = l_inst
except Exception as ex:
self.database.log("error",
"failed to load module: %s" % str(name),
str(ex))
return None
self.database.log("info", "module loaded: %s" % str(name))
return mod_details
0
Example 120
def begin(self):
"""
Create the test database and schema, if needed, and switch the
connection over to that database. Then call install() to install
all apps listed in the loaded settings module.
"""
for plugin in self.nose_config.plugins.plugins:
if getattr(plugin, 'django_plugin', False):
self.django_plugins.append(plugin)
os.environ['DJANGO_SETTINGS_MODULE'] = self.settings_module
if self.conf.addPaths:
map(add_path, self.conf.where)
try:
__import__(self.settings_module)
self.settings_path = self.settings_module
except ImportError:
# Settings module is not found in PYTHONPATH. Try to do
# some funky backwards crawling in directory tree, ie. add
# the working directory (and any package parents) to
# sys.path before trying to import django modules;
# otherwise, they won't be able to find project.settings
# if the working dir is project/ or project/..
self.settings_path = get_settings_path(self.settings_module)
if not self.settings_path:
# short circuit if no settings file can be found
raise RuntimeError("Can't find Django settings file!")
add_path(self.settings_path)
sys.path.append(self.settings_path)
from django.conf import settings
# Some Django code paths evaluate differently
# between DEBUG and not DEBUG. Example of this include the url
# dispatcher when 404's are hit. Django's own test runner forces DEBUG
# to be off.
settings.DEBUG = False
self.call_plugins_method('beforeConnectionSetup', settings)
from django.core import management
from django.test.utils import setup_test_environment
if hasattr(settings, 'DATABASES'):
self.old_db = settings.DATABASES['default']['NAME']
else:
self.old_db = settings.DATABASE_NAME
from django.db import connections
self._monkeypatch_test_classes()
for connection in connections.all():
self.call_plugins_method(
'beforeTestSetup', settings, setup_test_environment,
connection)
setup_test_environment()
import django
if hasattr(django, 'setup'):
django.setup()
self.call_plugins_method('afterTestSetup', settings)
management.get_commands()
# Ensure that nothing (eg. South) steals away our syncdb command
if self.django_version < self.DJANGO_1_7:
management._commands['syncdb'] = 'django.core'
for connection in connections.all():
self.call_plugins_method(
'beforeTestDb', settings, connection, management)
connection.creation.create_test_db(
verbosity=self.verbosity,
autoclobber=True,
)
logger.debug("Running syncdb")
self._num_syncdb_calls += 1
self.call_plugins_method('afterTestDb', settings, connection)
self.store_original_transaction_methods()
0
Example 121
def main():
global PROG_DIR, DRY_RUN
cmd_parser = OptionParser(version="%prog",description="RSS to Email Script", usage="%prog [options]")
cmd_parser.add_option("-d", "--dir", type="string", action="store", dest="dir", default="~/.rsstomail", help="Program config/save directort (default: ~/.rsstomail")
cmd_parser.add_option("-r", "--dry-run", action="store_true", dest="dryrun", default=False, help="Dry run - don't send mail, just show what would be sent on STDOUT")
cmd_parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Verbose output - show what's being sent")
cmd_parser.add_option("-D", "--debug", action="store_true", dest="debug", default=False, help="debug-level output of internal logic")
(cmd_options, cmd_args) = cmd_parser.parse_args()
log_level = logging.WARNING
if cmd_options.debug and cmd_options.debug is True:
log_level = logging.DEBUG
elif cmd_options.verbose and cmd_options.verbose is True:
log_level = logging.INFO
logging.basicConfig(level=log_level, format = '%(asctime)s %(levelname)s %(name)s %(message)s', datefmt = '%H:%M:%S')
if cmd_options.dryrun and cmd_options.dryrun is True:
DRY_RUN = cmd_options.dryrun
logging.warning("setting DRY_RUN to True, will not send mail")
if cmd_options.dir:
PROG_DIR = os.path.expanduser(cmd_options.dir)
logging.info("Setting PROG_DIR to '%s'", PROG_DIR)
if not os.path.exists("%s/config.py" % PROG_DIR):
logging.info("config file does not exist, calling do_config_setup()")
do_config_setup()
try:
sys.path.append(PROG_DIR)
import config
except:
logging.critical("could not import config.py")
sys.exit(1)
logging.debug("Imported config")
try:
config.EMAIL_TEXT_ONLY
except NameError:
config.EMAIL_TEXT_ONLY = False
check_feeds(config.FEEDS, config.EMAIL_TO, config.EMAIL_TEXT_ONLY, config.EMAIL_FROM)
0
Example 122
def __init__(self):
self.plugins = {}
self.brokerages = {}
# Load all plugins in user plugins path
prefsRoot = Prefs.prefsRootPath()
sys.path.append(prefsRoot + 'plugins')
path = os.path.join(prefsRoot, "plugins/plugin_*.py")
for file in glob.glob(path):
plugin = file.replace(prefsRoot + 'plugins', '')
plugin = plugin.replace('/', '')
plugin = plugin.replace('\\', '')
plugin = plugin.replace('.py', '')
__import__(plugin)
self.plugins[plugin] = sys.modules[plugin].Plugin()
self.plugins[plugin].builtin = False
# Load all plugins in builtin path
pluginPath = os.path.join(appGlobal.getPath(), 'builtinPlugins')
sys.path.append(pluginPath)
for file in glob.glob(pluginPath + '/plugin_*.py'):
plugin = os.path.basename(file)
plugin = plugin.replace('/', '')
plugin = plugin.replace('\\', '')
plugin = plugin.replace('.py', '')
__import__(plugin)
self.plugins[plugin] = sys.modules[plugin].Plugin()
self.plugins[plugin].builtin = True
# Check for duplicate names
names = {}
for p, plugin in self.plugins.items():
if plugin.name() in names:
print "Duplicate plugin name", plugin.name()
sys.exit()
names[plugin.name()] = True
for p, plugin in self.plugins.items():
plugin.initialize()
# Check for duplicate names
names = {}
for p, plugin in self.plugins.items():
if plugin.name() in names:
print "Duplicate plugin name", plugin.name()
sys.exit()
names[plugin.name()] = True
# Load all brokerages in user plugins path
path = os.path.join(prefsRoot, "plugins/brokerage_*.py")
for file in glob.glob(path):
brokerage = file.replace(prefsRoot + 'plugins', '')
brokerage = brokerage.replace('/', '')
brokerage = brokerage.replace('\\', '')
brokerage = brokerage.replace('.py', '')
__import__(brokerage)
self.brokerages[brokerage] = sys.modules[brokerage].Brokerage()
self.brokerages[brokerage].builtin = False
# Load all brokerages in builtin path
for file in glob.glob('builtinPlugins/brokerage_*.py'):
brokerage = file.replace('builtinPlugins', '')
brokerage = brokerage.replace('/', '')
brokerage = brokerage.replace('\\', '')
brokerage = brokerage.replace('.py', '')
__import__(brokerage)
self.brokerages[brokerage] = sys.modules[brokerage].Brokerage()
self.brokerages[brokerage].builtin = True
# Check for duplicate names
names = {}
for b, brokerage in self.brokerages.items():
if brokerage.getName() in names:
print "Duplicate brokerage name", brokerage.getName()
sys.exit()
names[brokerage.getName()] = True
0
Example 123
Project: databus Source File: regsetup.py
def SetupCore(searchPaths):
"""Setup the core Python information in the registry.
This function makes no assumptions about the current state of sys.path.
After this function has completed, you should have access to the standard
Python library, and the standard Win32 extensions
"""
import sys
for path in searchPaths:
sys.path.append(path)
import string, os
import regutil, win32api,win32con
installPath, corePaths = LocatePythonCore(searchPaths)
# Register the core Pythonpath.
print corePaths
regutil.RegisterNamedPath(None, string.join(corePaths,";"))
# Register the install path.
hKey = win32api.RegCreateKey(regutil.GetRootKey() , regutil.BuildDefaultPythonKey())
try:
# Core Paths.
win32api.RegSetValue(hKey, "InstallPath", win32con.REG_SZ, installPath)
finally:
win32api.RegCloseKey(hKey)
# Register the win32 core paths.
win32paths = os.path.abspath( os.path.split(win32api.__file__)[0]) + ";" + \
os.path.abspath( os.path.split(LocateFileName("win32con.py;win32con.pyc", sys.path ) )[0] )
# Python has builtin support for finding a "DLLs" directory, but
# not a PCBuild. Having it in the core paths means it is ignored when
# an EXE not in the Python dir is hosting us - so we add it as a named
# value
check = os.path.join(sys.prefix, "PCBuild")
if os.path.isdir(check):
regutil.RegisterNamedPath("PCBuild",check)
0
Example 124
Project: gitlint Source File: user_rules.py
def find_rule_classes(extra_path):
"""
Searches a given directory for rule classes. This is done by finding all python modules in the given directory,
adding them to the python path, importing them and then finding any Rule classes in those modules.
:param extra_path: absolute directory path to search for rule classes
:return: The list of rule classes that are found in the given directory.
"""
# Find all python files in the given path
modules = []
for filename in os.listdir(extra_path):
if fnmatch.fnmatch(filename, '*.py'):
modules.append(os.path.splitext(filename)[0])
# No need to continue if there are no modules specified
if len(modules) == 0:
return []
# Append the extra_path to the python path so that we can import the newly found rule modules
sys.path.append(extra_path)
# Find all the rule classes in the found python files
rule_classes = []
for module in modules:
# Import the module
importlib.import_module(module)
# Find all rule classes in the module. We do this my inspecting all members of the module and checking
# 1) is it a class, if not, skip
# 2) is the parent path the current module. If not, we are dealing with an imported class, skip
# 3) is it a subclass of rule
rule_classes.extend([clazz for _, clazz in inspect.getmembers(sys.modules[module])
if
inspect.isclass(clazz) and
clazz.__module__ == module and
issubclass(clazz, rules.Rule) and
assert_valid_rule_class(clazz)])
return rule_classes
0
Example 125
def start( config = myConfig , test_selection='Ganga.test.*', logger=myLogger):
"""
"""
import os
#rtconfig = getConfig('TestingFramework')
my_full_path = os.path.abspath(os.path.dirname(__file__))
#sys.stdout = UnbufferedStdout(sys.stdout)
#sys.stderr = UnbufferedStdout(sys.stderr)
##configure Ganga TestLoader
# enable XML reporting in release mode
pytf_reporting_opts=""
if config['ReleaseTesting']:
pytf_reporting_opts="--report-xml --report-outputdir=%(ReportsOutputDir)s --runid=%(RunID)s" % config
# output dir
if not os.path.exists(config['LogOutputDir']):
os.makedirs(config['LogOutputDir'])
# loader path
global gangaReleaseTopDir
pytf_loader_path = os.path.join(gangaReleaseTopDir,'python','GangaTest','Framework','loader.py')
# loader args
pytf_loader_args =[]
pytf_loader_args.append( '--loader-args=%s' % config['Config'])
#pytf_loader_args.append( '--loader-args=%s/python' % gangaReleaseTopDir)
pytf_loader_args.append( '--loader-args=%s' % gangaReleaseTopDir)
pytf_loader_args.append( '--loader-args=%s' % int(config['ReleaseTesting']))
#output_dir
pytf_loader_args.append( '--loader-args=%s' % config['LogOutputDir'])
#unit-testing: on/off
pytf_loader_args.append( '--loader-args=%s' % int(config['SearchLocalTests']))
#system-testing: on/off
pytf_loader_args.append( '--loader-args=%s' % int(config['SearchReleaseTests']))
#Pass the report(report path + runid) path
pytf_loader_args.append( '--loader-args=%s' % os.path.join(config['ReportsOutputDir'],config['RunID']))
#pass the schmema version (if any) to test
pytf_loader_args.append( '--loader-args=%s' % config['SchemaTesting'])
#print("PYTF path %s config: %s" % (pytf_loader_path, pytf_loader_args))
import sys
sys.path.append(os.getenv('PYTF_TOP_DIR','').split(':')[0])
sys.path.append(os.path.join(os.getenv('PYTF_TOP_DIR','').split(':')[0],'pytf'))
import runTests
runner_args = []
runner_args.extend(pytf_reporting_opts.split())
runner_args.extend(['--loader-path=%s' % pytf_loader_path])
runner_args.extend(pytf_loader_args)
runner_args.extend([test_selection])
try:
rc = runTests.main(logger,runner_args)
except:
rc = -9999
return rc
0
Example 126
def migrate_schema(self, version=None, rollback=False):
"""
migrate metadata schema from one version to another. If given a version number, attempt to migrate
schema to it. If rollback is True, attempt to revert to given schema number
"""
# current schema version
current = self.schema_version
if version is None:
version = __schema_version__
else:
version = int(version)
# if caller has provided a desired schema version, check if there
# is anything to be done
if current == version:
print "Schema up-to-date. Nothing to do!"
return
elif rollback:
if current < version:
print "Error: can't rollback to newer version!"
return
else:
if current > version:
print "Error: given version is behind current, use --rollback"
return
# grab current isolation level then set it to 'EXCLUSIVE'
current_isolation = self.__dbh.isolation_level
self.__dbh.isolation_level = 'EXCLUSIVE'
# add migration directory to sys.path
sys.path.append(self.resources)
file_list = os.listdir(os.path.join(self.resources, 'migration'))
file_list.sort()
if rollback:
file_list.reverse()
action = 'revert' if rollback else 'upgrade'
numeric_regex = re.compile("m(\d{3})", re.I)
for file in file_list:
if os.path.isfile(os.path.join(self.resources, 'migration', file)):
match = numeric_regex.match(file)
(name, ext) = os.path.splitext(file)
if match and ext == '.py':
num = int(match.group(1))
if rollback:
if num > current: continue
elif num < current:
raise SchemaMigrationError("unable to migrate schema! Missing script for schema version %d" % current)
num -= 1
else:
if num <= current: continue
elif num > current + 1:
raise SchemaMigrationError("unable to migrate schema! Missing script for schema version %d" % (current + 1,))
# import the migration script and
# call appropriate method
exec "import migration.%s" % name
module = getattr(migration, name)
if rollback:
print "reverting schema to version %d..." % num
else:
print "migrating schema to version %d..." % num
getattr(module, action)(self.__dbh)
# update schema version to num
# ATTENTION: this calls PRAGMA which will commit current transaction!
current = self.schema_version = num
# if current now equal to version, break out of loop
if current == version:
break
# all done, reset isolation_level
self.__dbh.isolation_level = current_isolation
print "Migration to schema version %d complete!" % version
0
Example 127
def get_cube(cube, init=False, pkgs=None, cube_paths=None, config=None,
backends=None, **kwargs):
'''
Dynamically locate and load a metrique cube
:param cube: name of the cube class to import from given module
:param init: flag to request initialized instance or uninitialized class
:param config: config dict to pass on initialization (implies init=True)
:param pkgs: list of package names to search for the cubes in
:param cube_path: additional paths to search for modules in (sys.path)
:param kwargs: additional kwargs to pass to cube during initialization
'''
pkgs = pkgs or ['cubes']
pkgs = [pkgs] if isinstance(pkgs, basestring) else pkgs
# search in the given path too, if provided
cube_paths = cube_paths or []
cube_paths_is_basestring = isinstance(cube_paths, basestring)
cube_paths = [cube_paths] if cube_paths_is_basestring else cube_paths
cube_paths = [os.path.expanduser(path) for path in cube_paths]
# append paths which don't already exist in sys.path to sys.path
[sys.path.append(path) for path in cube_paths if path not in sys.path]
pkgs = pkgs + DEFAULT_PKGS
err = False
for pkg in pkgs:
try:
_cube = _load_cube_pkg(pkg, cube)
except ImportError as err:
_cube = None
if _cube:
break
else:
logger.error(err)
raise RuntimeError('"%s" not found! %s; %s \n%s)' % (
cube, pkgs, cube_paths, sys.path))
if init:
_cube = _cube(config=config, **kwargs)
return _cube
0
Example 128
def getTestSuites(loc=gramps_root):
# in a developer's checkout, it is worth filtering-out .git
# and we only need to look inside test subdirs
# (this might change)
# this is not so performance critical that we can't afford
# a couple of function calls to make it readable
# TODO: handle parts of a module (see unittest.py)
ldr= unittest.defaultTestLoader
test_dirname = "test"
test_suffix = "_test.py"
def test_mod(p,ds):
""" test for path p=test dir; removes a dir '.git' in ds """
if ".git" in ds:
ds.remove(".git")
return os.path.basename(p) == test_dirname
def match_mod(fs):
""" test for any test modules; deletes all non-tests """
# NB: do not delete fs elements within a "for f in fs"
dels= [f for f in fs if not f.endswith(test_suffix)]
for f in dels:
fs.remove(f)
return len(fs) > 0
test_suites = []
perf_suites = []
# note that test_mod and match_mod modify passed-in lists
paths = [(path,files) for path,dirs,files in os.walk(loc)
if test_mod(path,dirs) and match_mod(files)]
## NO -- see explanation below
## oldpath = list(sys.path)
for (dir,test_modules) in paths:
sys.path.append(dir)
for module in test_modules:
if not module.endswith(test_suffix):
raise ValueError
mod = __import__(module[:-len(".py")])
if getattr(mod,"suite",None):
test_suites.append(mod.suite())
else:
test_suites.append(ldr.loadTestsFromModule(mod))
try:
perf_suites.append(mod.perfSuite())
except:
pass
# NO: was: remove temporary paths added
# this seems like it should be reasonable,
# but it causes failure in _GrampsDbWRFactories_test.py
# (I suspect it is an actual bug in the runner
# but the easiest fix is to keep the imports,
# which is what other loaders seem to do)
# ==> this aspect of test frameworks is *hard*
## NO -- do NOT:
## remove temporary paths added
## sys.path = list(oldpath)
return (test_suites,perf_suites)
0
Example 129
Project: modrana Source File: modrana.py
def _loadDeviceModule(self):
"""Load the device module"""
if self.dmod: # don't reload the module
return
# get the device module string
# (a unique device module string identificator)
if self.args.d:
device = self.args.d
else: # no device specified from CLI
# try to auto-detect the current device
from core import platform_detection
device = platform_detection.getBestDeviceModuleId()
device = device.lower() # convert to lowercase
self.initInfo["device"] = device
# get GUI ID from the CLI argument
if self.args.u:
try:
self.GUIString = self.args.u.split(":")[0].upper()
except Exception:
log.exception('splitting the GUI string failed')
else: # no ID specified
# the N900 device module needs the GUIString
# at startup
if device == "n900":
self.GUIString = "GTK"
# set the pre-import-visible GUIString
# for the device module
gs.GUIString = self.GUIString
## load the device specific module
# NOTE: other modules need the device and GUI modules
# during init
deviceModulesPath = os.path.join(MAIN_MODULES_FOLDER, "device_modules")
sys.path.append(deviceModulesPath)
dmod_instance = self._loadModule("device_%s" % device, "device")
if dmod_instance is None:
log.critical("!! device module failed to load !!\n"
"loading the Neo device module as fail-safe")
device = "neo"
dmod_instance = self._loadModule("device_%s" % device, "device")
self.dmod = dmod_instance
# if no GUIString was specified from CLI,
# get preferred GUI module strings from the device module
if self.GUIString == "":
ids = self.dmod.getSupportedGUIModuleIds()
if ids:
self.GUIString = ids[0]
else:
self.GUIString = "GTK" # fallback
# export the GUI string
# set the pre-import visible GUI string and subtype
splitGUIString = self.GUIString.split(":")
gs.GUIString = splitGUIString[0]
if len(splitGUIString) >= 2:
gs.GUISubtypeString = splitGUIString[1]
0
Example 130
Project: EventGhost Source File: BuildLibrary.py
def DoTask(self):
"""
Build the library and .exe files with py2exe.
"""
buildSetup = self.buildSetup
sys.path.append(EncodePath(buildSetup.pyVersionDir))
from distutils.core import setup
InstallPy2exePatch()
import py2exe
origIsSystemDLL = py2exe.build_exe.isSystemDLL
def isSystemDLL(path):
if basename(path).lower().startswith("api-ms-win-"):
return 1
else:
return origIsSystemDLL(path)
py2exe.build_exe.isSystemDLL = isSystemDLL
libraryDir = buildSetup.libraryDir
if exists(libraryDir):
for filename in os.listdir(libraryDir):
path = join(libraryDir, filename)
if not os.path.isdir(path):
os.remove(path)
setup(
script_args=["py2exe"],
windows=[Target(buildSetup)],
verbose=0,
zipfile=EncodePath(join(buildSetup.libraryName, self.zipName)),
options = dict(
build=dict(build_base=join(buildSetup.tmpDir, "build")),
py2exe=dict(
compressed=0,
includes=["encodings", "encodings.*", "Imports"],
excludes=buildSetup.excludeModules,
dll_excludes = DLL_EXCLUDES,
dist_dir = EncodePath(buildSetup.sourceDir),
custom_boot_script=join(
buildSetup.dataDir, "Py2ExeBootScript.py"
),
)
)
)
dllNames = [basename(name) for name in glob(join(libraryDir, "*.dll"))]
neededDlls = []
paths = [sys.prefix]
if hasattr(sys, "real_prefix"):
paths.append(sys.real_prefix)
for path in paths:
for _, _, files in os.walk(path):
for filename in files:
if filename in dllNames:
neededDlls.append(filename)
for dllName in dllNames:
if dllName not in neededDlls:
os.remove(join(libraryDir, dllName))
0
Example 131
Project: func Source File: hardware.py
def hw_info(with_devices=True):
# this may fail if smolt is not installed. That's ok. hal_info will
# still work.
# hack: smolt is not installed in site-packages
sys.path.append("/usr/share/smolt/client")
import smolt
hardware = smolt.Hardware()
host = hardware.host
# NOTE: casting is needed because these are DBusStrings, not real strings
data = {
'os' : str(host.os),
'defaultRunlevel' : str(host.defaultRunlevel),
'bogomips' : str(host.bogomips),
'cpuVendor' : str(host.cpuVendor),
'cpuModel' : str(host.cpuModel),
'numCpus' : str(host.numCpus),
'cpuSpeed' : str(host.cpuSpeed),
'systemMemory' : str(host.systemMemory),
'systemSwap' : str(host.systemSwap),
'kernelVersion' : str(host.kernelVersion),
'language' : str(host.language),
'platform' : str(host.platform),
'systemVendor' : str(host.systemVendor),
'systemModel' : str(host.systemModel),
'formfactor' : str(host.formfactor),
'selinux_enabled' : str(host.selinux_enabled),
'selinux_enforce' : str(host.selinux_enforce)
}
# if no hardware info requested, just return the above bits
if not with_devices:
return data
collection = data["devices"] = []
for item in hardware.deviceIter():
(VendorID,DeviceID,SubsysVendorID,SubsysDeviceID,Bus,Driver,Type,Description) = item
collection.append({
"VendorID" : str(VendorID),
"DeviceID" : str(DeviceID),
"SubsysVendorID" : str(SubsysVendorID),
"Bus" : str(Bus),
"Driver" : str(Driver),
"Type" : str(Type),
"Description" : str(Description)
})
return data
0
Example 132
Project: pizco Source File: __init__.py
def main(args=None):
import argparse
parser = argparse.ArgumentParser('Starts an server')
parser.add_argument('-g', '--gui', action='store_true',
help='Open a small window to display the server status.')
parser.add_argument('-v', '--verbose', action='store_true',
help='Print debug information to the console.')
parser.add_argument('-p', '--path', type=str,
help='Append this path to sys.path')
parser.add_argument('rep_endpoint',
help='REP endpoint of the Server')
parser.add_argument('pub_endpoint',
help='PUB endpoint of the Server')
args = parser.parse_args(args)
if args.path:
sys.path.append(args.path)
if args.verbose:
LOGGER.addHandler(logging.StreamHandler())
LOGGER.setLevel(logging.DEBUG)
from pizco import Server
s = Server(None, args.rep_endpoint, args.pub_endpoint)
print('Server started at {}'.format(s.rep_endpoint))
if args.gui:
if sys.version_info < (3, 0):
from Tkinter import Tk, Label
else:
from tkinter import Tk, Label
import time
while s.served_object is None:
time.sleep(.1)
name = s.served_object.__class__.__name__
root = Tk()
root.title('Pizco Server: {}'.format(name))
Label(root, text='{}'.format(name)).pack(padx=5)
Label(root, text='REP: {}'.format(s.rep_endpoint)).pack(padx=5)
Label(root, text='PUB: {}'.format(s.pub_endpoint)).pack(padx=5)
root.resizable(width=False, height=False)
root.mainloop()
else:
print('Press CTRL+c to stop ...')
s.serve_forever()
print('Server stopped')
0
Example 133
Project: mayaseed Source File: install_helper.py
def install():
cmds.confirmDialog(m='After clicking \'OK\' a file dialogue will open. Please navigate to and select the mayaseed directory and click \'select\'.', button=['ok'])
maya_app_dir = mel.eval('getenv MAYA_APP_DIR')
mayaseed_dir = cmds.fileDialog2(fm=2, okc='Select', cap='Select mayaseed directory')
if mayaseed_dir is not None:
mayaseed_dir = mayaseed_dir[0]
try:
sys.path.append(os.path.join(mayaseed_dir, 'scripts'))
import ms_commands
except:
print 'No valid mayaseed directory found.'
return False
modules_path = os.path.join(maya_app_dir, 'modules')
if not os.path.exists(modules_path):
print '{0} does not exist, creating...'.format(modules_path)
os.makedirs(modules_path)
if not os.path.exists(modules_path):
print 'Failed to create module directory.'
return False
module_file_path = os.path.join(modules_path, 'mayaseed.module')
if os.path.exists(module_file_path):
continue_return_value = cmds.confirmDialog(m='A mayaseed module file already exists from a previous installation, would you like to overwrite it?', button=['yes','no'])
if continue_return_value == 'no':
return False
try:
module_file_contents = "+ mayaseed {0} {1} \nMAYA_CUSTOM_TEMPLATE_PATH+:=scripts".format(ms_commands.MAYASEED_VERSION, mayaseed_dir)
file = open(module_file_path, 'w')
file.write(module_file_contents)
file.close()
except:
print 'Error creating the mayaseed module file'
return False
return True
0
Example 134
def setUp(self):
super(PatchTest, self).setUp()
self.zstorm = ZStorm()
self.patchdir = self.make_path()
self.pkgdir = os.path.join(self.patchdir, "mypackage")
os.makedirs(self.pkgdir)
f = open(os.path.join(self.pkgdir, "__init__.py"), "w")
f.write("shared_data = []")
f.close()
# Order of creation here is important to try to screw up the
# patch ordering, as os.listdir returns in order of mtime (or
# something).
for pname, data in [("patch_380.py", patch_test_1),
("patch_42.py", patch_test_0)]:
self.add_module(pname, data)
sys.path.append(self.patchdir)
self.filename = self.make_path()
self.uri = "sqlite:///%s" % self.filename
self.store = self.zstorm.create(None, self.uri)
self.store.execute("CREATE TABLE patch "
"(version INTEGER NOT NULL PRIMARY KEY)")
self.assertFalse(self.store.get(Patch, (42)))
self.assertFalse(self.store.get(Patch, (380)))
import mypackage
self.mypackage = mypackage
self.patch_applier = PatchApplier(self.store, self.mypackage)
# Create another store just to keep track of the state of the
# whole transaction manager. See the assertion functions below.
self.another_store = self.zstorm.create(None, "sqlite:")
self.another_store.execute("CREATE TABLE test (id INT)")
self.another_store.commit()
self.prepare_for_transaction_check()
0
Example 135
def __init__(self,
templates_path,
dest_path,
var_set_path=None,
use_env_vars=False,
variables=None,
filters_path=None,
ignores_path=None,
watch_command=None,
watch_dirs=None):
self.variables = variables or [] # Variable sets to apply
self.use_env_vars = use_env_vars # Whether or not to use env vars
self.watch_paths = set() # List of paths to watch
# Check required paths
if os.path.exists(templates_path):
self.templates_path = templates_path
self.templates_path_re = re.compile(
'^{}'.format(templates_path))
self.watch_paths.add(templates_path)
else:
raise NotFoundError(templates_path, "templates path")
if os.path.exists(dest_path):
self.dest_path = dest_path
else:
raise NotFoundError(dest_path, "destination path")
# Watchdog
self.observer = Observer()
self.watch_command = watch_command
# Jinja2
self.env = Environment(loader=FileSystemLoader(templates_path),
keep_trailing_newline=True,
undefined=StrictUndefined,
autoescape=False,
cache_size=0)
self.defaults = {
'filters': self.env.filters,
'globals': self.env.globals,
}
# Variables
if var_set_path:
if os.path.exists(var_set_path):
self.var_set_path = var_set_path
self.var_set_path_re = re.compile(
'^{}/?'.format(var_set_path or ''))
self.watch_paths.add(var_set_path)
else:
raise NotFoundError(var_set_path, "variable set path")
else:
self.var_set_path = None
# Filters
if filters_path:
if os.path.exists(filters_path):
sys.path.append(os.path.dirname(filters_path))
self.filters_module = os.path.splitext(
os.path.basename(filters_path))[0]
self.watch_paths.add(filters_path)
else:
raise NotFoundError(filters_path, "filters path")
else:
self.filters_module = None
# Ignores
if ignores_path:
if os.path.exists(ignores_path):
self.ignores_path = ignores_path
self.watch_paths.add(ignores_path)
else:
raise NotFoundError(ignores_path, "ignores file")
else:
self.ignores_path = None
# Override watch_paths?
if watch_dirs:
self.watch_paths = watch_dirs
# Initial setup
self.refresh()
0
Example 136
Project: Nova Source File: hubble.py
def _load_module(self, name):
'''
Override the module load code
'''
mod = None
fpath, suffix = self.file_mapping[name]
self.loaded_files.add(name)
if suffix == '.yaml':
try:
with open(fpath) as fh_:
data = yaml.safe_load(fh_)
except Exception as exc:
self.__missing_data__[name] = str(exc)
return False
self.__data__[name] = data
return True
try:
sys.path.append(os.path.dirname(fpath))
desc = self.suffix_map[suffix]
# if it is a directory, we don't open a file
with salt.utils.fopen(fpath, desc[1]) as fn_:
mod = imp.load_module(
'{0}.{1}.{2}.{3}'.format(
self.loaded_base_name,
self.mod_type_check(fpath),
self.tag,
name
), fn_, fpath, desc)
except IOError:
raise
except ImportError as error:
log.debug(
'Failed to import {0} {1}:\n'.format(
self.tag, name
),
exc_info=True
)
self.missing_modules[name] = str(error)
return False
except Exception as error:
log.error(
'Failed to import {0} {1}, this is due most likely to a '
'syntax error:\n'.format(
self.tag, name
),
exc_info=True
)
self.missing_modules[name] = str(error)
return False
except SystemExit as error:
log.error(
'Failed to import {0} {1} as the module called exit()\n'.format(
self.tag, name
),
exc_info=True
)
self.missing_modules[name] = str(error)
return False
finally:
sys.path.pop()
mod.__grains__ = __grains__
mod.__pillar__ = __pillar__
mod.__opts__ = __opts__
mod.__salt__ = __salt__
# pack whatever other globals we were asked to
for p_name, p_value in six.iteritems(self.pack):
setattr(mod, p_name, p_value)
module_name = name
# Call a module's initialization method if it exists
module_init = getattr(mod, '__init__', None)
if inspect.isfunction(module_init):
try:
module_init(self.opts)
except TypeError as e:
log.error(e)
except Exception:
err_string = '__init__ failed'
log.debug(
'Error loading {0}.{1}: {2}'.format(
self.tag,
module_name,
err_string),
exc_info=True)
self.missing_modules[name] = err_string
return False
# if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it.
if self.virtual_enable:
(virtual_ret, module_name, virtual_err) = self.process_virtual(
mod,
module_name,
)
if virtual_err is not None:
log.debug('Error loading {0}.{1}: {2}'.format(self.tag,
module_name,
virtual_err,
))
# if process_virtual returned a non-True value then we are
# supposed to not process this module
if virtual_ret is not True:
# If a module has information about why it could not be loaded, record it
self.missing_modules[name] = virtual_err
return False
# If this is a proxy minion then MOST modules cannot work. Therefore, require that
# any module that does work with salt-proxy-minion define __proxyenabled__ as a list
# containing the names of the proxy types that the module supports.
#
# Render modules and state modules are OK though
if 'proxy' in self.opts:
if self.tag in ['grains', 'proxy']:
if not hasattr(mod, '__proxyenabled__') or \
(self.opts['proxy']['proxytype'] not in mod.__proxyenabled__ and
'*' not in mod.__proxyenabled__):
err_string = 'not a proxy_minion enabled module'
self.missing_modules[name] = err_string
return False
if getattr(mod, '__load__', False) is not False:
log.info(
'The functions from module {0!r} are being loaded from the '
'provided __load__ attribute'.format(
module_name
)
)
mod_dict = salt.utils.odict.OrderedDict()
# In nova we only care about the audit() function, and we want to
# store it with directory structure in the name.
for attr in getattr(mod, '__load__', dir(mod)):
if attr != 'audit':
continue
func = getattr(mod, attr)
# Save many references for lookups
self._dict[name] = func
mod_dict[name] = func
self.loaded_modules[name] = mod_dict
return True
0
Example 137
Project: LO-PHI Source File: helper.py
def import_analysis_scripts(path):
"""
This is used to import analysis script files and extract their classes.
@path: Directory on the disk that contains module files that all
contain subclasses of LophiAnalysis
@return: dict of analysis classes (dict[name] = Class)
"""
# Append this path to import from it
sys.path.append(path)
analysis_classes = {}
# scan our path for suitable modules to import
if os.path.exists(path) and os.path.isdir(path):
for dirpath, _dirnames, filenames in os.walk(path):
for filename in filenames:
if filename.endswith(".py") and not filename.startswith("_"):
# get our full filename
path_filename = os.path.join(dirpath,filename)
# get our module name
split_path = dirpath.split(os.path.sep)
module = '.'.join(split_path)
module = filename[:-3]
logger.debug("Extracting analyses from %s..."%module)
try:
tmp_module = importlib.import_module(module)
analysis = extract_analysis(tmp_module)
except:
logger.error("Could not import module: %s"%module)
G.print_traceback()
continue
if analysis is not None:
if "NAME" in analysis.__dict__:
analysis_classes[analysis.NAME] = (analysis,path_filename)
else:
logger.warning("Found analysis with no NAME attribute")
analysis_classes[analysis.__class__.__name__] = (analysis,path_filename)
return analysis_classes
0
Example 138
def main():
"""
The main purpose of this function is to call the functions that setup and run sca workflow
This function on its own does the profiling of the workflow
Parameters
----------
None
Returns
-------
None
Notes
-----
sca_profiler reports Memory Usage, the CPU usage and the IO usage of the sca workflow
"""
parser = argparse.ArgumentParser(description="example: \
run sca_profiler.py -c config.py")
parser.add_argument('-c', '--config',
dest='config',
required=True,
help='location of config file'
)
args = parser.parse_args()
path, fname = os.path.split(os.path.realpath(args.config))
sys.path.append(path)
c = __import__(fname.split('.')[0])
p = psutil.Process(os.getpid())
p.get_cpu_percent(interval=0)
cpu_before = p.get_cpu_times()
prep_workflow(c)
print '\n\nSCA Memory CPU and IO Stats'
print '---------------------------'
print 'Memory Usage: ', p.get_memory_info()
print 'CPU Times before worlflow is run: ', cpu_before, ' & after workflow is run: ', p.get_cpu_times()
print 'CPU Usage: ', p.get_cpu_percent(interval=1)
print 'IO Usage: ', p.get_io_counters()
print '---------------------------\n\n'
0
Example 139
Project: popcorn_maker Source File: doctests.py
def loadTestsFromFile(self, filename):
"""Load doctests from the file.
Tests are loaded only if filename's extension matches
configured doctest extension.
"""
if self.extension and anyp(filename.endswith, self.extension):
name = os.path.basename(filename)
dh = open(filename)
try:
doc = dh.read()
finally:
dh.close()
fixture_context = None
globs = {'__file__': filename}
if self.fixtures:
base, ext = os.path.splitext(name)
dirname = os.path.dirname(filename)
sys.path.append(dirname)
fixt_mod = base + self.fixtures
try:
fixture_context = __import__(
fixt_mod, globals(), locals(), ["nop"])
except ImportError, e:
log.debug(
"Could not import %s: %s (%s)", fixt_mod, e, sys.path)
log.debug("Fixture module %s resolved to %s",
fixt_mod, fixture_context)
if hasattr(fixture_context, 'globs'):
globs = fixture_context.globs(globs)
parser = doctest.DocTestParser()
test = parser.get_doctest(
doc, globs=globs, name=name,
filename=filename, lineno=0)
if test.examples:
case = DocFileCase(
test,
setUp=getattr(fixture_context, 'setup_test', None),
tearDown=getattr(fixture_context, 'teardown_test', None),
result_var=self.doctest_result_var)
if fixture_context:
yield ContextList((case,), context=fixture_context)
else:
yield case
else:
yield False # no tests to load
0
Example 140
Project: api-samples Source File: 03_ArielAPISearchWorkFlow.py
def main():
import sys
import os
sys.path.append(os.path.realpath('../modules'))
import json
from arielapiclient import APIClient
# Creates instance of APIClient. It contains all of the API methods.
api_client = APIClient()
# This is the AQL expression to send for the search.
query_expression = "SELECT sourceIP from events"
# Use the query parameters above to call a method. This will call
# POST /searches on the Ariel API. (look at arielapiclient for more
# detail). A response object is returned. It contains
# successful or not successful search information.
# The search_id corresponding to this search is contained in
# the JSON object.
response = api_client.create_search(query_expression)
# Each response contains an HTTP response code.
# - Response codes in the 200 range indicate that your request succeeded.
# - Response codes in the 400 range indicate that your request failed due
# to incorrect input.
# - Response codes in the 500 range indicate that there was an error on
# the server side.
print(response.code)
# The search is asynchronous, so the response will not be the results of
# the search.
# The 2 lines below parse the body of the response (a JSON object)
# into a dictionary, so we can discern information, such as the search_id.
response_json = json.loads(response.read().decode('utf-8'))
# Prints the contents of the dictionary.
print(response_json)
# Retrieves the search_id of the query from the dictionary.
search_id = response_json['search_id']
# This block of code calls GET /searches/{search_id} on the Ariel API
# to determine if the search is complete. This block of code will repeat
# until the status of the search is 'COMPLETE' or there is an error.
response = api_client.get_search(search_id)
error = False
while (response_json['status'] != 'COMPLETED') and not error:
if (response_json['status'] == 'EXECUTE') | \
(response_json['status'] == 'SORTING') | \
(response_json['status'] == 'WAIT'):
response = api_client.get_search(search_id)
response_json = json.loads(response.read().decode('utf-8'))
else:
print(response_json['status'])
error = True
# After the search is complete, call the GET /searches/{search_id} to
# obtain the result of the search.
# Depending on whether the "application/json" or "application/csv"
# method is given, return search results will be in JSON form or CSV form.
response = api_client.get_search_results(
search_id, 'application/json', '1', '11')
body = response.read().decode('utf-8')
body_json = json.loads(body)
# This is for pretty printing the JSON object.
print(json.dumps(body_json, indent=2, separators=(',', ':')))
# This is the same call as before, but asks for a CSV object in return.
response = api_client.get_search_results(search_id, "application/csv")
print("\n" + response.read().decode('utf-8'))
# This method calls POST /searches/{search_id}. It saves the result of a
# search to a disk.
query_params = {"saveResults": "true"}
response = api_client.update_search(search_id, query_params)
0
Example 141
Project: mysql-utilities Source File: mut.py
def find_tests(path):
"""Find test files.
path[in] Path to find tests.
"""
test_files = []
suites_found = []
# Build the list of tests from suites to execute
for root, _, files in os.walk(path):
for f in files:
# Check --suites list. Skip if we have suites and dir is in list
start = len(path) + 1
if root[-2:] in ['/t', '\\t']:
end = len(root) - 2
else:
end = len(root)
directory = root[start:end]
if opt.suites:
if directory == "":
if "main" not in opt.suites:
continue
elif directory not in opt.suites:
continue
if directory == "":
directory = "main"
if not directory == "main" and not directory in suites_found:
suites_found.append(directory)
# Get file and extension
fname, ext = os.path.splitext(f)
# Skip template tests
if fname.endswith('_template'):
continue
# Check for suite.test as well as simply test
if (args and fname not in args and
"{0}.{1}".format(directory, fname) not in args):
continue
# See if test is to be skipped
if opt.skip_test:
if (fname in opt.skip_test or
"{0}.{1}".format(directory, fname) in opt.skip_test):
continue
# See if suite is to be skipped
if opt.skip_suites:
if directory in opt.skip_suites:
continue
# Include only tests that are .py files and ignore mut
# library files
if ext == ".py" and fname != "__init__" and fname != "mutlib":
test_ref = (directory, root, fname)
# Do selective tests based on matches for --do-test=
# Don't execute performance tests unless specifically
# told to do so.
if((opt.suites is not None and "performance" not in opt.suites
and directory == "performance")
or (directory == "performance" and
opt.suites is None)):
pass
else:
if opt.wildcard:
for wild in opt.wildcard:
if wild == fname[0:len(wild)]:
test_files.append(test_ref)
break
elif opt.like:
for like in opt.like:
if like in fname:
test_files.append(test_ref)
break
elif opt.skip_tests:
for skip in opt.skip_tests:
if skip != fname[0:len(skip)]:
test_files.append(test_ref)
break
# Add test if no wildcard and suite (dir) is included
else:
test_files.append(test_ref)
for suite in suites_found:
sys.path.append(os.path.join(SUITE_PATH, suite, "t"))
return test_files
0
Example 142
Project: decode-Django Source File: __init__.py
def setup_environ(settings_mod, original_settings_path=None):
"""
deprecated
Configures the runtime environment. This can also be used by external
scripts wanting to set up a similar environment to manage.py.
Returns the project directory (assuming the passed settings module is
directly in the project directory).
The "original_settings_path" parameter is optional, but recommended, since
trying to work out the original path from the module can be problematic.
"""
warnings.warn(
"The 'setup_environ' function is deprecated, "
"you likely need to update your 'manage.py'; "
"please see the Django 1.4 release notes "
"(https://docs.djangoproject.com/en/dev/releases/1.4/).",
DeprecationWarning)
# Add this project to sys.path so that it's importable in the conventional
# way. For example, if this file (manage.py) lives in a directory
# "myproject", this code would add "/path/to/myproject" to sys.path.
if '__init__.py' in upath(settings_mod.__file__):
p = os.path.dirname(upath(settings_mod.__file__))
else:
p = upath(settings_mod.__file__)
project_directory, settings_filename = os.path.split(p)
if project_directory == os.curdir or not project_directory:
project_directory = os.getcwd()
project_name = os.path.basename(project_directory)
# Strip filename suffix to get the module name.
settings_name = os.path.splitext(settings_filename)[0]
# Strip $py for Jython compiled files (like settings$py.class)
if settings_name.endswith("$py"):
settings_name = settings_name[:-3]
# Set DJANGO_SETTINGS_MODULE appropriately.
if original_settings_path:
os.environ['DJANGO_SETTINGS_MODULE'] = original_settings_path
else:
# If DJANGO_SETTINGS_MODULE is already set, use it.
os.environ['DJANGO_SETTINGS_MODULE'] = os.environ.get(
'DJANGO_SETTINGS_MODULE',
'%s.%s' % (project_name, settings_name)
)
# Import the project module. We add the parent directory to PYTHONPATH to
# avoid some of the path errors new users can have.
sys.path.append(os.path.join(project_directory, os.pardir))
import_module(project_name)
sys.path.pop()
return project_directory
0
Example 143
Project: spinalcordtoolbox Source File: msct_moco.py
def spline(folder_mat,nt,nz,verbose,index_b0 = [],graph=0):
# get path of the toolbox
status, path_sct = commands.getstatusoutput('echo $SCT_DIR')
# append path that contains scripts, to be able to load modules
sys.path.append(path_sct + '/scripts')
import sct_utils as sct
sct.printv('\n\n\n------------------------------------------------------------------------------',verbose)
sct.printv('Spline Regularization along T: Smoothing Patient Motion...',verbose)
file_mat = [[[] for i in range(nz)] for i in range(nt)]
for it in range(nt):
for iz in range(nz):
file_mat[it][iz] = folder_mat + 'mat.T' + str(it) + '_Z' + str(iz) + '.txt'
#Copying the existing Matrices to another folder
old_mat = folder_mat + 'old/'
if not os.path.exists(old_mat): os.makedirs(old_mat)
cmd = 'cp ' + folder_mat + '*.txt ' + old_mat
status, output = sct.run(cmd, verbose)
sct.printv('\nloading matrices...',verbose)
X = [[[] for i in range(nt)] for i in range(nz)]
Y = [[[] for i in range(nt)] for i in range(nz)]
X_smooth = [[[] for i in range(nt)] for i in range(nz)]
Y_smooth = [[[] for i in range(nt)] for i in range(nz)]
for iz in range(nz):
for it in range(nt):
file = open(file_mat[it][iz])
Matrix = np.loadtxt(file)
file.close()
X[iz][it] = Matrix[0,3]
Y[iz][it] = Matrix[1,3]
# Generate motion splines
sct.printv('\nGenerate motion splines...',verbose)
T = np.arange(nt)
if graph:
import pylab as pl
for iz in range(nz):
# frequency = scipy.fftpack.fftfreq(len(X[iz][:]), d=1)
# spectrum = np.abs(scipy.fftpack.fft(X[iz][:], n=None, axis=-1, overwrite_x=False))
# Wn = np.amax(frequency)/10
# N = 5 #Order of the filter
# b, a = scipy.signal.iirfilter(N, Wn, rp=None, rs=None, btype='low', analog=False, ftype='butter', output='ba')
# X_smooth[iz][:] = scipy.signal.filtfilt(b, a, X[iz][:], axis=-1, padtype=None)
spline = scipy.interpolate.UnivariateSpline(T, X[iz][:], w=None, bbox=[None, None], k=3, s=None)
X_smooth[iz][:] = spline(T)
if graph:
pl.plot(T,X_smooth[iz][:],label='spline_smoothing')
pl.plot(T,X[iz][:],marker='*',linestyle='None',label='original_val')
if len(index_b0)!=0:
T_b0 = [T[i_b0] for i_b0 in index_b0]
X_b0 = [X[iz][i_b0] for i_b0 in index_b0]
pl.plot(T_b0,X_b0,marker='D',linestyle='None',color='k',label='b=0')
pl.title('X')
pl.grid()
pl.legend()
pl.show()
# frequency = scipy.fftpack.fftfreq(len(Y[iz][:]), d=1)
# spectrum = np.abs(scipy.fftpack.fft(Y[iz][:], n=None, axis=-1, overwrite_x=False))
# Wn = np.amax(frequency)/10
# N = 5 #Order of the filter
# b, a = scipy.signal.iirfilter(N, Wn, rp=None, rs=None, btype='low', analog=False, ftype='butter', output='ba')
# Y_smooth[iz][:] = scipy.signal.filtfilt(b, a, Y[iz][:], axis=-1, padtype=None)
spline = scipy.interpolate.UnivariateSpline(T, Y[iz][:], w=None, bbox=[None, None], k=3, s=None)
Y_smooth[iz][:] = spline(T)
if graph:
pl.plot(T,Y_smooth[iz][:],label='spline_smoothing')
pl.plot(T,Y[iz][:],marker='*', linestyle='None',label='original_val')
if len(index_b0)!=0:
T_b0 = [T[i_b0] for i_b0 in index_b0]
Y_b0 = [Y[iz][i_b0] for i_b0 in index_b0]
pl.plot(T_b0,Y_b0,marker='D',linestyle='None',color='k',label='b=0')
pl.title('Y')
pl.grid()
pl.legend()
pl.show()
#Storing the final Matrices
sct.printv('\nStoring the final Matrices...',verbose)
for iz in range(nz):
for it in range(nt):
file = open(file_mat[it][iz])
Matrix = np.loadtxt(file)
file.close()
Matrix[0,3] = X_smooth[iz][it]
Matrix[1,3] = Y_smooth[iz][it]
file = open(file_mat[it][iz],'w')
np.savetxt(file_mat[it][iz], Matrix, fmt="%s", delimiter=' ', newline='\n')
file.close()
sct.printv('\n...Done. Patient motion has been smoothed', verbose)
sct.printv('------------------------------------------------------------------------------\n',verbose)
0
Example 144
Project: scrapy-cluster Source File: test_throttled_queue.py
def main():
import argparse
import redis
import time
import sys
from os import path
sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
from scutils.redis_queue import RedisPriorityQueue
from scutils.redis_throttled_queue import RedisThrottledQueue
parser = argparse.ArgumentParser(description="Throttled Queue Test Script."
" Start either a single or multiple processes to see the "
" throttled queue mechanism in action.")
parser.add_argument('-r', '--redis-host', action='store', required=True,
help="The Redis host ip")
parser.add_argument('-p', '--redis-port', action='store', default='6379',
help="The Redis port")
parser.add_argument('-m', '--moderate', action='store_const', const=True,
default=False, help="Moderate the outbound Queue")
parser.add_argument('-w', '--window', action='store', default=60,
help="The window time to test")
parser.add_argument('-n', '--num-hits', action='store', default=10,
help="The number of pops allowed in the given window")
parser.add_argument('-q', '--queue', action='store', default='testqueue',
help="The Redis queue name")
args = vars(parser.parse_args())
window = int(args['window'])
num = int(args['num_hits'])
host = args['redis_host']
port = args['redis_port']
mod = args['moderate']
queue = args['queue']
conn = redis.Redis(host=host, port=port)
q = RedisPriorityQueue(conn, queue)
t = RedisThrottledQueue(conn, q, window, num, mod)
def push_items(amount):
for i in range(0, amount):
t.push('item-'+str(i), i)
print "Adding", num * 2, "items for testing"
push_items(num * 2)
def read_items():
print "Kill when satisfied ^C"
ti = time.time()
count = 0
while True:
item = t.pop()
if item:
print "My item", item, "My time:", time.time() - ti
count += 1
try:
read_items()
except KeyboardInterrupt:
pass
t.clear()
print "Finished"
0
Example 145
Project: emulambda Source File: __init__.py
def main():
sys.path.append(os.getcwd())
sys.path.append("./lib")
args = parseargs()
# Get process peak RSS memory before execution
pre_rss = get_memory_usage()
# Import the lambda
lfunc = import_lambda(args.lambdapath)
# Build statistics dictionary
stats = {'clock': list(), 'rss': list()}
def execute(_event=None, _context=None):
"""
Encapsulation of _event-running code, with access to collectors and other variables in main() scope. Used
for both single-run and stream modes.
:param _event: A valid Lambda _event object.
:return: Void.
"""
# Invoke the lambda
# TODO consider refactoring to pass stats through function
result, exec_clock = invoke_lambda(lfunc, _event, _context, args.timeout, args.role)
# Get process peak RSS memory after execution
exec_rss = get_memory_usage() - pre_rss
# Store statistics
stats['clock'].append(exec_clock)
stats['rss'].append(exec_rss)
# Render the result
render_result(args.verbose, args.lambdapath, result, exec_clock, exec_rss)
if args.stream:
# Enter stream mode
emit_to_function(args.verbose, args.eventfile, execute)
render_summary(stats) if args.verbose else None
elif args.contextfile:
context = read_file_to_object(args.contextfile)
event = read_file_to_string(args.eventfile)
execute(parse_event(event), context)
else:
# Single event mode
event = read_file_to_string(args.eventfile)
execute(parse_event(event))
0
Example 146
Project: cylc Source File: jinja2support.py
def jinja2process(flines, dir_, template_vars=None):
"""Pass configure file through Jinja2 processor."""
env = Environment(
loader=FileSystemLoader(dir_),
undefined=StrictUndefined,
extensions=['jinja2.ext.do'])
# Load any custom Jinja2 filters in the suite definition directory
# Example: a filter to pad integer values some fill character:
# |(file SUITE_DEFINIION_DIRECTORY/Jinja2/foo.py)
# | #!/usr/bin/env python
# | def foo( value, length, fillchar ):
# | return str(value).rjust( int(length), str(fillchar) )
for fdir in [
os.path.join(os.environ['CYLC_DIR'], 'lib', 'Jinja2Filters'),
os.path.join(dir_, 'Jinja2Filters'),
os.path.join(os.environ['HOME'], '.cylc', 'Jinja2Filters')]:
if os.path.isdir(fdir):
sys.path.append(os.path.abspath(fdir))
for name in glob(os.path.join(fdir, '*.py')):
fname = os.path.splitext(os.path.basename(name))[0]
# TODO - EXCEPTION HANDLING FOR LOADING CUSTOM FILTERS
module = __import__(fname)
env.filters[fname] = getattr(module, fname)
# Import SUITE HOST USER ENVIRONMENT into template:
# (usage e.g.: {{environ['HOME']}}).
env.globals['environ'] = os.environ
# load file lines into a template, excluding '#!jinja2' so
# that '#!cylc-x.y.z' rises to the top.
# CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
# try:
# except Exception as exc:
# # This happens if we use an unknown Jinja2 filter, for example.
# # TODO - THIS IS CAUGHT BY VALIDATE BUT NOT BY VIEW COMMAND...
# raise TemplateError(exc)
if cylc.flags.verbose and template_vars:
print 'Setting Jinja2 template variables:'
for item in sorted(template_vars.items()):
print ' + %s=%s' % item
# CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
# AND TYPEERROR (e.g. for not using "|int" filter on number inputs.
# Convert unicode to plain str, ToDo - still needed for parsec?)
suiterc = []
template = env.from_string('\n'.join(flines[1:]))
for line in str(template.render(template_vars)).splitlines():
# Jinja2 leaves blank lines where source lines contain
# only Jinja2 code; this matters if line continuation
# markers are involved, so we remove blank lines here.
if not line.strip():
continue
# restoring newlines here is only necessary for display by
# the cylc view command:
# ##suiterc.append(line + '\n')
suiterc.append(line)
return suiterc
0
Example 147
Project: gcc-python-plugin Source File: testcpybuilder.py
def test_module_with_type(self):
# Verify an extension with a type
MODNAME = 'module_with_type'
sm = SimpleModule()
sm.cu.add_decl("""
struct PyExampleType {
PyObject_HEAD
int i;
};
""")
sm.cu.add_defn("PyObject *\n"
"example_Example_repr(PyObject * self)\n"
"{\n"
"#if PY_MAJOR_VERSION < 3\n"
" return PyString_FromString(\"example.ExampleType('')\");\n"
"#else\n"
" return PyUnicode_FromString(\"example.ExampleType('')\");\n"
"#endif\n"
"}\n")
sm.add_type_object(name = 'example_ExampleType',
localname = 'ExampleType',
tp_name = 'example.ExampleType',
struct_name = 'struct PyExampleType',
tp_repr = 'example_Example_repr')
sm.add_module_init(MODNAME, modmethods=None, moddoc='This is a doc string')
# print sm.cu.as_str()
# Build the module:
bm = BuiltModule(sm)
bm.build(MODNAME)
# Verify that it built:
sys.path.append(bm.tmpdir)
import module_with_type
self.assertEqual(repr(module_with_type.ExampleType()),
"example.ExampleType('')")
# Cleanup successful test runs:
bm.cleanup()
0
Example 148
Project: opengeode Source File: Asn1scc.py
def parse_asn1(*files, **options):
''' Call the ASN.1 parser on a number of files, and return the module
containing the AST
This function uses QProcess to launch the ASN.1 compiler because
the subprocess module from Python has issues on the Windows platform
'''
ast_version = options.get('ast_version', ASN1.UniqueEnumeratedNames)
rename_policy = options.get('rename_policy', ASN1.NoRename)
flags = options.get('flags', [ASN1.AstOnly])
pprint = options.get('pretty_print', False)
assert isinstance(ast_version, ASN1)
assert isinstance(rename_policy, ASN1)
assert isinstance(flags, list)
path_to_asn1scc = spawn.find_executable('asn1.exe')
if not path_to_asn1scc:
raise TypeError('ASN.1 Compiler not found in path')
if os.name == 'posix':
path_to_mono = spawn.find_executable('mono')
if not path_to_mono:
raise TypeErorr('"mono" not found in path. Please install it.')
binary = path_to_mono
arg0 = path_to_asn1scc
else:
binary = path_to_asn1scc
arg0 = ''
asn1scc_root = os.path.abspath(os.path.dirname(path_to_asn1scc))
# Create a temporary directory to store dataview.py and import it
tempdir = tempfile.mkdtemp()
sys.path.append(tempdir)
if os.name == 'nt':
# On windows, remove the drive letter, workaround to ASN1SCC bug
tempdir = tempdir[2:]
asn1scc_root = asn1scc_root[2:]
filename = str(uuid.uuid4()).replace('-', '_')
filepath = tempdir + os.sep + filename + '.py'
stg = asn1scc_root + os.sep + 'python.stg'
if pprint:
# Generate an html file with pretty-printed ASN.1 types
stg_qrc = QFile(':misc/pretty_print_asn1.stg')
stg_qrc.open(1)
content = stg_qrc.readAll()
stgfile = tempdir + os.sep + 'pretty_print_asn1.stg'
with open(stgfile, 'w') as tmpfile:
tmpfile.write(content.data())
out_html = tempdir + os.sep + 'dataview.html'
html = ['-customIcdUper', stgfile + '::' + out_html]
else:
html = []
args = [arg0, '-customStgAstVerion', str(ast_version.value),
'-customStg', stg + '::' + filepath,
'-renamePolicy', str(rename_policy.value)] + html + list(*files)
asn1scc = QProcess()
LOG.debug(os.getcwd())
LOG.debug(binary + ' ' + ' '.join(args))
asn1scc.start(binary, args)
_ = waitfor_qprocess(asn1scc, "ASN.1 Compiler")
if filename in AST.viewkeys():
# Re-import module if it was already loaded
ast = AST[filename]
reload(ast)
else:
ast = importlib.import_module(filename)
AST[filename] = ast
if pprint:
# add the path to the optionally-gernated pretty-printed HTML file
ast.html = out_html
return ast
0
Example 149
def test_reload(self):
# add test plugin to pythonpath
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp)
sys.path.append(tmp)
with open(os.path.join(tmp, 'p.py'), 'w') as fd:
fd.write(plugin)
bot = self.callFTU()
bot.include('p')
p = bot.get_plugin('p.P')
assert p.extend() is None
assert bot.extend() is None
crons = bot.get_plugin('irc3.plugins.cron.Crons')
assert len(crons) == 1
assert len(bot.registry.events['in']) == 6
bot.dispatch(':adm!user@host PRIVMSG #chan :!cmd')
self.assertSent(['PRIVMSG #chan :%s' % id(p)])
# modify test plugin
with open(os.path.join(tmp, 'p.py'), 'a') as fd:
fd.write('x = 1')
bot.reload()
# assume it's reloaded
np = bot.get_plugin('p.P')
assert np is not p
assert np.extend() is p
assert bot.extend() is p
assert crons == bot.get_plugin('irc3.plugins.cron.Crons')
assert len(crons) == 1
assert len(bot.registry.events['in']) == 6
bot.dispatch(':adm!user@host PRIVMSG #chan :!cmd')
self.assertSent(['PRIVMSG #chan :%s' % id(np)])
0
Example 150
Project: hooker Source File: StaticAnalysis.py
def execute(self):
"""
Executes the static analysis of APK using the androguard framework.
First parses the APK the get corresponding python object.
Then calls the androguard methods on the built object.
"""
sys.path.append(self.androguardPath)
from androguard.core.bytecodes import apk
self._logger.info("Analyzing {0} with androguard...".format(self.apkToAnalyze))
# Parse APK
parsedAPK = apk.APK(self.apkToAnalyze)
if parsedAPK is None:
raise Exception("Parsed APK is null, quitting StaticAnalysis.")
# Info: setters are dealing with None returns
try:
self.androidVersionCode = parsedAPK.get_androidversion_code()
self.androidVersionName = parsedAPK.get_androidversion_name()
self.mainActivity = parsedAPK.get_main_activity()
self.maxSDKVersion = parsedAPK.get_max_sdk_version()
if self.maxSDKVersion == "n/c":
self.maxSDKVersion = -1
self.minSDKVersion = parsedAPK.get_min_sdk_version()
if self.minSDKVersion == "n/c":
self.minSDKVersion = -1
self.packageName = parsedAPK.get_package()
self.timestamp = str(int(round(time.time() * 1000)))
self.activities = parsedAPK.get_activities()
self.permissions = parsedAPK.get_permissions()
self.providers = parsedAPK.get_providers()
self.receivers = parsedAPK.get_receivers()
self.services = parsedAPK.get_services()
self.libraries = parsedAPK.get_libraries()
except Exception,e:
self._logger.warn("An error occured while executing the static analysis of APK {0}: {1}".format(self.apkToAnalyze, e))
self.__saveResultsInESCluster()