sys.executable

Here are the examples of the python api sys.executable taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

135 Examples 7

Example 51

Project: ganga Source File: Batch.py
Function: submit
    def submit(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()

        inw = job.getInputWorkspace()
        outw = job.getOutputWorkspace()

        #scriptpath = self.preparejob(jobconfig,inw,outw)
        scriptpath = self.preparejob(jobconfig, master_input_sandbox)

        # FIX from Angelo Carbone
        # stderr_option = '-e '+str(outw.getPath())+'stderr'
        # stdout_option = '-o '+str(outw.getPath())+'stdout'

        # FIX from Alex Richards - see Savannah #87477
        stdout_option = self.config['stdoutConfig'] % str(outw.getPath())
        stderr_option = self.config['stderrConfig'] % str(outw.getPath())

        queue_option = ''
        if self.queue:
            queue_option = '-q ' + str(self.queue)

        try:
            jobnameopt = "-" + self.config['jobnameopt']
        except Exception, err:
            logger.debug("Unknown error: %s" % str(err))
            jobnameopt = False

        if self.extraopts:
            import re
            for opt in re.compile(r'(-\w+)').findall(self.extraopts):
                if opt in ('-o', '-e', '-oo', '-eo'):
                    logger.warning("option %s is forbidden", opt)
                    return False
                if self.queue and opt == '-q':
                    logger.warning("option %s is forbidden if queue is defined ( queue = '%s')", opt, self.queue)
                    return False
                if jobnameopt and opt == jobnameopt:
                    jobnameopt = False

            queue_option = queue_option + " " + self.extraopts

        if jobnameopt and job.name != '':
            # PBS doesn't like names with spaces
            tmp_name = job.name
            if isType(self, PBS):
                tmp_name = tmp_name.replace(" ", "_")
            queue_option = queue_option + " " + \
                jobnameopt + " " + "'%s'" % (tmp_name)

        # bugfix #16646
        if self.config['shared_python_executable']:
            import sys
            script_cmd = "%s %s" % (sys.executable, scriptpath)
        else:
            script_cmd = scriptpath

        command_str = self.config['submit_str'] % (inw.getPath(), queue_option, stderr_option, stdout_option, script_cmd)
        self.command_string = command_str
        rc, soutfile = self.command(command_str)
        with open(soutfile) as sout_file:
            sout = sout_file.read()
        import re
        m = re.compile(self.config['submit_res_pattern'], re.M).search(sout)
        if m is None:
            logger.warning('could not match the output and extract the Batch job identifier!')
            logger.warning('command output \n %s ', sout)
        else:
            self.id = m.group('id')
            try:
                queue = m.group('queue')
                if self.queue != queue:
                    if self.queue:
                        logger.warning('you requested queue "%s" but the job was submitted to queue "%s"', self.queue, queue)
                        logger.warning('command output \n %s ', sout)
                    else:
                        logger.info('using default queue "%s"', queue)
                    self.actualqueue = queue
            except IndexError:
                logger.info('could not match the output and extract the Batch queue name')

        # clean up the tmp file
        if os.path.exists(soutfile):
            os.remove(soutfile)

        return rc == 0

Example 52

Project: monaco Source File: monaco.py
def monaco():
    '''
    The management daemon for Monaco
    '''
    # Set up top-level console logger
    consolehandler = logging.StreamHandler()
    consolehandler.setLevel(logging.ERROR)
    consolehandler.setFormatter(logging.Formatter('%(name)s - %(levelname)s - %(message)s'))
    logger = logging.getLogger('monaco')
    logger.setLevel(logging.DEBUG)
    logger.addHandler(consolehandler)

    # Load config
    config.initLoggers()

    # Install reload handler
    def USR1_handler(signum, frame):
        ''' SIGUSR1 handler '''
        logger.info('Received SIGUSR1, reloading Monaco...')
        # this kills the current process tree and re-executes the
        # original call arguments in the same PID (good for upstart)
        os.execl(sys.executable, *([sys.executable]+sys.argv))
    signal.signal(signal.SIGUSR1, USR1_handler)

    # Setup scheduling
    sched = Scheduler(daemon=True)
    atexit.register(lambda: sched.shutdown(wait=False))
    sched.start()

    # Config and start slave thread to maintain app state
    slave = Slave()
    slave.update_subs()

    #slave.start() starting in apsched inherits daemon=True
    @sched.interval_schedule(seconds=1)
    def maintain_slave():
        '''
        This periodically ensures that the slave is subscribed to all the
        channels it should be. Again, this should be redundant
        '''
        if not hasattr(maintain_slave, 'started'):
            maintain_slave.started = True
            slave.start()
            logger.info('Slave thread started')
        if not slave.is_alive():
            logger.info('Slave thread died')
            slave.start()

    # lifeRaft - Use Raft algorithm to maintain management leader
    liferaft = LifeRaft()

    # Start master/monitor thread
    master = Master(liferaft)
    master.start()

    # maintain mgmt redis role from LifeRaft
    r = redis.StrictRedis(port=config.config['mgmt_port'])
    mastercli = None
    while True:
        time.sleep(1)
        try:
            master_tup = liferaft.value
            if not master_tup:
                logger.warn("Couldn't learn master value!")
            else:
                logger.debug('MGMT DB maintaining: %s', repr(master_tup))
                if master_tup[0] == config.config['IP']:
                    if r.info()['role'] == 'master':
                        # We're the master, and properly configured
                        continue
                    # Else set to master, rewrite config to persist
                    logger.debug('Promoting self to master')
                    r.slaveof()
                    r.config_rewrite()
                else:
                    if r.info()['role'] == 'slave' and r.info()['master_host'] == master_tup[0]:
                        # We're a slave and properly configured to the master
                        continue
                    for idx in xrange(3):
                        if idx != 0:
                            # Decongest on retry
                            time.sleep(0.5)
                        try:
                            # Try to get a connection to the mgmt master and verify it thinks it's master
                            if not mastercli:
                                mastercli = redis.StrictRedis(host=master_tup[0], port=config.config['mgmt_port'])
                            if mastercli.info()['role'] != 'master':
                                # If it doesn't think it's master, delete client and continue to retry
                                del mastercli
                                continue
                            break
                        except Exception:
                            continue
                    else:
                        # We didn't break
                        logger.debug('Assigned master (%s) has not yet assumed role', master_tup[0])
                        continue
                    # Set slave to proper master
                    logger.debug('Assigning self as slaveof %s:6379', master_tup[0])
                    r.slaveof(host=master_tup[0], port=config.config['mgmt_port'])
        except Exception:
            try:
                r = redis.StrictRedis(port=config.config['mgmt_port'])
            except Exception:
                logger.exception("Can't connect to mgmt db!")

Example 53

Project: tvnamer Source File: functional_runner.py
def run_tvnamer(with_files, with_flags = None, with_input = "", with_config = None, run_on_directory = False):
    """Runs tvnamer on list of file-names in with_files.
    with_files is a list of strings.
    with_flags is a list of command line arguments to pass to tvnamer.
    with_input is the sent to tvnamer's stdin
    with_config is a string containing the tvnamer to run tvnamer with.

    Returns a dict with stdout, stderr and a list of files created
    """
    # Create dummy files (config and episodes)
    tvnpath = get_tvnamer_path()
    episodes_location = make_temp_dir()
    dummy_files = make_dummy_files(with_files, episodes_location)

    if with_config is not None:
        configfname = make_temp_config(with_config)
        conf_args = ['-c', configfname]
    else:
        conf_args = []

    if with_flags is None:
        with_flags = []

    if run_on_directory:
        files = [episodes_location]
    else:
        files = dummy_files

    # Construct command
    cmd = [sys.executable, tvnpath] + conf_args + with_flags + files
    p("Running command:")
    p(" ".join(cmd))

    # Copy sys.path to PYTHONPATH so same modules are available as in
    # test environmen
    env = os.environ.copy()
    env['PYTHONPATH'] = ":".join(sys.path)

    proc = subprocess.Popen(
        cmd,
        stdout = subprocess.PIPE,
        stderr = subprocess.STDOUT, # All stderr to stdout
        stdin = subprocess.PIPE,
        env=env)

    proc.stdin.write(with_input.encode("utf-8"))
    output, _ = proc.communicate()

    if PY2:
        def unicodify(obj, encoding = "utf-8"):
            if isinstance(obj, basestring):
                if not isinstance(obj, unicode):
                    obj = unicode(obj, encoding)
            return obj
        output = unicodify(output)


    created_files = []

    for walkroot, walkdirs, walkfiles in os.walk(string_type(episodes_location)):
        curlist = [os.path.join(walkroot, name) for name in walkfiles]

        # Remove episodes_location from start of path
        curlist = [relpath(x, episodes_location) for x in curlist]

        created_files.extend(curlist)

    # Clean up dummy files and config
    clear_temp_dir(episodes_location)
    if with_config is not None:
        os.unlink(configfname)

    return {
        'output': output,
        'files': created_files,
        'returncode': proc.returncode}

Example 54

Project: EventGhost Source File: CheckDependencies.py
def CheckDependencies(buildSetup):
    failedDeps = []

    for dep in DEPENDENCIES:
        dep.buildSetup = buildSetup
        try:
            dep.Check()
        except (WrongVersion, MissingDependency):
            failedDeps.append(dep)

    if failedDeps and buildSetup.args.make_env and not os.environ.get("_REST"):
        if not IsAdmin():
            print WrapText(
                "ERROR: Can't create virtual environment from a command "
                "prompt without administrative privileges."
            )
            return False

        if not VirtualEnv.Running():
            if not VirtualEnv.Exists():
                print "Creating our virtual environment..."
                CreateVirtualEnv()
                print ""
            VirtualEnv.Activate()

        for dep in failedDeps[:]:
            print "Installing %s..." % dep.name
            try:
                if InstallDependency(dep):  #and dep.Check():
                    failedDeps.remove(dep)
                else:
                    print "ERROR: Installation of %s failed!" % dep.name
            except MissingChocolatey:
                print WrapText(
                    "ERROR: To complete installation of this package, I need "
                    "package manager Chocolatey, which wasn't found and "
                    "couldn't be installed automatically. Please install it "
                    "by hand and try again."
                )
            except MissingPip:
                print WrapText(
                    "ERROR: To complete installation of this package, I need "
                    "package manager pip, which wasn't found. Note that "
                    "all versions of Python capable of building EventGhost "
                    "come bundled with pip, so please install a supported "
                    "version of Python and try again."
                )
            except MissingPowerShell:
                print WrapText(
                    "ERROR: To complete installation of this package, I need "
                    "package manager Chocolatey, which can't be installed "
                    "without PowerShell. Please install PowerShell by hand "
                    "and try again."
                )
            print ""
        VirtualEnv.Restart()

    if failedDeps:
        print WrapText(
            "Before we can continue, the following dependencies must "
            "be installed:"
        )
        print ""
        for dep in failedDeps:
            print "  *", dep.name, dep.version
            if dep.url:
                print "    Link:", dep.url
            print ""
        print WrapText(
            "Dependencies without an associated URL can be installed via "
            "`pip install [package-name]`. Dependencies in .whl format "
            "can be installed via `pip install [url]`. All other dependencies "
            "will need to be installed manually or via Chocolatey "
            "<https://chocolatey.org/>."
        )
        if not buildSetup.args.make_env:
            print ""
            print WrapText(
                "Alternately, from a command prompt with administrative "
                "privileges, I can try to create a virtual environment for "
                "you that satisfies all dependencies via `%s %s --make-env`."
                % (basename(sys.executable).split(".")[0], sys.argv[0])
            )
    return not failedDeps

Example 55

Project: tfmesos Source File: scheduler.py
    def to_task_info(self, offer, master_addr, gpu_uuids=[],
                     gpu_resource_type=None):
        ti = Dict()
        ti.task_id.value = str(self.mesos_task_id)
        ti.agent_id.value = offer.agent_id.value
        ti.name = '/job:%s/task:%s' % (self.job_name, self.task_index)
        ti.resources = resources = []

        cpus = Dict()
        resources.append(cpus)
        cpus.name = 'cpus'
        cpus.type = 'SCALAR'
        cpus.scalar.value = self.cpus

        mem = Dict()
        resources.append(mem)
        mem.name = 'mem'
        mem.type = 'SCALAR'
        mem.scalar.value = self.mem

        image = os.environ.get('DOCKER_IMAGE')

        if image is not None:
            ti.container.type = 'DOCKER'
            ti.container.docker.image = image
            ti.container.volumes = volumes = []

            for path in ['/etc/passwd', '/etc/group']:
                v = Dict()
                volumes.append(v)
                v.host_path = v.container_path = path
                v.mode = 'RO'

            for src, dst in iteritems(self.volumes):
                v = Dict()
                volumes.append(v)
                v.container_path = dst
                v.host_path = src
                v.mode = 'RW'

            if self.gpus and gpu_uuids and gpu_resource_type is not None:
                if gpu_resource_type == 'SET':
                    hostname = offer.hostname
                    url = 'http://%s:3476/docker/cli?dev=%s' % (
                        hostname, urllib.parse.quote(
                            ' '.join(gpu_uuids)
                        )
                    )

                    try:
                        ti.container.docker.parameters = parameters = []
                        docker_args = urllib.request.urlopen(url).read()
                        for arg in docker_args.split():
                            k, v = arg.split('=')
                            assert k.startswith('--')
                            k = k[2:]
                            p = Dict()
                            parameters.append(p)
                            p.key = k
                            p.value = v

                        gpus = Dict()
                        resources.append(gpus)
                        gpus.name = 'gpus'
                        gpus.type = 'SET'
                        gpus.set.item = gpu_uuids
                    except Exception:
                        logger.exception(
                            'fail to determine remote device parameter,'
                            ' disable gpu resources'
                        )
                else:
                    gpus = Dict()
                    resources.append(gpus)
                    gpus.name = 'gpus'
                    gpus.type = 'SCALAR'
                    gpus.scalar.value = len(gpu_uuids)

        else:
            if self.gpus and gpu_uuids and gpu_resource_type is not None:
                if gpu_resource_type == 'SET':
                    gpus = Dict()
                    resources.append(gpus)
                    gpus.name = 'gpus'
                    gpus.type = 'SET'
                    gpus.set.item = gpu_uuids
                else:
                    gpus = Dict()
                    resources.append(gpus)
                    gpus.name = 'gpus'
                    gpus.type = 'SCALAR'
                    gpus.scalar.value = len(gpu_uuids)

        ti.command.shell = True
        cmd = [
            sys.executable, '-m', '%s.server' % __package__,
            str(self.mesos_task_id), master_addr
        ]
        ti.command.value = ' '.join(cmd)
        ti.command.environment.variables = variables = []
        env = Dict()
        variables.append(env)
        env.name = 'PYTHONPATH'
        env.value = ':'.join(sys.path)
        return ti

Example 56

Project: pygame_cffi Source File: run_tests.py
def run(*args, **kwds):
    """Run the Pygame unit test suite and return (total tests run, fails dict)

    Positional arguments (optional):
    The names of tests to include. If omitted then all tests are run. Test
    names need not include the trailing '_test'.

    Keyword arguments:
    incomplete - fail incomplete tests (default False)
    nosubprocess - run all test suites in the current process
                   (default False, use separate subprocesses)
    dump - dump failures/errors as dict ready to eval (default False)
    file - if provided, the name of a file into which to dump failures/errors
    timings - if provided, the number of times to run each individual test to
              get an average run time (default is run each test once)
    exclude - A list of TAG names to exclude from the run. The items may be
              comma or space separated.
    show_output - show silenced stderr/stdout on errors (default False)
    all - dump all results, not just errors (default False)
    randomize - randomize order of tests (default False)
    seed - if provided, a seed randomizer integer
    multi_thread - if provided, the number of THREADS in which to run
                   subprocessed tests
    time_out - if subprocess is True then the time limit in seconds before
               killing a test (default 30)
    fake - if provided, the name of the fake tests package in the
           run_tests__tests subpackage to run instead of the normal
           Pygame tests
    python - the path to a python executable to run subprocessed tests
             (default sys.executable)
    interative - allow tests tagged 'interative'.

    Return value:
    A tuple of total number of tests run, dictionary of error information. The
    dictionary is empty if no errors were recorded.

    By default individual test modules are run in separate subprocesses. This
    recreates normal Pygame usage where pygame.init() and pygame.quit() are
    called only once per program execution, and avoids unfortunate
    interactions between test modules. Also, a time limit is placed on test
    execution, so frozen tests are killed when there time allotment expired.
    Use the single process option if threading is not working properly or if
    tests are taking too long. It is not guaranteed that all tests will pass
    in single process mode.

    Tests are run in a randomized order if the randomize argument is True or a
    seed argument is provided. If no seed integer is provided then the system
    time is used.

    Individual test modules may have a corresponding *_tags.py module,
    defining a __tags__ attribute, a list of tag strings used to selectively
    omit modules from a run. By default only the 'interactive', 'ignore', and
    'subprocess_ignore' tags are ignored. 'interactive' is for modules that
    take user input, like cdrom_test.py. 'ignore' and 'subprocess_ignore' for
    for disabling modules for foreground and subprocess modes respectively.
    These are for disabling tests on optional modules or for experimental
    modules with known problems. These modules can be run from the console as
    a Python program.

    This function can only be called once per Python session. It is not
    reentrant.

    """

    global was_run

    if was_run:
        raise RuntimeError("run() was already called this session")
    was_run = True
                           
    options = kwds.copy()
    option_nosubprocess = options.get('nosubprocess', False)
    option_dump = options.pop('dump', False)
    option_file = options.pop('file', None)
    option_all = options.pop('all', False)
    option_randomize = options.get('randomize', False)
    option_seed = options.get('seed', None)
    option_multi_thread = options.pop('multi_thread', 1)
    option_time_out = options.pop('time_out', 120)
    option_fake = options.pop('fake', None)
    option_python = options.pop('python', sys.executable)
    option_exclude = options.pop('exclude', ())
    option_interactive = options.pop('interactive', False)

    if not option_interactive and 'interactive' not in option_exclude:
        option_exclude += ('interactive',)
    if not option_nosubprocess and 'subprocess_ignore' not in option_exclude:
        option_exclude += ('subprocess_ignore',)
    elif 'ignore' not in option_exclude:
        option_exclude += ('ignore',)
    if sys.version_info < (3, 0, 0):
        option_exclude += ('python2_ignore',)
    else:
        option_exclude += ('python3_ignore',)

    main_dir, test_subdir, fake_test_subdir = prepare_test_env()
    test_runner_py = os.path.join(test_subdir, "test_utils", "test_runner.py")
    cur_working_dir = os.path.abspath(os.getcwd())

    ###########################################################################
    # Compile a list of test modules. If fake, then compile list of fake
    # xxxx_test.py from run_tests__tests

    TEST_MODULE_RE = re.compile('^(.+_test)\.py$')

    test_mods_pkg_name = test_pkg_name
    
    if option_fake is not None:
        test_mods_pkg_name = '.'.join([test_mods_pkg_name,
                                       'run_tests__tests',
                                       option_fake])
        test_subdir = os.path.join(fake_test_subdir, option_fake)
        working_dir = test_subdir
    else:
        working_dir = main_dir


    # Added in because some machines will need os.environ else there will be
    # false failures in subprocess mode. Same issue as python2.6. Needs some
    # env vars.

    test_env = os.environ

    fmt1 = '%s.%%s' % test_mods_pkg_name
    fmt2 = '%s.%%s_test' % test_mods_pkg_name
    if args:
        test_modules = [
            m.endswith('_test') and (fmt1 % m) or (fmt2 % m) for m in args
        ]
    else:
        test_modules = []
        for f in sorted(os.listdir(test_subdir)):
            for match in TEST_MODULE_RE.findall(f):
                test_modules.append(fmt1 % (match,))

    ###########################################################################
    # Remove modules to be excluded.

    tmp = test_modules
    test_modules = []
    for name in tmp:
        tag_module_name = "%s_tags" % (name[0:-5],)
        try:
            tag_module = import_submodule(tag_module_name)
        except ImportError:
            test_modules.append(name)
        else:
            try:
                tags = tag_module.__tags__
            except AttributeError:
                print ("%s has no tags: ignoring" % (tag_module_name,))
                test_module.append(name)
            else:
                for tag in tags:
                    if tag in option_exclude:
                        print ("skipping %s (tag '%s')" % (name, tag))
                        break
                else:
                    test_modules.append(name)
    del tmp, tag_module_name, name

    ###########################################################################
    # Meta results

    results = {}
    meta_results = {'__meta__' : {}}
    meta = meta_results['__meta__']

    ###########################################################################
    # Randomization

    if option_randomize or option_seed is not None:
        if option_seed is None:
            option_seed = time.time()
        meta['random_seed'] = option_seed
        print ("\nRANDOM SEED USED: %s\n" % option_seed)
        random.seed(option_seed)
        random.shuffle(test_modules)

    ###########################################################################
    # Single process mode

    if option_nosubprocess:
        unittest_patch.patch(**options)

        options['exclude'] = option_exclude
        t = time.time()
        for module in test_modules:
            results.update(run_test(module, **options))
        t = time.time() - t

    ###########################################################################
    # Subprocess mode
    #

    if not option_nosubprocess:
        if is_pygame_pkg:
            from pygame.tests.test_utils.async_sub import proc_in_time_or_kill
        else:
            from test.test_utils.async_sub import proc_in_time_or_kill

        pass_on_args = ['--exclude', ','.join(option_exclude)]
        for option in ['timings', 'seed']:
            value = options.pop(option, None)
            if value is not None:
                pass_on_args.append('--%s' % option)
                pass_on_args.append(str(value))
        for option, value in options.items():
            option = option.replace('_', '-')
            if value:
                pass_on_args.append('--%s' % option)

        def sub_test(module):
            print ('loading %s' % module)

            cmd = [option_python, test_runner_py, module ] + pass_on_args

            return (module,
                    (cmd, test_env, working_dir),
                    proc_in_time_or_kill(cmd, option_time_out, env=test_env,
                                         wd=working_dir))

        if option_multi_thread > 1:
            def tmap(f, args):
                return pygame.threads.tmap (
                    f, args, stop_on_error = False,
                    num_workers = option_multi_thread
                )
        else:
            tmap = map

        t = time.time()

        for module, cmd, (return_code, raw_return) in tmap(sub_test,
                                                           test_modules):
            test_file = '%s.py' % os.path.join(test_subdir, module)
            cmd, test_env, working_dir = cmd

            test_results = get_test_results(raw_return)
            if test_results:
                results.update(test_results)
            else:
                results[module] = {}

            add_to_results = [
                'return_code', 'raw_return',  'cmd', 'test_file',
                'test_env', 'working_dir', 'module',
            ]

            results[module].update(from_namespace(locals(), add_to_results))

        t = time.time() - t

    ###########################################################################
    # Output Results
    #

    untrusty_total, combined = combine_results(results, t)
    total, fails = test_failures(results)

    meta['total_tests'] = total
    meta['combined'] = combined
    results.update(meta_results)

    if option_nosubprocess:
        assert total == untrusty_total

    if not option_dump:
        print (combined)
    else:
        results = option_all and results or fails
        print (TEST_RESULTS_START)
        print (pformat(results))

    if option_file is not None:
        results_file = open(option_file, 'w')
        try:
            results_file.write(pformat(results))
        finally:
            results_file.close()

    return total, fails

Example 57

Project: gonewilder Source File: serve.py
    def run_cgi(self):
        """Execute a CGI script."""
        path = self.path
        dir, rest = self.cgi_info

        i = path.find('/', len(dir) + 1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir) + 1)
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        print rest
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        # print str(ispy) + "am i python?"
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.getheader('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non Unix - use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)

            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin = subprocess.PIPE,
                                 stdout = subprocess.PIPE,
                                 stderr = subprocess.PIPE,
                                 env = env
                                )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 58

Project: PythonScript Source File: CGIHTTPServer.py
    def run_cgi(self):
        """Execute a CGI script."""
        dir, rest = self.cgi_info

        i = rest.find('/')
        while i >= 0:
            nextdir = rest[:i]
            nextrest = rest[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = rest.find('/')
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.getheader('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non Unix - use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)

            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin = subprocess.PIPE,
                                 stdout = subprocess.PIPE,
                                 stderr = subprocess.PIPE,
                                 env = env
                                )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 59

Project: ironpython3 Source File: platform.py
Function: platform
def platform(aliased=0, terse=0):

    """ Returns a single string identifying the underlying platform
        with as much useful information as possible (but no more :).

        The output is intended to be human readable rather than
        machine parseable. It may look different on different
        platforms and this is intended.

        If "aliased" is true, the function will use aliases for
        various platforms that report system names which differ from
        their common names, e.g. SunOS will be reported as
        Solaris. The system_alias() function is used to implement
        this.

        Setting terse to true causes the function to return only the
        absolute minimum information needed to identify the platform.

    """
    result = _platform_cache.get((aliased, terse), None)
    if result is not None:
        return result

    # Get uname information and then apply platform specific cosmetics
    # to it...
    system, node, release, version, machine, processor = uname()
    if machine == processor:
        processor = ''
    if aliased:
        system, release, version = system_alias(system, release, version)

    if system == 'Windows':
        # MS platforms
        rel, vers, csd, ptype = win32_ver(version)
        if terse:
            platform = _platform(system, release)
        else:
            platform = _platform(system, release, version, csd)

    elif system in ('Linux',):
        # Linux based systems
        distname, distversion, distid = dist('')
        if distname and not terse:
            platform = _platform(system, release, machine, processor,
                                 'with',
                                 distname, distversion, distid)
        else:
            # If the distribution name is unknown check for libc vs. glibc
            libcname, libcversion = libc_ver(sys.executable)
            platform = _platform(system, release, machine, processor,
                                 'with',
                                 libcname+libcversion)
    elif system == 'Java':
        # Java platforms
        r, v, vminfo, (os_name, os_version, os_arch) = java_ver()
        if terse or not os_name:
            platform = _platform(system, release, version)
        else:
            platform = _platform(system, release, version,
                                 'on',
                                 os_name, os_version, os_arch)

    elif system == 'MacOS':
        # MacOS platforms
        if terse:
            platform = _platform(system, release)
        else:
            platform = _platform(system, release, machine)

    else:
        # Generic handler
        if terse:
            platform = _platform(system, release)
        else:
            bits, linkage = architecture(sys.executable)
            platform = _platform(system, release, machine,
                                 processor, bits, linkage)

    _platform_cache[(aliased, terse)] = platform
    return platform

Example 60

Project: cgstudiomap Source File: test_traceback.py
Function: test_encoded_file
    def test_encoded_file(self):
        # Test that tracebacks are correctly printed for encoded source files:
        # - correct line number (Issue2384)
        # - respect file encoding (Issue3975)
        import tempfile, sys, subprocess, os

        # The spawned subprocess has its stdout redirected to a PIPE, and its
        # encoding may be different from the current interpreter, on Windows
        # at least.
        process = subprocess.Popen([sys.executable, "-c",
                                    "import sys; print(sys.stdout.encoding)"],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT)
        stdout, stderr = process.communicate()
        output_encoding = text_type(stdout, 'ascii').splitlines()[0]

        def do_test(firstlines, message, charset, lineno, output_encoding):
            # Raise the message in a subprocess, and catch the output
            with fixtures.TempDir() as d:
                TESTFN = d.path + '/fname'
                output = io.open(TESTFN, "w", encoding=charset)
                output.write(u("""{0}if 1:
                    import traceback;
                    raise RuntimeError('{1}')
                    """).format(firstlines, message))
                output.close()
                process = subprocess.Popen([sys.executable, TESTFN],
                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                stdout, stderr = process.communicate()
                if output_encoding == 'None':
                    output_encoding = charset
                stdout = stdout.decode(output_encoding).splitlines()

            # The source lines are encoded with the 'backslashreplace' handler
            encoded_message = message.encode(output_encoding,
                                             'backslashreplace')
            # and we just decoded them with the output_encoding.
            message_ascii = encoded_message.decode(output_encoding)

            err_line = u("raise RuntimeError('{0}')").format(message_ascii)
            err_msg = u("RuntimeError: {0}").format(message_ascii)

            if platform.python_implementation() == 'PyPy':
                # PyPy includes its own top level app_main.py in the traceback.
                del stdout[1]
            self.assertIn(("line %s" % lineno), stdout[1],
                "Invalid line number: {0!r} instead of {1}".format(
                    stdout[1], lineno))
            self.assertTrue(stdout[2].endswith(err_line),
                "Invalid traceback line: {0!r} instead of {1!r}".format(
                    stdout[2], err_line))
            self.assertTrue(stdout[3] == err_msg,
                "Invalid error message: {0!r} instead of {1!r}".format(
                    stdout[3], err_msg))

        do_test("", "foo", "ascii", 3, output_encoding)
        for charset in ("ascii", "iso-8859-1", "utf-8", "GBK"):
            if charset == "ascii":
                text = u("foo")
            elif charset == "GBK":
                text = u("\u4E02\u5100")
            else:
                text = u("h\xe9 ho")
            do_test("# coding: {0}\n".format(charset),
                    text, charset, 4, output_encoding)
            do_test("#!shebang\n# coding: {0}\n".format(charset),
                    text, charset, 5, output_encoding)
            do_test(" \t\f\n# coding: {0}\n".format(charset),
                    text, charset, 5, output_encoding)
        # Issue #18960: coding spec should has no effect
        # (Fixed in 3.4)
        if sys.version_info[:2] > (3, 3):
            do_test(
                "0\n# coding: GBK\n", u("h\xe9 ho"), 'utf-8', 5,
                output_encoding)

Example 61

Project: raster-functions Source File: setup.py
def main():
    pipURL = "http://bootstrap.pypa.io/get-pip.py"
    vcURL = "http://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi"

    pipExePath = path.join(path.dirname(sys.executable), r"Scripts\pip.exe")
    setupHome = path.join(path.abspath(path.dirname(__file__)), "scripts")
    distHome = path.join(path.abspath(path.dirname(__file__)), "dist")

    try:
        log("Installing PIP")
        pipPyPath = path.join(setupHome, "get-pip.py")
        locateFile(pipURL, pipPyPath)
        call([sys.executable, pipPyPath])

        if path.isfile(pipExePath):
            log("PIP installed successfully")
        else:
            raise Exception("PIP failed")

        call([pipExePath, "install", "--upgrade", "--no-index", "--find-links={0}".format(distHome), "pip"])
        call([pipExePath, "install", "--upgrade", "--no-index", "--find-links={0}".format(distHome), "wheel"])
    except:
        die("PIP installation failed!", 1)
    
    try:           
        if sys.version_info[0] == 2:
            log("Installing Microsoft Visual C++ Compiler")
            vcSetupPath = path.join(distHome, "VCForPython27.msi")
            locateFile(vcURL, vcSetupPath)
            c = ["msiexec", "/i", vcSetupPath, "/qb-"]
            log("Executing: {0}".format(" ".join(c)))
            call(c)
            log("C++ Compiler for Python installed successfully")
    except:
        die("VC++ Compiler for Python installation failed!.", 4)

    try:
        log("Installing Python dependencies")
        reqFilePath = path.join(setupHome, "requirements.txt")
        if not path.isfile(reqFilePath):
            die("Dependency listing file not found: {0}".format(reqFilePath), 5)

        c = [pipExePath, "install", "--upgrade", "--no-index", "--find-links={0}".format(distHome), "-r", reqFilePath]
        log("Executing: {0}".format(" ".join(c)))
        call(c)
    except:
        die("Dependency installation failed!", 6)
    
    try:
        arcpy = __import__('arcpy')
        info = arcpy.GetInstallInfo()

        bVersionOK = True
        
        minArcGISVersion = '10.3.1'
        if info['Version'].split(".")[0] == 10 and (tuple(map(int, (info['Version'].split(".")))) < tuple(map(int, (minArcGISVersion.split("."))))):
           bVersionOK = False

        minProVersion = '1.0'
        if info['Version'].split(".")[0] == 1 and (tuple(map(int, (info['Version'].split(".")))) < tuple(map(int, (minProVersion.split("."))))):
           bVersionOK = False

        if not bVersionOK:
           raise Exception("No ArcGIS")
               
        print("\n\n")
        if info['Version'][0] == 10:
            log("Python extensions for raster functions in ArcGIS {} {} build {} successfully installed.".format(
                info['ProductName'], info['Version'], info['BuildNumber']))
        else:
            log("Python extensions for raster functions in {} {} build {} successfully installed.".format(
                info['ProductName'], info['Version'], info['BuildNumber']))
    except:
        logging.warn("Unable to find ArcGIS 10.3.1/ArcGIS Pro 1.0 or above.")

    log("Done.")
    sleep(2)
    exit(0)

Example 62

Project: pip-update-requirements Source File: req_install.py
    def install(self, install_options, global_options=[], root=None,
                prefix=None):
        if self.editable:
            self.install_editable(
                install_options, global_options, prefix=prefix)
            return
        if self.is_wheel:
            version = pip.wheel.wheel_version(self.source_dir)
            pip.wheel.check_compatibility(version, self.name)

            self.move_wheel_files(self.source_dir, root=root, prefix=prefix)
            self.install_succeeded = True
            return

        # Extend the list of global and install options passed on to
        # the setup.py call with the ones from the requirements file.
        # Options specified in requirements file override those
        # specified on the command line, since the last option given
        # to setup.py is the one that is used.
        global_options += self.options.get('global_options', [])
        install_options += self.options.get('install_options', [])

        if self.isolated:
            global_options = list(global_options) + ["--no-user-cfg"]

        temp_location = tempfile.mkdtemp('-record', 'pip-')
        record_filename = os.path.join(temp_location, 'install-record.txt')
        try:
            install_args = [sys.executable, "-u"]
            install_args.append('-c')
            install_args.append(SETUPTOOLS_SHIM % self.setup_py)
            install_args += list(global_options) + \
                ['install', '--record', record_filename]

            if not self.as_egg:
                install_args += ['--single-version-externally-managed']

            if root is not None:
                install_args += ['--root', root]
            if prefix is not None:
                install_args += ['--prefix', prefix]

            if self.pycompile:
                install_args += ["--compile"]
            else:
                install_args += ["--no-compile"]

            if running_under_virtualenv():
                py_ver_str = 'python' + sysconfig.get_python_version()
                install_args += ['--install-headers',
                                 os.path.join(sys.prefix, 'include', 'site',
                                              py_ver_str, self.name)]
            msg = 'Running setup.py install for %s' % (self.name,)
            with open_spinner(msg) as spinner:
                with indent_log():
                    call_subprocess(
                        install_args + install_options,
                        cwd=self.setup_py_dir,
                        show_stdout=False,
                        spinner=spinner,
                    )

            if not os.path.exists(record_filename):
                logger.debug('Record file %s not found', record_filename)
                return
            self.install_succeeded = True
            if self.as_egg:
                # there's no --always-unzip option we can pass to install
                # command so we unable to save the installed-files.txt
                return

            def prepend_root(path):
                if root is None or not os.path.isabs(path):
                    return path
                else:
                    return change_root(root, path)

            with open(record_filename) as f:
                for line in f:
                    directory = os.path.dirname(line)
                    if directory.endswith('.egg-info'):
                        egg_info_dir = prepend_root(directory)
                        break
                else:
                    logger.warning(
                        'Could not find .egg-info directory in install record'
                        ' for %s',
                        self,
                    )
                    # FIXME: put the record somewhere
                    # FIXME: should this be an error?
                    return
            new_lines = []
            with open(record_filename) as f:
                for line in f:
                    filename = line.strip()
                    if os.path.isdir(filename):
                        filename += os.path.sep
                    new_lines.append(
                        os.path.relpath(
                            prepend_root(filename), egg_info_dir)
                    )
            inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
            with open(inst_files_path, 'w') as f:
                f.write('\n'.join(new_lines) + '\n')
        finally:
            if os.path.exists(record_filename):
                os.remove(record_filename)
            rmtree(temp_location)

Example 63

Project: Bento Source File: py3tool.py
Function: main
def main():
    p = OptionParser(usage=__doc__.strip())
    p.add_option("--clean", "-c", action="store_true",
                 help="clean source directory")
    options, args = p.parse_args()

    if not args:
        p.error('no submodules given')
    else:
        dirs = ['numpy/%s' % x for x in map(os.path.basename, args)]

    # Prepare
    if not os.path.isdir(TEMP):
        os.makedirs(TEMP)

    # Set up dummy files (for building only submodules)
    dummy_files = {
        '__init__.py': 'from numpy.version import version as __version__',
        'version.py': 'version = "1.4.0.dev"'
    }

    for fn, content in dummy_files.items():
        fn = os.path.join(TEMP, 'numpy', fn)
        if not os.path.isfile(fn):
            try:
                os.makedirs(os.path.dirname(fn))
            except OSError:
                pass
            f = open(fn, 'wb+')
            f.write(content.encode('ascii'))
            f.close()

    # Environment
    pp = [os.path.abspath(TEMP)]
    def getenv():
        env = dict(os.environ)
        env.update({'PYTHONPATH': ':'.join(pp)})
        return env

    # Copy
    for d in dirs:
        src = os.path.join(BASE, d)
        dst = os.path.join(TEMP, d)

        # Run 2to3
        sync_2to3(dst=dst,
                  src=src,
                  patchfile=os.path.join(TEMP, os.path.basename(d) + '.patch'),
                  clean=options.clean)

        # Run setup.py, falling back to Pdb post-mortem on exceptions
        setup_py = os.path.join(dst, 'setup.py')
        if os.path.isfile(setup_py):
            code = """\
import pdb, sys, traceback
p = pdb.Pdb()
try:
    import __main__
    __main__.__dict__.update({
        "__name__": "__main__", "__file__": "setup.py",
        "__builtins__": __builtins__})
    fp = open("setup.py", "rb")
    try:
        exec(compile(fp.read(), "setup.py", 'exec'))
    finally:
        fp.close()
except SystemExit:
    raise
except:
    traceback.print_exc()
    t = sys.exc_info()[2]
    p.interaction(None, t)
"""
            ret = subprocess.call([sys.executable, '-c', code,
                                   'build_ext', '-i'],
                                  cwd=dst,
                                  env=getenv())
            if ret != 0:
                raise RuntimeError("Build failed.")

        # Run nosetests
        subprocess.call(['nosetests3', '-v', d], cwd=TEMP)

Example 64

Project: fail2ban Source File: fail2banclienttestcase.py
	@with_tmpdir
	@with_kill_srv
	def testClientStartBackgroundCall(self, tmp):
		global INTERACT
		startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
		# if fast, start server process from client started direct here:
		if unittest.F2B.fast: # pragma: no cover
			self.execSuccess(startparams + ("start",))
		else:
			# start (in new process, using the same python version):
			cmd = (sys.executable, pjoin(BIN, CLIENT))
			logSys.debug('Start %s ...', cmd)
			cmd = cmd + startparams + ("--async", "start",)
			ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
			self.assertTrue(len(ret) and ret[0])
			# wait for server (socket and ready):
			self._wait_for_srv(tmp, True, startparams=cmd)
		self.assertLogged("Server ready")
		self.pruneLog()
		try:
			# echo from client (inside):
			self.execSuccess(startparams, "echo", "TEST-ECHO")
			self.assertLogged("TEST-ECHO")
			self.assertLogged("Exit with code 0")
			self.pruneLog()
			# test ping timeout:
			self.execSuccess(startparams, "ping", "0.1")
			self.assertLogged("Server replied: pong")
			self.pruneLog()
			# python 3 seems to bypass such short timeouts also, 
			# so suspend/resume server process and test between it...
			pid = _get_pid_from_file(pjoin(tmp, "f2b.pid"))
			try:
				# suspend:
				os.kill(pid, signal.SIGSTOP); # or SIGTSTP?
				time.sleep(Utils.DEFAULT_SHORT_INTERVAL)
				# test ping with short timeout:
				self.execFailed(startparams, "ping", "1e-10")
			finally:
				# resume:
				os.kill(pid, signal.SIGCONT)
			self.assertLogged("timed out")
			self.pruneLog()
			# interactive client chat with started server:
			INTERACT += [
				"echo INTERACT-ECHO",
				"status",
				"exit"
			]
			self.execSuccess(startparams, "-i")
			self.assertLogged("INTERACT-ECHO")
			self.assertLogged("Status", "Number of jail:")
			self.assertLogged("Exit with code 0")
			self.pruneLog()
			# test reload and restart over interactive client:
			INTERACT += [
				"reload",
				"restart",
				"exit"
			]
			self.execSuccess(startparams, "-i")
			self.assertLogged("Reading config files:")
			self.assertLogged("Shutdown successful")
			self.assertLogged("Server ready")
			self.assertLogged("Exit with code 0")
			self.pruneLog()
			# test reload missing jail (interactive):
			INTERACT += [
				"reload ~~unknown~jail~fail~~",
				"exit"
			]
			self.execSuccess(startparams, "-i")
			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
			self.pruneLog()
			# test reload missing jail (direct):
			self.execFailed(startparams, "reload", "~~unknown~jail~fail~~")
			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
			self.assertLogged("Exit with code -1")
			self.pruneLog()
		finally:
			self.pruneLog()
			# stop:
			self.execSuccess(startparams, "stop")
			self.assertLogged("Shutdown successful")
			self.assertLogged("Exit with code 0")

Example 65

Project: bde-tools Source File: configurehelper.py
    def _configure_external_libs(self):

        if len(self.build_config.external_dep) == 0:
            return

        self._configure_distribution_refroot()
        try:
            self.ctx.find_program('pkg-config', var='PKGCONFIG')
        except self.ctx.errors.ConfigurationError:
            Logs.warn('Could not find pkg-config on the PATH.  Using the'
                      'built-in python based pkg-config (pykg-config) '
                      'instead.')
            self.ctx.env['PKGCONFIG'] = [sys.executable, os.path.join(
                sysutil.repo_root_path(), 'bin', 'tools', 'pykg-config',
                'pykg-config.py')]
            self.ctx.find_program('pkg-config', var='PKGCONFIG')

        pkgconfig_args = ['--libs', '--cflags']

        if 'shr' not in self.ufid.flags:
            pkgconfig_args.append('--static')

        # If the static build is chosen (the default), waf assumes that all
        # libraries queried from pkg-config are to be built statically, which
        # is not true for some libraries. We work around this issue by manually
        # changing the affected libraries to be linked dynamically instead.
        dl_overrides = ['pthread', 'rt', 'nsl', 'socket']

        # If lib_suffix is set, we expect the pkgconfig files being depended on
        # to have the same suffix as well. Since the .dep files will not have
        # the suffix, we will remove the suffix from the names of the options
        # loaded into the waf environment.
        rename_keys = ['defines', 'includes', 'lib', 'libpath', 'stlib',
                       'stlibpath']
        lib_suffix = self.ctx.options.lib_suffix
        for lib in sorted(self.build_config.external_dep):
            actual_lib = lib + str(lib_suffix or '')
            help_str = """failed to find the library using pkg-config
Maybe "%s.pc" is missing from "PKG_CONFIG_PATH"? Inspect config.log in the
build output directory for details.""" % \
                actual_lib
            self.ctx.check_cfg(
                package=actual_lib,
                args=pkgconfig_args,
                errmsg=help_str)

            if lib_suffix:
                for k in rename_keys:
                    key_old = (k + '_' + actual_lib).upper()
                    key_new = (k + '_' + lib).upper()
                    self.ctx.env[key_new] = self.ctx.env[key_old]
                    del self.ctx.env[key_old]

            sl_key = ('stlib_' + lib).upper()
            dl_key = ('lib_' + lib).upper()

            # preserve the order of libraries
            for l in dl_overrides:
                if l in self.ctx.env[sl_key]:
                    if dl_key not in self.ctx.env:
                        self.ctx.env[dl_key] = []

                    self.ctx.env[sl_key].remove(l)
                    self.ctx.env[dl_key].append(l)

        if lib_suffix:
            defines_old = self.ctx.env['DEFINES']
            defines_new = []
            for d in defines_old:
                index = d.find('%s=1' % lib_suffix.upper())
                if index >= 0:
                    defines_new.append('%s=1' % d[0:index])
                else:
                    defines_new.append(d)

            self.ctx.env['DEFINES'] = defines_new

Example 66

Project: nikola Source File: plugin.py
    def do_install(self, url, name, show_install_notes=True):
        """Download and install a plugin."""
        data = self.get_json(url)
        if name in data:
            utils.makedirs(self.output_dir)
            url = data[name]
            LOGGER.info("Downloading '{0}'".format(url))
            try:
                zip_data = requests.get(url).content
            except requests.exceptions.SSLError:
                LOGGER.warning("SSL error, using http instead of https (press ^C to abort)")
                time.sleep(1)
                url = url.replace('https', 'http', 1)
                zip_data = requests.get(url).content

            zip_file = io.BytesIO()
            zip_file.write(zip_data)
            LOGGER.info('Extracting: {0} into {1}/'.format(name, self.output_dir))
            utils.extract_all(zip_file, self.output_dir)
            dest_path = os.path.join(self.output_dir, name)
        else:
            try:
                plugin_path = utils.get_plugin_path(name)
            except:
                LOGGER.error("Can't find plugin " + name)
                return 1

            utils.makedirs(self.output_dir)
            dest_path = os.path.join(self.output_dir, name)
            if os.path.exists(dest_path):
                LOGGER.error("{0} is already installed".format(name))
                return 1

            LOGGER.info('Copying {0} into plugins'.format(plugin_path))
            shutil.copytree(plugin_path, dest_path)

        reqpath = os.path.join(dest_path, 'requirements.txt')
        if os.path.exists(reqpath):
            LOGGER.notice('This plugin has Python dependencies.')
            LOGGER.info('Installing dependencies with pip...')
            try:
                subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath))
            except subprocess.CalledProcessError:
                LOGGER.error('Could not install the dependencies.')
                print('Contents of the requirements.txt file:\n')
                with io.open(reqpath, 'r', encoding='utf-8') as fh:
                    print(utils.indent(fh.read(), 4 * ' '))
                print('You have to install those yourself or through a '
                      'package manager.')
            else:
                LOGGER.info('Dependency installation succeeded.')

        reqnpypath = os.path.join(dest_path, 'requirements-nonpy.txt')
        if os.path.exists(reqnpypath):
            LOGGER.notice('This plugin has third-party '
                          'dependencies you need to install '
                          'manually.')
            print('Contents of the requirements-nonpy.txt file:\n')
            with io.open(reqnpypath, 'r', encoding='utf-8') as fh:
                for l in fh.readlines():
                    i, j = l.split('::')
                    print(utils.indent(i.strip(), 4 * ' '))
                    print(utils.indent(j.strip(), 8 * ' '))
                    print()

            print('You have to install those yourself or through a package '
                  'manager.')

        req_plug_path = os.path.join(dest_path, 'requirements-plugins.txt')
        if os.path.exists(req_plug_path):
            LOGGER.notice('This plugin requires other Nikola plugins.')
            LOGGER.info('Installing plugins...')
            try:
                with io.open(req_plug_path, 'r', encoding='utf-8') as inf:
                    for plugname in inf.readlines():
                        self.do_install(url, plugname, show_install_notes)
            except subprocess.CalledProcessError:
                LOGGER.error('Could not install a plugin.')
                print('Contents of the requirements-plugins.txt file:\n')
                with io.open(req_plug_path, 'r', encoding='utf-8') as fh:
                    print(utils.indent(fh.read(), 4 * ' '))
                print('You have to install those yourself manually.')
            else:
                LOGGER.info('Dependency installation succeeded.')

        confpypath = os.path.join(dest_path, 'conf.py.sample')
        if os.path.exists(confpypath) and show_install_notes:
            LOGGER.notice('This plugin has a sample config file.  Integrate it with yours in order to make this plugin work!')
            print('Contents of the conf.py.sample file:\n')
            with io.open(confpypath, 'r', encoding='utf-8') as fh:
                if self.site.colorful:
                    print(utils.indent(pygments.highlight(
                        fh.read(), PythonLexer(), TerminalFormatter()),
                        4 * ' '))
                else:
                    print(utils.indent(fh.read(), 4 * ' '))
        return 0

Example 67

Project: pybagit Source File: bagit.py
    def update(self, full=True):
        """ Scans the data directory, adds any new files it finds to the
            manifest, and removes any files from the manifest that it does not
            find in the directory.

        Args:
            full (bool): Only add new files and remove missing files and skip
            reindexing of previously indexed files
        """

        filelist = os.listdir(self.bag_directory)

        # Read old manifest so we can detect new, existing and deleted files
        md5_hashes = dict()
        sha1_hashes = dict()
        datamanifests = [f for f in filelist
                         if re.match(r"^manifest-(sha1|md5)\.txt", f)]
        for man in datamanifests:
            man = os.path.join(self.bag_directory, man)
            if 'manifest-md5.txt' in man:
                for line in codecs.open(man, 'rb', self.tag_file_encoding):
                    hash_, file_ = line.split(' ', 1)
                    md5_hashes[file_] = hash_
            elif 'manifest-sha1.txt' in man:
                for line in codecs.open(man, 'rb', self.tag_file_encoding):
                    hash_, file_ = line.split(' ', 1)
                    sha1_hashes[file_] = hash_

            if full:
                os.unlink(man)

        # clean up any old manifest files. We'll be regenerating them later.
        tagmanifests = [f for f in filelist
                        if re.match(r"^tagmanifest-(sha1|md5)\.txt", f)]
        for man in tagmanifests:
            man = os.path.join(self.bag_directory, man)
            os.unlink(man)

        self.new_files = set()
        self.existing_files = set()
        self.removed_files = set()

        for path, dirs, files in os.walk(self.data_directory):
            # add an empty .keep file in empty directories.
            if not dirs and not files:
                open(os.path.join(path, '.keep'), 'w').close()

            for name in files:
                self.new_filesfile = self._sanitize_filename(name)
                full_file = os.path.join(path, self.new_filesfile)
                if self.new_filesfile != name:
                    os.rename(os.path.join(path, name), full_file)

                relative_file = os.path.relpath(full_file, self.data_directory)
                if relative_file in md5_hashes:
                    self.existing_files.add(relative_file)
                elif relative_file in sha1_hashes:
                    self.existing_files.add(relative_file)
                else:
                    self.new_files.add(relative_file)

        known_files = set(md5_hashes) | set(sha1_hashes)
        self.removed_files = known_files - self.existing_files
        self.existing_files -= self.removed_files

        # checksum the data directory
        cmd = [
            sys.executable,
            self._path_to_multichecksum,
            "-a", self.hash_encoding,
            "-c", self.tag_file_encoding,
            self.data_directory,
        ]
        if not full:
            cmd.append('-u')
        p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE)

        while p.returncode is None:
            time.sleep(0.1)
            p.poll()

        # read in the manifest to an instance variable
        self._read_manifest_to_dict()

        # clean out any previous tag manifest contents.
        self.tag_manifest_contents = {}
        for f in ['bagit.txt', 'bag-info.txt', 'fetch.txt', self.manifest_file]:
            csum = self._calculate_checksum(os.path.join(self.bag_directory, f))
            relp = os.path.relpath(os.path.join(self.bag_directory, f), self.bag_directory)
            self.tag_manifest_contents[relp] = csum

        # write out the tag manifest
        self._write_dict_to_manifest(mode="t")

        # read it back in
        self._read_manifest_to_dict(mode="t")

Example 68

Project: easybuild-easyblocks Source File: pythonpackage.py
def pick_python_cmd(req_maj_ver=None, req_min_ver=None):
    """
    Pick 'python' command to use, based on specified version requirements.
    If the major version is specified, it must be an exact match (==).
    If the minor version is specified, it is considered a minimal minor version (>=).

    List of considered 'python' commands (in order)
    * 'python' available through $PATH
    * 'python<major_ver>' available through $PATH
    * 'python<major_ver>.<minor_ver>' available through $PATH
    * Python executable used in current session (sys.executable)
    """
    log = fancylogger.getLogger('pick_python_cmd', fname=False)

    def check_python_cmd(python_cmd):
        """Check whether specified Python command satisfies requirements."""

        # check whether specified Python command is available
        if os.path.isabs(python_cmd):
            if not os.path.isfile(python_cmd):
                log.debug("Python command '%s' does not exist", python_cmd)
                return False
        else:
            python_cmd_path = which(python_cmd)
            if python_cmd_path is None:
                log.debug("Python command '%s' not available through $PATH", python_cmd)
                return False

        if req_maj_ver is not None:
            if req_min_ver is None:
                req_majmin_ver = '%s.0' % req_maj_ver
            else:
                req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver)

            pycode = 'import sys; print("%s.%s" % sys.version_info[:2])'
            out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False)
            out = out.strip()

            # (strict) check for major version
            maj_ver = out.split('.')[0]
            if maj_ver != str(req_maj_ver):
                log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver)
                return False

            # check for minimal minor version
            if LooseVersion(out) < LooseVersion(req_majmin_ver):
                log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", out, req_majmin_ver)
                return False

        # all check passed
        log.debug("All check passed for Python command '%s'!", python_cmd)
        return True

    # compose list of 'python' commands to consider
    python_cmds = ['python']
    if req_maj_ver:
        python_cmds.append('python%s' % req_maj_ver)
        if req_min_ver:
            python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver))
    python_cmds.append(sys.executable)
    log.debug("Considering Python commands: %s", ', '.join(python_cmds))

    # try and find a 'python' command that satisfies the requirements
    res = None
    for python_cmd in python_cmds:
        if check_python_cmd(python_cmd):
            log.debug("Python command '%s' satisfies version requirements!", python_cmd)
            if os.path.isabs(python_cmd):
                res = python_cmd
            else:
                res = which(python_cmd)
            log.debug("Absolute path to retained Python command: %s", res)
            break
        else:
            log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on",
                      req_maj_ver, req_min_ver, python_cmd)

    return res

Example 69

Project: git-lockup Source File: test_git_lockup.py
    def test_setup(self):
        self.basedir = self.make_basedir("Create.setup")
        upstream = self.subpath("upstream")
        os.makedirs(upstream)
        one = self.subpath("one")
        self.git("init", "--bare", subdir="upstream")
        self.git("clone", os.path.abspath(upstream), os.path.abspath(one),
                 workdir=upstream)
        self.add_change(message="initial-unsigned")
        # first push needs to be explicit, since we haven't added a
        # remote.origin.push refspec in .git/config yet
        self.git("push", "origin", "master", subdir="one")
        out = self.run_command(["git-lockup", "setup-publish"], one)
        self.assertIn("the post-commit hook will now sign changes on branch 'master'", out)
        self.assertIn("verifykey: vk0-", out)
        vk_s = re.search(r"(vk0-\w+)", out).group(1)
        #self.assertIn("you should now commit the generated 'setup-lockup'", out)
        # setup-publish automatically adds setup-lockup and lockup.config

        # pyflakes one/.git/lockup-tool
        # pyflakes one/setup-lockup

        # now that the publishing repo is configured to sign commits, adding
        # a change should get a note with a signature
        self.add_change(message="first-signed")
        head = self.git("rev-parse", "HEAD", subdir="one")
        notes = self.git("notes", "list", head, subdir="one").split("\n")
        self.assertEqual(len(notes), 1, notes)

        # the updated refspec should push the notes along with the commits
        self.git("push", subdir="one")

        # so they should be present in the upstream (bare) repo
        notes = self.git("notes", "list", head, subdir="upstream").split("\n")
        self.assertEqual(len(notes), 1, notes)

        # cloning the repo doesn't get the notes by default
        two = self.subpath("two")
        self.git("clone", os.path.abspath(upstream), os.path.abspath(two),
                 workdir=upstream)
        notes = self.git("notes", subdir="two")
        self.assertEqual(notes, "")

        # run the downstream setup script
        out = self.run_command([sys.executable, "./setup-lockup"], two)
        self.assertIn("remote 'origin' configured to use verification proxy", out)
        self.assertIn("branch 'master' configured to verify with key %s" % vk_s, out)

        # now downstream pulls should work, fetch notes, and check signatures
        out = self.git("pull", subdir="two")
        #self.assertNotIn("Could not find local refs/notes/commits", out)
        #print "FIRST PULL", out

        self.add_change(message="second-signed")
        self.git("push", subdir="one")
        out = self.git("pull", subdir="two")
        #print "SECOND PULL", out
        one_head = self.git("rev-parse", "HEAD", subdir="one")
        two_head = self.git("rev-parse", "HEAD", subdir="two")
        self.assertEqual(one_head, two_head)

        # unsigned commits should be rejected by the downstream
        unsigned = self.subpath("unsigned")
        self.git("clone", os.path.abspath(upstream), os.path.abspath(unsigned),
                 workdir=upstream)
        self.add_change(subdir="unsigned", message="unsigned")
        unsigned_head = self.git("rev-parse", "HEAD", subdir="unsigned")
        self.assertNotEqual(unsigned, one_head)
        self.git("push", subdir="unsigned")

        rc,out,err = self.run_failing_command(["git", "pull"],
                                              self.subpath("two"))
        self.assertEqual(rc, 1)
        self.assertEqual(out, "")
        self.assertIn("\nno valid signature found for branch refs/heads/master (rev %s)\n" % unsigned_head, err)
        self.assertIn("\nfatal: Could not read from remote repository.\n", err)
        # should fail

        #print "THIRD PULL (unsigned)", out
        two_head = self.git("rev-parse", "HEAD", subdir="two")
        self.assertNotEqual(two_head, unsigned_head)

Example 70

Project: ironpython3 Source File: server.py
    def run_cgi(self):
        """Execute a CGI script."""
        dir, rest = self.cgi_info

        i = rest.find('/')
        while i >= 0:
            nextdir = rest[:i]
            nextrest = rest[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = rest.find('/')
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if self.have_fork or not ispy:
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.parse.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.get("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = authorization[1].encode('ascii')
                        authorization = base64.decodebytes(authorization).\
                                        decode('ascii')
                    except (binascii.Error, UnicodeError):
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.get('content-type') is None:
            env['CONTENT_TYPE'] = self.headers.get_content_type()
        else:
            env['CONTENT_TYPE'] = self.headers['content-type']
        length = self.headers.get('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.get('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.get('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.get_all('cookie', []))
        cookie_str = ', '.join(co)
        if cookie_str:
            env['HTTP_COOKIE'] = cookie_str
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")
        self.flush_headers()

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except OSError:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non-Unix -- use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)
            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 env = env
                                 )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 71

Project: deluge Source File: create_plugin.py
Function: create_plugin
def create_plugin():

    if not options.url:
        options.url = ''

    if not os.path.exists(options.basepath):
        print('basepath does not exist')
        return

    if not options.configdir:
        options.configdir = deluge.common.get_default_config_dir()

    options.configdir = os.path.realpath(options.configdir)

    real_name = options.name
    name = real_name.replace(' ', '_')
    safe_name = name.lower()
    if options.module_name:
        safe_name = options.module_name.lower()
    plugin_base = os.path.realpath(os.path.join(options.basepath, name))
    deluge_namespace = os.path.join(plugin_base, 'deluge')
    plugins_namespace = os.path.join(deluge_namespace, 'plugins')
    src = os.path.join(plugins_namespace, safe_name)
    data_dir = os.path.join(src, 'data')
    python_path = sys.executable

    if os.path.exists(plugin_base):
        print('the directory %s already exists, delete it first' % plugin_base)
        return

    def write_file(path, filename, template, include_gpl=True):
        plugin_args = {
            'author_name': options.author_name,
            'author_email': options.author_email,
            'name': name,
            'safe_name': safe_name,
            'filename': filename,
            'plugin_base': plugin_base,
            'python_path': python_path,
            'url': options.url,
            'configdir': options.configdir,
            'current_year': datetime.utcnow().year
        }

        filename = os.path.join(path, filename)
        with open(filename, 'w') as _file:
            if filename.endswith('.py') and include_gpl:
                _file.write(GPL % plugin_args)
            _file.write(template % plugin_args)

    print('creating folders..')
    os.mkdir(plugin_base)
    os.mkdir(deluge_namespace)
    os.mkdir(plugins_namespace)
    os.mkdir(src)
    os.mkdir(data_dir)

    print('creating files..')
    write_file(plugin_base, 'setup.py', SETUP)
    write_file(deluge_namespace, '__init__.py', NAMESPACE_INIT, False)
    write_file(plugins_namespace, '__init__.py', NAMESPACE_INIT, False)
    write_file(src, '__init__.py', INIT)
    write_file(src, 'gtkui.py', GTKUI)
    write_file(src, 'webui.py', WEBUI)
    write_file(src, 'core.py', CORE)
    write_file(src, 'common.py', COMMON)
    write_file(data_dir, 'config.glade', GLADE)
    write_file(data_dir, '%s.js' % safe_name, DEFAULT_JS)

    # add an input parameter for this?
    print('building dev-link..')
    write_file(plugin_base, 'create_dev_link.sh', CREATE_DEV_LINK)
    dev_link_path = os.path.join(plugin_base, 'create_dev_link.sh')
    os.system('chmod +x %s' % dev_link_path)  # lazy..
    os.system(dev_link_path)

Example 72

Project: djangobench Source File: main.py
Function: main
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--control',
        metavar='BRANCH',
        default='django-control',
        help="Django version to use as control. If --vcs git refers to a git"
             "branch name or commit-id. If --vcs none refers to a path."
    )
    parser.add_argument(
        '--experiment',
        metavar='BRANCH',
        default='django-experiment',
        help="Django version to use as experiment. If --vcs git refers to a git"
             "branch name or commit-id. If --vcs none refers to a path."
    )
    parser.add_argument(
        '--control-python',
        metavar='PATH',
        default=sys.executable,
        help="Python executable to use as control. Can be used to test Python "
             "2 vs 3 performance on the benchmarks."
    )
    parser.add_argument(
        '--experiment-python',
        metavar='PATH',
        default=sys.executable,
        help="Python executable to use as experiment. Can be used to test "
             "Python 2 vs 3 performance on the benchmarks."
    )
    parser.add_argument(
        '--vcs',
        choices=['git', 'hg', 'none'],
        default='git',
        help='Specify which VCS to use for control/experiment. Set to none to '
             'specify paths, not branch or commit-id\'s.'
    )
    parser.add_argument(
        '-t', '--trials',
        type=int,
        default=50,
        help='Number of times to run each benchmark.'
    )
    parser.add_argument(
        '-r', '--record',
        default=None,
        metavar='PATH',
        help='Directory to record detailed output as a series of JSON files.',
    )
    parser.add_argument(
        '--benchmark-dir',
        dest='benchmark_dir',
        metavar='PATH',
        default=DEFAULT_BENCHMARK_DIR,
        help='Directory to inspect for benchmarks. Defaults to the benchmarks '
             'included with djangobench.',
    )
    parser.add_argument(
        'benchmarks',
        metavar='name',
        default=None,
        help="Benchmarks to be run. Defaults to all.",
        nargs='*'
    )
    parser.add_argument(
        '-p',
        '--profile-dir',
        dest='profile_dir',
        default=None,
        metavar='PATH',
        help='Directory to record profiling statistics for the control and '
             'experimental run of each benchmark'
    )
    parser.add_argument(
        '--continue-on-error',
        dest='continue_on_error',
        action='store_true',
        help='Continue with the remaining benchmarks if any fail',
    )
    parser.add_argument(
        '-l',
        '--list',
        dest='list_benchmarks',
        action='store_true',
        help='List all available benchmarks and exit.',
    )
    parser.add_argument(
        '--log',
        dest='loglevel',
        default='WARNING',
        help='Define log level, set to INFO to show executed commands. Useful '
             'for debugging benchmarks.'
    )
    args = parser.parse_args()

    numeric_level = getattr(logging, args.loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % args.loglevel)
    logging.basicConfig(level=numeric_level)

    if args.list_benchmarks:
        print_benchmarks(args.benchmark_dir)
    else:
        run_benchmarks(
            control=args.control,
            experiment=args.experiment,
            benchmark_dir=args.benchmark_dir,
            benchmarks=args.benchmarks,
            trials=args.trials,
            vcs=None if args.vcs == 'none' else args.vcs,
            record_dir=args.record,
            profile_dir=args.profile_dir,
            continue_on_error=args.continue_on_error,
            control_python=args.control_python,
            experiment_python=args.experiment_python,
        )

Example 73

Project: stash Source File: subprocess.py
	def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=None):
		# vars
		self._fds = []
		self.returncode = None
		self._worker = None
		self._cwd = cwd
		self._environ = (env if env is not None else {})
		
		if isinstance(args, (str, unicode)):
			self.cmd = args
		else:
			if args[0] == sys.executable:
				# common use case
				args = ["python"] + list(args)
			self.cmd = l2c.list2cmdline(args)

		# === setup std* ===
		rfm = "rU" if universal_newlines else "rb"
		# setup stdout
		if stdout is None:
			# use own stdout
			self.stdout = None
			self._sp_stdout = None
		elif stdout == PIPE:
			# create new pipe
			rfd, wfd = os.pipe()
			self._fds += [rfd, wfd]
			self.stdout = os.fdopen(rfd, rfm, bufsize)
			self._sp_stdout = os.fdopen(wfd, "wb")
		elif isinstance(stdout, (int, long)):
			# use fd
			self.stdout = None
			self._fds.append(stdout)
			self._sp_stdout = os.fdopen(stdout, "wb")
		else:
			self.stdout = None
			self._sp_stdout = stdout
		
		# setup stderr
		if stderr is None:
			# use own stdout
			self.stderr = None
			self._sp_stderr = None
		elif stderr == PIPE:
			# create new pipe
			rfd, wfd = os.pipe()
			self._fds += [rfd, wfd]
			self.stderr = os.fdopen(rfd, rfm, bufsize)
			self._sp_stderr = os.fdopen(wfd, "wb")
		elif stderr == STDOUT:
			self.stderr = self.stdout
			self._sp_stderr = self._sp_stdout
		elif isinstance(stderr, (int, long)):
			# use fd
			self.stderr = None
			self._fds.append(stderr)
			self._sp_stderr = os.fdopen(stderr, "wb")
		else:
			self.stderr = None
			self._sp_stderr = stderr
		
		# setup stdin
		if stdin is None:
			# use own stdout
			self.stdin = None
			self._sp_stdin = None
		elif stdin == PIPE:
			# create new pipe
			rfd, wfd = os.pipe()
			self._fds += [rfd, wfd]
			self.stdin = os.fdopen(rfd, "wb")
			self._sp_stdin = os.fdopen(wfd, "rb")
		elif isinstance(stdin, (int, long)):
			# use fd
			self.stdin = None
			self._fds.append(stdin)
			self._sp_stdin = os.fdopen(stdin)
		else:
			self.stdin = None
			self._sp_stdin = stdin
		
		# run
		self._run()

Example 74

Project: HealthStarter Source File: req_install.py
    def install(self, install_options, global_options=[], root=None):
        if self.editable:
            self.install_editable(install_options, global_options)
            return
        if self.is_wheel:
            version = pip.wheel.wheel_version(self.source_dir)
            pip.wheel.check_compatibility(version, self.name)

            self.move_wheel_files(self.source_dir, root=root)
            self.install_succeeded = True
            return

        # Extend the list of global and install options passed on to
        # the setup.py call with the ones from the requirements file.
        # Options specified in requirements file override those
        # specified on the command line, since the last option given
        # to setup.py is the one that is used.
        global_options += self.options.get('global_options', [])
        install_options += self.options.get('install_options', [])

        if self.isolated:
            global_options = list(global_options) + ["--no-user-cfg"]

        temp_location = tempfile.mkdtemp('-record', 'pip-')
        record_filename = os.path.join(temp_location, 'install-record.txt')
        try:
            install_args = [sys.executable]
            install_args.append('-c')
            install_args.append(
                "import setuptools, tokenize;__file__=%r;"
                "exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
                ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py
            )
            install_args += list(global_options) + \
                ['install', '--record', record_filename]

            if not self.as_egg:
                install_args += ['--single-version-externally-managed']

            if root is not None:
                install_args += ['--root', root]

            if self.pycompile:
                install_args += ["--compile"]
            else:
                install_args += ["--no-compile"]

            if running_under_virtualenv():
                py_ver_str = 'python' + sysconfig.get_python_version()
                install_args += ['--install-headers',
                                 os.path.join(sys.prefix, 'include', 'site',
                                              py_ver_str, self.name)]
            logger.info('Running setup.py install for %s', self.name)
            with indent_log():
                call_subprocess(
                    install_args + install_options,
                    cwd=self.source_dir,
                    show_stdout=False,
                )

            if not os.path.exists(record_filename):
                logger.debug('Record file %s not found', record_filename)
                return
            self.install_succeeded = True
            if self.as_egg:
                # there's no --always-unzip option we can pass to install
                # command so we unable to save the installed-files.txt
                return

            def prepend_root(path):
                if root is None or not os.path.isabs(path):
                    return path
                else:
                    return change_root(root, path)

            with open(record_filename) as f:
                for line in f:
                    directory = os.path.dirname(line)
                    if directory.endswith('.egg-info'):
                        egg_info_dir = prepend_root(directory)
                        break
                else:
                    logger.warning(
                        'Could not find .egg-info directory in install record'
                        ' for %s',
                        self,
                    )
                    # FIXME: put the record somewhere
                    # FIXME: should this be an error?
                    return
            new_lines = []
            with open(record_filename) as f:
                for line in f:
                    filename = line.strip()
                    if os.path.isdir(filename):
                        filename += os.path.sep
                    new_lines.append(
                        make_path_relative(
                            prepend_root(filename), egg_info_dir)
                    )
            inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
            with open(inst_files_path, 'w') as f:
                f.write('\n'.join(new_lines) + '\n')
        finally:
            if os.path.exists(record_filename):
                os.remove(record_filename)
            rmtree(temp_location)

Example 75

Project: bcbio-nextgen Source File: tophat.py
def tophat_align(fastq_file, pair_file, ref_file, out_base, align_dir, data,
                 names=None):
    """
    run alignment using Tophat v2
    """
    config = data["config"]
    options = get_in(config, ("resources", "tophat", "options"), {})
    options = _set_fusion_mode(options, config)
    options = _set_quality_flag(options, data)
    options = _set_transcriptome_option(options, data, ref_file)
    options = _set_cores(options, config)
    options = _set_rg_options(options, names)
    options = _set_stranded_flag(options, config)

    ref_file, runner = _determine_aligner_and_reference(ref_file, config)

    # fusion search does not work properly with Bowtie2
    if options.get("fusion-search", False):
        ref_file = ref_file.replace("/bowtie2", "/bowtie")

    if _tophat_major_version(config) == 1:
        raise NotImplementedError("Tophat versions < 2.0 are not supported, please "
                                  "download the newest version of Tophat here: "
                                  "http://tophat.cbcb.umd.edu")

    if _ref_version(ref_file) == 1 or options.get("fusion-search", False):
        options["bowtie1"] = True

    out_dir = os.path.join(align_dir, "%s_tophat" % out_base)
    final_out = os.path.join(out_dir, "{0}.bam".format(names["sample"]))
    if file_exists(final_out):
        return final_out

    out_file = os.path.join(out_dir, "accepted_hits.bam")
    unmapped = os.path.join(out_dir, "unmapped.bam")
    files = [ref_file, fastq_file]
    if not file_exists(out_file):
        with file_transaction(config, out_dir) as tx_out_dir:
            safe_makedir(tx_out_dir)
            if pair_file and not options.get("mate-inner-dist", None):
                d, d_stdev = _estimate_paired_innerdist(fastq_file, pair_file,
                                                        ref_file, out_base,
                                                        tx_out_dir, data)
                options["mate-inner-dist"] = d
                options["mate-std-dev"] = d_stdev
                files.append(pair_file)
            options["output-dir"] = tx_out_dir
            options["no-coverage-search"] = True
            options["no-mixed"] = True
            tophat_runner = sh.Command(config_utils.get_program("tophat",
                                                                config))
            ready_options = {}
            for k, v in options.iteritems():
                ready_options[k.replace("-", "_")] = v
            # tophat requires options before arguments,
            # otherwise it silently ignores them
            tophat_ready = tophat_runner.bake(**ready_options)
            cmd = "%s %s" % (sys.executable, str(tophat_ready.bake(*files)))
            do.run(cmd, "Running Tophat on %s and %s." % (fastq_file, pair_file), None)
    if pair_file and _has_alignments(out_file):
        fixed = _fix_mates(out_file, os.path.join(out_dir, "%s-align.bam" % out_base),
                           ref_file, config)
    else:
        fixed = out_file
    fixed_unmapped = _fix_unmapped(fixed, unmapped, data)
    fixed = merge_unmapped(fixed, fixed_unmapped, config)
    fixed = _add_rg(fixed, config, names)
    fixed = bam.sort(fixed, config)
    picard = broad.runner_from_path("picard", config)
    # set the contig order to match the reference file so GATK works
    fixed = picard.run_fn("picard_reorder", fixed, data["sam_ref"],
                          os.path.splitext(fixed)[0] + ".picard.bam")
    fixed = fix_insert_size(fixed, config)
    if not file_exists(final_out):
        symlink_plus(fixed, final_out)
    return final_out

Example 76

Project: qibuild Source File: venv.py
def configure_virtualenv(config, python_worktree,  build_worktree=None,
                         remote_packages=None, site_packages=True,
                         python_executable=None, env=None):
    """ Main entry point. Called by ``qipy bootstrap``

    :param: remote_packages List of third-party packages to add in the virtualenv
    :param: site_packages Allow access to global site packages

    """
    ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
    if not remote_packages:
        remote_packages = list()

    # create a new virtualenv
    python_worktree.config = config
    venv_path = python_worktree.venv_path
    pip = python_worktree.pip

    if not python_executable:
        python_executable = sys.executable
    virtualenv_py = virtualenv.__file__
    if virtualenv_py.endswith(".pyc"):
        virtualenv_py = virtualenv_py[:-1]
    cmd = [python_executable, virtualenv_py]
    cmd.append(venv_path)
    if site_packages:
        cmd.append("--system-site-packages")
    if not env:
        env = os.environ.copy()
    # Make sure PYTHONHOME is never set when we call bootstrap
    env.pop("PYTHONHOME", None)
    try:
        qisys.command.call(cmd, env=env)
    except qisys.command.CommandFailedException:
        ui.error("Failed to create virtualenv")
        return False

    ui.info(ui.blue, "::", ui.reset, "Adding python projects")
    # Write a qi.pth file containing path to C/C++ extensions and
    # path to pure python modules or packages
    pure_python_ok = handle_pure_python(venv_path, python_worktree, env=env)
    if build_worktree:
        handle_extensions(venv_path, python_worktree, build_worktree)
    handle_modules(venv_path, python_worktree)
    if not pure_python_ok:
        ui.info(ui.red, "Failed to add some python projects")
    ui.info()

    binaries_path = virtualenv.path_locations(venv_path)[-1]
    pip_binary = os.path.join(binaries_path, "pip")
    remote_ok = True
    if remote_packages:
        ui.info(ui.blue, "::", ui.reset,
                "Adding other requirements: " + ", ".join(remote_packages))
        cmd = [pip_binary, "install"]
        if not ui.CONFIG["verbose"]:
            cmd.append("--quiet")
        cmd.extend(remote_packages)
        rc = qisys.command.call(cmd, ignore_ret_code=True, env=env)
        remote_ok = (rc == 0)
    if not remote_ok:
        ui.info(ui.red, "Failed to add some third party requirements")

    ui.info()
    projects_with_requirements = list()
    for project in python_worktree.python_projects:
        path = os.path.join(project.path, "requirements.txt")
        if os.path.isfile(path):
            projects_with_requirements.append(project)

    ui.info(ui.blue, "::", ui.reset,
            "Installing deps from requirements.txt files")
    requirements_ok = True
    for i, project in enumerate(projects_with_requirements):
        ui.info_count(i, len(projects_with_requirements),
                      ui.blue, project.name)
        cmd = [pip_binary, "install"]
        if not ui.CONFIG["verbose"]:
            cmd.append("--quiet")
        path = os.path.join(project.path, "requirements.txt")
        cmd.extend(["--requirement", path])
        rc = qisys.command.call(cmd, ignore_ret_code=True, env=env)
        requirements_ok = (rc == 0)
    ui.info()
    res = (pure_python_ok and remote_ok and requirements_ok)
    if res:
        ui.info(ui.green, "Done")
    else:
        ui.error("Bootstrap failed")
    return res

Example 77

Project: comtypes Source File: register.py
    def _registry_entries(self, cls):
        """Return a sequence of tuples containing registry entries.

        The tuples must be (key, subkey, name, value).

        Required entries:
        =================
        _reg_clsid_ - a string or GUID instance
        _reg_clsctx_ - server type(s) to register

        Optional entries:
        =================
        _reg_desc_ - a string
        _reg_progid_ - a string naming the progid, typically 'MyServer.MyObject.1'
        _reg_novers_progid_ - version independend progid, typically 'MyServer.MyObject'
        _reg_typelib_ - an tuple (libid, majorversion, minorversion) specifying a typelib.
        _reg_threading_ - a string specifying the threading model

        Note that the first part of the progid string is typically the
        IDL library name of the type library containing the coclass.
        """
        HKCR = _winreg.HKEY_CLASSES_ROOT

        # table format: rootkey, subkey, valuename, value
        table = []
        append = lambda *args: table.append(args)

        # basic entry - names the comobject
        reg_clsid = str(cls._reg_clsid_) # that's the only required attribute for registration
        reg_desc = getattr(cls, "_reg_desc_", "")
        if not reg_desc:
            # Simple minded algorithm to construct a description from
            # the progid:
            reg_desc = getattr(cls, "_reg_novers_progid_", "") or \
                       getattr(cls, "_reg_progid_", "")
            if reg_desc:
                reg_desc = reg_desc.replace(".", " ")
        append(HKCR, "CLSID\\%s" % reg_clsid, "", reg_desc)

        reg_progid = getattr(cls, "_reg_progid_", None)
        if reg_progid:
            # for ProgIDFromCLSID:
            append(HKCR, "CLSID\\%s\\ProgID" % reg_clsid, "", reg_progid) # 1

            # for CLSIDFromProgID
            if reg_desc:
                append(HKCR, reg_progid, "", reg_desc) # 2
            append(HKCR, "%s\\CLSID" % reg_progid, "", reg_clsid) # 3

            reg_novers_progid = getattr(cls, "_reg_novers_progid_", None)
            if reg_novers_progid:
                append(HKCR, "CLSID\\%s\\VersionIndependentProgID" % reg_clsid, # 1a
                       "", reg_novers_progid)
                if reg_desc:
                    append(HKCR, reg_novers_progid, "", reg_desc) # 2a
                append(HKCR, "%s\\CurVer" % reg_novers_progid, "", reg_progid) #
                append(HKCR, "%s\\CLSID" % reg_novers_progid, "", reg_clsid) # 3a

        clsctx = getattr(cls, "_reg_clsctx_", 0)

        if clsctx & comtypes.CLSCTX_LOCAL_SERVER \
               and not hasattr(sys, "frozendllhandle"):
            exe = sys.executable
            if " " in exe:
                exe = '"%s"' % exe
            if not hasattr(sys, "frozen"):
                if not __debug__:
                    exe = "%s -O" % exe
                script = os.path.abspath(sys.modules[cls.__module__].__file__)
                if " " in script:
                    script = '"%s"' % script
                append(HKCR, "CLSID\\%s\\LocalServer32" % reg_clsid, "", "%s %s" % (exe, script))
            else:
                append(HKCR, "CLSID\\%s\\LocalServer32" % reg_clsid, "", "%s" % exe)

        # Register InprocServer32 only when run from script or from
        # py2exe dll server, not from py2exe exe server.
        if clsctx & comtypes.CLSCTX_INPROC_SERVER \
               and getattr(sys, "frozen", None) in (None, "dll"):
            append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
                   "", self._get_serverdll())
            # only for non-frozen inproc servers the PythonPath/PythonClass is needed.
            if not hasattr(sys, "frozendllhandle") \
                   or not comtypes.server.inprocserver._clsid_to_class:
                append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
                       "PythonClass", self._get_full_classname(cls))
                append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
                       "PythonPath", self._get_pythonpath(cls))

            reg_threading = getattr(cls, "_reg_threading_", None)
            if reg_threading is not None:
                append(HKCR, "CLSID\\%s\\InprocServer32" % reg_clsid,
                       "ThreadingModel", reg_threading)

        reg_tlib = getattr(cls, "_reg_typelib_", None)
        if reg_tlib is not None:
            append(HKCR, "CLSID\\%s\\Typelib" % reg_clsid, "", reg_tlib[0])

        return table

Example 78

Project: coala Source File: LinterTest.py
    def test_process_output_regex(self):
        # Also test the case when an unknown severity is matched.
        test_output = ("12:4-14:0-Serious issue (error) -> ORIGIN=X -> D\n"
                       "0:0-0:1-This is a warning (warning) -> ORIGIN=Y -> A\n"
                       "813:77-1024:32-Just a note (info) -> ORIGIN=Z -> C\n"
                       "0:0-0:0-Some unknown sev (???) -> ORIGIN=W -> B\n")
        regex = (r"(?P<line>\d+):(?P<column>\d+)-"
                 r"(?P<end_line>\d+):(?P<end_column>\d+)-"
                 r"(?P<message>.*) \((?P<severity>.*)\) -> "
                 r"ORIGIN=(?P<origin>.*) -> (?P<additional_info>.*)")

        uut = (linter(sys.executable,
                      output_format="regex",
                      output_regex=regex)
               (self.EmptyTestLinter)
               (self.section, None))
        uut.warn = Mock()

        sample_file = "some-file.xtx"
        results = list(uut.process_output(test_output, sample_file, [""]))
        expected = [Result.from_values("EmptyTestLinter (X)",
                                       "Serious issue",
                                       sample_file,
                                       12, 4, 14, 0,
                                       RESULT_SEVERITY.MAJOR,
                                       additional_info="D"),
                    Result.from_values("EmptyTestLinter (Y)",
                                       "This is a warning",
                                       sample_file,
                                       0, 0, 0, 1,
                                       RESULT_SEVERITY.NORMAL,
                                       additional_info="A"),
                    Result.from_values("EmptyTestLinter (Z)",
                                       "Just a note",
                                       sample_file,
                                       813, 77, 1024, 32,
                                       RESULT_SEVERITY.INFO,
                                       additional_info="C"),
                    Result.from_values("EmptyTestLinter (W)",
                                       "Some unknown sev",
                                       sample_file,
                                       0, 0, 0, 0,
                                       RESULT_SEVERITY.NORMAL,
                                       additional_info="B")]

        self.assertEqual(results, expected)
        uut.warn.assert_called_once_with(
            "'???' not found in severity-map. Assuming "
            "`RESULT_SEVERITY.NORMAL`.")

        # Test when providing a sequence as output.
        test_output = ["",
                       "12:4-14:0-Serious issue (error) -> ORIGIN=X -> XYZ\n"]
        results = list(uut.process_output(test_output, sample_file, [""]))
        expected = [Result.from_values("EmptyTestLinter (X)",
                                       "Serious issue",
                                       sample_file,
                                       12, 4, 14, 0,
                                       RESULT_SEVERITY.MAJOR,
                                       additional_info="XYZ")]

        self.assertEqual(results, expected)

        # Test with using `result_message` parameter.
        uut = (linter(sys.executable,
                      output_format="regex",
                      output_regex=regex,
                      result_message="Hello world")
               (self.EmptyTestLinter)
               (self.section, None))

        results = list(uut.process_output(test_output, sample_file, [""]))
        expected = [Result.from_values("EmptyTestLinter (X)",
                                       "Hello world",
                                       sample_file,
                                       12, 4, 14, 0,
                                       RESULT_SEVERITY.MAJOR,
                                       additional_info="XYZ")]

        self.assertEqual(results, expected)

Example 79

Project: evennia Source File: evennia_launcher.py
def server_operation(mode, service, interactive, profiler, logserver=False):
    """
    Handle argument options given on the command line.

    Args:
        mode (str): Start/stop/restart and so on.
        service (str): "server", "portal" or "all".
        interactive (bool). Use interactive mode or daemon.
        profiler (bool): Run the service under the profiler.
        logserver (bool, optional): Log Server data to logfile
            specified by settings.SERVER_LOG_FILE.

    """

    cmdstr = [sys.executable, EVENNIA_RUNNER]
    errmsg = "The %s does not seem to be running."

    if mode == 'start':

        # launch the error checker. Best to catch the errors already here.
        error_check_python_modules()

        # starting one or many services
        if service == 'server':
            if profiler:
                cmdstr.append('--pserver')
            if interactive:
                cmdstr.append('--iserver')
            if logserver:
                cmdstr.append('--logserver')
            cmdstr.append('--noportal')
        elif service == 'portal':
            if profiler:
                cmdstr.append('--pportal')
            if interactive:
                cmdstr.append('--iportal')
            cmdstr.append('--noserver')
            django.core.management.call_command(
                'collectstatic', verbosity=1, interactive=False)
        else:
            # all
            # for convenience we don't start logging of
            # portal, only of server with this command.
            if profiler:
                # this is the common case
                cmdstr.append('--pserver')
            if interactive:
                cmdstr.append('--iserver')
            if logserver:
                cmdstr.append('--logserver')
            django.core.management.call_command(
                'collectstatic', verbosity=1, interactive=False)
        cmdstr.extend([
            GAMEDIR, TWISTED_BINARY, SERVER_LOGFILE,
            PORTAL_LOGFILE, HTTP_LOGFILE])
        # start the server
        process = Popen(cmdstr, env=getenv())

        if interactive:
            try:
                process.wait()
            except KeyboardInterrupt:
                server_operation("stop", "portal", False, False)
                return
            finally:
                print(NOTE_KEYBOARDINTERRUPT)

    elif mode == 'reload':
        # restarting services
        if os.name == 'nt':
            print(
                "Restarting from command line is not supported under Windows. "
                "Log into the game to restart.")
            return
        if service == 'server':
            kill(SERVER_PIDFILE, SIG, "Server reloaded.",
                 errmsg % 'Server', SERVER_RESTART, restart=True)
        elif service == 'portal':
            print(
                "Note: Portal usually doesnt't need to be reloaded unless you "
                "are debugging in interactive mode. If Portal was running in "
                "default Daemon mode, it cannot be restarted. In that case "
                "you have to restart it manually with 'evennia.py "
                "start portal'")
            kill(PORTAL_PIDFILE, SIG,
                 "Portal reloaded (or stopped, if it was in daemon mode).",
                 errmsg % 'Portal', PORTAL_RESTART, restart=True)
        else:
            # all
            # default mode, only restart server
            kill(SERVER_PIDFILE, SIG,
                 "Server reload.",
                 errmsg % 'Server', SERVER_RESTART, restart=True)

    elif mode == 'stop':
        # stop processes, avoiding reload
        if service == 'server':
            kill(SERVER_PIDFILE, SIG,
                 "Server stopped.", errmsg % 'Server', SERVER_RESTART)
        elif service == 'portal':
            kill(PORTAL_PIDFILE, SIG,
                 "Portal stopped.", errmsg % 'Portal', PORTAL_RESTART)
        else:
            kill(PORTAL_PIDFILE, SIG,
                 "Portal stopped.", errmsg % 'Portal', PORTAL_RESTART)
            kill(SERVER_PIDFILE, SIG,
                 "Server stopped.", errmsg % 'Server', SERVER_RESTART)

Example 80

Project: burp-ui Source File: setup.py
Function: run
    def run(self):
        os.chdir(ROOT)
        log.info("getting revision number")
        call('{} ./burpui -m manage compile_translation'.format(sys.executable).split())
        rev = 'stable'
        try:
            branch = check_output('sed s@^.*/@@g .git/HEAD'.split()).rstrip()
            ver = open(os.path.join('burpui', 'VERSION')).read().rstrip()
            if branch and 'dev' in ver:
                rev = branch
            try:
                with open('burpui/RELEASE', 'w') as f:
                    f.write(rev)
            except:
                pass
        except:
            pass
        # Not sure bower was a great idea...
        keep = [
            'burpui/static/vendor/bootswatch/slate/bootstrap.min.css',
            'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.eot',
            'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.svg',
            'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.ttf',
            'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.woff',
            'burpui/static/vendor/bootswatch/fonts/glyphicons-halflings-regular.woff2',
            'burpui/static/vendor/nvd3/build/nv.d3.min.css',
            'burpui/static/vendor/datatables.net-bs/css/dataTables.bootstrap.min.css',
            'burpui/static/vendor/datatables.net-responsive-bs/css/responsive.bootstrap.min.css',
            'burpui/static/vendor/jquery.fancytree/dist/skin-bootstrap/ui.fancytree.min.css',
            'burpui/static/vendor/bootstrap-switch/dist/css/bootstrap3/bootstrap-switch.min.css',
            'burpui/static/vendor/ui-select/dist/select.min.css',
            'burpui/static/vendor/jquery/dist/jquery.min.js',
            'burpui/static/vendor/jquery-ui/jquery-ui.min.js',
            'burpui/static/vendor/bootstrap/dist/js/bootstrap.min.js',
            'burpui/static/vendor/typeahead.js/dist/typeahead.bundle.min.js',
            'burpui/static/vendor/d3/d3.min.js',
            'burpui/static/vendor/nvd3/build/nv.d3.min.js',
            'burpui/static/vendor/datatables.net/js/jquery.dataTables.min.js',
            'burpui/static/vendor/datatables.net-bs/js/dataTables.bootstrap.min.js',
            'burpui/static/vendor/datatables.net-responsive/js/dataTables.responsive.js',
            'burpui/static/vendor/datatables.net-responsive-bs/js/responsive.bootstrap.js',
            'burpui/static/vendor/jquery.fancytree/dist/jquery.fancytree-all.min.js',
            'burpui/static/vendor/jquery-file-download/src/Scripts/jquery.fileDownload.js',
            'burpui/static/vendor/lodash/dist/lodash.min.js',
            'burpui/static/vendor/angular/angular.min.js',
            'burpui/static/vendor/angular-route/angular-route.min.js',
            'burpui/static/vendor/angular-sanitize/angular-sanitize.min.js',
            'burpui/static/vendor/angular-resource/angular-resource.min.js',
            'burpui/static/vendor/angular-animate/angular-animate.min.js',
            'burpui/static/vendor/bootstrap-switch/dist/js/bootstrap-switch.min.js',
            'burpui/static/vendor/angular-bootstrap-switch/dist/angular-bootstrap-switch.min.js',
            'burpui/static/vendor/ui-select/dist/select.min.js',
            'burpui/static/vendor/angular-strap/dist/angular-strap.min.js',
            'burpui/static/vendor/angular-strap/dist/angular-strap.tpl.min.js',
            'burpui/static/vendor/angular-onbeforeunload/build/angular-onbeforeunload.js',
            'burpui/static/vendor/moment/min/moment.min.js',
            'burpui/static/vendor/moment/locale/fr.js',
            'burpui/static/vendor/angular-ui-calendar/src/calendar.js',
            'burpui/static/vendor/fullcalendar/dist/fullcalendar.min.css',
            'burpui/static/vendor/fullcalendar/dist/fullcalendar.print.css',
            'burpui/static/vendor/fullcalendar/dist/fullcalendar.min.js',
            'burpui/static/vendor/fullcalendar/dist/gcal.js',
            'burpui/static/vendor/fullcalendar/dist/lang/fr.js',
            'burpui/static/vendor/angular-bootstrap/ui-bootstrap.min.js',
            'burpui/static/vendor/angular-bootstrap/ui-bootstrap-tpls.min.js',
            'burpui/static/vendor/components-font-awesome/css/font-awesome.min.css',
            'burpui/static/vendor/components-font-awesome/fonts/FontAwesome.otf',
            'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.eot',
            'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.svg',
            'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.ttf',
            'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.woff',
            'burpui/static/vendor/components-font-awesome/fonts/fontawesome-webfont.woff2',
        ]
        dirlist = []
        for dirname, subdirs, files in os.walk('burpui/static/vendor'):
            for filename in files:
                path = os.path.join(dirname, filename)
                _, ext = os.path.splitext(path)
                if os.path.isfile(path) and path not in keep and filename not in ['bower.json', 'package.json']:
                    if (rev == 'stable' and ext == '.map') or ext != '.map':
                        os.unlink(path)
                elif os.path.isdir(path):
                    dirlist.append(path)
        dirlist.sort(reverse=True)
        for d in dirlist:
            if os.path.isdir(d) and not os.listdir(d):
                os.rmdir(d)

Example 81

Project: eventlet Source File: server.py
    def run_cgi(self):
        """Execute a CGI script."""
        dir, rest = self.cgi_info
        path = dir + '/' + rest
        i = path.find('/', len(dir)+1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir)+1)
            else:
                break

        # find an explicit query string, if present.
        rest, _, query = rest.partition('?')

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(
                HTTPStatus.NOT_FOUND,
                "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(
                HTTPStatus.FORBIDDEN,
                "CGI script is not a plain file (%r)" % scriptname)
            return
        ispy = self.is_python(scriptname)
        if self.have_fork or not ispy:
            if not self.is_executable(scriptfile):
                self.send_error(
                    HTTPStatus.FORBIDDEN,
                    "CGI script is not executable (%r)" % scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.parse.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.get("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = authorization[1].encode('ascii')
                        authorization = base64.decodebytes(authorization).\
                                        decode('ascii')
                    except (binascii.Error, UnicodeError):
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.get('content-type') is None:
            env['CONTENT_TYPE'] = self.headers.get_content_type()
        else:
            env['CONTENT_TYPE'] = self.headers['content-type']
        length = self.headers.get('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.get('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.get('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.get_all('cookie', []))
        cookie_str = ', '.join(co)
        if cookie_str:
            env['HTTP_COOKIE'] = cookie_str
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(HTTPStatus.OK, "Script output follows")
        self.flush_headers()

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except OSError:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non-Unix -- use subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)
            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 env = env
                                 )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 82

Project: crossbar Source File: process.py
    def _start_native_worker(self, wtype, id, options=None, details=None):

        assert(wtype in ['router', 'container', 'websocket-testee'])

        # prohibit starting a worker twice
        #
        if id in self._workers:
            emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(id)
            self.log.error(emsg)
            raise ApplicationError(u'crossbar.error.worker_already_running', emsg)

        # check worker options
        #
        options = options or {}
        try:
            if wtype == 'router':
                checkconfig.check_router_options(options)
            elif wtype == 'container':
                checkconfig.check_container_options(options)
            elif wtype == 'websocket-testee':
                checkconfig.check_websocket_testee_options(options)
            else:
                raise Exception("logic error")
        except Exception as e:
            emsg = "Could not start native worker: invalid configuration ({})".format(e)
            self.log.error(emsg)
            raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)

        # allow override Python executable from options
        #
        if 'python' in options:
            exe = options['python']

            # the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy
            #
            if not os.path.isabs(exe):
                emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format(exe)
                self.log.error(emsg)
                raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)

            # of course the path must exist and actually be executable
            #
            if not (os.path.isfile(exe) and os.access(exe, os.X_OK)):
                emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format(exe)
                self.log.error(emsg)
                raise ApplicationError(u'crossbar.error.invalid_configuration', emsg)
        else:
            exe = sys.executable

        # assemble command line for forking the worker
        #
        # all native workers (routers and containers for now) start
        # from the same script in crossbar/worker/process.py -- we're
        # invoking via "-m" so that .pyc files, __pycache__ etc work
        # properly.

        args = [exe, "-u", "-m", "crossbar.worker.process"]
        args.extend(["--cbdir", self._node._cbdir])
        args.extend(["--worker", str(id)])
        args.extend(["--realm", self._realm])
        args.extend(["--type", wtype])
        args.extend(["--loglevel", get_global_log_level()])

        # allow override worker process title from options
        #
        if options.get('title', None):
            args.extend(['--title', options['title']])

        # forward explicit reactor selection
        #
        if 'reactor' in options and sys.platform in options['reactor']:
            args.extend(['--reactor', options['reactor'][sys.platform]])
        # FIXME
        # elif self._node.options.reactor:
        #    args.extend(['--reactor', self._node.options.reactor])

        # create worker process environment
        #
        worker_env = create_process_env(options)

        # We need to use the same PYTHONPATH we were started with, so we can
        # find the Crossbar we're working with -- it may not be the same as the
        # one on the default path
        worker_env["PYTHONPATH"] = os.pathsep.join(sys.path)

        # log name of worker
        #
        worker_logname = {
            'router': 'Router',
            'container': 'Container',
            'websocket-testee': 'WebSocketTestee'
        }.get(wtype, 'Worker')

        # topic URIs used (later)
        #
        if wtype == 'router':
            starting_topic = 'crossbar.node.on_router_starting'
            started_topic = 'crossbar.node.on_router_started'
        elif wtype == 'container':
            starting_topic = 'crossbar.node.on_container_starting'
            started_topic = 'crossbar.node.on_container_started'
        elif wtype == 'websocket-testee':
            starting_topic = 'crossbar.node.on_websocket_testee_starting'
            started_topic = 'crossbar.node.on_websocket_testee_started'
        else:
            raise Exception("logic error")

        # add worker tracking instance to the worker map ..
        #
        if wtype == 'router':
            worker = RouterWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))
        elif wtype == 'container':
            worker = ContainerWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))
        elif wtype == 'websocket-testee':
            worker = WebSocketTesteeWorkerProcess(self, id, details.caller, keeplog=options.get('traceback', None))
        else:
            raise Exception("logic error")

        self._workers[id] = worker

        # create a (custom) process endpoint.
        #
        if platform.isWindows():
            childFDs = None  # Use the default Twisted ones
        else:
            # The communication between controller and container workers is
            # using WAMP running over 2 pipes.
            # For controller->container traffic this runs over FD 0 (`stdin`)
            # and for the container->controller traffic, this runs over FD 3.
            #
            # Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for
            # container->controller traffic, so that components running in the
            # container which happen to write to `stdout` or `stderr` do not
            # interfere with the container-controller communication.
            childFDs = {0: "w", 1: "r", 2: "r", 3: "r"}

        ep = WorkerProcessEndpoint(
            self._node._reactor, exe, args, env=worker_env, worker=worker,
            childFDs=childFDs)

        # ready handling
        #
        def on_ready_success(id):
            self.log.info("{worker} with ID '{id}' and PID {pid} started",
                          worker=worker_logname, id=worker.id, pid=worker.pid)

            self._node._reactor.addSystemEventTrigger(
                'before', 'shutdown',
                self._cleanup_worker, self._node._reactor, worker,
            )

            worker.status = 'started'
            worker.started = datetime.utcnow()

            started_info = {
                u'id': worker.id,
                u'status': worker.status,
                u'started': utcstr(worker.started),
                u'who': worker.who,
            }

            # FIXME: make start of stats printer dependent on log level ..
            worker.log_stats(5.)

            self.publish(started_topic, started_info, options=PublishOptions(exclude=details.caller))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]
            emsg = 'Failed to start native worker: {}'.format(err.value)
            self.log.error(emsg)
            raise ApplicationError(u"crossbar.error.cannot_start", emsg, worker.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(_):
            self.log.info("Node worker {worker.id} ended successfully", worker=worker)
            worker.log_stats(0)
            del self._workers[worker.id]
            return True

        def on_exit_error(err):
            self.log.info("Node worker {worker.id} ended with error ({err})", worker=worker, err=err)
            worker.log_stats(0)
            del self._workers[worker.id]
            return False

        def check_for_shutdown(was_successful):
            shutdown = False

            # automatically shutdown node whenever a worker ended (successfully, or with error)
            #
            if checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers:
                self.log.info("Node worker ended, and trigger '{trigger}' active", trigger=checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT)
                shutdown = True

            # automatically shutdown node when worker ended with error
            #
            if not was_successful and checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers:
                self.log.info("Node worker ended with error, and trigger '{trigger}' active", trigger=checkconfig.NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR)
                shutdown = True

            # automatically shutdown node when no more workers are left
            #
            if len(self._workers) == 0 and checkconfig.NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers:
                self.log.info("No more node workers running, and trigger '{trigger}' active", trigger=checkconfig.NODE_SHUTDOWN_ON_LAST_WORKER_EXIT)
                shutdown = True

            # initiate shutdown (but only if we are not already shutting down)
            #
            if shutdown:
                if not self._shutdown_requested:
                    self.log.info("Node shutting down ..")
                    self.shutdown()
                else:
                    # ignore: shutdown already initiated ..
                    self.log.info("Node is already shutting down.")
            else:
                self.log.info(
                    "Node will continue to run (node shutdown triggers active: {triggers})",
                    triggers=self._node._node_shutdown_triggers,
                )

        d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error)
        d_on_exit.addBoth(check_for_shutdown)

        # create a transport factory for talking WAMP to the native worker
        #
        transport_factory = create_native_worker_client_factory(self._node._router_session_factory, worker.ready, worker.exit)
        transport_factory.noisy = False
        self._workers[id].factory = transport_factory

        # now (immediately before actually forking) signal the starting of the worker
        #
        starting_info = {
            u'id': id,
            u'status': worker.status,
            u'created': utcstr(worker.created),
            u'who': worker.who,
        }

        # the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        # .. while all others get an event
        self.publish(starting_topic, starting_info, options=PublishOptions(exclude=details.caller))

        # now actually fork the worker ..
        #
        self.log.info("Starting {worker} with ID '{id}'...",
                      worker=worker_logname, id=id)
        self.log.debug("{worker} '{id}' command line is '{cmdline}'",
                       worker=worker_logname, id=id, cmdline=' '.join(args))

        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            # this seems to be called immediately when the child process
            # has been forked. even if it then immediately fails because
            # e.g. the executable doesn't even exist. in other words,
            # I'm not sure under what conditions the deferred will errback ..

            pid = proto.transport.pid
            self.log.debug("Native worker process connected with PID {pid}",
                           pid=pid)

            # note the PID of the worker
            worker.pid = pid

            # proto is an instance of NativeWorkerClientProtocol
            worker.proto = proto

            worker.status = 'connected'
            worker.connected = datetime.utcnow()

        def on_connect_error(err):

            # not sure when this errback is triggered at all ..
            self.log.error("Interal error: connection to forked native worker failed ({err})", err=err)

            # in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready

Example 83

Project: datafari Source File: test_gdb.py
Function: get_stack_trace
    def get_stack_trace(self, source=None, script=None,
                        breakpoint='PyObject_Print',
                        cmds_after_breakpoint=None,
                        import_site=False):
        '''
        Run 'python -c SOURCE' under gdb with a breakpoint.

        Support injecting commands after the breakpoint is reached

        Returns the stdout from gdb

        cmds_after_breakpoint: if provided, a list of strings: gdb commands
        '''
        # We use "set breakpoint pending yes" to avoid blocking with a:
        #   Function "foo" not defined.
        #   Make breakpoint pending on future shared library load? (y or [n])
        # error, which typically happens python is dynamically linked (the
        # breakpoints of interest are to be found in the shared library)
        # When this happens, we still get:
        #   Function "PyObject_Print" not defined.
        # emitted to stderr each time, alas.

        # Initially I had "--eval-command=continue" here, but removed it to
        # avoid repeated print breakpoints when traversing hierarchical data
        # structures

        # Generate a list of commands in gdb's language:
        commands = ['set breakpoint pending yes',
                    'break %s' % breakpoint,

                    # The tests assume that the first frame of printed
                    #  backtrace will not contain program counter,
                    #  that is however not guaranteed by gdb
                    #  therefore we need to use 'set print address off' to
                    #  make sure the counter is not there. For example:
                    # #0 in PyObject_Print ...
                    #  is assumed, but sometimes this can be e.g.
                    # #0 0x00003fffb7dd1798 in PyObject_Print ...
                    'set print address off',

                    'run']

        # GDB as of 7.4 onwards can distinguish between the
        # value of a variable at entry vs current value:
        #   http://sourceware.org/gdb/onlinedocs/gdb/Variables.html
        # which leads to the selftests failing with errors like this:
        #   AssertionError: 'v@entry=()' != '()'
        # Disable this:
        if (gdb_major_version, gdb_minor_version) >= (7, 4):
            commands += ['set print entry-values no']

        if cmds_after_breakpoint:
            commands += cmds_after_breakpoint
        else:
            commands += ['backtrace']

        # print commands

        # Use "commands" to generate the arguments with which to invoke "gdb":
        args = ["gdb", "--batch", "-nx"]
        args += ['--eval-command=%s' % cmd for cmd in commands]
        args += ["--args",
                 sys.executable]

        if not import_site:
            # -S suppresses the default 'import site'
            args += ["-S"]

        if source:
            args += ["-c", source]
        elif script:
            args += [script]

        # print args
        # print ' '.join(args)

        # Use "args" to invoke gdb, capturing stdout, stderr:
        out, err = run_gdb(*args, PYTHONHASHSEED='0')

        errlines = err.splitlines()
        unexpected_errlines = []

        # Ignore some benign messages on stderr.
        ignore_patterns = (
            'Function "%s" not defined.' % breakpoint,
            "warning: no loadable sections found in added symbol-file"
            " system-supplied DSO",
            "warning: Unable to find libthread_db matching"
            " inferior's thread library, thread debugging will"
            " not be available.",
            "warning: Cannot initialize thread debugging"
            " library: Debugger service failed",
            'warning: Could not load shared library symbols for '
            'linux-vdso.so',
            'warning: Could not load shared library symbols for '
            'linux-gate.so',
            'warning: Could not load shared library symbols for '
            'linux-vdso64.so',
            'Do you need "set solib-search-path" or '
            '"set sysroot"?',
            'warning: Source file is more recent than executable.',
            # Issue #19753: missing symbols on System Z
            'Missing separate debuginfo for ',
            'Try: zypper install -C ',
            )
        for line in errlines:
            if not line.startswith(ignore_patterns):
                unexpected_errlines.append(line)

        # Ensure no unexpected error messages:
        self.assertEqual(unexpected_errlines, [])
        return out

Example 84

Project: VisTrails Source File: engine_manager.py
    def start_engines(self, nb=None, prompt="Number of engines to start"):
        """Start some engines locally
        """
        c = self.ensure_controller()
        if c is None:
            if qt_available:
                QtGui.QMessageBox.warning(
                        None,
                        "No controller",
                        "Can't start engines: couldn't connect to a "
                        "controller")
            print "parallelflow: no controller, not starting engines"
        else:
            if not nb and qt_available:
                nb, res = QtGui.QInputDialog.getInt(
                        None,
                        "Start engines",
                        prompt,
                        1,  # value
                        1,  # min
                        16) # max
                if not res:
                    return
            elif nb is None:
                nb = 1
            print "parallelflow: about to start %d engines" % nb
            if qt_available:
                bar = QtGui.QProgressDialog(
                        "Starting engines...",
                        None,
                        0, nb)
                def progress(n):
                    bar.setValue(n)
                bar.show()
            else:
                def progress(n): pass
            progress(0)

            init_engines = set(c.ids)
            # Start the processes
            starting = set()
            for i in xrange(nb):
                proc, res = self.start_process(
                        None,
                        sys.executable,
                        '-m',
                        'IPython.parallel.apps.ipengineapp',
                        '--profile=%s' % self.profile)
                starting.add(proc)
            # Wait for each one to either fail or connect
            failed = []
            connected = 0
            while connected < len(starting):
                connected = len(set(c.ids) - init_engines)
                progress(len(failed) + connected)
                time.sleep(0.5)
                for p in list(starting):
                    res = p.poll()
                    if res is not None:
                        failed.append(res)
                        starting.remove(p)
            if failed:
                nb_failed = len(failed)
                if nb_failed > 3:
                    failed = "%s, ..." % (', '.join('%d' % f for f in failed))
                else:
                    failed = ', '.join('%d' % f for f in failed)
                if qt_available:
                    QtGui.QMessageBox.critical(
                        None,
                        "Error",
                        "%d engine(s) exited with codes: %s" % (
                        nb_failed, failed))
                print "parallelflow: %d engine(s) exited with codes: %s" % (
                        nb_failed, failed)
            self.started_engines.update(starting)

            if qt_available:
                bar.hide()
                bar.deleteLater()
            print "parallelflow: %d engines started" % nb

Example 85

Project: ganga Source File: Batch.py
Function: resubmit
    def resubmit(self):

        job = self.getJobObject()

        inw = job.getInputWorkspace()
        outw = job.getOutputWorkspace()

        statusfilename = outw.getPath('__jobstatus__')
        try:
            os.remove(statusfilename)
        except OSError as x:
            if x.errno != 2:
                logger.warning("OSError:" + str(x))

        scriptpath = inw.getPath('__jobscript__')
        #stderr_option = '-e '+str(outw.getPath())+'stderr'
        #stdout_option = '-o '+str(outw.getPath())+'stdout'

        # FIX from Alex Richards - see Savannah #87477
        stdout_option = self.config['stdoutConfig'] % str(outw.getPath())
        stderr_option = self.config['stderrConfig'] % str(outw.getPath())

        queue_option = ''
        if self.queue:
            queue_option = '-q ' + str(self.queue)

        try:
            jobnameopt = "-" + self.config['jobnameopt']
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            jobnameopt = False

        if self.extraopts:
            import re
            for opt in re.compile(r'(-\w+)').findall(self.extraopts):
                if opt in ('-o', '-e', '-oo', '-eo'):
                    logger.warning("option %s is forbidden", opt)
                    return False
                if self.queue and opt == '-q':
                    logger.warning("option %s is forbidden if queue is defined ( queue = '%s')", opt, self.queue)
                    return False
                if jobnameopt and opt == jobnameopt:
                    jobnameopt = False

            queue_option = queue_option + " " + self.extraopts

        if jobnameopt and job.name != '':
            # PBS doesn't like names with spaces
            tmp_name = job.name
            if isType(self, PBS):
                tmp_name = tmp_name.replace(" ", "_")
            queue_option = queue_option + " " + \
                jobnameopt + " " + "'%s'" % (tmp_name)

        # bugfix #16646
        if self.config['shared_python_executable']:
            import sys
            script_cmd = "%s %s" % (sys.executable, scriptpath)
        else:
            script_cmd = scriptpath

        command_str = self.config['submit_str'] % (
            inw.getPath(), queue_option, stderr_option, stdout_option, script_cmd)
        self.command_string = command_str
        rc, soutfile = self.command(command_str)
        logger.debug('from command get rc: "%d"', rc)
        if rc == 0:
            with open(soutfile) as sout_file:
                sout = sout_file.read()
            import re
            m = re.compile(
                self.config['submit_res_pattern'], re.M).search(sout)
            if m is None:
                logger.warning('could not match the output and extract the Batch job identifier!')
                logger.warning('command output \n %s ', sout)
            else:
                self.id = m.group('id')
                try:
                    queue = m.group('queue')
                    if self.queue != queue:
                        if self.queue:
                            logger.warning('you requested queue "%s" but the job was submitted to queue "%s"', self.queue, queue)
                            logger.warning('command output \n %s ', sout)
                        else:
                            logger.info('using default queue "%s"', queue)
                        self.actualqueue = queue
                except IndexError:
                    logger.info('could not match the output and extract the Batch queue name')
        else:
            with open(soutfile) as sout_file:
                logger.warning(sout_file.read())

        return rc == 0

Example 86

Project: vent Source File: menu_launcher.py
def processmenu(path_dirs, menu, parent=None):
    """ processes the execution of the interaction sent to the menu """
    optioncount = len(menu['options'])
    exitmenu = False
    while not exitmenu:
        getin = runmenu(menu, parent)
        if getin == optioncount:
            exitmenu = True
        elif menu['options'][getin]['type'] in [PROMPT]:
            curses.def_prog_mode()
            os.system('reset')
            screen.clear()
            if prompt():
                os.system(menu['options'][getin]['command'])

            screen.clear()
            curses.reset_prog_mode()
            try:
                curses.curs_set(1)
                curses.curs_set(0)
            except Exception as e:
                pass
        elif menu['options'][getin]['type'] in [COMMAND, CONFIRM]:
            curses.def_prog_mode()
            os.system('reset')
            screen.clear()
            os.system(menu['options'][getin]['command'])

            if menu['options'][getin]['type'] == CONFIRM:
                if menu['title'] == "Remove Plugins":
                    exitmenu = True
                confirm()

            screen.clear()
            curses.reset_prog_mode()
            try:
                curses.curs_set(1)
                curses.curs_set(0)
            except Exception as e:
                pass
        elif menu['options'][getin]['type'] in [INFO, DISPLAY]:
            pass
        elif menu['options'][getin]['type'] == INPUT:
            if menu['options'][getin]['title'] == "Add Plugins":
                plugin_url = get_param("Enter the HTTPS Git URL that contains the new plugins, e.g. https://github.com/CyberReboot/vent-plugins.git:")
                curses.def_prog_mode()
                os.system('reset')
                screen.clear()
                if not "https://" in plugin_url:
                    os.system("echo No plugins added, url is not formatted correctly.")
                    os.system("echo Please use a git url, e.g. https://github.com/CyberReboot/vent-plugins.git")
                else:
                    os.system("python2.7 "+path_dirs.data_dir+"plugin_parser.py add_plugins "+plugin_url+" "+path_dirs.base_dir+" "+path_dirs.data_dir)
                confirm()
                screen.clear()
                os.execl(sys.executable, sys.executable, *sys.argv)
            elif menu['options'][getin]['title'] == "Files":
                filename = get_param("Enter the name of the processed file to lookup logs for:")
                curses.def_prog_mode()
                os.system('reset')
                os.system("clear")
                screen.clear()
                try:
                    # ensure directory exists
                    os.system("if [ ! -d /tmp/vent_logs ]; then mkdir /tmp/vent_logs; fi;")
                    # check logs exist for that file
                    found = call("python2.7 "+path_dirs.info_dir+"get_logs.py -f "+filename+" | grep "+filename, shell=True)
                    if found == 0:
                        # print logs
                        os.system("python2.7 "+path_dirs.info_dir+"get_logs.py -f "+filename+" | tee /tmp/vent_logs/vent_file_"+filename+" | less")
                    else:
                        os.system("echo \"No logs found for that file.\" | less")
                except Exception as e:
                    os.system("echo \"Error retrieving logs for that file.\" | less")
                screen.clear()
                curses.reset_prog_mode()
                try:
                    curses.curs_set(1)
                    curses.curs_set(0)
                except Exception as e:
                    pass
        elif menu['options'][getin]['type'] == MENU:
            if menu['options'][getin]['title'] == "Remove Plugins":
                screen.clear()
                installed_plugins = get_installed_plugin_repos(path_dirs, CONFIRM, "remove")
                processmenu(path_dirs, installed_plugins, menu)
                screen.clear()
            elif menu['options'][getin]['title'] == "Show Installed Plugins":
                screen.clear()
                installed_plugins = get_installed_plugin_repos(path_dirs, DISPLAY, "")
                processmenu(path_dirs, installed_plugins, menu)
                screen.clear()
            elif menu['options'][getin]['title'] == "Update Plugins":
                screen.clear()
                installed_plugins = get_installed_plugin_repos(path_dirs, CONFIRM, "update")
                processmenu(path_dirs, installed_plugins, menu)
                screen.clear()
            elif menu['options'][getin]['title'] == "Status":
                screen.clear()
                plugins = get_plugin_status(path_dirs)
                processmenu(path_dirs, plugins, menu)
                screen.clear()
            elif menu['options'][getin]['title'] == "Services":
                screen.clear()
                containers = get_container_menu(path_dirs)
                processmenu(path_dirs, containers, menu)
                screen.clear()
            elif menu['options'][getin]['title'] == "Namespaces":
                screen.clear()
                namespaces = get_namespace_menu(path_dirs)
                processmenu(path_dirs, namespaces, menu)
                screen.clear()
            else:
                screen.clear()
                processmenu(path_dirs, menu['options'][getin], menu)
                screen.clear()
        elif menu['options'][getin]['type'] == EXITMENU:
            exitmenu = True

Example 87

Project: gsutil Source File: test_update.py
  @unittest.skipUnless(CERTIFICATE_VALIDATION_ENABLED,
                       'Test requires https certificate validation enabled.')
  def test_update(self):
    """Tests that the update command works or raises proper exceptions."""
    if os.environ.get('CLOUDSDK_WRAPPER') == '1':
      stderr = self.RunGsUtil(['update'], stdin='n',
                              return_stderr=True, expected_status=1)
      self.assertIn('update command is disabled for Cloud SDK', stderr)
      return

    if gslib.IS_PACKAGE_INSTALL:
      # The update command is not present when installed via package manager.
      stderr = self.RunGsUtil(['update'], return_stderr=True, expected_status=1)
      self.assertIn('Invalid command', stderr)
      return

    # Create two temp directories, one of which we will run 'gsutil update' in
    # to pull the changes from the other.
    tmpdir_src = self.CreateTempDir()
    tmpdir_dst = self.CreateTempDir()

    # Copy gsutil to both source and destination directories.
    gsutil_src = os.path.join(tmpdir_src, 'gsutil')
    gsutil_dst = os.path.join(tmpdir_dst, 'gsutil')
    # Path when executing from tmpdir (Windows doesn't support in-place rename)
    gsutil_relative_dst = os.path.join('gsutil', 'gsutil')

    shutil.copytree(GSUTIL_DIR, gsutil_src)
    # Copy specific files rather than all of GSUTIL_DIR so we don't pick up temp
    # working files left in top-level directory by gsutil developers (like tags,
    # .git*, etc.)
    os.makedirs(gsutil_dst)
    for comp in ('CHANGES.md', 'CHECKSUM', 'COPYING', 'gslib', 'gsutil',
                 'gsutil.py', 'MANIFEST.in', 'README.md', 'setup.py', 'test',
                 'third_party', 'VERSION'):
      if os.path.isdir(os.path.join(GSUTIL_DIR, comp)):
        func = shutil.copytree
      else:
        func = shutil.copyfile
      func(os.path.join(GSUTIL_DIR, comp), os.path.join(gsutil_dst, comp))

    # Create a fake version number in the source so we can verify it in the
    # destination.
    expected_version = '17.25'
    src_version_file = os.path.join(gsutil_src, 'VERSION')
    self.assertTrue(os.path.exists(src_version_file))
    with open(src_version_file, 'w') as f:
      f.write(expected_version)

    # Create a tarball out of the source directory and copy it to a bucket.
    src_tarball = os.path.join(tmpdir_src, 'gsutil.test.tar.gz')

    normpath = os.path.normpath
    try:
      # We monkey patch os.path.normpath here because the tarfile module
      # normalizes the ./gsutil path, but the update command expects the tar
      # file to be prefixed with . This preserves the ./gsutil path.
      os.path.normpath = lambda fname: fname
      tar = tarfile.open(src_tarball, 'w:gz')
      tar.add(gsutil_src, arcname='./gsutil')
      tar.close()
    finally:
      os.path.normpath = normpath

    prefix = [sys.executable] if sys.executable else []

    # Run with an invalid gs:// URI.
    p = subprocess.Popen(prefix + ['gsutil', 'update', 'gs://pub'],
                         cwd=gsutil_dst, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    (_, stderr) = p.communicate()
    p.stdout.close()
    p.stderr.close()
    self.assertEqual(p.returncode, 1)
    self.assertIn('update command only works with tar.gz', stderr)

    # Run with non-existent gs:// URI.
    p = subprocess.Popen(
        prefix + ['gsutil', 'update', 'gs://pub/Jdjh38)(;.tar.gz'],
        cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (_, stderr) = p.communicate()
    p.stdout.close()
    p.stderr.close()
    self.assertEqual(p.returncode, 1)
    self.assertIn('NotFoundException', stderr)

    # Run with file:// URI wihout -f option.
    p = subprocess.Popen(prefix + ['gsutil', 'update', suri(src_tarball)],
                         cwd=gsutil_dst, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    (_, stderr) = p.communicate()
    p.stdout.close()
    p.stderr.close()
    self.assertEqual(p.returncode, 1)
    self.assertIn('command does not support', stderr)

    # Run with a file present that was not distributed with gsutil.
    with open(os.path.join(gsutil_dst, 'userdata.txt'), 'w') as fp:
      fp.write('important data\n')
    p = subprocess.Popen(prefix + ['gsutil', 'update', '-f', suri(src_tarball)],
                         cwd=gsutil_dst, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE, stdin=subprocess.PIPE)
    (_, stderr) = p.communicate()
    p.stdout.close()
    p.stderr.close()
    # Clean up before next test, and before assertions so failure doesn't leave
    # this file around.
    os.unlink(os.path.join(gsutil_dst, 'userdata.txt'))
    self.assertEqual(p.returncode, 1)
    self.assertIn(
        'The update command cannot run with user data in the gsutil directory',
        stderr.replace(os.linesep, ' '))

    # Determine whether we'll need to decline the analytics prompt.
    analytics_prompt = not (
        os.path.exists(_UUID_FILE_PATH) or
        boto.config.get_value('GSUtil', 'disable_analytics_prompt'))

    update_input = 'n\r\ny\r\n' if analytics_prompt else 'y\r\n'

    # Now do the real update, which should succeed.
    p = subprocess.Popen(prefix + [gsutil_relative_dst, 'update', '-f',
                                   suri(src_tarball)],
                         cwd=tmpdir_dst, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE, stdin=subprocess.PIPE)
    (_, stderr) = p.communicate(input=update_input)
    p.stdout.close()
    p.stderr.close()
    self.assertEqual(p.returncode, 0, msg=(
        'Non-zero return code (%d) from gsutil update. stderr = \n%s' %
        (p.returncode, stderr)))

    # Verify that version file was updated.
    dst_version_file = os.path.join(tmpdir_dst, 'gsutil', 'VERSION')
    with open(dst_version_file, 'r') as f:
      self.assertEqual(f.read(), expected_version)

    # If the analytics prompt was given, that means we disabled analytics. We
    # should reset to the default by deleting the UUID file.
    if analytics_prompt:
      os.unlink(_UUID_FILE_PATH)

Example 88

Project: ClusterRunner Source File: base_config_loader.py
    def configure_defaults(self, conf):
        """
        This is the base configuration. All default configuration values belong here. These values may be overridden by
        other configurations.
        :type conf: Configuration
        """
        if getattr(sys, 'frozen', False):
            root_directory = dirname(sys.executable)  # frozen
        else:
            root_directory = dirname(dirname(dirname(dirname(realpath(__file__)))))  # unfrozen

        conf.set('secret', None)  # This must be overridden by conf or the service will not start for security reasons

        conf.set('root_directory', root_directory)  # the root directory of the application
        conf.set('main_executable_path', sys.argv[0])
        conf.set('version', autoversioning.get_version())

        # If the user installed ClusterRunner manually, the directories used by the manual install should take
        # precedence over the default install location (the home directory).
        static_configured_base_directory = join('/var', 'lib', 'clusterrunner')

        if isdir(static_configured_base_directory):
            base_directory = static_configured_base_directory
        else:
            base_directory = join(expanduser('~/'), '.clusterrunner')

        # where all of the clusterrunner specific files will be stored (other than source code)
        conf.set('base_directory', base_directory)
        # the path to the clusterrunner config file. We have to specify this in defaults since it cannot depend on
        # values in the file it refers to.
        conf.set('config_file', join(base_directory, 'clusterrunner.conf'))

        # Where the clusterrunner service will save the process id to. These settings are set in base_config_loader
        # and not in master_config_loader and slave_config_loader because CLI tools such as "clusterrunner stop"
        # needs to read in these settings.
        conf.set('master_pid_file', join(base_directory, '.clusterrunner_master.pid'))
        conf.set('slave_pid_file', join(base_directory, '.clusterrunner_slave.pid'))

        # contains symlinks to build-specific repos
        conf.set('build_symlink_directory', join('/tmp', 'clusterrunner_build_symlinks'))
        # where the repos are cloned to
        conf.set('repo_directory', None)

        conf.set('project_yaml_filename', 'clusterrunner.yaml')

        conf.set('log_file', None)
        conf.set('log_level', 'DEBUG')
        conf.set('max_log_file_size', 1024 * 1024 * 50)  # 50mb
        conf.set('max_log_file_backups', 5)

        # set eventlog file conf values to None to disable eventlogs by default
        conf.set('log_filename', 'clusterrunner_default.log')
        conf.set('eventlog_filename', 'eventlog_default.log')
        conf.set('eventlog_file', None)
        conf.set('max_eventlog_file_size', 1024 * 1024 * 50)  # 50mb
        conf.set('max_eventlog_file_backups', 5)
        conf.set('hostname', platform.node())
        conf.set('master_hostname', 'localhost')
        conf.set('master_port', '43000')
        conf.set('slaves', ['localhost'])

        # Strict host key checking on git remote operations, disabled by default
        conf.set('git_strict_host_key_checking', False)

        # CORS support - a regex to match against allowed API request origins, or None to disable CORS
        conf.set('cors_allowed_origins_regex', None)

        # Helper executables
        bin_dir = join(root_directory, 'bin')
        conf.set('git_askpass_exe', join(bin_dir, 'git_askpass.sh'))
        conf.set('git_ssh_exe', join(bin_dir, 'git_ssh.sh'))

        # How slaves get the project
        # Slaves would get the project from master if set to True. Otherwise it would just get the project in
        # the same way how the master gets the project.
        conf.set('get_project_from_master', True)

        # Should we have shallow or full clones of the repository?
        # The master must have full clones, as slaves fetch from the master, and one cannot fetch from a shallow clone.
        conf.set('shallow_clones', False)

Example 89

Project: sonospy Source File: cron.py
Function: crondance
def crondance(apppath, ctype='soft',startup=False):
    cron_path = os.path.join(apppath,'admin','cron')
    token = Token(cron_path)
    cronmaster = token.acquire(startup=startup)
    if not cronmaster:
        return
    now_s = time.localtime()
    checks=(('min',now_s.tm_min),
            ('hr',now_s.tm_hour),
            ('mon',now_s.tm_mon),
            ('dom',now_s.tm_mday),
            ('dow',now_s.tm_wday))
    
    apps = [x for x in os.listdir(apppath)
            if os.path.isdir(os.path.join(apppath, x))]
        
    for app in apps:
        apath = os.path.join(apppath,app)
        cronpath = os.path.join(apath, 'cron')
        crontab = os.path.join(cronpath, 'crontab')
        if not os.path.exists(crontab):
            continue
        try:
            f = open(crontab, 'rt')
            cronlines = f.readlines()
            lines = [x for x in cronlines if x.strip() and x[0]!='#']
            tasks = [parsecronline(cline) for cline in lines]
        except Exception, e:
            logging.error('WEB2PY CRON: crontab read error %s' % e)
            continue

        for task in tasks:
            commands = [sys.executable]
            if os.path.exists('web2py.py'):
                commands.append('web2py.py')
            citems = [(k in task and not v in task[k]) for k,v in checks]
            task_min= task.get('min',[])
            if not task:
                continue
            elif not startup and task_min == [-1]:
                continue
            elif task_min != [-1] and reduce(lambda a,b: a or b, citems):
                continue
            logging.info('WEB2PY CRON (%s): %s executing %s in %s at %s' \
                             % (ctype, app, task.get('cmd'),
                                os.getcwd(), datetime.datetime.now()))
            action, command, models = False, task['cmd'], ''
            if command.startswith('**'):
                (action,models,command) = (True,'',command[2:])
            elif command.startswith('*'):
                (action,models,command) = (True,'-M',command[1:])
            else:
                action=False
            if action and command.endswith('.py'):
                commands.extend(('-P',
                                 '-N',models,
                                 '-S',app,
                                 '-a','"<recycle>"',
                                 '-R',command))
                shell = True
            elif action:
                commands.extend(('-P',
                                 '-N',models,
                                 '-S',app+'/'+command,
                                 '-a','"<recycle>"'))
                shell = True
            else:
                commands = command
                shell = False
            try:
                print time.ctime()+' '+ctype+' CRON RUNNING %s' % commands
                cronlauncher(commands, shell=shell).start()
            except Exception, e:
                logging.warning(
                    'WEB2PY CRON: Execution error for %s: %s' \
                        % (task.get('cmd'), e))
    token.release()

Example 90

Project: gcc-python-plugin Source File: run-test-suite.py
def run_test(testdir):
    # Compile each 'input.c', using 'script.py'
    # Assume success and empty stdout; compare against expected stderr, or empty if file not present
    inputfiles = get_source_files(testdir)
    outfile = os.path.join(testdir, 'output.o')
    script_py = os.path.join(testdir, 'script.py')
    out = TestStream(os.path.join(testdir, 'stdout.txt'))
    err = TestStream(os.path.join(testdir, 'stderr.txt'))

    cp = configparser.SafeConfigParser()
    metadatapath = os.path.join(testdir, 'metadata.ini')
    cp.read([metadatapath])

    if cp.has_section('WhenToRun'):
        if cp.has_option('WhenToRun', 'required_features'):
            required_features = cp.get('WhenToRun', 'required_features').split()
            for feature in required_features:
                if feature not in features:
                    raise ValueError('%s in %s not found in %s'
                                     % (feature, metadatapath, config_h))
                if not features[feature]:
                    raise SkipTest('required feature %s not available in %s'
                                   % (feature, config_h))

    env = dict(os.environ)
    env['LC_ALL'] = 'C'

    # Generate the command-line for invoking gcc:
    args = [CC]
    if len(inputfiles) == 1:
        args += ['-c'] # (don't run the linker)
    else:
        args += ['-fPIC', '-shared']
        # Force LTO when there's more than one source file:
        args += ['-flto', '-flto-partition=none']

    if GCC_VERSION >= 4008:
        # GCC 4.8 started showing the source line where the problem is,
        # followed by another line showing a caret indicating column.
        # This is a great usability feature, but totally breaks our "gold"
        # output, so turn it off for running tests:
        args += ['-fno-diagnostics-show-caret']

        # Similarly, the macro expansion tracking is great for usability,
        # but breaks the "gold" output, so we disable it during tests:
        args += ['-ftrack-macro-expansion=0']

    args += ['-o', outfile]
    args += ['-fplugin=%s' % os.path.abspath('%s.so' % PLUGIN_NAME),
             '-fplugin-arg-%s-script=%s' % (PLUGIN_NAME, script_py)]

    # Force the signedness of char so that the tests have consistent
    # behavior across all archs:
    args += ['-fsigned-char']

    # Special-case: add the python include dir (for this runtime) if the C code
    # uses Python.h:
    def uses_python_headers():
        for inputfile in inputfiles:
            with open(inputfile, 'r') as f:
                code = f.read()
            if '#include <Python.h>' in code:
                return True

    if uses_python_headers():
        args += ['-I' + get_python_inc()]

    # If there's a getopts.py, run it to get additional test-specific
    # command-line options:
    getopts_py = os.path.join(testdir, 'getopts.py')
    if os.path.exists(getopts_py):
        p = Popen([sys.executable, getopts_py], stdout=PIPE, stderr=PIPE)
        opts_out, opts_err = p.communicate()
        if six.PY3:
            opts_out = opts_out.decode()
            opts_err = opts_err.decode()
        c = p.wait()
        if c != 0:
            raise CommandError()
        args += opts_out.split()

    # and the source files go at the end:
    args += inputfiles

    if options.show:
        # Show the gcc invocation:
        print(' '.join(args))

    # Invoke the compiler:
    p = Popen(args, env=env, stdout=PIPE, stderr=PIPE)
    out.actual, err.actual = p.communicate()
    if six.PY3:
        out.actual = out.actual.decode()
        err.actual = err.actual.decode()
    #print 'out: %r' % out.actual
    #print 'err: %r' % err.actual
    exitcode_actual = p.wait()

    if options.show:
        # then the user wants to see the gcc invocation directly
        sys.stdout.write(out.actual)
        sys.stderr.write(err.actual)

    # Expected exit code
    # By default, we expect success if the expected stderr is empty, and
    # and failure if it's non-empty.
    # This can be overridden if the test has a metadata.ini, by setting
    # exitcode within the [ExpectedBehavior] section:
    if err.expdata == '':
        exitcode_expected = 0
    else:
        exitcode_expected = 1
    if cp.has_section('ExpectedBehavior'):
        if cp.has_option('ExpectedBehavior', 'exitcode'):
            exitcode_expected = cp.getint('ExpectedBehavior', 'exitcode')

    # Check exit code:
    if exitcode_actual != exitcode_expected:
        sys.stderr.write(out.diff('stdout'))
        sys.stderr.write(err.diff('stderr'))
        raise CompilationError(out.actual, err.actual, p, args)

    if exitcode_expected == 0:
        assert os.path.exists(outfile)
    
    out.check_for_diff(out.actual, err.actual, p, args, 'stdout', WRITEBACK)
    err.check_for_diff(out.actual, err.actual, p, args, 'stderr', WRITEBACK)

Example 91

Project: python-3parclient Source File: HPE3ParClient_base.py
    def setUp(self, withSSH=False, withFilePersona=False):

        self.withSSH = withSSH
        self.withFilePersona = withFilePersona

        cwd = os.path.dirname(os.path.abspath(
            inspect.getfile(inspect.currentframe())))

        if self.unitTest:
            self.printHeader('Using flask ' + self.flask_url)
            parsed_url = urlparse(self.flask_url)
            userArg = '-user=%s' % self.user
            passwordArg = '-password=%s' % self.password
            portArg = '-port=%s' % parsed_url.port

            script = 'HPE3ParMockServer_flask.py'
            path = "%s/%s" % (cwd, script)
            try:
                self.mockServer = subprocess.Popen([sys.executable,
                                                    path,
                                                    userArg,
                                                    passwordArg,
                                                    portArg],
                                                   stdout=subprocess.PIPE,
                                                   stderr=subprocess.PIPE,
                                                   stdin=subprocess.PIPE
                                                   )
            except Exception:
                pass

            time.sleep(1)
            if self.withFilePersona:
                self.cl = file_client.HPE3ParFilePersonaClient(self.flask_url)
            else:
                self.cl = client.HPE3ParClient(self.flask_url)

            if self.withSSH:

                self.printHeader('Using paramiko SSH server on port %s' %
                                 self.ssh_port)

                ssh_script = 'HPE3ParMockServer_ssh.py'
                ssh_path = "%s/%s" % (cwd, ssh_script)

                self.mockSshServer = subprocess.Popen([sys.executable,
                                                       ssh_path,
                                                       str(self.ssh_port)],
                                                      stdout=subprocess.PIPE,
                                                      stderr=subprocess.PIPE,
                                                      stdin=subprocess.PIPE)
                time.sleep(1)

        else:
            if withFilePersona:
                self.printHeader('Using 3PAR %s with File Persona' %
                                 self.url_3par)
                self.cl = file_client.HPE3ParFilePersonaClient(self.url_3par)
            else:
                self.printHeader('Using 3PAR ' + self.url_3par)
                self.cl = client.HPE3ParClient(self.url_3par)

        if self.withSSH:
            # This seems to slow down the test cases, so only use this when
            # requested
            if self.unitTest:
                # The mock SSH server can be accessed at 0.0.0.0.
                ip = '0.0.0.0'
            else:
                parsed_3par_url = urlparse(self.url_3par)
                ip = parsed_3par_url.hostname.split(':').pop()
            try:
                # Now that we don't do keep-alive, the conn_timeout needs to
                # be set high enough to avoid sometimes slow response in
                # the File Persona tests.
                self.cl.setSSHOptions(
                    ip,
                    self.user,
                    self.password,
                    port=self.ssh_port,
                    conn_timeout=500,
                    known_hosts_file=self.known_hosts_file,
                    missing_key_policy=self.missing_key_policy)
            except Exception as ex:
                print(ex)
                self.fail("failed to start ssh client")

        # Setup remote copy target
        if self.run_remote_copy:
            parsed_3par_url = urlparse(self.secondary_url_3par)
            ip = parsed_3par_url.hostname.split(':').pop()
            self.secondary_cl = client.HPE3ParClient(self.secondary_url_3par)
            try:
                self.secondary_cl.setSSHOptions(
                    ip,
                    self.secondary_user,
                    self.secondary_password,
                    port=self.ssh_port,
                    conn_timeout=500,
                    known_hosts_file=self.known_hosts_file,
                    missing_key_policy=self.missing_key_policy)
            except Exception as ex:
                print(ex)
                self.fail("failed to start ssh client")
            self.secondary_cl.login(self.secondary_user,
                                    self.secondary_password)

        if self.debug:
            self.cl.debug_rest(True)

        self.cl.login(self.user, self.password)

        if not self.port:
            ports = self.cl.getPorts()
            ports = [p for p in ports['members']
                     if p['linkState'] == 4 and  # Ready
                     ('device' not in p or not p['device']) and
                     p['mode'] == self.cl.PORT_MODE_TARGET]
            self.port = ports[0]['portPos']

Example 92

Project: visionegg Source File: Configuration.py
    def __init__(self):
        """Load global Vision Egg configuration information."""
        cfg = ConfigParser.ConfigParser()

        if hasattr(sys,'argv') and sys.executable == sys.argv[0]: # Windows binary
            self.VISIONEGG_SYSTEM_DIR = os.curdir
            self.VISIONEGG_USER_DIR = os.curdir
        else:
            # non-standard VisionEgg installations
            try:
                self.VISIONEGG_SYSTEM_DIR = os.environ['VISIONEGG_SYSTEM_DIR']
            except KeyError:
                self.VISIONEGG_SYSTEM_DIR = os.path.split(__file__)[0]
            user_dir = os.path.expanduser("~")
            self.VISIONEGG_USER_DIR = os.path.join(user_dir,"VisionEgg")

        # See if there's an environment variable for the config file
        if 'VISIONEGG_CONFIG_FILE' in os.environ.keys():
            configFile = os.environ['VISIONEGG_CONFIG_FILE']
        else:
            # Is there one in VISIONEGG_USER_DIR?
            configFile = os.path.join(self.VISIONEGG_USER_DIR,"VisionEgg.cfg")
            if not os.path.isfile(configFile):
                configFile = os.path.join(self.VISIONEGG_SYSTEM_DIR,"VisionEgg.cfg")
                if not os.path.isfile(configFile):
                    configFile = None # No file, use defaults specified in environment variables then here

        if configFile:
            cfg.read(configFile)
        else:
            # pretend we have a config file
            cfg.add_section('General')
            for key in defaults.keys():
                cfg.set('General',key,str(defaults[key]))
            if sys.platform == 'darwin':
                cfg.add_section('darwin')
                for key in extra_darwin_defaults.keys():
                    cfg.set('darwin',key,str(extra_darwin_defaults[key]))

        # Do the general stuff first
        # Set the default values
        for name in defaults.keys():
            if name in os.environ.keys():
                value = os.environ[name]
            else:
                value = defaults[name]
            if isinstance(defaults[name], int):
		if value == 'False':
		    value = 0
		elif value == 'True':
		    value = 1
                setattr(self,name,int(value))
            elif isinstance(defaults[name], float):
                setattr(self,name,float(value))
            else:
                setattr(self,name,value)

        # Get the values from the configFile
        general_options = cfg.options('General')

        self._delayed_configuration_log_warnings = [] # chick and egg problem
        # set defaults from config file
        for option in general_options:
            name = option.upper()
            if name not in defaults.keys():
                self._delayed_configuration_log_warnings.append(
                    "While reading %s: The variable \"%s\" is not (anymore) a Vision Egg variable."%(os.path.abspath(configFile),option))
                continue
            value = cfg.get('General',option)
            if name in os.environ.keys():
                value = os.environ[name]
            if isinstance(defaults[name], int):
		if value == 'False':
		    value = 0
		elif value == 'True':
		    value = 1
                setattr(self,name,int(value))
            elif isinstance(defaults[name], float):
                setattr(self,name,float(value))
            else:
                setattr(self,name,value)

        # Do platform specific stuff
        # Set the default values
        platform_name = sys.platform
        extra_name = "extra_%s_defaults"%(platform_name,)
        if extra_name in globals().keys():
            extra_defaults = globals()[extra_name]
            for name in extra_defaults.keys():
                setattr(self,name,extra_defaults[name])

            # Get the values from the configFile
            platform_options = cfg.options(platform_name)
            for option in platform_options:
                name = option.upper()
                if name not in extra_defaults.keys():
                    raise KeyError("No Vision Egg configuration variable \"%s\""%option)
                value = cfg.get(platform_name,option)
                if name in os.environ.keys():
                    value = os.environ[name]
                if isinstance(extra_defaults[name], int):
		    if value == 'False':
		        value = 0
    		    elif value == 'True':
		        value = 1
                    setattr(self,name,int(value))
                elif isinstance(extra_defaults[name], float):
                    setattr(self,name,float(value))
                else:
                    setattr(self,name,value)

        if(configFile):
            self.VISIONEGG_CONFIG_FILE = os.path.abspath(configFile)
        else:
            self.VISIONEGG_CONFIG_FILE = None

Example 93

Project: python-hunter Source File: test_hunter.py
def test_pth_sample2(LineMatcher):
    env = dict(os.environ, PYTHONHUNTER="module='__main__'")
    env.pop('COVERAGE_PROCESS_START', None)
    env.pop('COV_CORE_SOURCE', None)
    output = subprocess.check_output(
        [sys.executable, os.path.join(os.path.dirname(__file__), 'sample2.py')],
        env=env,
        stderr=subprocess.STDOUT,
    )
    lm = LineMatcher(output.decode('utf-8').splitlines())
    lm.fnmatch_lines([
        '*tests*sample2.py:* call      if __name__ == "__main__":  #*',
        '*tests*sample2.py:* line      if __name__ == "__main__":  #*',
        '*tests*sample2.py:* line          import functools',
        '*tests*sample2.py:* line          def deco(opt):',
        '*tests*sample2.py:* line          @deco(1)',
        '*tests*sample2.py:* call          def deco(opt):',
        '*tests*sample2.py:* line              def decorator(func):',
        '*tests*sample2.py:* line              return decorator',
        '*tests*sample2.py:* return            return decorator',
        '*                 * ...       return value: <function deco*',
        '*tests*sample2.py:* line          @deco(2)',
        '*tests*sample2.py:* call          def deco(opt):',
        '*tests*sample2.py:* line              def decorator(func):',
        '*tests*sample2.py:* line              return decorator',
        '*tests*sample2.py:* return            return decorator',
        '*                 * ...       return value: <function deco*',
        '*tests*sample2.py:* line          @deco(3)',
        '*tests*sample2.py:* call          def deco(opt):',
        '*tests*sample2.py:* line              def decorator(func):',
        '*tests*sample2.py:* line              return decorator',
        '*tests*sample2.py:* return            return decorator',
        '*                 * ...       return value: <function deco*',
        '*tests*sample2.py:* call              def decorator(func):',
        '*tests*sample2.py:* line                  @functools.wraps(func)',
        '*tests*sample2.py:* line                  return wrapper',
        '*tests*sample2.py:* return                return wrapper',
        '*                 * ...       return value: <function foo *',
        '*tests*sample2.py:* call              def decorator(func):',
        '*tests*sample2.py:* line                  @functools.wraps(func)',
        '*tests*sample2.py:* line                  return wrapper',
        '*tests*sample2.py:* return                return wrapper',
        '*                 * ...       return value: <function foo *',
        '*tests*sample2.py:* call              def decorator(func):',
        '*tests*sample2.py:* line                  @functools.wraps(func)',
        '*tests*sample2.py:* line                  return wrapper',
        '*tests*sample2.py:* return                return wrapper',
        '*                 * ...       return value: <function foo *',
        '*tests*sample2.py:* line          foo(',
        "*tests*sample2.py:* line              'a*',",
        "*tests*sample2.py:* line              'b'",
        '*tests*sample2.py:* call                  @functools.wraps(func)',
        '*                 *    |                  def wrapper(*args):',
        '*tests*sample2.py:* line                      return func(*args)',
        '*tests*sample2.py:* call                  @functools.wraps(func)',
        '*                 *    |                  def wrapper(*args):',
        '*tests*sample2.py:* line                      return func(*args)',
        '*tests*sample2.py:* call                  @functools.wraps(func)',
        '*                 *    |                  def wrapper(*args):',
        '*tests*sample2.py:* line                      return func(*args)',
        '*tests*sample2.py:* call          @deco(1)',
        '*                 *    |          @deco(2)',
        '*                 *    |          @deco(3)',
        '*                 *    |          def foo(*args):',
        '*tests*sample2.py:* line              return args',
        '*tests*sample2.py:* return            return args',
        "*                 * ...       return value: ('a*', 'b')",
        "*tests*sample2.py:* return                    return func(*args)",
        "*                 * ...       return value: ('a*', 'b')",
        "*tests*sample2.py:* return                    return func(*args)",
        "*                 * ...       return value: ('a*', 'b')",
        "*tests*sample2.py:* return                    return func(*args)",
        "*                 * ...       return value: ('a*', 'b')",
        "*tests*sample2.py:* line          try:",
        "*tests*sample2.py:* line              None(",
        "*tests*sample2.py:* line                  'a',",
        "*tests*sample2.py:* line                  'b'",
        "*tests*sample2.py:* exception             'b'",
        "*                 * ...       exception value: *",
        "*tests*sample2.py:* line          except:",
        "*tests*sample2.py:* line              pass",
        "*tests*sample2.py:* return            pass",
        "*                   ...       return value: None",
    ])

Example 94

Project: databus Source File: CGIHTTPServer.py
    def run_cgi(self):
        """Execute a CGI script."""
        path = self.path
        dir, rest = self.cgi_info

        i = path.find('/', len(dir) + 1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir) + 1)
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = {}
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.getheader('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")
        os.environ.update(env)

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, os.environ)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        elif self.have_popen2 or self.have_popen3:
            # Windows -- use popen2 or popen3 to create a subprocess
            import shutil
            if self.have_popen3:
                popenx = os.popen3
            else:
                popenx = os.popen2
            cmdline = scriptfile
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = "%s -u %s" % (interp, cmdline)
            if '=' not in query and '"' not in query:
                cmdline = '%s "%s"' % (cmdline, query)
            self.log_message("command: %s", cmdline)
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            files = popenx(cmdline, 'b')
            fi = files[0]
            fo = files[1]
            if self.have_popen3:
                fe = files[2]
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
                fi.write(data)
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            fi.close()
            shutil.copyfileobj(fo, self.wfile)
            if self.have_popen3:
                errors = fe.read()
                fe.close()
                if errors:
                    self.log_error('%s', errors)
            sts = fo.close()
            if sts:
                self.log_error("CGI script exit status %#x", sts)
            else:
                self.log_message("CGI script exited OK")

        else:
            # Other O.S. -- execute script in this process
            save_argv = sys.argv
            save_stdin = sys.stdin
            save_stdout = sys.stdout
            save_stderr = sys.stderr
            try:
                save_cwd = os.getcwd()
                try:
                    sys.argv = [scriptfile]
                    if '=' not in decoded_query:
                        sys.argv.append(decoded_query)
                    sys.stdout = self.wfile
                    sys.stdin = self.rfile
                    execfile(scriptfile, {"__name__": "__main__"})
                finally:
                    sys.argv = save_argv
                    sys.stdin = save_stdin
                    sys.stdout = save_stdout
                    sys.stderr = save_stderr
                    os.chdir(save_cwd)
            except SystemExit, sts:
                self.log_error("CGI script exit status %s", str(sts))
            else:
                self.log_message("CGI script exited OK")

Example 95

Project: brython Source File: test_gdb.py
Function: get_stack_trace
    def get_stack_trace(self, source=None, script=None,
                        breakpoint=BREAKPOINT_FN,
                        cmds_after_breakpoint=None,
                        import_site=False):
        '''
        Run 'python -c SOURCE' under gdb with a breakpoint.

        Support injecting commands after the breakpoint is reached

        Returns the stdout from gdb

        cmds_after_breakpoint: if provided, a list of strings: gdb commands
        '''
        # We use "set breakpoint pending yes" to avoid blocking with a:
        #   Function "foo" not defined.
        #   Make breakpoint pending on future shared library load? (y or [n])
        # error, which typically happens python is dynamically linked (the
        # breakpoints of interest are to be found in the shared library)
        # When this happens, we still get:
        #   Function "textiowrapper_write" not defined.
        # emitted to stderr each time, alas.

        # Initially I had "--eval-command=continue" here, but removed it to
        # avoid repeated print breakpoints when traversing hierarchical data
        # structures

        # Generate a list of commands in gdb's language:
        commands = ['set breakpoint pending yes',
                    'break %s' % breakpoint,
                    'run']
        if cmds_after_breakpoint:
            commands += cmds_after_breakpoint
        else:
            commands += ['backtrace']

        # print commands

        # Use "commands" to generate the arguments with which to invoke "gdb":
        args = ["gdb", "--batch"]
        args += ['--eval-command=%s' % cmd for cmd in commands]
        args += ["--args",
                 sys.executable]

        if not import_site:
            # -S suppresses the default 'import site'
            args += ["-S"]

        if source:
            args += ["-c", source]
        elif script:
            args += [script]

        # print args
        # print ' '.join(args)

        # Use "args" to invoke gdb, capturing stdout, stderr:
        out, err = run_gdb(*args, PYTHONHASHSEED='0')

        errlines = err.splitlines()
        unexpected_errlines = []

        # Ignore some benign messages on stderr.
        ignore_patterns = (
            'Function "%s" not defined.' % breakpoint,
            "warning: no loadable sections found in added symbol-file"
            " system-supplied DSO",
            "warning: Unable to find libthread_db matching"
            " inferior's thread library, thread debugging will"
            " not be available.",
            "warning: Cannot initialize thread debugging"
            " library: Debugger service failed",
            'warning: Could not load shared library symbols for '
            'linux-vdso.so',
            'warning: Could not load shared library symbols for '
            'linux-gate.so',
            'Do you need "set solib-search-path" or '
            '"set sysroot"?',
            )
        for line in errlines:
            if not line.startswith(ignore_patterns):
                unexpected_errlines.append(line)

        # Ensure no unexpected error messages:
        self.assertEqual(unexpected_errlines, [])
        return out

Example 96

Project: reprozip Source File: functional.py
@in_temp_dir
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
    rpz_python = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
    rpuz_python = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]

    # Can't match on the SignalWarning category here because of a Python bug
    # http://bugs.python.org/issue22543
    if raise_warnings:
        rpz_python.extend(['-W', 'error:signal'])
        rpuz_python.extend(['-W', 'error:signal'])

    if 'COVER' in os.environ:
        rpz_python.extend(['-m'] + os.environ['COVER'].split(' '))
        rpuz_python.extend(['-m'] + os.environ['COVER'].split(' '))

    reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
    reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'

    verbose = ['-v'] * 3
    rpz = rpz_python + [reprozip_main.absolute().path] + verbose
    rpuz = rpuz_python + [reprounzip_main.absolute().path] + verbose

    print("Command lines are:\n%r\n%r" % (rpz, rpuz))

    # ########################################
    # testrun /bin/echo
    #

    output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
    assert any(b' 1 | /bin/echo outputhere ' in l
               for l in output.splitlines())

    output = check_output(rpz + ['testrun', '-a', '/fake/path/echo',
                                 '/bin/echo', 'outputhere'])
    assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
               for l in output.splitlines())

    # ########################################
    # testrun multiple commands
    #

    check_call(rpz + ['testrun', 'bash', '-c',
                      'cat ../../../../../etc/passwd;'
                      'cd /var/lib;'
                      'cat ../../etc/group'])
    check_call(rpz + ['trace', '--overwrite',
                      'bash', '-c', 'cat /etc/passwd;echo'])
    check_call(rpz + ['trace', '--continue',
                      'sh', '-c', 'cat /etc/group;/usr/bin/id'])
    check_call(rpz + ['pack'])
    check_call(rpuz + ['graph', 'graph.dot'])
    check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])

    sudo = ['sudo', '-E']  # -E to keep REPROZIP_USAGE_STATS

    # ########################################
    # 'simple' program: trace, pack, info, unpack
    #

    def check_simple(args, stream, infile=1):
        output = check_output(args, stream).splitlines()
        try:
            first = output.index(b"Read 6 bytes")
        except ValueError:
            stderr.write("output = %r\n" % output)
            raise
        if infile == 1:
            assert output[first + 1] == b"a = 29, b = 13"
            assert output[first + 2] == b"result = 42"
        else:  # infile == 2
            assert output[first + 1] == b"a = 25, b = 11"
            assert output[first + 2] == b"result = 36"

    # Build
    build('simple', ['simple.c'])
    # Trace
    check_call(rpz + ['trace', '--overwrite', '-d', 'rpz-simple',
                      './simple',
                      (tests / 'simple_input.txt').path,
                      'simple_output.txt'])
    orig_output_location = Path('simple_output.txt').absolute()
    assert orig_output_location.is_file()
    with orig_output_location.open(encoding='utf-8') as fp:
        assert fp.read().strip() == '42'
    orig_output_location.remove()
    # Read config
    with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
        conf = yaml.safe_load(fp)
    other_files = set(Path(f).absolute() for f in conf['other_files'])
    expected = [Path('simple'), (tests / 'simple_input.txt')]
    assert other_files.issuperset([f.resolve() for f in expected])
    # Check input and output files
    inputs_outputs = conf['inputs_outputs']
    # Exactly one input: "arg1", "...simple_input.txt"
    # Output: 'arg2', "...simple_output.txt"
    # There might be more output files: the C coverage files
    found = 0
    for fdict in inputs_outputs:
        if Path(fdict['path']).name == b'simple_input.txt':
            assert fdict['name'] == 'arg1'
            assert fdict['read_by_runs'] == [0]
            assert not fdict.get('written_by_runs')
            found |= 0x01
        elif Path(fdict['path']).name == b'simple_output.txt':
            assert fdict['name'] == 'arg2'
            assert not fdict.get('read_by_runs')
            assert fdict['written_by_runs'] == [0]
            found |= 0x02
        else:
            # No other inputs
            assert not fdict.get('read_by_runs')
    assert found == 0x03
    # Pack
    check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
    Path('simple').rename('simple.orig')
    # Info
    check_call(rpuz + ['info', 'simple.rpz'])
    # Show files
    check_call(rpuz + ['showfiles', 'simple.rpz'])
    # Lists packages
    check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
    # Unpack directory
    check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
    # Run directory
    check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
    output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
    with output_in_dir.open(encoding='utf-8') as fp:
        assert fp.read().strip() == '42'
    # Delete with wrong command (should fail)
    p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
                         stderr=subprocess.PIPE)
    out, err = p.communicate()
    assert p.poll() != 0
    err = err.splitlines()
    assert b"Wrong unpacker used" in err[0]
    assert err[1].startswith(b"usage: ")
    # Delete directory
    check_call(rpuz + ['directory', 'destroy', 'simpledir'])
    # Unpack chroot
    check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
                              'simple.rpz', 'simplechroot'])
    try:
        output_in_chroot = join_root(Path('simplechroot/root'),
                                     orig_output_location)
        # Run chroot
        check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
        with output_in_chroot.open(encoding='utf-8') as fp:
            assert fp.read().strip() == '42'
        # Get output file
        check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
                                  'arg2:output1.txt'])
        with Path('output1.txt').open(encoding='utf-8') as fp:
            assert fp.read().strip() == '42'
        # Get random file
        check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
                                  '%s:binc.bin' % (Path.cwd() / 'simple')])
        assert same_files('simple.orig', 'binc.bin')
        # Replace input file
        check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
                                  '%s:arg1' % (tests / 'simple_input2.txt')])
        check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
        # Run again
        check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
        with output_in_chroot.open(encoding='utf-8') as fp:
            assert fp.read().strip() == '36'
        # Reset input file
        check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
        # Run again
        check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
        with output_in_chroot.open(encoding='utf-8') as fp:
            assert fp.read().strip() == '42'
        # Replace input file via path
        check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
                                  '%s:%s' % (tests / 'simple_input2.txt',
                                             tests / 'simple_input.txt')])
        check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
        # Run again
        check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
        # Delete with wrong command (should fail)
        p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
                             stderr=subprocess.PIPE)
        out, err = p.communicate()
        assert p.poll() != 0
        err = err.splitlines()
        assert b"Wrong unpacker used" in err[0]
        assert err[1].startswith(b"usage:")
    finally:
        # Delete chroot
        check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])

    # Use reprounzip-vistrails with chroot
    check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
                              'simple.rpz', 'simplechroot_vt'])
    try:
        output_in_chroot = join_root(Path('simplechroot_vt/root'),
                                     orig_output_location)
        # Run using reprounzip-vistrails
        check_simple(
            sudo + rpuz_python +
            ['-m', 'reprounzip.plugins.vistrails', '1',
             'chroot', 'simplechroot_vt', '0',
             '--input-file', 'arg1:%s' % (tests / 'simple_input2.txt'),
             '--output-file', 'arg2:output_vt.txt'],
            'err', 2)
        with output_in_chroot.open(encoding='utf-8') as fp:
            assert fp.read().strip() == '36'
    finally:
        # Delete chroot
        check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot_vt'])

    if not (tests / 'vagrant').exists():
        check_call(['sudo', 'sh', '-c',
                    'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'}])

    # Unpack Vagrant-chroot
    check_call(rpuz + ['vagrant', 'setup/create', '--memory', '512',
                       '--use-chroot', 'simple.rpz',
                       (tests / 'vagrant/simplevagrantchroot').path])
    print("\nVagrant project set up in simplevagrantchroot")
    try:
        if run_vagrant:
            check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
                                 (tests / 'vagrant/simplevagrantchroot').path],
                         'out')
            # Get output file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrantchroot').path,
                               'arg2:voutput1.txt'])
            with Path('voutput1.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '42'
            # Get random file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrantchroot').path,
                               '%s:binvc.bin' % (Path.cwd() / 'simple')])
            assert same_files('simple.orig', 'binvc.bin')
            # Replace input file
            check_call(rpuz + ['vagrant', 'upload',
                               (tests / 'vagrant/simplevagrantchroot').path,
                               '%s:arg1' % (tests / 'simple_input2.txt')])
            check_call(rpuz + ['vagrant', 'upload',
                               (tests / 'vagrant/simplevagrantchroot').path])
            # Run again
            check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
                                 (tests / 'vagrant/simplevagrantchroot').path],
                         'out', 2)
            # Get output file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrantchroot').path,
                               'arg2:voutput2.txt'])
            with Path('voutput2.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '36'
            # Reset input file
            check_call(rpuz + ['vagrant', 'upload',
                               (tests / 'vagrant/simplevagrantchroot').path,
                               ':arg1'])
            # Run again
            check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
                                 (tests / 'vagrant/simplevagrantchroot').path],
                         'out')
            # Get output file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrantchroot').path,
                               'arg2:voutput1.txt'])
            with Path('voutput1.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '42'
            # Replace input file via path
            check_call(rpuz + ['vagrant', 'upload',
                               (tests / 'vagrant/simplevagrantchroot').path,
                               '%s:%s' % (tests / 'simple_input2.txt',
                                          tests / 'simple_input.txt')])
            # Run again
            check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
                                 (tests / 'vagrant/simplevagrantchroot').path],
                         'out', 2)
            # Destroy
            check_call(rpuz + ['vagrant', 'destroy',
                               (tests / 'vagrant/simplevagrantchroot').path])
        elif interactive:
            print("Test and press enter")
            sys.stdin.readline()
    finally:
        if (tests / 'vagrant/simplevagrantchroot').exists():
            (tests / 'vagrant/simplevagrantchroot').rmtree()
    # Unpack Vagrant without chroot
    check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
                       'simple.rpz',
                       (tests / 'vagrant/simplevagrant').path])
    print("\nVagrant project set up in simplevagrant")
    try:
        if run_vagrant:
            check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
                                 (tests / 'vagrant/simplevagrant').path],
                         'out')
            # Get output file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrant').path,
                               'arg2:woutput1.txt'])
            with Path('woutput1.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '42'
            # Get random file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrant').path,
                               '%s:binvs.bin' % (Path.cwd() / 'simple')])
            assert same_files('simple.orig', 'binvs.bin')
            # Replace input file
            check_call(rpuz + ['vagrant', 'upload',
                               (tests / 'vagrant/simplevagrant').path,
                               '%s:arg1' % (tests / 'simple_input2.txt')])
            check_call(rpuz + ['vagrant', 'upload',
                               (tests / 'vagrant/simplevagrant').path])
            # Run again
            check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
                                 (tests / 'vagrant/simplevagrant').path],
                         'out', 2)
            # Get output file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrant').path,
                               'arg2:woutput2.txt'])
            with Path('woutput2.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '36'
            # Reset input file
            check_call(rpuz + ['vagrant', 'upload',
                               (tests / 'vagrant/simplevagrant').path,
                               ':arg1'])
            # Run again
            check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
                                 (tests / 'vagrant/simplevagrant').path],
                         'out')
            # Get output file
            check_call(rpuz + ['vagrant', 'download',
                               (tests / 'vagrant/simplevagrant').path,
                               'arg2:voutput1.txt'])
            with Path('voutput1.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '42'
            # Destroy
            check_call(rpuz + ['vagrant', 'destroy',
                               (tests / 'vagrant/simplevagrant').path])
        elif interactive:
            print("Test and press enter")
            sys.stdin.readline()
    finally:
        if (tests / 'vagrant/simplevagrant').exists():
            (tests / 'vagrant/simplevagrant').rmtree()

    # Unpack Docker
    check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
    print("\nDocker project set up in simpledocker")
    try:
        if run_docker:
            check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
            check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
            # Get output file
            check_call(rpuz + ['docker', 'download', 'simpledocker',
                               'arg2:doutput1.txt'])
            with Path('doutput1.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '42'
            # Get random file
            check_call(rpuz + ['docker', 'download', 'simpledocker',
                               '%s:bind.bin' % (Path.cwd() / 'simple')])
            assert same_files('simple.orig', 'bind.bin')
            # Replace input file
            check_call(rpuz + ['docker', 'upload', 'simpledocker',
                               '%s:arg1' % (tests / 'simple_input2.txt')])
            check_call(rpuz + ['docker', 'upload', 'simpledocker'])
            check_call(rpuz + ['showfiles', 'simpledocker'])
            # Run again
            check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
            # Get output file
            check_call(rpuz + ['docker', 'download', 'simpledocker',
                               'arg2:doutput2.txt'])
            with Path('doutput2.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '36'
            # Reset input file
            check_call(rpuz + ['docker', 'upload', 'simpledocker',
                               ':arg1'])
            # Run again
            check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
            # Get output file
            check_call(rpuz + ['docker', 'download', 'simpledocker',
                               'arg2:doutput1.txt'])
            with Path('doutput1.txt').open(encoding='utf-8') as fp:
                assert fp.read().strip() == '42'
            # Replace input file via path
            check_call(rpuz + ['docker', 'upload', 'simpledocker',
                               '%s:%s' % (tests / 'simple_input2.txt',
                                          tests / 'simple_input.txt')])
            # Run again
            check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
            # Destroy
            check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
        elif interactive:
            print("Test and press enter")
            sys.stdin.readline()
    finally:
        if Path('simpledocker').exists():
            Path('simpledocker').rmtree()

    # ########################################
    # 'threads' program: testrun
    #

    # Build
    build('threads', ['threads.c'], ['-lpthread'])
    # Trace
    output = check_output(rpz + ['testrun', './threads'], 'err')
    assert any(b'successfully exec\'d /bin/./echo' in l
               for l in output.splitlines())

    # ########################################
    # 'threads2' program: testrun
    #

    # Build
    build('threads2', ['threads2.c'], ['-lpthread'])
    # Trace
    output = check_output(rpz + ['testrun', './threads2'], 'err')
    assert any(b'successfully exec\'d /bin/echo' in l
               for l in output.splitlines())

    # ########################################
    # 'segv' program: testrun
    #

    # Build
    build('segv', ['segv.c'])
    # Trace
    check_call(rpz + ['testrun', './segv'])

    # ########################################
    # 'exec_echo' program: trace, pack, run --cmdline
    #

    # Build
    build('exec_echo', ['exec_echo.c'])
    # Trace
    check_call(rpz + ['trace', '--overwrite',
                      './exec_echo', 'originalexecechooutput'])
    # Pack
    check_call(rpz + ['pack', 'exec_echo.rpz'])
    # Unpack chroot
    check_call(sudo + rpuz + ['chroot', 'setup',
                              'exec_echo.rpz', 'echochroot'])
    try:
        # Run original command-line
        output = check_output(sudo + rpuz + ['chroot', 'run',
                                             'echochroot'])
        assert output == b'originalexecechooutput\n'
        # Prints out command-line
        output = check_output(sudo + rpuz + ['chroot', 'run',
                                             'echochroot', '--cmdline'])
        assert any(b'./exec_echo originalexecechooutput' == s.strip()
                   for s in output.split(b'\n'))
        # Run with different command-line
        output = check_output(sudo + rpuz + [
            'chroot', 'run', 'echochroot',
            '--cmdline', './exec_echo', 'changedexecechooutput'])
        assert output == b'changedexecechooutput\n'
    finally:
        check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])

    # ########################################
    # 'exec_echo' program: testrun
    # This is built with -m32 so that we transition:
    #   python (x64) -> exec_echo (i386) -> echo (x64)
    #

    if sys.maxsize > 2 ** 32:
        # Build
        build('exec_echo32', ['exec_echo.c'], ['-m32'])
        # Trace
        check_call(rpz + ['testrun', './exec_echo32 42'])
    else:
        print("Can't try exec_echo transitions: not running on 64bits")

    # ########################################
    # Tracing non-existing program
    #

    check_call(rpz + ['testrun', './doesntexist'])

    # ########################################
    # 'connect' program: testrun
    #

    # Build
    build('connect', ['connect.c'])
    # Trace
    err = check_output(rpz + ['testrun', './connect'], 'err')
    err = err.split(b'\n')
    assert not any(b'program exited with non-zero code' in l for l in err)
    assert any(re.search(br'process connected to [0-9.]+:80', l)
               for l in err)

    # ########################################
    # 'vfork' program: testrun
    #

    # Build
    build('vfork', ['vfork.c'])
    # Trace
    err = check_output(rpz + ['testrun', './vfork'], 'err')
    err = err.split(b'\n')
    assert not any(b'program exited with non-zero code' in l for l in err)

    # ########################################
    # 'rename' program: trace
    #

    # Build
    build('rename', ['rename.c'])
    # Trace
    check_call(rpz + ['trace', '--overwrite', '-d', 'rename-trace',
                      './rename'])
    with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
        config = yaml.safe_load(fp)
    # Check that written files were logged
    database = Path.cwd() / 'rename-trace/trace.sqlite3'
    if PY3:
        # On PY3, connect() only accepts unicode
        conn = sqlite3.connect(str(database))
    else:
        conn = sqlite3.connect(database.path)
    conn.row_factory = sqlite3.Row
    rows = conn.execute(
        '''
        SELECT name FROM opened_files
        ''')
    files = set(Path(r[0]) for r in rows)
    for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
        if (Path.cwd() / n) not in files:
            raise AssertionError("Missing file: %s" % (Path.cwd() / n))
    conn.close()
    # Check that created files won't be packed
    for f in config.get('other_files'):
        if 'dir2' in Path(f).parent.components:
            raise AssertionError("Created file shouldn't be packed: %s" %
                                 Path(f))

    # ########################################
    # Test shebang corner-cases
    #

    Path('a').symlink('b')
    with Path('b').open('w') as fp:
        fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c'))
    Path('b').chmod(0o744)
    Path('c').symlink('d')
    with Path('d').open('w') as fp:
        fp.write('#!e')
    Path('d').chmod(0o744)
    with Path('e').open('w') as fp:
        fp.write('#!/bin/echo')
    Path('e').chmod(0o744)

    # Trace
    out = check_output(rpz + ['trace', '--overwrite', '-d', 'shebang-trace',
                              '--dont-identify-packages', './a', '1', '2'])
    out = out.splitlines()[0]
    assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii')

    # Check config
    with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp:
        config = yaml.safe_load(fp)
    other_files = set(Path(f) for f in config['other_files']
                      if f.startswith('%s/' % Path.cwd()))

    # Check database
    database = Path.cwd() / 'shebang-trace/trace.sqlite3'
    if PY3:
        # On PY3, connect() only accepts unicode
        conn = sqlite3.connect(str(database))
    else:
        conn = sqlite3.connect(database.path)
    conn.row_factory = sqlite3.Row
    rows = conn.execute(
        '''
        SELECT name FROM opened_files
        ''')
    opened = [Path(r[0]) for r in rows
              if r[0].startswith('%s/' % Path.cwd())]
    rows = conn.execute(
        '''
        SELECT name, argv FROM executed_files
        ''')
    executed = [(Path(r[0]), r[1]) for r in rows
                if Path(r[0]).lies_under(Path.cwd())]

    print("other_files: %r" % sorted(other_files))
    print("opened: %r" % opened)
    print("executed: %r" % executed)

    assert other_files == set(Path.cwd() / p
                              for p in ('a', 'b', 'c', 'd', 'e'))
    assert opened == [Path.cwd() / 'c', Path.cwd() / 'e']
    assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')]

    # ########################################
    # Test old packages
    #

    old_packages = [
        ('simple-0.4.0.rpz',
         'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V'
         'eDhXNTQ'),
        ('simple-0.6.0.rpz',
         'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr'
         'cUdtbGs'),
        ('simple-0.7.1.rpz',
         'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V'
         'QVpWOGs'),
    ]
    for name, url in old_packages:
        print("Testing old package %s" % name)
        f = Path(name)
        if not f.exists():
            download_file(url, f)
        # Info
        check_call(rpuz + ['info', name])
        # Show files
        check_call(rpuz + ['showfiles', name])
        # Lists packages
        check_call(rpuz + ['installpkgs', '--summary', name])
        # Unpack directory
        check_call(rpuz + ['directory', 'setup', name, 'simpledir'])
        # Run directory
        check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
        output_in_dir = Path('simpledir/root/tmp')
        output_in_dir = output_in_dir.listdir('reprozip_*')[0]
        output_in_dir = output_in_dir / 'simple_output.txt'
        with output_in_dir.open(encoding='utf-8') as fp:
            assert fp.read().strip() == '42'
        # Delete with wrong command (should fail)
        p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
                             stderr=subprocess.PIPE)
        out, err = p.communicate()
        assert p.poll() != 0
        err = err.splitlines()
        assert b"Wrong unpacker used" in err[0]
        assert err[1].startswith(b"usage: ")
        # Delete directory
        check_call(rpuz + ['directory', 'destroy', 'simpledir'])

    # ########################################
    # Copies back coverage report
    #

    coverage = Path('.coverage')
    if coverage.exists():
        coverage.copyfile(tests.parent / '.coverage.runpy')

Example 97

Project: vispy Source File: _runners.py
def _unit(mode, extra_arg_string, coverage=False):
    """Run unit tests using a particular mode"""
    import_dir = _get_import_dir()[0]
    cwd = op.abspath(op.join(import_dir, '..'))
    extra_args = [''] + extra_arg_string.split(' ')
    del extra_arg_string
    use_pytest = False
    try:
        import pytest  # noqa, analysis:ignore
        use_pytest = True
    except ImportError:
        try:
            import nose  # noqa, analysis:ignore
        except ImportError:
            raise SkipTest('Skipping unit tests, neither pytest nor nose '
                           'installed')

    if mode == 'nobackend':
        msg = 'Running tests with no backend'
        if use_pytest:
            extra_args += ['-m', '"not vispy_app_test"']
        else:
            extra_args += ['-a', '"!vispy_app_test"']
    else:
        # check to make sure we actually have the backend of interest
        invalid = run_subprocess([sys.executable, '-c',
                                  'import vispy.app; '
                                  'vispy.app.use_app("%s"); exit(0)' % mode],
                                 return_code=True)[2]
        if invalid:
            print('%s\n%s\n%s' % (_line_sep, 'Skipping backend %s, not '
                                  'installed or working properly' % mode,
                                  _line_sep))
            raise SkipTest()
        msg = 'Running tests with %s backend' % mode
        if use_pytest:
            extra_args += ['-m', 'vispy_app_test']
        else:
            extra_args += ['-a', 'vispy_app_test']
    if coverage and use_pytest:
        extra_args += ['--cov', 'vispy', '--no-cov-on-fail']
    # make a call to "python" so that it inherits whatever the system
    # thinks is "python" (e.g., virtualenvs)
    extra_arg_string = ' '.join(extra_args)
    insert = extra_arg_string if use_pytest else extra_args
    extra_args += [import_dir]  # positional argument
    cmd = [sys.executable, '-c', _unit_script % insert]
    env = deepcopy(os.environ)

    # We want to set this for all app backends plus "nobackend" to
    # help ensure that app tests are appropriately decorated
    env.update(dict(_VISPY_TESTING_APP=mode, VISPY_IGNORE_OLD_VERSION='true'))
    env_str = '_VISPY_TESTING_APP=%s ' % mode
    if len(msg) > 0:
        msg = ('%s\n%s:\n%s%s'
               % (_line_sep, msg, env_str, extra_arg_string))
        print(msg)
    sys.stdout.flush()
    return_code = run_subprocess(cmd, return_code=True, cwd=cwd,
                                 env=env, stdout=None, stderr=None)[2]
    if return_code:
        raise RuntimeError('unit failure (%s)' % return_code)
    if coverage:
        # Running a py.test with coverage will wipe out any files that
        # exist as .coverage or .coverage.*. It should work to pass
        # COVERAGE_FILE env var when doing run_subprocess above, but
        # it does not. Therefore we instead use our own naming scheme,
        # and in Travis when we combine them, use COVERAGE_FILE with the
        # `coverage combine` command.
        out_name = op.join(cwd, '.vispy-coverage.%s' % mode)
        if op.isfile(out_name):
            os.remove(out_name)
        os.rename(op.join(cwd, '.coverage'), out_name)

Example 98

Project: zest.releaser Source File: utils.py
def _execute_command(command, input_value=''):
    """commands.getoutput() replacement that also works on windows"""
    logger.debug("Running command: %r", command)
    if command.startswith(sys.executable):
        env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
        if ' upload ' in command or ' register ' in command:
            # We really do want to see the stderr here, otherwise a
            # failed upload does not even show up in the output.
            show_stderr = True
        else:
            show_stderr = False
    else:
        env = None
        show_stderr = True
    p = subprocess.Popen(command,
                         shell=True,
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         close_fds=MUST_CLOSE_FDS,
                         env=env)
    i, o, e = (p.stdin, p.stdout, p.stderr)
    if input_value:
        i.write(input_value.encode(INPUT_ENCODING))
    i.close()
    stdout_output = o.read()
    stderr_output = e.read()
    # We assume that the output from commands we're running is text.
    if not isinstance(stdout_output, six.text_type):
        stdout_output = stdout_output.decode(OUTPUT_ENCODING)
    if not isinstance(stderr_output, six.text_type):
        stderr_output = stderr_output.decode(OUTPUT_ENCODING)
    # TODO.  Note that the returncode is always None, also after
    # running p.kill().  The shell=True may be tripping us up.  For
    # some ideas, see http://stackoverflow.com/questions/4789837
    if p.returncode or show_stderr or 'Traceback' in stderr_output:
        # Some error occured
        # print(Fore.RED + stderr_output)
        stderr_output = stderr_output.strip()
        if stderr_output:
            # Make sure every error line is marked red.  The stderr
            # output also catches some warnings though.  It would be
            # really irritating if we start treating a line like this
            # as an error: warning: no previously-included files
            # matching '*.pyc' found anywhere in distribution.  Same
            # for empty lines.  So try to be smart about it.
            errors = []
            for line in stderr_output.split('\n'):
                line = line.strip()
                if not line:
                    # Keep it in the errors, but do not mark it with a color.
                    errors.append(line)
                    continue
                for known in KNOWN_WARNINGS:
                    if line.lower().startswith(known):
                        # Not a real error, so mark it as a warning.
                        errors.append(Fore.MAGENTA + line)
                        break
                else:
                    # Not found in known warnings, so mark it as an error.
                    errors.append(Fore.RED + line)
            errors = '\n'.join(errors)
        else:
            errors = ''
        result = stdout_output + errors
    else:
        # Only return the stdout. Stderr only contains possible
        # weird/confusing warnings that might trip up extraction of version
        # numbers and so.
        result = stdout_output
        if stderr_output:
            logger.debug("Stderr of running command '%s':\n%s",
                         command, stderr_output)
    o.close()
    e.close()
    return result

Example 99

Project: django-nonrel Source File: debug.py
    def get_traceback_html(self):
        "Return HTML code for traceback."

        if self.exc_type and issubclass(self.exc_type, TemplateDoesNotExist):
            from django.template.loader import template_source_loaders
            self.template_does_not_exist = True
            self.loader_debug_info = []
            for loader in template_source_loaders:
                try:
                    module = import_module(loader.__module__)
                    if hasattr(loader, '__class__'):
                        source_list_func = loader.get_template_sources
                    else: # NOTE: Remember to remove this branch when we deprecate old template loaders in 1.4
                        source_list_func = module.get_template_sources
                    # NOTE: This assumes exc_value is the name of the template that
                    # the loader attempted to load.
                    template_list = [{'name': t, 'exists': os.path.exists(t)} \
                        for t in source_list_func(str(self.exc_value))]
                except (ImportError, AttributeError):
                    template_list = []
                if hasattr(loader, '__class__'):
                    loader_name = loader.__module__ + '.' + loader.__class__.__name__
                else: # NOTE: Remember to remove this branch when we deprecate old template loaders in 1.4
                    loader_name = loader.__module__ + '.' + loader.__name__
                self.loader_debug_info.append({
                    'loader': loader_name,
                    'templates': template_list,
                })
        if (settings.TEMPLATE_DEBUG and hasattr(self.exc_value, 'source') and
            isinstance(self.exc_value, TemplateSyntaxError)):
            self.get_template_exception_info()

        frames = self.get_traceback_frames()
        for i, frame in enumerate(frames):
            if 'vars' in frame:
                frame['vars'] = [(k, force_escape(pprint(v))) for k, v in frame['vars']]
            frames[i] = frame

        unicode_hint = ''
        if self.exc_type and issubclass(self.exc_type, UnicodeError):
            start = getattr(self.exc_value, 'start', None)
            end = getattr(self.exc_value, 'end', None)
            if start is not None and end is not None:
                unicode_str = self.exc_value.args[1]
                unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
        from django import get_version
        t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
        c = Context({
            'is_email': self.is_email,
            'unicode_hint': unicode_hint,
            'frames': frames,
            'request': self.request,
            'settings': get_safe_settings(),
            'sys_executable': sys.executable,
            'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
            'server_time': datetime.datetime.now(),
            'django_version_info': get_version(),
            'sys_path' : sys.path,
            'template_info': self.template_info,
            'template_does_not_exist': self.template_does_not_exist,
            'loader_debug_info': self.loader_debug_info,
        })
        # Check whether exception info is available
        if self.exc_type:
            c['exception_type'] = self.exc_type.__name__
        if self.exc_value:
            c['exception_value'] = smart_unicode(self.exc_value, errors='replace')
        if frames:
            c['lastframe'] = frames[-1]
        return t.render(c)

Example 100

Project: envisage Source File: client.py
Function: run
    def run(self):
        # Get the server port, spawning it if necessary
        server_port = get_server_port()
        if server_port == -1 or not Server.ping(server_port, timeout=5):
            if len(self.client.server_prefs):
                # Spawn the server
                logger.info("Client spawning Server...")
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.settimeout(5)
                sock.bind(('localhost', 0))
                sock.listen(1)
                port = sock.getsockname()[1]
                args = self.client.server_prefs + ( port, )
                code = "from envisage.plugins.remote_editor.communication." \
                    "server import main; main(r'%s', '%s', %i)" % args
                spawn_independent([sys.executable, '-c', code])

                # Await a reponse from the server
                try:
                    server, address = accept_no_intr(sock)
                    try:
                        command, arguments = receive(server)
                        if command == "__port__":
                            self.client._server_port = int(arguments)
                        else:
                            raise socket.error
                    finally:
                        # Use try...except to handle timeouts
                        try:
                            server.shutdown(socket.SHUT_RD)
                        except:
                            pass
                except socket.error as e:
                    logger.error(repr(e))
                    logger.error("Client spawned a non-responsive Server! " \
                                     "Unregistering...")
                    self.client.error = True
                    self.client.unregister()
                    return
                finally:
                    sock.close()

            else:
                logger.error("Client could not contact the Server and no " \
                                 "spawn command is defined. Unregistering...")
                self.client.error = True
                self.client.unregister()
                return
        else:
            self.client._server_port = server_port

        # Create the socket that will receive commands from the server
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.bind(('localhost', 0))
        sock.listen(1)
        self.client._port = sock.getsockname()[1]

        # Register with the server
        port = str(self.client._port)
        arguments = MESSAGE_SEP.join((port, self.client.self_type,
                                      self.client.other_type))
        self.client.error = not send_port(self.client._server_port, 'register',
                                          arguments)
        self.client.registered = True

        # Send queued commands (these can only exist if we spawned the server)
        for command, args in self.client._queue:
            arguments = MESSAGE_SEP.join((port, command, args))
            self.client.error = not send_port(self.client._server_port, 'send',
                                              arguments)
        self.client._queue = []

        # Start the loop to listen for commands from the Server
        logger.info("Client listening on port %i..." % self.client._port)
        try:
            while not self._finished:
                server, address = accept_no_intr(sock)

                # Reject non-local connections
                if address[0] != '127.0.0.1':
                    msg = "Client on port %s received connection from a " \
                        "non-local party (port %s). Ignoring..."
                    logger.warning(msg % (port, address[0]))
                    continue

                # Receive the command from the server
                self.client.error = False
                command, arguments = receive(server)
                msg = r"Client on port %s received: %s %s"
                logger.debug(msg, port, command, arguments)

                # Handle special commands from the server
                if command == "__orphaned__":
                    self.client.orphaned = bool(int(arguments))

                elif command == "__error__":
                    error_status = arguments[0]
                    error_message = ''
                    if len(arguments) > 0:
                        error_message = arguments[1:]
                    logger.warning("Error status received from the server: " \
                                       "%s\n%s" % (error_status, error_message))
                    self.client.error = bool(int(error_status))

                # Handle other commands through Client interface
                else:
                    if self.client.ui_dispatch == 'off':
                        self.client.handle_command(command, arguments)
                    else:
                        if self.client.ui_dispatch == 'auto':
                            from pyface.gui import GUI
                        else:
                            exec('from pyface.ui.%s.gui import GUI' %
                                 self.client.ui_dispatch)
                        GUI.invoke_later(self.client.handle_command,
                                         command, arguments)
        finally:
            self.client.unregister()
See More Examples - Go to Next Page
Page 1 Page 2 Selected Page 3