sys.executable

Here are the examples of the python api sys.executable taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: asv
Source File: test_subprocess.py
View license
def test_timeout():
    timeout_codes = []
    timeout_codes.append(r"""
import sys
import time

sys.stdout.write("Stdout before waiting\n")
sys.stderr.write("Stderr before waiting\n")
sys.stdout.flush()
sys.stderr.flush()
time.sleep(60)
sys.stdout.write("Stdout after waiting\n")
sys.stderr.write("Stderr after waiting\n")
    """)

    # Another example, where timeout is due to a hanging sub-subprocess
    if getattr(os, 'setpgid', None):
        # only on posix
        timeout_codes.append(r"""
import sys
import time
import subprocess

sys.stdout.write("Stdout before waiting\n")
sys.stderr.write("Stderr before waiting\n")
sys.stdout.flush()
sys.stderr.flush()
subprocess.call([sys.executable, "-c",
    "import sys, subprocess; subprocess.call([sys.executable, '-c', 'import time; time.sleep(60)'])"])
sys.stdout.write("Stdout after waiting\n")
sys.stderr.write("Stderr after waiting\n")
        """)

    for timeout_code in timeout_codes:
        t = time.time()
        try:
            util.check_output([
                sys.executable, "-c", timeout_code], timeout=1)
        except util.ProcessError as e:
            assert len(e.stdout.strip().split('\n')) == 1
            assert len(e.stderr.strip().split('\n')) == 1
            print(e.stdout)
            assert e.stdout.strip() == "Stdout before waiting"
            assert e.stderr.strip() == "Stderr before waiting"
            assert e.retcode == util.TIMEOUT_RETCODE
            assert "timed out" in str(e)
        else:
            assert False, "Expected timeout exception"
        # Make sure the timeout is triggered in a sufficiently short amount of time
        assert time.time() - t < 5.0

Example 2

Project: tahoe-lafs
Source File: conftest.py
View license
@pytest.fixture(scope='session')
def tor_network(reactor, temp_dir, chutney, request):
    # this is the actual "chutney" script at the root of a chutney checkout
    chutney_dir = chutney
    chut = join(chutney_dir, 'chutney')

    # now, as per Chutney's README, we have to create the network
    # ./chutney configure networks/basic
    # ./chutney start networks/basic

    proto = _DumpOutputProtocol(None)
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'chutney.TorNet', 'configure',
            join(chutney_dir, 'networks', 'basic'),
        ),
        path=join(chutney_dir),
        env={"PYTHONPATH": join(chutney_dir, "lib")},
    )
    pytest.blockon(proto.done)

    proto = _DumpOutputProtocol(None)
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'chutney.TorNet', 'start',
            join(chutney_dir, 'networks', 'basic'),
        ),
        path=join(chutney_dir),
        env={"PYTHONPATH": join(chutney_dir, "lib")},
    )
    pytest.blockon(proto.done)

    # print some useful stuff
    proto = _CollectOutputProtocol()
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'chutney.TorNet', 'status',
            join(chutney_dir, 'networks', 'basic'),
        ),
        path=join(chutney_dir),
        env={"PYTHONPATH": join(chutney_dir, "lib")},
    )
    pytest.blockon(proto.done)

    def cleanup():
        print("Tearing down Chutney Tor network")
        proto = _CollectOutputProtocol()
        reactor.spawnProcess(
            proto,
            sys.executable,
            (
                sys.executable, '-m', 'chutney.TorNet', 'stop',
                join(chutney_dir, 'networks', 'basic'),
            ),
            path=join(chutney_dir),
            env={"PYTHONPATH": join(chutney_dir, "lib")},
        )
        pytest.blockon(proto.done)
    request.addfinalizer(cleanup)

    return chut

Example 3

Project: AntiRansom
Source File: monit.py
View license
def ScanRansom():
	
	global safepids
	global safeprog
	
	while True: 
			
		offpid = ownhandle()
			
		if offpid :
			
			p = psutil.Process(int(offpid))
			exefile = p.exe()
		
		
			if exefile in safeprog :
			
				continue 
			
			else :
			
				p.suspend()
				
				safepids.append(offpid)
			
				if langlocal[0] == "es_ES" :
				
					splashtext = os.path.dirname(sys.executable) + "\\" + 'splash.exe ' + '\"Anti Ransom ha detectado acceso a la carpeta trampa por parte del programa\n\n' + exefile + '\n\n' + 'Desea detener dicho proceso?\"'
			
				else :
				
					splashtext = os.path.dirname(sys.executable) + "\\" + 'splash.exe ' +  '\"Anti Ransom has detected access to the Honey Folder by process\n\n' + exefile + '\n\n' + 'Do you want to Stop it?\"'
				print splashtext
				splash = subprocess.Popen(splashtext, stdout=subprocess.PIPE)
				streamdata = splash.communicate()[0]
				rc = splash.returncode
			
				if rc == 10:
				
					randdump = str(time.time()) + str(offpid) + ".dmp" ;
				
					dumpcmd = os.path.dirname(sys.executable) + "\\" + 'procdump.exe -ma ' + "\"" + str(offpid) + "\"" + ' -accepteula ' + randdump
				
					cmdblock =subprocess.Popen(dumpcmd, stdout=subprocess.PIPE)
					cmdblock.wait()
				
					p.kill()
				
				else :
				
					safeprog.append(exefile)
					p.resume()
			
			
				return(0)
			
		time.sleep(1)	

Example 4

Project: esky
Source File: sudo_osx.py
View license
def spawn_sudo(proxy):
    """Spawn the sudo slave process, returning proc and a pipe to message it."""

    pipe = SecureStringPipe()
    c_pipe = pipe.connect()

    if not getattr(sys,"frozen",False):
        exe = [sys.executable,"-c","import esky; esky.run_startup_hooks()"]
    elif os.path.basename(sys.executable).lower() in ("python","pythonw"):
        exe = [sys.executable,"-c","import esky; esky.run_startup_hooks()"]
    else:
        if not esky._startup_hooks_were_run:
            raise OSError(None,"unable to sudo: startup hooks not run")
        exe = [sys.executable]
    args = ["--esky-spawn-sudo"]
    args.append(base.b64pickle(proxy))
    args.append(base.b64pickle(c_pipe))

    # Make it a slave process so it dies if we die
    exe = exe + esky.slaveproc.get_slave_process_args() + args

    auth = ctypes.c_void_p()

    right = AuthorizationRight()
    if sys.version_info[0] < 3:
        right.name = "py.esky.sudo.{}".format(proxy.name)
    else:
        right.name = "py.esky.sudo.{}".format(proxy.name).encode('utf8')
    right.valueLength = 0
    right.value = None
    right.flags = 0

    rights = AuthorizationRights()
    rights.count = 1
    rights.items[0] = right

    r_auth = byref(auth)
    err = sec.AuthorizationCreate(None,None,kAuthorizationFlagDefaults,r_auth)
    if err:
        raise OSError(errno.EACCES,"could not sudo: %d" % (err,))

    try:

        kAuthFlags = kAuthorizationFlagDefaults \
                     | kAuthorizationFlagPreAuthorize \
                     | kAuthorizationFlagInteractionAllowed \
                     | kAuthorizationFlagExtendRights
        
        err = sec.AuthorizationCopyRights(auth,None,None,kAuthFlags,None)
        if err:
            raise OSError(errno.EACCES,"could not sudo: %d" % (err,))

        args = (ctypes.c_char_p * len(exe))()
        for i,arg in enumerate(exe[1:]):
            if sys.version_info[0] < 3:
                args[i] = arg
            else:
                args[i] = arg.encode('utf8')

        args[len(exe)-1] = None
        io = ctypes.c_void_p()
        if sys.version_info[0] < 3:
            err = sec.AuthorizationExecuteWithPrivileges(auth,exe[0],0,args,byref(io))
        else:
            err = sec.AuthorizationExecuteWithPrivileges(auth,exe[0].encode('utf8'),0,args,byref(io))
        if err:
            raise OSError(errno.EACCES,"could not sudo: %d" %(err,))
        
        buf = ctypes.create_string_buffer(8)
        read = libc.fread(byref(buf),1,4,io)
        if read != 4:
            libc.fclose(io)
            raise OSError(errno.EACCES,"could not sudo: child failed")
        pid = struct.unpack("I",buf.raw[:4])[0]
        pipe.fp = io
        return (FakePopen(pid),pipe)
    finally:
        sec.AuthorizationFree(auth,kAuthorizationFlagDestroyRights)

Example 5

Project: reflectrpc
Source File: cmdline-tests.py
View license
    def test_cmdline_expect_connection_fails(self):
        for cmd in self.cmdline_programs:
            # connect although no server is running
            try:
                python = sys.executable
                outfile = ''
                if cmd != 'rpcsh':
                    outfile = 'outfile.html'

                child = pexpect.spawn('%s ../%s localhost 5500 %s' % (python, cmd, outfile))

                child.expect('NetworkError: \[Errno \d+\] Connection refused\r\n')
                child.expect('\r\n')
                child.expect('Failed to connect to localhost on TCP port 5500\r\n')
            finally:
                child.close(True)

            # connect to a TLS server without enabling TLS
            try:
                server = ServerRunner('../examples/servertls.py', 5500)
                server.run()

                python = sys.executable
                outfile = ''
                if cmd != 'rpcsh':
                    outfile = 'outfile.html'

                child = pexpect.spawn('%s ../%s localhost 5500 %s' % (python, cmd, outfile))

                child.expect('NetworkError: Non-JSON content received\r\n')
                child.expect('\r\n')
                child.expect('Failed to connect to localhost on TCP port 5500\r\n')
            finally:
                child.close(True)
                server.stop()

            # connect to a Non-TLS server with enabled TLS
            try:
                server = ServerRunner('../examples/server.py', 5500)
                server.run()

                python = sys.executable
                outfile = ''
                if cmd != 'rpcsh':
                    outfile = 'outfile.html'

                child = pexpect.spawn('%s ../%s localhost 5500 %s --tls' % (python, cmd, outfile))

                child.expect('NetworkError: EOF occurred in violation of protocol \(_ssl.c:\d+\)\r\n')
                child.expect('\r\n')
                child.expect('Failed to connect to localhost on TCP port 5500\r\n')
            finally:
                child.close(True)
                server.stop()

            # connect to a TLS server but fail the hostname check
            try:
                server = ServerRunner('../examples/servertls.py', 5500)
                server.run()

                python = sys.executable
                outfile = ''
                if cmd != 'rpcsh':
                    outfile = 'outfile.html'

                child = pexpect.spawn('%s ../%s localhost 5500 %s --tls --ca ../examples/certs/rootCA.crt --check-hostname' % (python, cmd, outfile))

                child.expect("NetworkError: TLSHostnameError: Host name 'localhost' doesn't match certificate host 'reflectrpc'\r\n")
                child.expect('\r\n')
                child.expect('Failed to connect to localhost on TCP port 5500\r\n')
            finally:
                child.close(True)
                server.stop()

Example 6

Project: gdal2mbtiles
Source File: test_scripts.py
View license
    def test_colors(self):
        null = open('/dev/null', 'rw')

        with NamedTemporaryFile(suffix='.mbtiles') as output:
            # Valid
            command = [sys.executable, self.script,
                       '--coloring', 'palette',
                       '--color', '0:#00f',
                       '--color', '1:green',
                       self.inputfile, output.name]
            check_call(command, env=self.environ)

            # Invalid color
            command = [sys.executable, self.script,
                       '--coloring', 'palette',
                       '--color', 'invalid',
                       self.inputfile, output.name]
            self.assertRaises(CalledProcessError,
                              check_call, command, env=self.environ,
                              stderr=null)

            command = [sys.executable, self.script,
                       '--coloring', 'palette',
                       '--color', '0:1',
                       self.inputfile, output.name]
            self.assertRaises(CalledProcessError,
                              check_call, command, env=self.environ,
                              stderr=null)

            command = [sys.executable, self.script,
                       '--coloring', 'palette',
                       '--color', 's:#000',
                       self.inputfile, output.name]
            self.assertRaises(CalledProcessError,
                              check_call, command, env=self.environ,
                              stderr=null)

            # Missing --color
            command = [sys.executable, self.script,
                       '--coloring', 'palette',
                       self.inputfile, output.name]
            self.assertRaises(CalledProcessError,
                              check_call, command, env=self.environ,
                              stderr=null)

            # Invalid --coloring
            command = [sys.executable, self.script,
                       '--coloring', 'invalid',
                       self.inputfile, output.name]
            self.assertRaises(CalledProcessError,
                              check_call, command, env=self.environ,
                              stderr=null)

            # Missing --coloring
            command = [sys.executable, self.script,
                       '--color', '0:#00f',
                       self.inputfile, output.name]
            self.assertRaises(CalledProcessError,
                              check_call, command, env=self.environ,
                              stderr=null)

            # Valid multi-band
            command = [sys.executable, self.script,
                       '--coloring', 'gradient',
                       '--color', '0:#00f',
                       '--color', '1:green',
                       '--colorize-band', '2',
                       self.inputfile, output.name]
            check_call(command, env=self.environ)

            # Invalid band
            command = [sys.executable, self.script,
                       '--coloring', 'palette',
                       '--color', '0:#00f',
                       '--color', '1:green',
                       '--colorize-band', '-2',
                       self.inputfile, output.name]
            self.assertRaises(CalledProcessError,
                              check_call, command, env=self.environ,
                              stderr=null)

Example 7

Project: fail2ban
Source File: misctestcase.py
View license
	def testSetupInstallRoot(self):
		if not self.setup:
			return			  # if verbose skip didn't work out
		tmp = tempfile.mkdtemp()
		# suppress stdout (and stderr) if not heavydebug
		supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
		try:
			os.system("%s %s install --disable-2to3 --dry-run --root=%s%s"
					  % (sys.executable, self.setup, tmp, supdbgout))

			def strippath(l):
				return [x[len(tmp)+1:] for x in l]

			got = strippath(sorted(glob('%s/*' % tmp)))
			need = ['etc', 'usr', 'var']

			# if anything is missing
			if set(need).difference(got): # pragma: no cover
				#  below code was actually to print out not missing but
				#  rather files in 'excess'.  Left in place in case we
				#  decide to revert to such more strict test

				# based on
				# http://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python
				def recursive_glob(treeroot, pattern):
					results = []
					for base, dirs, files in os.walk(treeroot):
						goodfiles = fnmatch.filter(dirs + files, pattern)
						results.extend(os.path.join(base, f) for f in goodfiles)
					return results

				files = {}
				for missing in set(got).difference(need):
					missing_full = os.path.join(tmp, missing)
					files[missing] = os.path.exists(missing_full) \
						and strippath(recursive_glob(missing_full, '*')) or None

				self.assertEqual(
					got, need,
					msg="Got: %s Needed: %s under %s. Files under new paths: %s"
					% (got, need, tmp, files))

			# Assure presence of some files we expect to see in the installation
			for f in ('etc/fail2ban/fail2ban.conf',
					  'etc/fail2ban/jail.conf'):
				self.assertTrue(os.path.exists(os.path.join(tmp, f)),
								msg="Can't find %s" % f)
			# Because the install (test) path in virtual-env differs from some development-env,
			# it is not a `tmp + '/usr/local/bin/'`, so search for it:
			installedPath = _sh_call('find ' + tmp+ ' -name fail2ban-python').split('\n')
			self.assertTrue(len(installedPath) > 0)
			for installedPath in installedPath:
				self.assertEqual(
					os.path.realpath(installedPath), os.path.realpath(sys.executable))

		finally:
			# clean up
			shutil.rmtree(tmp)
			# remove build directory
			os.system("%s %s clean --all%s"
					  % (sys.executable, self.setup, (supdbgout + ' 2>&1') if supdbgout else ''))

Example 8

Project: meson
Source File: gnome.py
View license
    def yelp(self, state, args, kwargs):
        if len(args) < 1:
            raise MesonException('Yelp requires a project id')

        project_id = args[0]
        sources = mesonlib.stringlistify(kwargs.pop('sources', []))
        if not sources:
            if len(args) > 1:
                sources = mesonlib.stringlistify(args[1:])
            if not sources:
                raise MesonException('Yelp requires a list of sources')
        source_str = '@@'.join(sources)

        langs = mesonlib.stringlistify(kwargs.pop('languages', []))
        media = mesonlib.stringlistify(kwargs.pop('media', []))
        symlinks = kwargs.pop('symlink_media', False)

        if not isinstance(symlinks, bool):
            raise MesonException('symlink_media must be a boolean')

        if kwargs:
            raise MesonException('Unknown arguments passed: {}'.format(', '.join(kwargs.keys())))

        install_cmd = [
            sys.executable,
            state.environment.get_build_command(),
            '--internal',
            'yelphelper',
            'install',
            '--subdir=' + state.subdir,
            '--id=' + project_id,
            '--installdir=' + os.path.join(state.environment.get_datadir(), 'help'),
            '--sources=' + source_str,
        ]
        if symlinks:
            install_cmd.append('--symlinks=true')
        if media:
            install_cmd.append('--media=' + '@@'.join(media))
        if langs:
            install_cmd.append('--langs=' + '@@'.join(langs))
        inscript = build.InstallScript(install_cmd)

        potargs = [state.environment.get_build_command(), '--internal', 'yelphelper', 'pot',
                   '--subdir=' + state.subdir,
                   '--id=' + project_id,
                   '--sources=' + source_str]
        pottarget = build.RunTarget('help-' + project_id + '-pot', sys.executable,
                                     potargs, [], state.subdir)

        poargs = [state.environment.get_build_command(), '--internal', 'yelphelper', 'update-po',
                   '--subdir=' + state.subdir,
                   '--id=' + project_id,
                   '--sources=' + source_str,
                   '--langs=' + '@@'.join(langs)]
        potarget = build.RunTarget('help-' + project_id + '-update-po', sys.executable,
                                     poargs, [], state.subdir)

        return [inscript, pottarget, potarget]

Example 9

Project: Qt.py
Source File: tests.py
View license
def test_vendoring():
    """Qt.py may be bundled along with another library/project

    Create toy project

    from project.vendor import Qt  # Absolute
    from .vendor import Qt         # Relative

    project/
        vendor/
            __init__.py
        __init__.py

    """

    project = os.path.join(self.tempdir, "myproject")
    vendor = os.path.join(project, "vendor")

    os.makedirs(vendor)

    # Make packages out of folders
    with open(os.path.join(project, "__init__.py"), "w") as f:
        f.write("from .vendor.Qt import QtWidgets")

    with open(os.path.join(vendor, "__init__.py"), "w") as f:
        f.write("\n")

    # Copy real Qt.py into myproject
    shutil.copy(os.path.join(os.path.dirname(__file__), "Qt.py"),
                os.path.join(vendor, "Qt.py"))

    print("Testing relative import..")
    assert subprocess.call(
        [sys.executable, "-c", "import myproject"],
        cwd=self.tempdir,
        stdout=subprocess.PIPE,    # With nose process isolation, buffer can
        stderr=subprocess.STDOUT,  # easily get full and throw an error.
    ) == 0

    print("Testing absolute import..")
    assert subprocess.call(
        [sys.executable, "-c", "from myproject.vendor.Qt import QtWidgets"],
        cwd=self.tempdir,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    ) == 0

    print("Testing direct import..")
    assert subprocess.call(
        [sys.executable, "-c", "import myproject.vendor.Qt"],
        cwd=self.tempdir,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    ) == 0

Example 10

Project: talk.org
Source File: debug.py
View license
def get_traceback_html(request, exc_type, exc_value, tb):
    "Return HTML code for traceback."
    template_info = None
    template_does_not_exist = False
    loader_debug_info = None

    # Handle deprecated string exceptions
    if isinstance(exc_type, basestring):
        exc_value = Exception('Deprecated String Exception: %r' % exc_type)
        exc_type = type(exc_value)

    if issubclass(exc_type, TemplateDoesNotExist):
        from django.template.loader import template_source_loaders
        template_does_not_exist = True
        loader_debug_info = []
        for loader in template_source_loaders:
            try:
                source_list_func = getattr(__import__(loader.__module__, {}, {}, ['get_template_sources']), 'get_template_sources')
                # NOTE: This assumes exc_value is the name of the template that
                # the loader attempted to load.
                template_list = [{'name': t, 'exists': os.path.exists(t)} \
                    for t in source_list_func(str(exc_value))]
            except (ImportError, AttributeError):
                template_list = []
            loader_debug_info.append({
                'loader': loader.__module__ + '.' + loader.__name__,
                'templates': template_list,
            })
    if settings.TEMPLATE_DEBUG and hasattr(exc_value, 'source'):
        exc_type, exc_value, tb, template_info = get_template_exception_info(exc_type, exc_value, tb)
    frames = []
    while tb is not None:
        # support for __traceback_hide__ which is used by a few libraries
        # to hide internal frames.
        if tb.tb_frame.f_locals.get('__traceback_hide__'):
            tb = tb.tb_next
            continue
        filename = tb.tb_frame.f_code.co_filename
        function = tb.tb_frame.f_code.co_name
        lineno = tb.tb_lineno - 1
        loader = tb.tb_frame.f_globals.get('__loader__')
        module_name = tb.tb_frame.f_globals.get('__name__')
        pre_context_lineno, pre_context, context_line, post_context = _get_lines_from_file(filename, lineno, 7, loader, module_name)
        if pre_context_lineno is not None:
            frames.append({
                'tb': tb,
                'filename': filename,
                'function': function,
                'lineno': lineno + 1,
                'vars': tb.tb_frame.f_locals.items(),
                'id': id(tb),
                'pre_context': pre_context,
                'context_line': context_line,
                'post_context': post_context,
                'pre_context_lineno': pre_context_lineno + 1,
            })
        tb = tb.tb_next

    if not frames:
        frames = [{
            'filename': '<unknown>',
            'function': '?',
            'lineno': '?',
        }]

    unicode_hint = ''
    if issubclass(exc_type, UnicodeError):
        start = getattr(exc_value, 'start', None)
        end = getattr(exc_value, 'end', None)
        if start is not None and end is not None:
            unicode_str = exc_value.args[1]
            unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace')
    from django import get_version
    t = Template(TECHNICAL_500_TEMPLATE, name='Technical 500 template')
    c = Context({
        'exception_type': exc_type.__name__,
        'exception_value': smart_unicode(exc_value, errors='replace'),
        'unicode_hint': unicode_hint,
        'frames': frames,
        'lastframe': frames[-1],
        'request': request,
        'request_protocol': request.is_secure() and "https" or "http",
        'settings': get_safe_settings(),
        'sys_executable': sys.executable,
        'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
        'server_time': datetime.datetime.now(),
        'django_version_info': get_version(),
        'sys_path' : sys.path,
        'template_info': template_info,
        'template_does_not_exist': template_does_not_exist,
        'loader_debug_info': loader_debug_info,
    })
    return t.render(c)

Example 11

View license
    def run_cgi(self):
        """Execute a CGI script."""
        dir, rest = self.cgi_info
        path = dir + '/' + rest
        i = path.find('/', len(dir)+1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir)+1)
            else:
                break

        # find an explicit query string, if present.
        rest, _, query = rest.partition('?')

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.getheader('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non Unix - use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)

            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin = subprocess.PIPE,
                                 stdout = subprocess.PIPE,
                                 stderr = subprocess.PIPE,
                                 env = env
                                )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 12

Project: terrarium
Source File: terrarium.py
View license
    def install(self):
        logger.debug('Running install')

        old_target = self.get_target_location()
        old_target_backup = self.get_backup_location()
        new_target = old_target
        prompt = os.path.basename(new_target)

        # Are we building a new environment, or replacing an existing one?
        old_target_exists = self.environment_exists(old_target)
        if old_target_exists:
            new_target = tempfile.mkdtemp(
                prefix='%s.' % os.path.basename(old_target),
                dir=os.path.dirname(old_target),
            )

        # Can the requested environment be downloaded?
        downloaded = False
        if self.args.download:
            downloaded = self.download(new_target)

        if not downloaded:
            if self.args.require_download:
                logger.error(
                    'Failed to download bundle and download is '
                    'required. Refusing to build a new bundle.'
                )
                return 1

            # Create a self-contained script to create a virtual environment
            # and install all of the requested requirements
            logger.info('Building new environment')
            fd, bootstrap = tempfile.mkstemp(
                prefix='terrarium_bootstrap-',
                suffix='.py',
            )
            self.create_bootstrap(bootstrap)

            # Run the bootstrap script which pip installs everything that has
            # been defined as a requirement
            call_subprocess([
                sys.executable,
                bootstrap,
                '--prompt=(%s)' % prompt,
                new_target
            ])

            # Do we want to copy the bootstrap into the environment for future
            # use?
            if self.args.bootstrap:
                logger.info('Copying bootstrap script to new environment')
                dest = os.path.join(
                    new_target, 'bin', 'terrarium_bootstrap.py')
                shutil.copyfile(bootstrap, dest)
                os.chmod(dest, 0744)
            os.close(fd)
            os.unlink(bootstrap)

            if self.args.upload:
                self.upload(new_target)

        if old_target_exists:
            logger.info('Moving old environment out of the way')
            if os.path.exists(old_target_backup):
                if not rmtree(old_target_backup):
                    old_target_backup = tempfile.mkdtemp(
                        prefix='terrarium_old_backup_target-'
                    )
                    old_target_backup = os.path.join(old_target_backup, prompt)
                    logger.info(
                        'Backing environment up to %s' % old_target_backup)
            try:
                os.rename(old_target, old_target_backup)
            except OSError, why:
                logger.error(
                    'Failed to move environment out of the way. '
                    'Check that you have the correct permissions. '
                    '%s' % why
                )
                return 1

            # Fix paths
            Terrarium.replace_all_in_directory(
                os.path.join(new_target, 'bin'),
                new_target,
                old_target,
            )

        try:
            # move the new environment into the target's place
            os.rename(new_target, old_target)
        except OSError, why:
            logger.error(
                'Failed to move the new environment into the correct path. '
                'Check that you have the correct permissions. '
                '%s' % why
            )
            return 1

        # Do we keep a backup of the old environment around or wipe it?
        if os.path.isdir(old_target_backup) and not self.args.backup:
            logger.info('Deleting old environment')
            rmtree(old_target_backup)
        logger.info('Terrarium is finished')
        return 0

Example 13

Project: fades
Source File: main.py
View license
def go(argv):
    """Make the magic happen."""
    parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog, usage=help_usage,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-V', '--version', action='store_true',
                        help="show version and info about the system, and exit")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="send all internal debugging lines to stderr, which may be very "
                             "useful to debug any problem that may arise.")
    parser.add_argument('-q', '--quiet', action='store_true',
                        help="don't show anything (unless it has a real problem), so the "
                             "original script stderr is not polluted at all.")
    parser.add_argument('-d', '--dependency', action='append',
                        help="specify dependencies through command line (this option can be "
                             "used multiple times)")
    parser.add_argument('-r', '--requirement',
                        help="indicate from which file read the dependencies")
    parser.add_argument('-p', '--python', action='store',
                        help=("Specify the Python interpreter to use.\n"
                              " Default is: %s") % (sys.executable,))
    parser.add_argument('-x', '--exec', dest='executable', action='store_true',
                        help=("Indicate that the child_program should be looked up in the "
                              "virtualenv."))
    parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell.")
    parser.add_argument('--system-site-packages', action='store_true', default=False,
                        help=("Give the virtual environment access to the "
                              "system site-packages dir."))
    parser.add_argument('--virtualenv-options', action='append', default=[],
                        help=("Extra options to be supplied to virtualenv. (this option can be "
                              "used multiple times)"))
    parser.add_argument('--check-updates', action='store_true',
                        help=("check for packages updates"))
    parser.add_argument('--pip-options', action='append', default=[],
                        help=("Extra options to be supplied to pip. (this option can be "
                              "used multiple times)"))
    parser.add_argument('--rm', dest='remove', metavar='UUID',
                        help=("Remove a virtualenv by UUID."))
    parser.add_argument('--clean-unused-venvs', action='store',
                        help=("This option remove venvs that haven't been used for more than "
                              "CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
                              "stats file.\n"
                              "When this option is present, the cleaning takes place at the "
                              "beginning of the execution."))
    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)

    # support the case when executed from a shell-bang, where all the
    # parameters come in sys.argv[1] in a single string separated
    # by spaces (in this case, the third parameter is what is being
    # executed)
    if len(sys.argv) > 1 and " " in sys.argv[1]:
        real_args = sys.argv[1].split() + sys.argv[2:]
        cli_args = parser.parse_args(real_args)
    else:
        cli_args = parser.parse_args()

    # update args from config file (if needed).
    args = file_options.options_from_file(cli_args)

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", fades.__version__)
        print("    Python:", sys.version_info)
        print("    System:", sys.platform)
        sys.exit(0)

    # set up logger and dump basic version info
    l = logger.set_up(args.verbose, args.quiet)
    l.debug("Running Python %s on %r", sys.version_info, sys.platform)
    l.debug("Starting fades v. %s", fades.__version__)
    l.debug("Arguments: %s", args)

    # verify that the module is NOT being used from a virtualenv
    if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None),
                                getattr(sys, 'base_prefix', None)):
        l.warning("fades is running from a virtualenv (%r), which is not supported", sys.prefix)

    if args.verbose and args.quiet:
        l.warning("Overriding 'quiet' option ('verbose' also requested)")

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx'))
    # start usage manager
    usage_manager = envbuilder.UsageManager(os.path.join(helpers.get_basedir(), 'usage_stats'),
                                            venvscache)

    rc = 0
    if args.clean_unused_venvs:
        try:
            max_days_to_keep = int(args.clean_unused_venvs)
            usage_manager.clean_unused_venvs(max_days_to_keep)
        except:
            rc = 1
            l.debug("CLEAN_UNUSED_VENVS must be an integer.")
            raise
        finally:
            sys.exit(rc)

    uuid = args.remove
    if uuid:
        venv_data = venvscache.get_venv(uuid=uuid)
        if venv_data:
            # remove this venv from the cache
            env_path = venv_data.get('env_path')
            if env_path:
                envbuilder.destroy_venv(env_path, venvscache)
            else:
                l.warning("Invalid 'env_path' found in virtualenv metadata: %r. "
                          "Not removing virtualenv.", env_path)
        else:
            l.warning('No virtualenv found with uuid: %s.', uuid)
        return

    # parse file and get deps
    if args.ipython:
        l.debug("Adding ipython dependency because --ipython was detected")
        ipython_dep = parsing.parse_manual(['ipython'])
    else:
        ipython_dep = {}

    if args.executable:
        indicated_deps = {}
        docstring_deps = {}
    else:
        indicated_deps = parsing.parse_srcfile(args.child_program)
        l.debug("Dependencies from source file: %s", indicated_deps)
        docstring_deps = parsing.parse_docstring(args.child_program)
        l.debug("Dependencies from docstrings: %s", docstring_deps)
    reqfile_deps = parsing.parse_reqfile(args.requirement)
    l.debug("Dependencies from requirements file: %s", reqfile_deps)
    manual_deps = parsing.parse_manual(args.dependency)
    l.debug("Dependencies from parameters: %s", manual_deps)
    indicated_deps = _merge_deps(ipython_dep, indicated_deps, docstring_deps,
                                 reqfile_deps, manual_deps)

    # Check for packages updates
    if args.check_updates:
        helpers.check_pypi_updates(indicated_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # options
    pip_options = args.pip_options  # pip_options mustn't store.
    options = {}
    options['pyvenv_options'] = []
    options['virtualenv_options'] = args.virtualenv_options
    if args.system_site_packages:
        options['virtualenv_options'].append("--system-site-packages")
        options['pyvenv_options'] = ["--system-site-packages"]

    create_venv = False
    venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
    if venv_data:
        env_path = venv_data['env_path']
        # A venv was found in the cache check if its valid or re-generate it.
        if not os.path.exists(env_path):
            l.warning("Missing directory (the virtualenv will be re-created): %r", env_path)
            venvscache.remove(env_path)
            create_venv = True
    else:
        create_venv = True

    if create_venv:
        # Create a new venv
        venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current,
                                                      options, pip_options)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter, options)

    # run forest run!!
    python_exe = 'ipython' if args.ipython else 'python'
    python_exe = os.path.join(venv_data['env_bin_path'], python_exe)

    # store usage information
    usage_manager.store_usage_stat(venv_data, venvscache)

    if args.child_program is None:
        interactive = True
        l.debug("Calling the interactive Python interpreter")
        p = subprocess.Popen([python_exe])

    else:
        interactive = False
        if args.executable:
            cmd = [os.path.join(venv_data['env_bin_path'], args.child_program)]
        else:
            cmd = [python_exe, args.child_program]
        l.debug("Calling the child program %r with options %s",
                args.child_program, args.child_options)
        p = subprocess.Popen(cmd + args.child_options)

    def _signal_handler(signum, _):
        """Handle signals received by parent process, send them to child.

        The only exception is CTRL-C, that is generated *from* the interactive
        interpreter (it's a keyboard combination!), so we swallow it for the
        interpreter to not see it twice.
        """
        if interactive and signum == signal.SIGINT:
            l.debug("Swallowing signal %s", signum)
        else:
            l.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

    # redirect the useful signals
    for s in REDIRECTED_SIGNALS:
        signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        l.debug("Child process not finished correctly: returncode=%d", rc)
    sys.exit(rc)

Example 14

Project: pymo
Source File: CGIHTTPServer.py
View license
    def run_cgi(self):
        """Execute a CGI script."""
        path = self.path
        dir, rest = self.cgi_info

        i = path.find('/', len(dir) + 1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir) + 1)
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.getheader('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non Unix - use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)

            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin = subprocess.PIPE,
                                 stdout = subprocess.PIPE,
                                 stderr = subprocess.PIPE,
                                 env = env
                                )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 15

Project: pymo
Source File: platform.py
View license
def architecture(executable=sys.executable,bits='',linkage=''):

    """ Queries the given executable (defaults to the Python interpreter
        binary) for various architecture information.

        Returns a tuple (bits,linkage) which contains information about
        the bit architecture and the linkage format used for the
        executable. Both values are returned as strings.

        Values that cannot be determined are returned as given by the
        parameter presets. If bits is given as '', the sizeof(pointer)
        (or sizeof(long) on Python version < 1.5.2) is used as
        indicator for the supported pointer size.

        The function relies on the system's "file" command to do the
        actual work. This is available on most if not all Unix
        platforms. On some non-Unix platforms where the "file" command
        does not exist and the executable is set to the Python interpreter
        binary defaults from _default_architecture are used.

    """
    # Use the sizeof(pointer) as default number of bits if nothing
    # else is given as default.
    if not bits:
        import struct
        try:
            size = struct.calcsize('P')
        except struct.error:
            # Older installations can only query longs
            size = struct.calcsize('l')
        bits = str(size*8) + 'bit'

    # Get data from the 'file' system command
    output = _syscmd_file(executable,'')

    if not output and \
       executable == sys.executable:
        # "file" command did not return anything; we'll try to provide
        # some sensible defaults then...
        if _default_architecture.has_key(sys.platform):
            b,l = _default_architecture[sys.platform]
            if b:
                bits = b
            if l:
                linkage = l
        return bits,linkage

    # Split the output into a list of strings omitting the filename
    fileout = _architecture_split(output)[1:]

    if 'executable' not in fileout:
        # Format not supported
        return bits,linkage

    # Bits
    if '32-bit' in fileout:
        bits = '32bit'
    elif 'N32' in fileout:
        # On Irix only
        bits = 'n32bit'
    elif '64-bit' in fileout:
        bits = '64bit'

    # Linkage
    if 'ELF' in fileout:
        linkage = 'ELF'
    elif 'PE' in fileout:
        # E.g. Windows uses this format
        if 'Windows' in fileout:
            linkage = 'WindowsPE'
        else:
            linkage = 'PE'
    elif 'COFF' in fileout:
        linkage = 'COFF'
    elif 'MS-DOS' in fileout:
        linkage = 'MSDOS'
    else:
        # XXX the A.OUT format also falls under this class...
        pass

    return bits,linkage

Example 16

Project: pyqtgraph
Source File: processes.py
View license
    def __init__(self, name=None, target=None, executable=None, copySysPath=True, debug=False, timeout=20, wrapStdout=None):
        """
        ==============  =============================================================
        **Arguments:**
        name            Optional name for this process used when printing messages
                        from the remote process.
        target          Optional function to call after starting remote process.
                        By default, this is startEventLoop(), which causes the remote
                        process to process requests from the parent process until it
                        is asked to quit. If you wish to specify a different target,
                        it must be picklable (bound methods are not).
        copySysPath     If True, copy the contents of sys.path to the remote process
        debug           If True, print detailed information about communication
                        with the child process.
        wrapStdout      If True (default on windows) then stdout and stderr from the
                        child process will be caught by the parent process and
                        forwarded to its stdout/stderr. This provides a workaround
                        for a python bug: http://bugs.python.org/issue3905
                        but has the side effect that child output is significantly
                        delayed relative to the parent output.
        ==============  =============================================================
        """
        if target is None:
            target = startEventLoop
        if name is None:
            name = str(self)
        if executable is None:
            executable = sys.executable
        self.debug = 7 if debug is True else False  # 7 causes printing in white
        
        ## random authentication key
        authkey = os.urandom(20)

        ## Windows seems to have a hard time with hmac 
        if sys.platform.startswith('win'):
            authkey = None

        #print "key:", ' '.join([str(ord(x)) for x in authkey])
        ## Listen for connection from remote process (and find free port number)
        l = multiprocessing.connection.Listener(('localhost', 0), authkey=authkey)
        port = l.address[1]

        ## start remote process, instruct it to run target function
        sysPath = sys.path if copySysPath else None
        bootstrap = os.path.abspath(os.path.join(os.path.dirname(__file__), 'bootstrap.py'))
        self.debugMsg('Starting child process (%s %s)' % (executable, bootstrap))

        # Decide on printing color for this process
        if debug:
            procDebug = (Process._process_count%6) + 1  # pick a color for this process to print in
            Process._process_count += 1
        else:
            procDebug = False
        
        if wrapStdout is None:
            wrapStdout = sys.platform.startswith('win')

        if wrapStdout:
            ## note: we need all three streams to have their own PIPE due to this bug:
            ## http://bugs.python.org/issue3905
            stdout = subprocess.PIPE
            stderr = subprocess.PIPE
            self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE, stdout=stdout, stderr=stderr)
            ## to circumvent the bug and still make the output visible, we use 
            ## background threads to pass data from pipes to stdout/stderr
            self._stdoutForwarder = FileForwarder(self.proc.stdout, "stdout", procDebug)
            self._stderrForwarder = FileForwarder(self.proc.stderr, "stderr", procDebug)
        else:
            self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE)

        targetStr = pickle.dumps(target)  ## double-pickle target so that child has a chance to 
                                          ## set its sys.path properly before unpickling the target
        pid = os.getpid() # we must send pid to child because windows does not have getppid
        
        ## Send everything the remote process needs to start correctly
        data = dict(
            name=name+'_child', 
            port=port, 
            authkey=authkey, 
            ppid=pid, 
            targetStr=targetStr, 
            path=sysPath, 
            pyside=USE_PYSIDE,
            debug=procDebug
            )
        pickle.dump(data, self.proc.stdin)
        self.proc.stdin.close()
        
        ## open connection for remote process
        self.debugMsg('Listening for child process on port %d, authkey=%s..' % (port, repr(authkey)))
        while True:
            try:
                conn = l.accept()
                break
            except IOError as err:
                if err.errno == 4:  # interrupted; try again
                    continue
                else:
                    raise

        RemoteEventHandler.__init__(self, conn, name+'_parent', pid=self.proc.pid, debug=self.debug)
        self.debugMsg('Connected to child process.')
        
        atexit.register(self.join)

Example 17

Project: pyrocore
Source File: pyroadmin.py
View license
    def mainloop(self):
        """ The main loop.
        """
        if self.options.create_config:
            # Create configuration
            config_loader = load_config.ConfigLoader(self.options.config_dir)
            config_loader.create()

            # Create directories
            for dirname in self.CONFIG_DIRS:
                dirpath = os.path.join(config_loader.config_dir, dirname)
                if not os.path.isdir(dirpath):
                    self.LOG.info("Creating %r..." % (dirpath,))
                    os.mkdir(dirpath)

            # Initialize webserver stuff
            if matching.truth(getattr(config, "torque", {}).get("httpd.active", "False"), "httpd.active"):
                self.download_resource(config.torque["httpd.download_url.smoothie"], "htdocs/js", "smoothie.js")

        elif self.options.dump_config or self.options.output:
            # Get public config attributes
            public = dict((key, val)
                for key, val in vars(config).items()
                if not key.startswith('_') and (self.options.reveal or not (
                    callable(val) or key in config._PREDEFINED
                ))
            )

            if self.options.dump_config:
                # Dump configuration
                pprinter = (pprint.PrettyPrinter if self.options.reveal else metafile.MaskingPrettyPrinter)()
                pprinter.pprint(public)
            else:
                def splitter(fields):
                    "Yield single names for a list of comma-separated strings."
                    for flist in fields:
                        for field in flist.split(','):
                            yield field.strip()

                values = []
                for field in splitter(self.options.output):
                    default = None
                    if '=' in field:
                        field, default = field.split('=', 1)

                    try:
                        val = public
                        for key in field.split('.'):
                            if key in val:
                                val = val[key]
                            elif isinstance(val, list) and key.isdigit():
                                val = val[int(key, 10)]
                            else:
                                matches = [i for i in val.keys() if i.lower() == key.lower()]
                                if matches:
                                    val = val[matches[0]]
                                else:
                                    raise KeyError(key)
                    except (IndexError, KeyError), exc:
                        if default is None:
                            self.LOG.error("Field %r not found (%s)" % (field, exc))
                            break
                        values.append(default)
                    else:
                        values.append(str(val))
                else:
                    print '\t'.join(values)

        elif self.options.screenlet:
            # Create screenlet stub
            stub_dir = os.path.expanduser("~/.screenlets/PyroScope")
            if os.path.exists(stub_dir):
                self.fatal("Screenlet stub %r already exists" % stub_dir)

            stub_template = os.path.join(os.path.dirname(config.__file__), "data", "screenlet")
            shutil.copytree(stub_template, stub_dir)

            py_stub= os.path.join(stub_dir, "PyroScopeScreenlet.py")
            with closing(open(py_stub, "w")) as handle:
                handle.write('\n'.join([
                    "#! %s" % sys.executable,
                    "from pyrocore.screenlet.rtorrent import PyroScopeScreenlet, run",
                    "if __name__ == '__main__':",
                    "    run()",
                    "",
                ]))
            os.chmod(py_stub, 0755)

        else:
            # Print usage
            self.parser.print_help()
            self.parser.exit()

Example 18

Project: pyspace
Source File: big_bang.py
View license
    def load_configuration(self, conf_file_name=None):
        """ Load a configuration from the specified :ref:`YAML<yaml>` configuration file.
        
        Overwrites default directories and path with directories set
        in the specified YAML file.
        
        **Parameters**
        
            :conf_file_name: the name of the conf file that lies in the 'conf_dir'
        """
        if conf_file_name is None:
            conf_file_name = 'config.yaml'
            logging.debug("No configuration file given in %s. Defaulting to %s." %
                         (self.conf_dir,conf_file_name))

        conf_file_name = os.sep.join([self.conf_dir, conf_file_name])
        self.conf_file_name = conf_file_name    # store for later reference
        logging.debug("Configuration file: %s"%conf_file_name)
        print("--> Using configuration file: \n\t\t %s."%conf_file_name)
        
        self.python_path = None
        try:
            conf_file = open(conf_file_name, 'r')
            conf = yaml.load(conf_file)
            for k, v in conf.iteritems():
                if v is not None or (isinstance(v, str) and v.isspace()):
                    if isinstance(v,str) and '~' in v:
                        v = os.path.expanduser(v)
                    self.__dict__[k] = v
            conf_file.close()
            # check for first call
            if "first_call" in self.__dict__.keys() and self.first_call:
                conf_file = open(conf_file_name, 'r')
                lines = conf_file.readlines()
                conf_file.close()
                for line in lines:
                    if line.startswith("first_call : True"):
                        lines.remove(line)
                        print self.first_call_epilog
                        conf_file = open(conf_file_name, 'w')
                        conf_file.writelines(lines)
                        conf_file.close()
                        break
        except IOError, _:
            msg = "Unable to open configuration file " + str(conf_file_name)
            msg += " Put it to the pySPACEcenter folder or change "
            msg += "the 'PYSPACE_CONF_DIR' parameter in your shell."
            warnings.warn(msg)
        except AttributeError, _:
            warnings.warn("Your configuration file  is empty. Please check the installation documentation! Using defaults.")

        print("--> Using Python: \n\t\t %s"%sys.executable)
        # Setup python path as specified
        if self.python_path is not None:
            for i in range(len(self.python_path)):
                if '~' in self.python_path[i]:
                    self.python_path[i]=os.path.expanduser(self.python_path[i])
            new_python_path = deepcopy(self.python_path)
            python_path_set = set(new_python_path)
            unique_sys_path = [i for i in sys.path if i not in python_path_set and not python_path_set.add(i)]
            new_python_path.extend(unique_sys_path)
            sys.path = new_python_path
        else:
            self.python_path=[]
        print("--> Prepending to your PYTHONPATH: \n\t\t %s"%str(self.python_path))

        if len(self.external_nodes) > 0:
            print("--> Using external nodes: \n\t\t %s" %
                  str(self.external_nodes))

        # Append root directory to PYTHONPATH anew
        if not self.parent_dir in sys.path:
            sys.path.append(self.parent_dir)

        if "resources_dir" in self.__dict__.keys():
            self.__dict__["storage"] = self.__dict__["resources_dir"]
            warnings.warn("Change config parameter 'resources_dir' to 'storage'.")
        return self

Example 19

Project: pytest
Source File: test_assertion.py
View license
    @pytest.mark.parametrize('mode', ['plain', 'rewrite'])
    @pytest.mark.parametrize('plugin_state', ['development', 'installed'])
    def test_installed_plugin_rewrite(self, testdir, mode, plugin_state):
        # Make sure the hook is installed early enough so that plugins
        # installed via setuptools are re-written.
        testdir.tmpdir.join('hampkg').ensure(dir=1)
        contents = {
            'hampkg/__init__.py': """
                import pytest

                @pytest.fixture
                def check_first2():
                    def check(values, value):
                        assert values.pop(0) == value
                    return check
            """,
            'spamplugin.py': """
            import pytest
            from hampkg import check_first2

            @pytest.fixture
            def check_first():
                def check(values, value):
                    assert values.pop(0) == value
                return check
            """,
            'mainwrapper.py': """
            import pytest, pkg_resources

            plugin_state = "{plugin_state}"

            class DummyDistInfo:
                project_name = 'spam'
                version = '1.0'

                def _get_metadata(self, name):
                    # 'RECORD' meta-data only available in installed plugins
                    if name == 'RECORD' and plugin_state == "installed":
                        return ['spamplugin.py,sha256=abc,123',
                                'hampkg/__init__.py,sha256=abc,123']
                    # 'SOURCES.txt' meta-data only available for plugins in development mode
                    elif name == 'SOURCES.txt' and plugin_state == "development":
                        return ['spamplugin.py',
                                'hampkg/__init__.py']
                    return []

            class DummyEntryPoint:
                name = 'spam'
                module_name = 'spam.py'
                attrs = ()
                extras = None
                dist = DummyDistInfo()

                def load(self, require=True, *args, **kwargs):
                    import spamplugin
                    return spamplugin

            def iter_entry_points(name):
                yield DummyEntryPoint()

            pkg_resources.iter_entry_points = iter_entry_points
            pytest.main()
            """.format(plugin_state=plugin_state),
            'test_foo.py': """
            def test(check_first):
                check_first([10, 30], 30)

            def test2(check_first2):
                check_first([10, 30], 30)
            """,
        }
        testdir.makepyfile(**contents)
        result = testdir.run(sys.executable, 'mainwrapper.py', '-s', '--assert=%s' % mode)
        if mode == 'plain':
            expected = 'E       AssertionError'
        elif mode == 'rewrite':
            expected = '*assert 10 == 30*'
        else:
            assert 0
        result.stdout.fnmatch_lines([expected])

Example 20

Project: performance
Source File: cli.py
View license
def parse_args():
    parser = argparse.ArgumentParser(
        description=("Compares the performance of baseline_python with" +
                     " changed_python and prints a report."))

    subparsers = parser.add_subparsers(dest='action')
    cmds = []

    # run
    cmd = subparsers.add_parser(
        'run', help='Run benchmarks on the running python')
    cmds.append(cmd)
    cmd.add_argument("-r", "--rigorous", action="store_true",
                     help=("Spend longer running tests to get more" +
                           " accurate results"))
    cmd.add_argument("-f", "--fast", action="store_true",
                     help="Get rough answers quickly")
    cmd.add_argument("--debug-single-sample", action="store_true",
                     help="Debug: fastest mode, only collect a single sample")
    cmd.add_argument("-v", "--verbose", action="store_true",
                     help="Print more output")
    cmd.add_argument("-m", "--track-memory", action="store_true",
                     help="Track memory usage. This only works on Linux.")
    cmd.add_argument("--affinity", metavar="CPU_LIST", default=None,
                     help=("Specify CPU affinity for benchmark runs. This "
                           "way, benchmarks can be forced to run on a given "
                           "CPU to minimize run to run variation."))
    cmd.add_argument("-o", "--output", metavar="FILENAME",
                     help="Run the benchmarks on only one interpreter and "
                           "write benchmark into FILENAME. "
                           "Provide only baseline_python, not changed_python.")
    cmd.add_argument("--append", metavar="FILENAME",
                     help="Add runs to an existing file, or create it "
                     "if it doesn't exist")
    filter_opts(cmd)

    # show
    cmd = subparsers.add_parser('show', help='Display a benchmark file')
    cmd.add_argument("filename", metavar="FILENAME")
    cmds.append(cmd)

    # compare
    cmd = subparsers.add_parser('compare', help='Compare two benchmark files')
    cmds.append(cmd)
    cmd.add_argument("-v", "--verbose", action="store_true",
                     help="Print more output")
    cmd.add_argument("-O", "--output_style", metavar="STYLE",
                     choices=("normal", "table"),
                     default="normal",
                     help=("What style the benchmark output should take."
                           " Valid options are 'normal' and 'table'."
                           " Default is normal."))
    cmd.add_argument("--csv", metavar="CSV_FILE",
                     action="store", default=None,
                     help=("Name of a file the results will be written to,"
                           " as a three-column CSV file containing minimum"
                           " runtimes for each benchmark."))
    cmd.add_argument("baseline_filename", metavar="baseline_file.json")
    cmd.add_argument("changed_filename", metavar="changed_file.json")

    # list
    cmd = subparsers.add_parser(
        'list', help='List benchmarks of the running Python')
    cmds.append(cmd)
    filter_opts(cmd)

    # list_groups
    cmd = subparsers.add_parser(
        'list_groups', help='List benchmark groups of the running Python')
    cmds.append(cmd)

    # venv
    cmd = subparsers.add_parser('venv',
                                help='Actions on the virtual environment')
    cmd.add_argument("venv_action", nargs="?",
                     choices=('show', 'create', 'recreate', 'remove'),
                     default='show')
    cmds.append(cmd)

    for cmd in cmds:
        cmd.add_argument("--inherit-environ", metavar="VAR_LIST",
                         type=comma_separated,
                         help=("Comma-separated list of environment variable "
                               "names that are inherited from the parent "
                               "environment when running benchmarking "
                               "subprocesses."))
        cmd.add_argument("--inside-venv", action="store_true",
                         help=("Option for internal usage only, don't use "
                               "it directly. Notice that we are already "
                               "inside the virtual environment."))
        cmd.add_argument("-p", "--python",
                         help="Python executable (default: use running Python)",
                         default=sys.executable)
        cmd.add_argument("--venv",
                         help="Path to the virtual environment")

    options = parser.parse_args()

    if options.action == 'run' and options.debug_single_sample:
        options.fast = True

    if not options.action:
        # an action is mandatory
        parser.print_help()
        sys.exit(1)

    # Replace "~" with the user home directory
    options.python = os.path.expanduser(options.python)
    # Try to the absolute path to the binary
    abs_python = which(options.python)
    if not abs_python:
        print("ERROR: Unable to locate the Python executable: %r" %
              options.python)
        sys.exit(1)
    options.python = os.path.realpath(abs_python)

    return (parser, options)

Example 21

Project: python-future
Source File: server.py
View license
    def run_cgi(self):
        """Execute a CGI script."""
        path = self.path
        dir, rest = self.cgi_info

        i = path.find('/', len(dir) + 1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir) + 1)
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if self.have_fork or not ispy:
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib_parse.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.get("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = authorization[1].encode('ascii')
                        if utils.PY3:
                            # In Py3.3, was:
                            authorization = base64.decodebytes(authorization).\
                                            decode('ascii')
                        else:
                            # Backport to Py2.7:
                            authorization = base64.decodestring(authorization).\
                                            decode('ascii')
                    except (binascii.Error, UnicodeError):
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.get('content-type') is None:
            env['CONTENT_TYPE'] = self.headers.get_content_type()
        else:
            env['CONTENT_TYPE'] = self.headers['content-type']
        length = self.headers.get('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.get('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.get('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.get_all('cookie', []))
        cookie_str = ', '.join(co)
        if cookie_str:
            env['HTTP_COOKIE'] = cookie_str
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")
        self.flush_headers()

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non-Unix -- use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)
            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 env = env
                                 )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 22

Project: AvsPmod
Source File: build.py
View license
def main():
    # Define names and paths
    pythonexe = sys.executable
    x86_64 = sys.maxsize > 2**32
    zipname = '{0}_v{1}_({2}_{3}).zip'.format(global_vars.name, 
        global_vars.version, platform.system(), 'x86-64' if x86_64 else 'x86-32')
    upx = os.path.join(os.environ['PROGRAMFILES'], 'upx', 'upx.exe')
    exe7z = os.path.join(os.environ['PROGRAMFILES'], '7-Zip', '7z.exe')
    
    tempdir = tempfile.mkdtemp()
    atexit.register(shutil.rmtree, tempdir)
    programdirname = os.path.join(tempdir, global_vars.name)
    
    # Create/update the master translation file
    print '\nCreating translation file...'
    if not i18n.main():
        return
    
    # Create the program executable using py2exe
    if os.system('""%s" -O setup.py py2exe -d %s"' % (pythonexe, programdirname)):
        return
    
    # Update the translation files in the temporal subdirectory
    i18n.UpdateTranslationFile(os.path.join(programdirname, 'translations'), version=global_vars.version)
    
    # Create/update 'macros_readme.txt' in the macros subdirectory
    avsp.GenerateMacroReadme(os.path.join(programdirname, 'macros', 'macros_readme.txt'))
    
    # Set the large address aware flag in the executable
    if not x86_64:
        re_vs = re.compile('VS\d+COMNTOOLS', re.I)
        for key, value in os.environ.iteritems():
            match = re_vs.match(key)
            if match:
                vs_path = value.rsplit(os.sep, 3)[0]
                editbin = os.path.join(vs_path, 'VC', 'bin', 'editbin.exe')
                if os.path.isfile(editbin):
                    dir = os.path.join(vs_path, 'Common7', 'IDE')
                    if os.path.isdir(dir):
                        os.environ['PATH'] += os.pathsep + dir
                        print '\nSetting large address aware flag...'
                        if os.system('""%s" /LARGEADDRESSAWARE "%s""' % 
                                (editbin, os.path.join(programdirname, 'run.exe'))):
                            print 'Failed'
                        else: break
        else:
            print "\neditbin not found.  Large address aware flag not set"
    
    # Compress the files with UPX, if available
    if x86_64:
        print "\nSkipping UPX'ing on x86-64 builds"
    elif  __debug__:
        print "\nDebug mode, skipping UPX'ing"
    else:
        if not os.path.isfile(upx):
            upx = isinpath('upx.exe')
        if upx:
            args = [upx, '--best', '--no-progress']
            for root, dirs, files in os.walk(programdirname):
                for file in files:
                    if os.path.splitext(file)[1] in ('.exe','.dll','.pyd'):
                        args.append(os.path.join(root, file))
            subprocess.call(args)
        else:
            print "\nUPX not found"
    
    # Manage the files
    os.rename(os.path.join(programdirname, 'run.exe'), 
              os.path.join(programdirname, '{0}.exe'.format(global_vars.name)))
    os.rename(os.path.join(programdirname, 'README.md'), 
              os.path.join(programdirname, 'readme.txt'))
    atexit.register(shutil.rmtree, 'build')

    # Create the zip file for distribution.  Use 7-Zip if available
    if not os.path.isfile(exe7z):
        exe7z = isinpath('7z.exe')
    if exe7z:
        print '\nCreating 7z archive...'
        zipname = zipname.replace('.zip', '.7z')
        if os.path.isfile(zipname):
            os.remove(zipname)
        subprocess.call([exe7z, 'a', '-bd', '-r', zipname, programdirname])
    else:
        print '\nCreating ZIP archive...'
        if os.path.isfile(zipname):
            os.remove(zipname)
        zip = zipfile.ZipFile(zipname, 'w', zipfile.ZIP_DEFLATED)
        for root, dirs, files in os.walk(programdirname):
            for name in files:
                real_path = os.path.join(root, name)
                archive_path = os.path.relpath(real_path, tempdir)
                zip.write(real_path, archive_path)
        zip.close()

Example 23

Project: headphones
Source File: Headphones.py
View license
def main():
    """
    Headphones application entry point. Parses arguments, setups encoding and
    initializes the application.
    """

    # Fixed paths to Headphones
    if hasattr(sys, 'frozen'):
        headphones.FULL_PATH = os.path.abspath(sys.executable)
    else:
        headphones.FULL_PATH = os.path.abspath(__file__)

    headphones.PROG_DIR = os.path.dirname(headphones.FULL_PATH)
    headphones.ARGS = sys.argv[1:]

    # From sickbeard
    headphones.SYS_PLATFORM = sys.platform
    headphones.SYS_ENCODING = None

    try:
        locale.setlocale(locale.LC_ALL, "")
        headphones.SYS_ENCODING = locale.getpreferredencoding()
    except (locale.Error, IOError):
        pass

    # for OSes that are poorly configured I'll just force UTF-8
    if not headphones.SYS_ENCODING or headphones.SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
        headphones.SYS_ENCODING = 'UTF-8'

    # Set up and gather command line arguments
    parser = argparse.ArgumentParser(
        description='Music add-on for SABnzbd+, Transmission and more.')

    parser.add_argument(
        '-v', '--verbose', action='store_true', help='Increase console logging verbosity')
    parser.add_argument(
        '-q', '--quiet', action='store_true', help='Turn off console logging')
    parser.add_argument(
        '-d', '--daemon', action='store_true', help='Run as a daemon')
    parser.add_argument(
        '-p', '--port', type=int, help='Force Headphones to run on a specified port')
    parser.add_argument(
        '--datadir', help='Specify a directory where to store your data files')
    parser.add_argument('--config', help='Specify a config file to use')
    parser.add_argument('--nolaunch', action='store_true',
                        help='Prevent browser from launching on startup')
    parser.add_argument(
        '--pidfile', help='Create a pid file (only relevant when running as a daemon)')

    args = parser.parse_args()

    if args.verbose:
        headphones.VERBOSE = True
    if args.quiet:
        headphones.QUIET = True

    # Do an intial setup of the logger.
    logger.initLogger(console=not headphones.QUIET, log_dir=False,
        verbose=headphones.VERBOSE)

    if args.daemon:
        if sys.platform == 'win32':
            sys.stderr.write(
                "Daemonizing not supported under Windows, starting normally\n")
        else:
            headphones.DAEMON = True
            headphones.QUIET = True

    if args.pidfile:
        headphones.PIDFILE = str(args.pidfile)

        # If the pidfile already exists, headphones may still be running, so
        # exit
        if os.path.exists(headphones.PIDFILE):
            raise SystemExit("PID file '%s' already exists. Exiting." %
                headphones.PIDFILE)

        # The pidfile is only useful in daemon mode, make sure we can write the
        # file properly
        if headphones.DAEMON:
            headphones.CREATEPID = True

            try:
                with open(headphones.PIDFILE, 'w') as fp:
                    fp.write("pid\n")
            except IOError as e:
                raise SystemExit("Unable to write PID file: %s", e)
        else:
            logger.warn("Not running in daemon mode. PID file creation " \
                "disabled.")

    # Determine which data directory and config file to use
    if args.datadir:
        headphones.DATA_DIR = args.datadir
    else:
        headphones.DATA_DIR = headphones.PROG_DIR

    if args.config:
        config_file = args.config
    else:
        config_file = os.path.join(headphones.DATA_DIR, 'config.ini')

    # Try to create the DATA_DIR if it doesn't exist
    if not os.path.exists(headphones.DATA_DIR):
        try:
            os.makedirs(headphones.DATA_DIR)
        except OSError:
            raise SystemExit(
                'Could not create data directory: ' + headphones.DATA_DIR + '. Exiting....')

    # Make sure the DATA_DIR is writeable
    if not os.access(headphones.DATA_DIR, os.W_OK):
        raise SystemExit(
            'Cannot write to the data directory: ' + headphones.DATA_DIR + '. Exiting...')

    # Put the database in the DATA_DIR
    headphones.DB_FILE = os.path.join(headphones.DATA_DIR, 'headphones.db')

    # Read config and start logging
    try:
        headphones.initialize(config_file)
    except headphones.exceptions.SoftChrootError as e:
        raise SystemExit('FATAL ERROR')

    if headphones.DAEMON:
        headphones.daemonize()

    # Configure the connection to the musicbrainz database
    headphones.mb.startmb()

    # Force the http port if neccessary
    if args.port:
        http_port = args.port
        logger.info('Using forced web server port: %i', http_port)
    else:
        http_port = int(headphones.CONFIG.HTTP_PORT)

    # Check if pyOpenSSL is installed. It is required for certificate generation
    # and for CherryPy.
    if headphones.CONFIG.ENABLE_HTTPS:
        try:
            import OpenSSL
        except ImportError:
            logger.warn("The pyOpenSSL module is missing. Install this " \
                "module to enable HTTPS. HTTPS will be disabled.")
            headphones.CONFIG.ENABLE_HTTPS = False

    # Try to start the server. Will exit here is address is already in use.
    web_config = {
        'http_port': http_port,
        'http_host': headphones.CONFIG.HTTP_HOST,
        'http_root': headphones.CONFIG.HTTP_ROOT,
        'http_proxy': headphones.CONFIG.HTTP_PROXY,
        'enable_https': headphones.CONFIG.ENABLE_HTTPS,
        'https_cert': headphones.CONFIG.HTTPS_CERT,
        'https_key': headphones.CONFIG.HTTPS_KEY,
        'http_username': headphones.CONFIG.HTTP_USERNAME,
        'http_password': headphones.CONFIG.HTTP_PASSWORD,
    }
    webstart.initialize(web_config)

    # Start the background threads
    headphones.start()

    # Open webbrowser
    if headphones.CONFIG.LAUNCH_BROWSER and not args.nolaunch:
        headphones.launch_browser(headphones.CONFIG.HTTP_HOST, http_port,
                                  headphones.CONFIG.HTTP_ROOT)

    # Wait endlessy for a signal to happen
    while True:
        if not headphones.SIGNAL:
            try:
                time.sleep(1)
            except KeyboardInterrupt:
                headphones.SIGNAL = 'shutdown'
        else:
            logger.info('Received signal: %s', headphones.SIGNAL)

            if headphones.SIGNAL == 'shutdown':
                headphones.shutdown()
            elif headphones.SIGNAL == 'restart':
                headphones.shutdown(restart=True)
            else:
                headphones.shutdown(restart=True, update=True)

            headphones.SIGNAL = None

Example 24

Project: pygame_sdl2
Source File: run_tests.py
View license
def run(*args, **kwds):
    """Run the Pygame unit test suite and return (total tests run, fails dict)

    Positional arguments (optional):
    The names of tests to include. If omitted then all tests are run. Test
    names need not include the trailing '_test'.

    Keyword arguments:
    incomplete - fail incomplete tests (default False)
    nosubprocess - run all test suites in the current process
                   (default False, use separate subprocesses)
    dump - dump failures/errors as dict ready to eval (default False)
    file - if provided, the name of a file into which to dump failures/errors
    timings - if provided, the number of times to run each individual test to
              get an average run time (default is run each test once)
    exclude - A list of TAG names to exclude from the run. The items may be
              comma or space separated.
    show_output - show silenced stderr/stdout on errors (default False)
    all - dump all results, not just errors (default False)
    randomize - randomize order of tests (default False)
    seed - if provided, a seed randomizer integer
    multi_thread - if provided, the number of THREADS in which to run
                   subprocessed tests
    time_out - if subprocess is True then the time limit in seconds before
               killing a test (default 30)
    fake - if provided, the name of the fake tests package in the
           run_tests__tests subpackage to run instead of the normal
           Pygame tests
    python - the path to a python executable to run subprocessed tests
             (default sys.executable)
    interative - allow tests tagged 'interative'.

    Return value:
    A tuple of total number of tests run, dictionary of error information. The
    dictionary is empty if no errors were recorded.

    By default individual test modules are run in separate subprocesses. This
    recreates normal Pygame usage where pygame.init() and pygame.quit() are
    called only once per program execution, and avoids unfortunate
    interactions between test modules. Also, a time limit is placed on test
    execution, so frozen tests are killed when there time allotment expired.
    Use the single process option if threading is not working properly or if
    tests are taking too long. It is not guaranteed that all tests will pass
    in single process mode.

    Tests are run in a randomized order if the randomize argument is True or a
    seed argument is provided. If no seed integer is provided then the system
    time is used.

    Individual test modules may have a corresponding *_tags.py module,
    defining a __tags__ attribute, a list of tag strings used to selectively
    omit modules from a run. By default only the 'interactive', 'ignore', and
    'subprocess_ignore' tags are ignored. 'interactive' is for modules that
    take user input, like cdrom_test.py. 'ignore' and 'subprocess_ignore' for
    for disabling modules for foreground and subprocess modes respectively.
    These are for disabling tests on optional modules or for experimental
    modules with known problems. These modules can be run from the console as
    a Python program.

    This function can only be called once per Python session. It is not
    reentrant.

    """

    global was_run

    if was_run:
        raise RuntimeError("run() was already called this session")
    was_run = True
                           
    options = kwds.copy()
    option_nosubprocess = options.get('nosubprocess', False)
    option_dump = options.pop('dump', False)
    option_file = options.pop('file', None)
    option_all = options.pop('all', False)
    option_randomize = options.get('randomize', False)
    option_seed = options.get('seed', None)
    option_multi_thread = options.pop('multi_thread', 1)
    option_time_out = options.pop('time_out', 120)
    option_fake = options.pop('fake', None)
    option_python = options.pop('python', sys.executable)
    option_exclude = options.pop('exclude', ())
    option_interactive = options.pop('interactive', False)

    if not option_interactive and 'interactive' not in option_exclude:
        option_exclude += ('interactive',)
    if not option_nosubprocess and 'subprocess_ignore' not in option_exclude:
        option_exclude += ('subprocess_ignore',)
    elif 'ignore' not in option_exclude:
        option_exclude += ('ignore',)
    if sys.version_info < (3, 0, 0):
        option_exclude += ('python2_ignore',)
    else:
        option_exclude += ('python3_ignore',)

    main_dir, test_subdir, fake_test_subdir = prepare_test_env()
    test_runner_py = os.path.join(test_subdir, "test_utils", "test_runner.py")
    cur_working_dir = os.path.abspath(os.getcwd())

    ###########################################################################
    # Compile a list of test modules. If fake, then compile list of fake
    # xxxx_test.py from run_tests__tests

    TEST_MODULE_RE = re.compile('^(.+_test)\.py$')

    test_mods_pkg_name = test_pkg_name
    
    if option_fake is not None:
        test_mods_pkg_name = '.'.join([test_mods_pkg_name,
                                       'run_tests__tests',
                                       option_fake])
        test_subdir = os.path.join(fake_test_subdir, option_fake)
        working_dir = test_subdir
    else:
        working_dir = main_dir


    # Added in because some machines will need os.environ else there will be
    # false failures in subprocess mode. Same issue as python2.6. Needs some
    # env vars.

    test_env = os.environ

    fmt1 = '%s.%%s' % test_mods_pkg_name
    fmt2 = '%s.%%s_test' % test_mods_pkg_name
    if args:
        test_modules = [
            m.endswith('_test') and (fmt1 % m) or (fmt2 % m) for m in args
        ]
    else:
        test_modules = []
        for f in sorted(os.listdir(test_subdir)):
            for match in TEST_MODULE_RE.findall(f):
                test_modules.append(fmt1 % (match,))

    ###########################################################################
    # Remove modules to be excluded.

    tmp = test_modules
    test_modules = []
    for name in tmp:
        tag_module_name = "%s_tags" % (name[0:-5],)
        try:
            tag_module = import_submodule(tag_module_name)
        except ImportError:
            test_modules.append(name)
        else:
            try:
                tags = tag_module.__tags__
            except AttributeError:
                print ("%s has no tags: ignoring" % (tag_module_name,))
                test_module.append(name)
            else:
                for tag in tags:
                    if tag in option_exclude:
                        print ("skipping %s (tag '%s')" % (name, tag))
                        break
                else:
                    test_modules.append(name)
    del tmp, tag_module_name, name

    ###########################################################################
    # Meta results

    results = {}
    meta_results = {'__meta__' : {}}
    meta = meta_results['__meta__']

    ###########################################################################
    # Randomization

    if option_randomize or option_seed is not None:
        if option_seed is None:
            option_seed = time.time()
        meta['random_seed'] = option_seed
        print ("\nRANDOM SEED USED: %s\n" % option_seed)
        random.seed(option_seed)
        random.shuffle(test_modules)

    ###########################################################################
    # Single process mode

    if option_nosubprocess:
        unittest_patch.patch(**options)

        options['exclude'] = option_exclude
        t = time.time()
        for module in test_modules:
            results.update(run_test(module, **options))
        t = time.time() - t

    ###########################################################################
    # Subprocess mode
    #

    if not option_nosubprocess:
        if is_pygame_pkg:
            from pygame.tests.test_utils.async_sub import proc_in_time_or_kill
        else:
            from test.test_utils.async_sub import proc_in_time_or_kill

        pass_on_args = ['--exclude', ','.join(option_exclude)]
        for option in ['timings', 'seed']:
            value = options.pop(option, None)
            if value is not None:
                pass_on_args.append('--%s' % option)
                pass_on_args.append(str(value))
        for option, value in options.items():
            if value:
                pass_on_args.append('--%s' % option)

        def sub_test(module):
            print ('loading %s' % module)

            cmd = [option_python, test_runner_py, module ] + pass_on_args

            return (module,
                    (cmd, test_env, working_dir),
                    proc_in_time_or_kill(cmd, option_time_out, env=test_env,
                                         wd=working_dir))

        if option_multi_thread > 1:
            def tmap(f, args):
                return pygame.threads.tmap (
                    f, args, stop_on_error = False,
                    num_workers = option_multi_thread
                )
        else:
            tmap = map

        t = time.time()

        for module, cmd, (return_code, raw_return) in tmap(sub_test,
                                                           test_modules):
            test_file = '%s.py' % os.path.join(test_subdir, module)
            cmd, test_env, working_dir = cmd

            test_results = get_test_results(raw_return)
            if test_results:
                results.update(test_results)
            else:
                results[module] = {}

            add_to_results = [
                'return_code', 'raw_return',  'cmd', 'test_file',
                'test_env', 'working_dir', 'module',
            ]

            results[module].update(from_namespace(locals(), add_to_results))

        t = time.time() - t

    ###########################################################################
    # Output Results
    #

    untrusty_total, combined = combine_results(results, t)
    total, fails = test_failures(results)

    meta['total_tests'] = total
    meta['combined'] = combined
    results.update(meta_results)

    if option_nosubprocess:
        assert total == untrusty_total

    if not option_dump:
        print (combined)
    else:
        results = option_all and results or fails
        print (TEST_RESULTS_START)
        print (pformat(results))

    if option_file is not None:
        results_file = open(option_file, 'w')
        try:
            results_file.write(pformat(results))
        finally:
            results_file.close()

    return total, fails

Example 25

Project: django-supervisor
Source File: config.py
View license
def get_merged_config(**options):
    """Get the final merged configuration for supvervisord, as a string.

    This is the top-level function exported by this module.  It combines
    the config file from the main project with default settings and those
    specified in the command-line, processes various special section names,
    and returns the resulting configuration as a string.
    """
    #  Find and load the containing project module.
    #  This can be specified explicity using the --project-dir option.
    #  Otherwise, we attempt to guess by looking for the manage.py file.
    project_dir = options.get("project_dir")
    if project_dir is None:
        project_dir = guess_project_dir()
    # Find the config file to load.
    # Default to <project-dir>/supervisord.conf.
    config_file = options.get("config_file")
    if config_file is None:
        config_file = os.path.join(project_dir,CONFIG_FILE)
    #  Build the default template context variables.
    #  This is mostly useful information about the project and environment.
    ctx = {
        "PROJECT_DIR": project_dir,
        "PYTHON": os.path.realpath(os.path.abspath(sys.executable)),
        "SUPERVISOR_OPTIONS": rerender_options(options),
        "settings": settings,
        "environ": os.environ,
    }
    #  Initialise the ConfigParser.
    #  Fortunately for us, ConfigParser has merge-multiple-config-files
    #  functionality built into it.  You just read each file in turn, and
    #  values from later files overwrite values from former.
    cfg = RawConfigParser()
    #  Start from the default configuration options.
    data = render_config(DEFAULT_CONFIG,ctx)
    cfg.readfp(StringIO(data))
    #  Add in the project-specific config file.
    with open(config_file,"r") as f:
        data = render_config(f.read(),ctx)
    cfg.readfp(StringIO(data))
    #  Add in the options specified on the command-line.
    cfg.readfp(StringIO(get_config_from_options(**options)))
    #  Add options from [program:__defaults__] to each program section
    #  if it happens to be missing that option.
    PROG_DEFAULTS = "program:__defaults__"
    if cfg.has_section(PROG_DEFAULTS):
        for option in cfg.options(PROG_DEFAULTS):
            default = cfg.get(PROG_DEFAULTS,option)
            for section in cfg.sections():
                if section.startswith("program:"):
                    if not cfg.has_option(section,option):
                        cfg.set(section,option,default)
        cfg.remove_section(PROG_DEFAULTS)
    #  Add options from [program:__overrides__] to each program section
    #  regardless of whether they already have that option.
    PROG_OVERRIDES = "program:__overrides__"
    if cfg.has_section(PROG_OVERRIDES):
        for option in cfg.options(PROG_OVERRIDES):
            override = cfg.get(PROG_OVERRIDES,option)
            for section in cfg.sections():
                if section.startswith("program:"):
                    cfg.set(section,option,override)
        cfg.remove_section(PROG_OVERRIDES)
    #  Make sure we've got a port configured for supervisorctl to
    #  talk to supervisord.  It's passworded based on secret key.
    #  If they have configured a unix socket then use that, otherwise
    #  use an inet server on localhost at fixed-but-randomish port.
    username = hashlib.md5(settings.SECRET_KEY).hexdigest()[:7]
    password = hashlib.md5(username).hexdigest()
    if cfg.has_section("unix_http_server"):
        set_if_missing(cfg,"unix_http_server","username",username)
        set_if_missing(cfg,"unix_http_server","password",password)
        serverurl = "unix://" + cfg.get("unix_http_server","file")
    else:
        #  This picks a "random" port in the 9000 range to listen on.
        #  It's derived from the secret key, so it's stable for a given
        #  project but multiple projects are unlikely to collide.
        port = int(hashlib.md5(password).hexdigest()[:3],16) % 1000
        addr = "127.0.0.1:9%03d" % (port,)
        set_if_missing(cfg,"inet_http_server","port",addr)
        set_if_missing(cfg,"inet_http_server","username",username)
        set_if_missing(cfg,"inet_http_server","password",password)
        serverurl = "http://" + cfg.get("inet_http_server","port")
    set_if_missing(cfg,"supervisorctl","serverurl",serverurl)
    set_if_missing(cfg,"supervisorctl","username",username)
    set_if_missing(cfg,"supervisorctl","password",password)
    set_if_missing(cfg,"rpcinterface:supervisor",
                       "supervisor.rpcinterface_factory",
                       "supervisor.rpcinterface:make_main_rpcinterface")
    #  Remove any [program:] sections with exclude=true
    for section in cfg.sections():
        try:
            if cfg.getboolean(section,"exclude"):
                cfg.remove_section(section)
        except NoOptionError:
            pass
    #  Sanity-check to give better error messages.
    for section in cfg.sections():
        if section.startswith("program:"):
            if not cfg.has_option(section,"command"):
                msg = "Process name '%s' has no command configured"
                raise ValueError(msg % (section.split(":",1)[-1]))
    #  Write it out to a StringIO and return the data
    s = StringIO()
    cfg.write(s)
    return s.getvalue()

Example 26

Project: youtube-dl
Source File: update.py
View license
def update_self(to_screen, verbose, opener):
    """Update the program file with the latest version from the repository"""

    UPDATE_URL = 'https://rg3.github.io/youtube-dl/update/'
    VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
    JSON_URL = UPDATE_URL + 'versions.json'
    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, 'frozen'):
        to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
        return

    # Check if there is a new version
    try:
        newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
    except Exception:
        if verbose:
            to_screen(encode_compat_str(traceback.format_exc()))
        to_screen('ERROR: can\'t find the current version. Please try again later.')
        return
    if newversion == __version__:
        to_screen('youtube-dl is up-to-date (' + __version__ + ')')
        return

    # Download and check versions info
    try:
        versions_info = opener.open(JSON_URL).read().decode('utf-8')
        versions_info = json.loads(versions_info)
    except Exception:
        if verbose:
            to_screen(encode_compat_str(traceback.format_exc()))
        to_screen('ERROR: can\'t obtain versions info. Please try again later.')
        return
    if 'signature' not in versions_info:
        to_screen('ERROR: the versions file is not signed or corrupted. Aborting.')
        return
    signature = versions_info['signature']
    del versions_info['signature']
    if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
        to_screen('ERROR: the versions file signature is invalid. Aborting.')
        return

    version_id = versions_info['latest']

    def version_tuple(version_str):
        return tuple(map(int, version_str.split('.')))
    if version_tuple(__version__) >= version_tuple(version_id):
        to_screen('youtube-dl is up to date (%s)' % __version__)
        return

    to_screen('Updating to version ' + version_id + ' ...')
    version = versions_info['versions'][version_id]

    print_notes(to_screen, versions_info['versions'])

    # sys.executable is set to the full pathname of the exe-file for py2exe
    filename = sys.executable if hasattr(sys, 'frozen') else sys.argv[0]

    if not os.access(filename, os.W_OK):
        to_screen('ERROR: no write permissions on %s' % filename)
        return

    # Py2EXE
    if hasattr(sys, 'frozen'):
        exe = filename
        directory = os.path.dirname(exe)
        if not os.access(directory, os.W_OK):
            to_screen('ERROR: no write permissions on %s' % directory)
            return

        try:
            urlh = opener.open(version['exe'][0])
            newcontent = urlh.read()
            urlh.close()
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to download latest version')
            return

        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
        if newcontent_hash != version['exe'][1]:
            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
            return

        try:
            with open(exe + '.new', 'wb') as outf:
                outf.write(newcontent)
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to write the new version')
            return

        try:
            bat = os.path.join(directory, 'youtube-dl-updater.bat')
            with io.open(bat, 'w') as batfile:
                batfile.write('''
@echo off
echo Waiting for file handle to be closed ...
ping 127.0.0.1 -n 5 -w 1000 > NUL
move /Y "%s.new" "%s" > NUL
echo Updated youtube-dl to version %s.
start /b "" cmd /c del "%%~f0"&exit /b"
                \n''' % (exe, exe, version_id))

            subprocess.Popen([bat])  # Continues to run in the background
            return  # Do not show premature success messages
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to overwrite current version')
            return

    # Zip unix package
    elif isinstance(globals().get('__loader__'), zipimporter):
        try:
            urlh = opener.open(version['bin'][0])
            newcontent = urlh.read()
            urlh.close()
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to download latest version')
            return

        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
        if newcontent_hash != version['bin'][1]:
            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
            return

        try:
            with open(filename, 'wb') as outf:
                outf.write(newcontent)
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to overwrite current version')
            return

    to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.')

Example 27

Project: imagrium
Source File: CGIHTTPServer.py
View license
    def run_cgi(self):
        """Execute a CGI script."""
        path = self.path
        dir, rest = self.cgi_info

        i = path.find('/', len(dir) + 1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir) + 1)
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.getheader('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non Unix - use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)

            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin = subprocess.PIPE,
                                 stdout = subprocess.PIPE,
                                 stderr = subprocess.PIPE,
                                 env = env
                                )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 28

View license
def run(stdout, stderr, argv, theano_nose, batch_size, time_profile,
        display_batch_output):

    # Setting aside current working directory for later saving
    sav_dir = os.getcwd()
    # The first argument is the called script.
    argv = argv[1:]

    # It seems safer to fully regenerate the list of tests on each call.
    if os.path.isfile('.noseids'):
        os.remove('.noseids')

    # Collect test IDs.
    print("""\
####################
# COLLECTING TESTS #
####################""")
    stdout.flush()
    stderr.flush()
    dummy_in = open(os.devnull)
    # We need to call 'python' on Windows, because theano-nose is not a
    # native Windows app; and it does not hurt to call it on Unix.
    # Using sys.executable, so that the same Python version is used.
    python = sys.executable
    rval = subprocess.call(
        ([python, theano_nose, '--collect-only', '--with-id']
         + argv),
        stdin=dummy_in.fileno(),
        stdout=stdout.fileno(),
        stderr=stderr.fileno())
    stdout.flush()
    stderr.flush()
    assert rval == 0
    noseids_file = '.noseids'

    with open(noseids_file, 'rb') as f:
        data = pickle.load(f)

    ids = data['ids']
    n_tests = len(ids)
    if n_tests == 0:
        raise Exception("0 test selected")
    assert n_tests == max(ids)

    # Standard batch testing is called for
    if not time_profile:
        failed = set()
        print("""\
###################################
# RUNNING TESTS IN BATCHES OF %s #
###################################""" % batch_size)
        # When `display_batch_output` is False, we suppress all output because
        # we want the user to focus only on the failed tests, which are re-run
        # (with output) below.
        dummy_out = open(os.devnull, 'w')
        for test_id in xrange(1, n_tests + 1, batch_size):
            stdout.flush()
            stderr.flush()
            test_range = list(range(test_id,
                                    min(test_id + batch_size, n_tests + 1)))
            cmd = ([python, theano_nose, '--with-id'] +
                   list(map(str, test_range)) +
                   argv)
            subprocess_extra_args = dict(stdin=dummy_in.fileno())
            if not display_batch_output:
                # Use quiet mode in nosetests.
                cmd.append('-q')
                # Suppress all output.
                subprocess_extra_args.update(dict(
                    stdout=dummy_out.fileno(),
                    stderr=dummy_out.fileno()))
            t0 = time.time()
            subprocess.call(cmd, **subprocess_extra_args)
            t1 = time.time()
            # Recover failed test indices from the 'failed' field of the
            # '.noseids' file. We need to do it after each batch because
            # otherwise this field may get erased. We use a set because it
            # seems like it is not systematically erased though, and we want
            # to avoid duplicates.
            with open(noseids_file, 'rb') as f:
                failed = failed.union(pickle.load(f)['failed'])

            print('%s%% done in %.3fs (failed: %s)' % (
                (test_range[-1] * 100) // n_tests, t1 - t0, len(failed)))
        # Sort for cosmetic purpose only.
        failed = sorted(failed)
        if failed:
            # Re-run only failed tests
            print("""\
################################
# RE-RUNNING FAILED TESTS ONLY #
################################""")
            stdout.flush()
            stderr.flush()
            subprocess.call(
                ([python, theano_nose, '-v', '--with-id']
                 + failed
                 + argv),
                stdin=dummy_in.fileno(),
                stdout=stdout.fileno(),
                stderr=stderr.fileno())
            stdout.flush()
            stderr.flush()
            return 0
        else:
            print("""\
####################
# ALL TESTS PASSED #
####################""")

    # Time-profiling is called for
    else:
        print("""\
########################################
# RUNNING TESTS IN TIME-PROFILING MODE #
########################################""")

        # finds first word of list l containing string s
        def getIndexOfFirst(l, s):
            for pos, word in enumerate(l):
                if s in word:
                    return pos

        # finds last word of list l containing string s
        def getIndexOfLast(l, s):
            for pos, word in enumerate(reversed(l)):
                if s in word:
                    return (len(l) - pos - 1)

        # iterating through tests
        # initializing master profiling list and raw log
        prof_master_nosort = []
        prof_rawlog = []
        dummy_out = open(os.devnull, 'w')
        path_rawlog = os.path.join(sav_dir, 'timeprof_rawlog')
        stamp = str(datetime.datetime.now()) + '\n\n'
        f_rawlog = open(path_rawlog, 'w')
        f_rawlog.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
                       ' (raw log)\n\n' + stamp)
        f_rawlog.flush()

        stamp = str(datetime.datetime.now()) + '\n\n'
        fields = ('Fields: computation time; nosetests sequential id;'
                  ' test name; parent class (if any); outcome\n\n')
        path_nosort = os.path.join(sav_dir, 'timeprof_nosort')
        # probably this part can be extracted for function with many args
        with open(path_nosort, 'w') as f_nosort:
            # begin of saving nosort
            f_nosort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
                           ' (by sequential id)\n\n' + stamp + fields)
            f_nosort.flush()
            for test_floor in xrange(1, n_tests + 1, batch_size):
                for test_id in xrange(test_floor, min(test_floor + batch_size,
                                                     n_tests + 1)):
                    # Print the test we will start in the raw log to help
                    # debug tests that are too long.
                    f_rawlog.write("\n%s Will run test #%d %s\n" % (
                        time.ctime(), test_id, data["ids"][test_id]))
                    f_rawlog.flush()

                    p_out = output_subprocess_Popen(
                        ([python, theano_nose, '-v', '--with-id']
                         + [str(test_id)] + argv +
                         ['--disabdocstring']))
                        # the previous option calls a custom Nosetests plugin
                        # precluding automatic sustitution of doc. string for
                        # test name in display
                        # (see class 'DisabDocString' in file theano-nose)

                    # recovering and processing data from pipe
                    err = p_out[1]
                    # print the raw log
                    f_rawlog.write(err)
                    f_rawlog.flush()

                    # parsing the output
                    l_err = err.split()
                    try:
                        pos_id = getIndexOfFirst(l_err, '#')
                        prof_id = l_err[pos_id]
                        pos_dot = getIndexOfFirst(l_err, '...')
                        prof_test = ''
                        for s in l_err[pos_id + 1: pos_dot]:
                            prof_test += s + ' '
                        if 'OK' in err:
                            pos_ok = getIndexOfLast(l_err, 'OK')
                            if len(l_err) == pos_ok + 1:
                                prof_time = float(l_err[pos_ok - 1][0:-1])
                                prof_pass = 'OK'
                            elif 'SKIP' in l_err[pos_ok + 1]:
                                prof_time = 0.
                                prof_pass = 'SKIPPED TEST'
                            elif 'KNOWNFAIL' in l_err[pos_ok + 1]:
                                prof_time = float(l_err[pos_ok - 1][0:-1])
                                prof_pass = 'OK'
                            else:
                                prof_time = 0.
                                prof_pass = 'FAILED TEST'
                        else:
                            prof_time = 0.
                            prof_pass = 'FAILED TEST'
                    except Exception:
                        prof_time = 0
                        prof_id = '#' + str(test_id)
                        prof_test = ('FAILED PARSING, see raw log for details'
                                     ' on test')
                        prof_pass = ''
                    prof_tuple = (prof_time, prof_id, prof_test, prof_pass)

                    # appending tuple to master list
                    prof_master_nosort.append(prof_tuple)

                    # write the no sort file
                    s_nosort = ((str(prof_tuple[0]) + 's').ljust(10) +
                     " " + prof_tuple[1].ljust(7) + " " +
                     prof_tuple[2] + prof_tuple[3] +
                     "\n")
                    f_nosort.write(s_nosort)
                    f_nosort.flush()

                print('%s%% time-profiled' % ((test_id * 100) // n_tests))
            f_rawlog.close()

            # sorting tests according to running-time
            prof_master_sort = sorted(prof_master_nosort,
                                      key=lambda test: test[0], reverse=True)

            # saving results to readable files
            path_sort = os.path.join(sav_dir, 'timeprof_sort')
            with open(path_sort, 'w') as f_sort:
                f_sort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
                             ' (sorted by computation time)\n\n' + stamp + fields)
                for i in xrange(len(prof_master_nosort)):
                    s_sort = ((str(prof_master_sort[i][0]) + 's').ljust(10) +
                         " " + prof_master_sort[i][1].ljust(7) + " " +
                         prof_master_sort[i][2] + prof_master_sort[i][3] +
                         "\n")
                    f_sort.write(s_sort)

Example 29

Project: babble
Source File: CGIHTTPServer.py
View license
    def run_cgi(self):
        """Execute a CGI script."""
        path = self.path
        dir, rest = self.cgi_info

        i = path.find('/', len(dir) + 1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir) + 1)
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = {}
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE'):
            env.setdefault(k, "")
        os.environ.update(env)

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, os.environ)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        elif self.have_popen2 or self.have_popen3:
            # Windows -- use popen2 or popen3 to create a subprocess
            import shutil
            if self.have_popen3:
                popenx = os.popen3
            else:
                popenx = os.popen2
            cmdline = scriptfile
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = "%s -u %s" % (interp, cmdline)
            if '=' not in query and '"' not in query:
                cmdline = '%s "%s"' % (cmdline, query)
            self.log_message("command: %s", cmdline)
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            files = popenx(cmdline, 'b')
            fi = files[0]
            fo = files[1]
            if self.have_popen3:
                fe = files[2]
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
                fi.write(data)
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            fi.close()
            shutil.copyfileobj(fo, self.wfile)
            if self.have_popen3:
                errors = fe.read()
                fe.close()
                if errors:
                    self.log_error('%s', errors)
            sts = fo.close()
            if sts:
                self.log_error("CGI script exit status %#x", sts)
            else:
                self.log_message("CGI script exited OK")

        else:
            # Other O.S. -- execute script in this process
            save_argv = sys.argv
            save_stdin = sys.stdin
            save_stdout = sys.stdout
            save_stderr = sys.stderr
            try:
                save_cwd = os.getcwd()
                try:
                    sys.argv = [scriptfile]
                    if '=' not in decoded_query:
                        sys.argv.append(decoded_query)
                    sys.stdout = self.wfile
                    sys.stdin = self.rfile
                    execfile(scriptfile, {"__name__": "__main__"})
                finally:
                    sys.argv = save_argv
                    sys.stdin = save_stdin
                    sys.stdout = save_stdout
                    sys.stderr = save_stderr
                    os.chdir(save_cwd)
            except SystemExit, sts:
                self.log_error("CGI script exit status %s", str(sts))
            else:
                self.log_message("CGI script exited OK")

Example 30

Project: babble
Source File: util.py
View license
def byte_compile (py_files,
                  optimize=0, force=0,
                  prefix=None, base_dir=None,
                  verbose=1, dry_run=0,
                  direct=None):
    """Byte-compile a collection of Python source files to either .pyc
    or .pyo files in the same directory.  'py_files' is a list of files
    to compile; any files that don't end in ".py" are silently skipped.
    'optimize' must be one of the following:
      0 - don't optimize (generate .pyc)
      1 - normal optimization (like "python -O")
      2 - extra optimization (like "python -OO")
    If 'force' is true, all files are recompiled regardless of
    timestamps.

    The source filename encoded in each bytecode file defaults to the
    filenames listed in 'py_files'; you can modify these with 'prefix' and
    'basedir'.  'prefix' is a string that will be stripped off of each
    source filename, and 'base_dir' is a directory name that will be
    prepended (after 'prefix' is stripped).  You can supply either or both
    (or neither) of 'prefix' and 'base_dir', as you wish.

    If 'dry_run' is true, doesn't actually do anything that would
    affect the filesystem.

    Byte-compilation is either done directly in this interpreter process
    with the standard py_compile module, or indirectly by writing a
    temporary script and executing it.  Normally, you should let
    'byte_compile()' figure out to use direct compilation or not (see
    the source for details).  The 'direct' flag is used by the script
    generated in indirect mode; unless you know what you're doing, leave
    it set to None.
    """

    # First, if the caller didn't force us into direct or indirect mode,
    # figure out which mode we should be in.  We take a conservative
    # approach: choose direct mode *only* if the current interpreter is
    # in debug mode and optimize is 0.  If we're not in debug mode (-O
    # or -OO), we don't know which level of optimization this
    # interpreter is running with, so we can't do direct
    # byte-compilation and be certain that it's the right thing.  Thus,
    # always compile indirectly if the current interpreter is in either
    # optimize mode, or if either optimization level was requested by
    # the caller.
    if direct is None:
        direct = (__debug__ and optimize == 0)

    # "Indirect" byte-compilation: write a temporary script and then
    # run it with the appropriate flags.
    if not direct:
        try:
            from tempfile import mkstemp
            (script_fd, script_name) = mkstemp(".py")
        except ImportError:
            from tempfile import mktemp
            (script_fd, script_name) = None, mktemp(".py")
        log.info("writing byte-compilation script '%s'", script_name)
        if not dry_run:
            if script_fd is not None:
                script = os.fdopen(script_fd, "w")
            else:
                script = open(script_name, "w")

            script.write("""\
from distutils.util import byte_compile
files = [
""")

            # XXX would be nice to write absolute filenames, just for
            # safety's sake (script should be more robust in the face of
            # chdir'ing before running it).  But this requires abspath'ing
            # 'prefix' as well, and that breaks the hack in build_lib's
            # 'byte_compile()' method that carefully tacks on a trailing
            # slash (os.sep really) to make sure the prefix here is "just
            # right".  This whole prefix business is rather delicate -- the
            # problem is that it's really a directory, but I'm treating it
            # as a dumb string, so trailing slashes and so forth matter.

            #py_files = map(os.path.abspath, py_files)
            #if prefix:
            #    prefix = os.path.abspath(prefix)

            script.write(string.join(map(repr, py_files), ",\n") + "]\n")
            script.write("""
byte_compile(files, optimize=%r, force=%r,
             prefix=%r, base_dir=%r,
             verbose=%r, dry_run=0,
             direct=1)
""" % (optimize, force, prefix, base_dir, verbose))

            script.close()

        cmd = [sys.executable, script_name]
        if optimize == 1:
            cmd.insert(1, "-O")
        elif optimize == 2:
            cmd.insert(1, "-OO")
        spawn(cmd, dry_run=dry_run)
        execute(os.remove, (script_name,), "removing %s" % script_name,
                dry_run=dry_run)

    # "Direct" byte-compilation: use the py_compile module to compile
    # right here, right now.  Note that the script generated in indirect
    # mode simply calls 'byte_compile()' in direct mode, a weird sort of
    # cross-process recursion.  Hey, it works!
    else:
        from py_compile import compile

        for file in py_files:
            if file[-3:] != ".py":
                # This lets us be lazy and not filter filenames in
                # the "install_lib" command.
                continue

            # Terminology from the py_compile module:
            #   cfile - byte-compiled file
            #   dfile - purported source filename (same as 'file' by default)
            if sys.platform.startswith('java'):
                cfile = file[:-3] + '$py.class'
            else:
                cfile = file + (__debug__ and "c" or "o")
            dfile = file
            if prefix:
                if file[:len(prefix)] != prefix:
                    raise ValueError, \
                          ("invalid prefix: filename %r doesn't start with %r"
                           % (file, prefix))
                dfile = dfile[len(prefix):]
            if base_dir:
                dfile = os.path.join(base_dir, dfile)

            cfile_base = os.path.basename(cfile)
            if direct:
                if force or newer(file, cfile):
                    log.info("byte-compiling %s to %s", file, cfile_base)
                    if not dry_run:
                        compile(file, cfile, dfile)
                else:
                    log.debug("skipping byte-compilation of %s to %s",
                              file, cfile_base)

Example 31

View license
def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Generate a 'release' script")
    add_argument_config_url(parser)
    add_argument_rosdistro_name(parser)
    add_argument_build_name(parser, 'release')
    add_argument_package_name(parser)
    add_argument_os_name(parser)
    add_argument_os_code_name(parser)
    add_argument_arch(parser)
    parser.add_argument(
        '--skip-install',
        action='store_true',
        help='Skip trying to install binarydeb')
    args = parser.parse_args(argv)

    # collect all template snippets of specific types
    class IncludeHook(Hook):

        def __init__(self):
            Hook.__init__(self)
            self.scripts = []

        def beforeFile(self, *args, **kwargs):
            template_path = kwargs['file'].name
            if template_path.endswith('/release/binarydeb_job.xml.em'):
                self.scripts.append('--')
            if template_path.endswith('/snippet/builder_shell.xml.em'):
                self.scripts.append(kwargs['locals']['script'])

    hook = IncludeHook()
    from ros_buildfarm import templates
    templates.template_hooks = [hook]

    configure_release_job(
        args.config_url, args.rosdistro_name, args.release_build_name,
        args.package_name, args.os_name, args.os_code_name,
        jenkins=False, views=[], generate_import_package_job=False,
        generate_sync_packages_jobs=False, filter_arches=args.arch)

    templates.template_hooks = None

    source_job_name = get_sourcedeb_job_name(
        args.rosdistro_name, args.release_build_name,
        args.package_name, args.os_name, args.os_code_name)

    binary_job_name = get_binarydeb_job_name(
        args.rosdistro_name, args.release_build_name,
        args.package_name, args.os_name, args.os_code_name, args.arch)

    separator_index = hook.scripts.index('--')
    source_scripts = hook.scripts[:separator_index]
    binary_scripts = hook.scripts[separator_index + 1:]

    # inject additional argument to skip fetching sourcedeb from repo
    script_name = '/run_binarydeb_job.py '
    additional_argument = '--skip-download-sourcedeb '
    for i, script in enumerate(binary_scripts):
        offset = script.find(script_name)
        if offset != -1:
            offset += len(script_name)
            script = script[:offset] + additional_argument + script[offset:]
            binary_scripts[i] = script
            break

    # remove rm command for sourcedeb location
    rm_command = 'rm -fr $WORKSPACE/binarydeb'
    for i, script in enumerate(binary_scripts):
        offset = script.find(rm_command)
        if offset != -1:
            script = script[:offset] + script[offset + len(rm_command):]
            binary_scripts[i] = script
            break

    if args.skip_install:
        # remove install step
        script_name = '/create_binarydeb_install_task_generator.py '
        for i, script in enumerate(binary_scripts):
            offset = script.find(script_name)
            if offset != -1:
                del binary_scripts[i]
                break

    value = expand_template(
        'release/release_script.sh.em', {
            'source_job_name': source_job_name,
            'binary_job_name': binary_job_name,
            'source_scripts': source_scripts,
            'binary_scripts': binary_scripts},
        options={BANGPATH_OPT: False})
    value = value.replace('python3', sys.executable)
    print(value)

Example 32

Project: rst2pdf
Source File: styles.py
View license
    def __init__(self, flist, font_path=None, style_path=None, def_dpi=300):
        log.info('Using stylesheets: %s' % ','.join(flist))
        # find base path
        if hasattr(sys, 'frozen'):
            self.PATH = abspath(dirname(sys.executable))
        else:
            self.PATH = abspath(dirname(__file__))

        # flist is a list of stylesheet filenames.
        # They will be loaded and merged in order.
        # but the two default stylesheets will always
        # be loaded first
        flist = [join(self.PATH, 'styles', 'styles.style'),
                join(self.PATH, 'styles', 'default.style')] + flist

        self.def_dpi=def_dpi
        if font_path is None:
            font_path=[]
        font_path+=['.', os.path.join(self.PATH, 'fonts')]
        self.FontSearchPath = list(map(os.path.expanduser, font_path))

        if style_path is None:
            style_path=[]
        style_path+=['.', os.path.join(self.PATH, 'styles'),
                      '~/.rst2pdf/styles']
        self.StyleSearchPath = list(map(os.path.expanduser, style_path))
        self.FontSearchPath=list(set(self.FontSearchPath))
        self.StyleSearchPath=list(set(self.StyleSearchPath))

        log.info('FontPath:%s'%self.FontSearchPath)
        log.info('StylePath:%s'%self.StyleSearchPath)

        findfonts.flist = self.FontSearchPath
        # Page width, height
        self.pw = 0
        self.ph = 0

        # Page size [w,h]
        self.ps = None

        # Margins (top,bottom,left,right,gutter)
        self.tm = 0
        self.bm = 0
        self.lm = 0
        self.rm = 0
        self.gm = 0

        #text width
        self.tw = 0

        # Default emsize, later it will be the fontSize of the base style
        self.emsize=10

        self.languages = []

        ssdata = self.readSheets(flist)

        # Get pageSetup data from all stylessheets in order:
        self.ps = pagesizes.A4
        self.page={}
        for data, ssname in ssdata:
            page = data.get('pageSetup', {})
            if page:
                self.page.update(page)
                pgs=page.get('size', None)
                if pgs: # A standard size
                    pgs=pgs.upper()
                    if pgs in pagesizes.__dict__:
                        self.ps = list(pagesizes.__dict__[pgs])
                        self.psname = pgs
                        if 'width' in self.page: del(self.page['width'])
                        if 'height' in self.page: del(self.page['height'])
                    elif pgs.endswith('-LANDSCAPE'):
                        self.psname = pgs.split('-')[0]
                        self.ps = list(pagesizes.landscape(pagesizes.__dict__[self.psname]))
                        if 'width' in self.page: del(self.page['width'])
                        if 'height' in self.page: del(self.page['height'])
                    else:
                        log.critical('Unknown page size %s in stylesheet %s'%\
                            (page['size'], ssname))
                        continue
                else: #A custom size
                    if 'size'in self.page:
                        del(self.page['size'])
                    # The sizes are expressed in some unit.
                    # For example, 2cm is 2 centimeters, and we need
                    # to do 2*cm (cm comes from reportlab.lib.units)
                    if 'width' in page:
                        self.ps[0] = self.adjustUnits(page['width'])
                    if 'height' in page:
                        self.ps[1] = self.adjustUnits(page['height'])
                self.pw, self.ph = self.ps
                if 'margin-left' in page:
                    self.lm = self.adjustUnits(page['margin-left'])
                if 'margin-right' in page:
                    self.rm = self.adjustUnits(page['margin-right'])
                if 'margin-top' in page:
                    self.tm = self.adjustUnits(page['margin-top'])
                if 'margin-bottom' in page:
                    self.bm = self.adjustUnits(page['margin-bottom'])
                if 'margin-gutter' in page:
                    self.gm = self.adjustUnits(page['margin-gutter'])
                if 'spacing-header' in page:
                    self.ts = self.adjustUnits(page['spacing-header'])
                if 'spacing-footer' in page:
                    self.bs = self.adjustUnits(page['spacing-footer'])
                if 'firstTemplate' in page:
                    self.firstTemplate = page['firstTemplate']

                # tw is the text width.
                # We need it to calculate header-footer height
                # and compress literal blocks.
                self.tw = self.pw - self.lm - self.rm - self.gm

        # Get page templates from all stylesheets
        self.pageTemplates = {}
        for data, ssname in ssdata:
            templates = data.get('pageTemplates', {})
            # templates is a dictionary of pageTemplates
            for key in templates:
                template = templates[key]
                # template is a dict.
                # template[┬┤frames'] is a list of frames
                if key in self.pageTemplates:
                    self.pageTemplates[key].update(template)
                else:
                    self.pageTemplates[key] = template

        # Get font aliases from all stylesheets in order
        self.fontsAlias = {}
        for data, ssname in ssdata:
            self.fontsAlias.update(data.get('fontsAlias', {}))

        embedded_fontnames = []
        self.embedded = []
        # Embed all fonts indicated in all stylesheets
        for data, ssname in ssdata:
            embedded = data.get('embeddedFonts', [])

            for font in embedded:
                try:
                    # Just a font name, try to embed it
                    if isinstance(font, str):
                        # See if we can find the font
                        fname, pos = findfonts.guessFont(font)
                        if font in embedded_fontnames:
                            pass
                        else:
                            fontList = findfonts.autoEmbed(font)
                            if fontList:
                                embedded_fontnames.append(font)
                        if not fontList:
                            if (fname, pos) in embedded_fontnames:
                                fontList = None
                            else:
                                fontList = findfonts.autoEmbed(fname)
                        if fontList is not None:
                            self.embedded += fontList
                            # Maybe the font we got is not called
                            # the same as the one we gave
                            # so check that out
                            suff = ["", "-Oblique", "-Bold", "-BoldOblique"]
                            if not fontList[0].startswith(font):
                                # We need to create font aliases, and use them
                                for fname, aliasname in zip(
                                        fontList,
                                        [font + suffix for suffix in suff]):
                                    self.fontsAlias[aliasname] = fname
                        continue

                    # Each "font" is a list of four files, which will be
                    # used for regular / bold / italic / bold+italic
                    # versions of the font.
                    # If your font doesn't have one of them, just repeat
                    # the regular font.

                    # Example, using the Tuffy font from
                    # http://tulrich.com/fonts/
                    # "embeddedFonts" : [
                    #                    ["Tuffy.ttf",
                    #                     "Tuffy_Bold.ttf",
                    #                     "Tuffy_Italic.ttf",
                    #                     "Tuffy_Bold_Italic.ttf"]
                    #                   ],

                    # The fonts will be registered with the file name,
                    # minus the extension.

                    if font[0].lower().endswith('.ttf'): # A True Type font
                        for variant in font:
                            location=self.findFont(variant)
                            pdfmetrics.registerFont(
                                TTFont(str(variant.split('.')[0]),
                                location))
                            log.info('Registering font: %s from %s'%\
                                (str(variant.split('.')[0]),location))
                            self.embedded.append(str(variant.split('.')[0]))

                        # And map them all together
                        regular, bold, italic, bolditalic = [
                            variant.split('.')[0] for variant in font]
                        addMapping(regular, 0, 0, regular)
                        addMapping(regular, 0, 1, italic)
                        addMapping(regular, 1, 0, bold)
                        addMapping(regular, 1, 1, bolditalic)
                    else: # A Type 1 font
                        # For type 1 fonts we require
                        # [FontName,regular,italic,bold,bolditalic]
                        # where each variant is a (pfbfile,afmfile) pair.
                        # For example, for the URW palladio from TeX:
                        # ["Palatino",("uplr8a.pfb","uplr8a.afm"),
                        #             ("uplri8a.pfb","uplri8a.afm"),
                        #             ("uplb8a.pfb","uplb8a.afm"),
                        #             ("uplbi8a.pfb","uplbi8a.afm")]
                        faceName = font[0]
                        regular = pdfmetrics.EmbeddedType1Face(*font[1])
                        italic = pdfmetrics.EmbeddedType1Face(*font[2])
                        bold = pdfmetrics.EmbeddedType1Face(*font[3])
                        bolditalic = pdfmetrics.EmbeddedType1Face(*font[4])

                except Exception:
                    _, e, _ = sys.exc_info()
                    try:
                        if isinstance(font, list):
                            fname = font[0]
                        else:
                            fname = font
                        log.error("Error processing font %s: %s",
                            os.path.splitext(fname)[0], str(e))
                        log.error("Registering %s as Helvetica alias", fname)
                        self.fontsAlias[fname] = 'Helvetica'
                    except Exception:
                        _, e, _ = sys.exc_info()
                        log.critical("Error processing font %s: %s",
                            fname, str(e))
                        continue

        # Go though all styles in all stylesheets and find all fontNames.
        # Then decide what to do with them
        for data, ssname in ssdata:
            for [skey, style] in self.stylepairs(data):
                for key in style:
                    if key == 'fontName' or key.endswith('FontName'):
                        # It's an alias, replace it
                        if style[key] in self.fontsAlias:
                            style[key] = self.fontsAlias[style[key]]
                        # Embedded already, nothing to do
                        if style[key] in self.embedded:
                            continue
                        # Standard font, nothing to do
                        if style[key] in (
                                    "Courier",
                                    "Courier-Bold",
                                    "Courier-BoldOblique",
                                    "Courier-Oblique",
                                    "Helvetica",
                                    "Helvetica-Bold",
                                    "Helvetica-BoldOblique",
                                    "Helvetica-Oblique",
                                    "Symbol",
                                    "Times-Bold",
                                    "Times-BoldItalic",
                                    "Times-Italic",
                                    "Times-Roman",
                                    "ZapfDingbats"):
                            continue
                        # Now we need to do something
                        # See if we can find the font
                        fname, pos = findfonts.guessFont(style[key])

                        if style[key] in embedded_fontnames:
                            pass
                        else:
                            fontList = findfonts.autoEmbed(style[key])
                            if fontList:
                                embedded_fontnames.append(style[key])
                        if not fontList:
                            if (fname, pos) in embedded_fontnames:
                                fontList = None
                            else:
                                fontList = findfonts.autoEmbed(fname)
                            if fontList:
                                embedded_fontnames.append((fname, pos))
                        if fontList:
                            self.embedded += fontList
                            # Maybe the font we got is not called
                            # the same as the one we gave so check that out
                            suff = ["", "-Bold", "-Oblique", "-BoldOblique"]
                            if not fontList[0].startswith(style[key]):
                                # We need to create font aliases, and use them
                                basefname=style[key].split('-')[0]
                                for fname, aliasname in zip(
                                        fontList,
                                        [basefname + suffix for
                                        suffix in suff]):
                                    self.fontsAlias[aliasname] = fname
                                style[key] = self.fontsAlias[basefname +\
                                             suff[pos]]
                        else:
                            log.error("Unknown font: \"%s\","
                                      "replacing with Helvetica", style[key])
                            style[key] = "Helvetica"

        #log.info('FontList: %s'%self.embedded)
        #log.info('FontAlias: %s'%self.fontsAlias)
        # Get styles from all stylesheets in order
        self.stylesheet = {}
        self.styles = []
        self.linkColor = 'navy'
        # FIXME: linkColor should probably not be a global
        #        style, and tocColor should probably not
        #        be a special case, but for now I'm going
        #        with the flow...
        self.tocColor = None
        for data, ssname in ssdata:
            self.linkColor = data.get('linkColor') or self.linkColor
            self.tocColor = data.get('tocColor') or self.tocColor
            for [skey, style] in self.stylepairs(data):
                sdict = {}
                # FIXME: this is done completely backwards
                for key in style:
                    # Handle color references by name
                    if key == 'color' or key.endswith('Color') and style[key]:
                        style[key] = formatColor(style[key])

                    # Yet another workaround for the unicode bug in
                    # reportlab's toColor
                    elif key == 'commands':
                        style[key]=validateCommands(style[key])
                        #for command in style[key]:
                            #c=command[0].upper()
                            #if c=='ROWBACKGROUNDS':
                                #command[3]=[str(c) for c in command[3]]
                            #elif c in ['BOX','INNERGRID'] or c.startswith('LINE'):
                                #command[4]=str(command[4])

                    # Handle alignment constants
                    elif key == 'alignment':
                        style[key] = dict(TA_LEFT=0,
                                          LEFT=0,
                                          TA_CENTER=1,
                                          CENTER=1,
                                          TA_CENTRE=1,
                                          CENTRE=1,
                                          TA_RIGHT=2,
                                          RIGHT=2,
                                          TA_JUSTIFY=4,
                                          JUSTIFY=4,
                                          DECIMAL=8, )[style[key].upper()]

                    elif key == 'language':
                        if not style[key] in self.languages:
                            self.languages.append(style[key])

                    # Make keys str instead of unicode (required by reportlab)
                    sdict[str(key)] = style[key]
                    sdict['name'] = skey
                # If the style already exists, update it
                if skey in self.stylesheet:
                    self.stylesheet[skey].update(sdict)
                else: # New style
                    self.stylesheet[skey] = sdict
                    self.styles.append(sdict)

        # If the stylesheet has a style name docutils won't reach
        # make a copy with a sanitized name.
        # This may make name collisions possible but that should be
        # rare (who would have custom_name and custom-name in the
        # same stylesheet? ;-)
        # Issue 339

        styles2=[]
        for s in self.styles:
            if not re.match("^[a-z](-?[a-z0-9]+)*$", s['name']):
                s2 = copy(s)
                s2['name'] = docutils.nodes.make_id(s['name'])
                log.warning('%s is an invalid docutils class name, adding alias %s'%(s['name'], s2['name']))
                styles2.append(s2)
        self.styles.extend(styles2)

        # And create  reportlabs stylesheet
        self.StyleSheet = StyleSheet1()
        # Patch to make the code compatible with reportlab from SVN 2.4+ and
        # 2.4
        if not hasattr(self.StyleSheet, 'has_key'):
            self.StyleSheet.__class__.has_key = lambda s, k : k in s
        for s in self.styles:
            if 'parent' in s:
                if s['parent'] is None:
                    if s['name'] != 'base':
                        s['parent'] = self.StyleSheet['base']
                    else:
                        del(s['parent'])
                else:
                    s['parent'] = self.StyleSheet[s['parent']]
            else:
                if s['name'] != 'base':
                    s['parent'] = self.StyleSheet['base']

            # If the style has no bulletFontName but it has a fontName, set it
            if ('bulletFontName' not in s) and ('fontName' in s):
                s['bulletFontName'] = s['fontName']

            hasFS = True
            # Adjust fontsize units
            if 'fontSize' not in s:
                s['fontSize'] = s['parent'].fontSize
                s['trueFontSize']=None
                hasFS = False
            elif 'parent' in s:
                # This means you can set the fontSize to
                # "2cm" or to "150%" which will be calculated
                # relative to the parent style
                s['fontSize'] = self.adjustUnits(s['fontSize'],
                                    s['parent'].fontSize)
                s['trueFontSize']=s['fontSize']
            else:
                # If s has no parent, it's base, which has
                # an explicit point size by default and %
                # makes no sense, but guess it as % of 10pt
                s['fontSize'] = self.adjustUnits(s['fontSize'], 10)

            # If the leading is not set, but the size is, set it
            if 'leading' not in s and hasFS:
                s['leading'] = 1.2*s['fontSize']

            # If the bullet font size is not set, set it as fontSize
            if ('bulletFontSize' not in s) and ('fontSize' in s):
                s['bulletFontSize'] = s['fontSize']

            # If the borderPadding is a list and wordaxe <=0.3.2,
            # convert it to an integer. Workaround for Issue
            if 'borderPadding' in s and ((HAS_WORDAXE and \
                    wordaxe_version <='wordaxe 0.3.2') or
                    reportlab.Version < "2.3" )\
                    and isinstance(s['borderPadding'], list):
                log.warning('Using a borderPadding list in '\
                    'style %s with wordaxe <= 0.3.2 or Reportlab < 2.3. That is not '\
                    'supported, so it will probably look wrong'%s['name'])
                s['borderPadding']=s['borderPadding'][0]

            self.StyleSheet.add(ParagraphStyle(**s))


        self.emsize=self['base'].fontSize
        # Make stdFont the basefont, for Issue 65
        reportlab.rl_config.canvas_basefontname = self['base'].fontName
        # Make stdFont the default font for table cell styles (Issue 65)
        reportlab.platypus.tables.CellStyle.fontname=self['base'].fontName

Example 33

Project: script.module.youtube.dl
Source File: update.py
View license
def update_self(to_screen, verbose, opener):
    """Update the program file with the latest version from the repository"""

    UPDATE_URL = 'https://rg3.github.io/youtube-dl/update/'
    VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
    JSON_URL = UPDATE_URL + 'versions.json'
    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, 'frozen'):
        to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
        return

    # Check if there is a new version
    try:
        newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
    except Exception:
        if verbose:
            to_screen(encode_compat_str(traceback.format_exc()))
        to_screen('ERROR: can\'t find the current version. Please try again later.')
        return
    if newversion == __version__:
        to_screen('youtube-dl is up-to-date (' + __version__ + ')')
        return

    # Download and check versions info
    try:
        versions_info = opener.open(JSON_URL).read().decode('utf-8')
        versions_info = json.loads(versions_info)
    except Exception:
        if verbose:
            to_screen(encode_compat_str(traceback.format_exc()))
        to_screen('ERROR: can\'t obtain versions info. Please try again later.')
        return
    if 'signature' not in versions_info:
        to_screen('ERROR: the versions file is not signed or corrupted. Aborting.')
        return
    signature = versions_info['signature']
    del versions_info['signature']
    if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
        to_screen('ERROR: the versions file signature is invalid. Aborting.')
        return

    version_id = versions_info['latest']

    def version_tuple(version_str):
        return tuple(map(int, version_str.split('.')))
    if version_tuple(__version__) >= version_tuple(version_id):
        to_screen('youtube-dl is up to date (%s)' % __version__)
        return

    to_screen('Updating to version ' + version_id + ' ...')
    version = versions_info['versions'][version_id]

    print_notes(to_screen, versions_info['versions'])

    # sys.executable is set to the full pathname of the exe-file for py2exe
    filename = sys.executable if hasattr(sys, 'frozen') else sys.argv[0]

    if not os.access(filename, os.W_OK):
        to_screen('ERROR: no write permissions on %s' % filename)
        return

    # Py2EXE
    if hasattr(sys, 'frozen'):
        exe = filename
        directory = os.path.dirname(exe)
        if not os.access(directory, os.W_OK):
            to_screen('ERROR: no write permissions on %s' % directory)
            return

        try:
            urlh = opener.open(version['exe'][0])
            newcontent = urlh.read()
            urlh.close()
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to download latest version')
            return

        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
        if newcontent_hash != version['exe'][1]:
            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
            return

        try:
            with open(exe + '.new', 'wb') as outf:
                outf.write(newcontent)
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to write the new version')
            return

        try:
            bat = os.path.join(directory, 'youtube-dl-updater.bat')
            with io.open(bat, 'w') as batfile:
                batfile.write('''
@echo off
echo Waiting for file handle to be closed ...
ping 127.0.0.1 -n 5 -w 1000 > NUL
move /Y "%s.new" "%s" > NUL
echo Updated youtube-dl to version %s.
start /b "" cmd /c del "%%~f0"&exit /b"
                \n''' % (exe, exe, version_id))

            subprocess.Popen([bat])  # Continues to run in the background
            return  # Do not show premature success messages
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to overwrite current version')
            return

    # Zip unix package
    elif isinstance(globals().get('__loader__'), zipimporter):
        try:
            urlh = opener.open(version['bin'][0])
            newcontent = urlh.read()
            urlh.close()
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to download latest version')
            return

        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
        if newcontent_hash != version['bin'][1]:
            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
            return

        try:
            with open(filename, 'wb') as outf:
                outf.write(newcontent)
        except (IOError, OSError):
            if verbose:
                to_screen(encode_compat_str(traceback.format_exc()))
            to_screen('ERROR: unable to overwrite current version')
            return

    to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.')

Example 34

Project: pangyp
Source File: win_tool.py
View license
  def ExecLinkWithManifests(self, arch, embed_manifest, out, ldcmd, resname,
                            mt, rc, intermediate_manifest, *manifests):
    """A wrapper for handling creating a manifest resource and then executing
    a link command."""
    # The 'normal' way to do manifests is to have link generate a manifest
    # based on gathering dependencies from the object files, then merge that
    # manifest with other manifests supplied as sources, convert the merged
    # manifest to a resource, and then *relink*, including the compiled
    # version of the manifest resource. This breaks incremental linking, and
    # is generally overly complicated. Instead, we merge all the manifests
    # provided (along with one that includes what would normally be in the
    # linker-generated one, see msvs_emulation.py), and include that into the
    # first and only link. We still tell link to generate a manifest, but we
    # only use that to assert that our simpler process did not miss anything.
    variables = {
      'python': sys.executable,
      'arch': arch,
      'out': out,
      'ldcmd': ldcmd,
      'resname': resname,
      'mt': mt,
      'rc': rc,
      'intermediate_manifest': intermediate_manifest,
      'manifests': ' '.join(manifests),
    }
    add_to_ld = ''
    if manifests:
      subprocess.check_call(
          '%(python)s gyp-win-tool manifest-wrapper %(arch)s %(mt)s -nologo '
          '-manifest %(manifests)s -out:%(out)s.manifest' % variables)
      if embed_manifest == 'True':
        subprocess.check_call(
            '%(python)s gyp-win-tool manifest-to-rc %(arch)s %(out)s.manifest'
          ' %(out)s.manifest.rc %(resname)s' % variables)
        subprocess.check_call(
            '%(python)s gyp-win-tool rc-wrapper %(arch)s %(rc)s '
            '%(out)s.manifest.rc' % variables)
        add_to_ld = ' %(out)s.manifest.res' % variables
    subprocess.check_call(ldcmd + add_to_ld)

    # Run mt.exe on the theoretically complete manifest we generated, merging
    # it with the one the linker generated to confirm that the linker
    # generated one does not add anything. This is strictly unnecessary for
    # correctness, it's only to verify that e.g. /MANIFESTDEPENDENCY was not
    # used in a #pragma comment.
    if manifests:
      # Merge the intermediate one with ours to .assert.manifest, then check
      # that .assert.manifest is identical to ours.
      subprocess.check_call(
          '%(python)s gyp-win-tool manifest-wrapper %(arch)s %(mt)s -nologo '
          '-manifest %(out)s.manifest %(intermediate_manifest)s '
          '-out:%(out)s.assert.manifest' % variables)
      assert_manifest = '%(out)s.assert.manifest' % variables
      our_manifest = '%(out)s.manifest' % variables
      # Load and normalize the manifests. mt.exe sometimes removes whitespace,
      # and sometimes doesn't unfortunately.
      with open(our_manifest, 'rb') as our_f:
        with open(assert_manifest, 'rb') as assert_f:
          our_data = our_f.read().translate(None, string.whitespace)
          assert_data = assert_f.read().translate(None, string.whitespace)
      if our_data != assert_data:
        os.unlink(out)
        def dump(filename):
          sys.stderr.write('%s\n-----\n' % filename)
          with open(filename, 'rb') as f:
            sys.stderr.write(f.read() + '\n-----\n')
        dump(intermediate_manifest)
        dump(our_manifest)
        dump(assert_manifest)
        sys.stderr.write(
            'Linker generated manifest "%s" added to final manifest "%s" '
            '(result in "%s"). '
            'Were /MANIFEST switches used in #pragma statements? ' % (
              intermediate_manifest, our_manifest, assert_manifest))
        return 1

Example 35

Project: Django--an-app-at-a-time
Source File: debug.py
View license
    def get_traceback_data(self):
        """Return a dictionary containing traceback information."""
        try:
            default_template_engine = Engine.get_default()
        except Exception:
            # Since the debug view must never crash, catch all exceptions.
            # If Django can't find a default template engine, get_default()
            # raises ImproperlyConfigured. If some template engines fail to
            # load, any exception may be raised.
            default_template_engine = None

        # TODO: add support for multiple template engines (#24120).
        # TemplateDoesNotExist should carry all the information.
        # Replaying the search process isn't a good design.
        if self.exc_type and issubclass(self.exc_type, TemplateDoesNotExist):
            if default_template_engine is None:
                template_loaders = []
            else:
                self.template_does_not_exist = True
                self.loader_debug_info = []
                # If Django fails in get_template_loaders, provide an empty list
                # for the following loop to not fail.
                try:
                    template_loaders = default_template_engine.template_loaders
                except Exception:
                    template_loaders = []

            for loader in template_loaders:
                try:
                    source_list_func = loader.get_template_sources
                    # NOTE: This assumes exc_value is the name of the template that
                    # the loader attempted to load.
                    template_list = [{
                        'name': t,
                        'status': self.format_path_status(t),
                    } for t in source_list_func(str(self.exc_value))]
                except AttributeError:
                    template_list = []
                loader_name = loader.__module__ + '.' + loader.__class__.__name__
                self.loader_debug_info.append({
                    'loader': loader_name,
                    'templates': template_list,
                })

        # TODO: add support for multiple template engines (#24119).
        if (default_template_engine is not None
                and default_template_engine.debug
                and hasattr(self.exc_value, 'django_template_source')):
            self.get_template_exception_info()

        frames = self.get_traceback_frames()
        for i, frame in enumerate(frames):
            if 'vars' in frame:
                frame_vars = []
                for k, v in frame['vars']:
                    v = pprint(v)
                    # The force_escape filter assume unicode, make sure that works
                    if isinstance(v, six.binary_type):
                        v = v.decode('utf-8', 'replace')  # don't choke on non-utf-8 input
                    # Trim large blobs of data
                    if len(v) > 4096:
                        v = '%s... <trimmed %d bytes string>' % (v[0:4096], len(v))
                    frame_vars.append((k, force_escape(v)))
                frame['vars'] = frame_vars
            frames[i] = frame

        unicode_hint = ''
        if self.exc_type and issubclass(self.exc_type, UnicodeError):
            start = getattr(self.exc_value, 'start', None)
            end = getattr(self.exc_value, 'end', None)
            if start is not None and end is not None:
                unicode_str = self.exc_value.args[1]
                unicode_hint = smart_text(
                    unicode_str[max(start - 5, 0):min(end + 5, len(unicode_str))],
                    'ascii', errors='replace'
                )
        from django import get_version
        c = {
            'is_email': self.is_email,
            'unicode_hint': unicode_hint,
            'frames': frames,
            'request': self.request,
            'filtered_POST': self.filter.get_post_parameters(self.request),
            'settings': get_safe_settings(),
            'sys_executable': sys.executable,
            'sys_version_info': '%d.%d.%d' % sys.version_info[0:3],
            'server_time': timezone.now(),
            'django_version_info': get_version(),
            'sys_path': sys.path,
            'template_info': self.template_info,
            'template_does_not_exist': self.template_does_not_exist,
            'loader_debug_info': self.loader_debug_info,
        }
        # Check whether exception info is available
        if self.exc_type:
            c['exception_type'] = self.exc_type.__name__
        if self.exc_value:
            c['exception_value'] = smart_text(self.exc_value, errors='replace')
        if frames:
            c['lastframe'] = frames[-1]
        return c

Example 36

Project: sfepy
Source File: ls.py
View license
    @standard_call
    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 i_max=None, mtx=None, status=None, **kwargs):
        import os, sys, shutil, tempfile
        from sfepy import base_dir
        from sfepy.base.ioutils import ensure_path

        eps_a = get_default(eps_a, self.conf.eps_a)
        eps_r = get_default(eps_r, self.conf.eps_r)
        i_max = get_default(i_max, self.conf.i_max)
        eps_d = self.conf.eps_d

        petsc = self.petsc

        ksp, pmtx, psol, prhs = self.set_matrix(mtx)

        ksp.setFromOptions() # PETSc.Options() not used yet...
        ksp.setTolerances(atol=eps_a, rtol=eps_r, divtol=eps_d, max_it=i_max)

        output_dir = tempfile.mkdtemp()

        # Set PETSc rhs, solve, get solution from PETSc solution.
        if x0 is not None:
            psol[...] = x0
            sol0_filename = os.path.join(output_dir, 'sol0.dat')

        else:
            sol0_filename = ''

        prhs[...] = rhs

        script_filename = os.path.join(base_dir, 'solvers/petsc_worker.py')

        mtx_filename = os.path.join(output_dir, 'mtx.dat')
        rhs_filename = os.path.join(output_dir, 'rhs.dat')
        sol_filename = os.path.join(output_dir, 'sol.dat')
        status_filename = os.path.join(output_dir, 'status.txt')

        log_filename = os.path.join(self.conf.log_dir, 'sol.log')
        ensure_path(log_filename)

        output('storing system to %s...' % output_dir)
        tt = time.clock()
        view_mtx = petsc.Viewer().createBinary(mtx_filename, mode='w')
        view_rhs = petsc.Viewer().createBinary(rhs_filename, mode='w')
        pmtx.view(view_mtx)
        prhs.view(view_rhs)
        if sol0_filename:
            view_sol0 = petsc.Viewer().createBinary(sol0_filename, mode='w')
            psol.view(view_sol0)
        output('...done in %.2f s' % (time.clock() - tt))

        command = [
            'mpiexec -n %d' % self.conf.n_proc,
            sys.executable, script_filename,
            '-mtx %s' % mtx_filename, '-rhs %s' % rhs_filename,
            '-sol0 %s' % sol0_filename, '-sol %s' % sol_filename,
            '-status %s' % status_filename,
            '-ksp_type %s' % self.conf.method,
            '-pc_type %s' % self.conf.precond,
            '-sub_pc_type %s' % self.conf.sub_precond,
            '-ksp_atol %.3e' % self.conf.eps_a,
            '-ksp_rtol %.3e' % self.conf.eps_r,
            '-ksp_max_it %d' % self.conf.i_max,
            '-ksp_monitor %s' % log_filename,
            '-ksp_view %s' % log_filename,
        ]
        if self.conf.precond_side is not None:
            command.append('-ksp_pc_side %s' % self.conf.precond_side)

        out = os.system(" ".join(command))
        assert_(out == 0)

        output('reading solution...')
        tt = time.clock()
        view_sol = self.petsc.Viewer().createBinary(sol_filename, mode='r')
        psol = petsc.Vec().load(view_sol)

        fd = open(status_filename, 'r')
        line = fd.readline().split()
        reason = int(line[0])
        elapsed = float(line[1])
        fd.close()
        output('...done in %.2f s' % (time.clock() - tt))

        sol = psol[...].copy()
        output('%s(%s, %s/proc) convergence: %s (%s)'
               % (self.conf.method, self.conf.precond, self.conf.sub_precond,
                  reason, self.converged_reasons[reason]))
        output('elapsed: %.2f [s]' % elapsed)

        shutil.rmtree(output_dir)

        return sol

Example 37

Project: OpenSesame
Source File: qtopensesame.py
View license
	def resume_init(self):

		"""Resume GUI initialization"""

		from libopensesame import misc, metadata
		from libqtopensesame.misc import theme
		from libqtopensesame.extensions import extension_manager
		import random

		# Set some initial variables
		self.current_path = None
		self.version = metadata.__version__
		self.codename = metadata.codename
		self.lock_refresh = False
		self.unsaved_changes = False
		self._run_status = u'inactive'
		self.block_close_event = False

		# Make sure that icons are shown in context menu, regardless of the
		# system settings. This is necessary, because Ubuntu doesn't show menu
		# icons by default.
		QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_DontShowIconsInMenus,
			False)
		# Add the Qt plugin folders to the library path, if they exists. Where
		# these folders are depends on the version of Qt4, but these are two
		# possible locations.
		qt_plugin_path = os.path.join(
			os.path.dirname(sys.executable), 'Library', 'plugins')
		if os.path.isdir(qt_plugin_path):
			QtCore.QCoreApplication.addLibraryPath(qt_plugin_path)
		qt_plugin_path = os.path.join(
			os.path.dirname(sys.executable), 'Library', 'lib', 'qt4', 'plugins')
		if os.path.isdir(qt_plugin_path):
			QtCore.QCoreApplication.addLibraryPath(qt_plugin_path)
		# Do a few things to customize QProgEdit behavior:
		# - Register the bundled monospace font (Droid Sans Mono)
		# - Make sure that QProgEdit doesn't complain about some standard names
		# - Ignore undefined name warnings, which don't play well with
		#   OpenSesame's single workspace
		QtGui.QFontDatabase.addApplicationFont(misc.resource(u'mono.ttf'))
		from QProgEdit import validate
		validate.addPythonBuiltins([u'exp', u'win', u'self'])
		if hasattr(validate, u'setPyFlakesFilter'):
			validate.setPyFlakesFilter(
				lambda msg: msg.message == u'undefined name %r')
		# Initialize random number generator
		random.seed()

		# Check the filesystem encoding for debugging purposes
		debug.msg(u'filesystem encoding: %s' % misc.filesystem_encoding())

		# Parse the command line
		self.parse_command_line()

		# Restore the configuration
		self.restore_config()
		self.set_style()
		self.set_warnings()

		# Setup the UI
		self.load_ui(u'misc.main_window')
		self.theme = theme.theme(self, self.options._theme)
		self.ui.itemtree.setup(self)
		self.ui.console.setup(self)
		self.ui.tabwidget.main_window = self

		# Determine the home folder
		self.home_folder = libopensesame.misc.home_folder()

		# Create .opensesame folder if it doesn't exist yet
		if not os.path.exists(os.path.join(self.home_folder, u".opensesame")):
			os.mkdir(os.path.join(self.home_folder, u".opensesame"))

		# Set the filter-string for opening and saving files
		self.save_file_filter =u'OpenSesame files (*.osexp)'
		self.open_file_filter = \
			u'OpenSesame files (*.osexp *.opensesame.tar.gz *.opensesame)'

		# Set the window message
		self.window_message(_(u"New experiment"))

		# Set the window icon
		self.setWindowIcon(self.theme.qicon(u"opensesame"))

		# Make the connections
		self.ui.itemtree.structure_change.connect(self.update_overview_area)
		self.ui.action_quit.triggered.connect(self.close)
		self.ui.action_open.triggered.connect(self.open_file)
		self.ui.action_save.triggered.connect(self.save_file)
		self.ui.action_save_as.triggered.connect(self.save_file_as)
		self.ui.action_run.triggered.connect(self.run_experiment)
		self.ui.action_run_in_window.triggered.connect(
			self.run_experiment_in_window)
		self.ui.action_run_quick.triggered.connect(self.run_quick)
		self.ui.action_enable_auto_response.triggered.connect(
			self.set_auto_response)
		self.ui.action_close_current_tab.triggered.connect(
			self.ui.tabwidget.close_current)
		self.ui.action_close_all_tabs.triggered.connect(
			self.ui.tabwidget.close_all)
		self.ui.action_close_other_tabs.triggered.connect(
			self.ui.tabwidget.close_other)
		self.ui.action_onetabmode.triggered.connect(
			self.ui.tabwidget.toggle_onetabmode)
		self.ui.action_show_overview.triggered.connect(self.toggle_overview)
		self.ui.action_show_pool.triggered.connect(
			self.toggle_pool)
		self.ui.action_show_stdout.triggered.connect(self.refresh_stdout)
		self.ui.action_preferences.triggered.connect(
			self.ui.tabwidget.open_preferences)

		# Setup console
		self.ui.button_help_console.clicked.connect(
			self.ui.tabwidget.open_stdout_help)
		self.ui.button_reset_console.clicked.connect(
			self.ui.console.reset)

		# Setup the overview area
		self.ui.dock_overview.show()
		self.ui.dock_overview.visibilityChanged.connect(
			self.ui.action_show_overview.setChecked)

		# Setup the file pool
		from libqtopensesame.widgets.pool_widget import pool_widget
		self.ui.dock_pool.hide()
		self.ui.dock_pool.visibilityChanged.connect(
			self.ui.action_show_pool.setChecked)
		self.ui.pool_widget = pool_widget(self)
		self.ui.dock_pool.setWidget(self.ui.pool_widget)

		# Uncheck the debug window button on debug window close
		self.ui.dock_stdout.hide()
		self.ui.dock_stdout.visibilityChanged.connect(
			self.ui.action_show_stdout.setChecked)

		# Initialize keyboard shortcuts
		self.ui.shortcut_itemtree = QtWidgets.QShortcut(QtGui.QKeySequence(), self,
			self.focus_overview_area)
		self.ui.shortcut_tabwidget = QtWidgets.QShortcut(
			QtGui.QKeySequence(), self, self.ui.tabwidget.focus)
		self.ui.shortcut_stdout = QtWidgets.QShortcut(QtGui.QKeySequence(), self,
			self.focus_debug_window)
		self.ui.shortcut_pool = QtWidgets.QShortcut(QtGui.QKeySequence(), self,
			self.focus_file_pool)

		# Create the initial experiment, which is the default template. Because
		# not all backends are supported under Python 3, we use a different
		# backend for each.
		if py3:
			tmpl = u'default-py3.osexp'
		else:
			tmpl = u'default.osexp'
		with open(misc.resource(os.path.join(u'templates', tmpl)), u'r') as fd:
			self.experiment = experiment.experiment(self, u'New experiment',
				fd.read())
		self.experiment.build_item_tree()
		self.ui.itemtree.default_fold_state()

		# Miscellaneous initialization
		self.restore_state()
		self.update_recent_files()
		self.set_unsaved(False)
		self.init_custom_fonts()

		# Initialize extensions
		self.extension_manager = extension_manager(self)
		self.extension_manager.fire(u'startup')

Example 38

Project: scoop
Source File: launcher.py
View license
def makeParser():
    """Create the SCOOP module arguments parser."""
    # TODO: Add environment variable (all + selection)
    parser = argparse.ArgumentParser(
        description="Starts a parallel program using SCOOP.",
        prog="{0} -m scoop".format(sys.executable),
    )
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--hosts', '--host',
                       help="The list of hosts. The first host will execute "
                            "the origin. (default is 127.0.0.1)",
                       metavar="Address",
                       nargs='*')
    group.add_argument('--hostfile',
                       help="The hostfile name",
                       metavar="FileName")
    parser.add_argument('--path', '-p',
                        help="The path to the executable on remote hosts "
                             "(default is local directory)",
                        default=os.getcwd())
    parser.add_argument('--nice',
                        type=int,
                        metavar="NiceLevel",
                        help="*nix niceness level (-20 to 19) to run the "
                             "executable")
    parser.add_argument('--verbose', '-v',
                        action='count',
                        help="Verbosity level of this launch script (-vv for "
                             "more)",
                        default=1)
    parser.add_argument('--quiet', '-q',
                        action='store_true')
    parser.add_argument('-n',
                        help="Total number of workers to launch on the hosts. "
                             "Workers are spawned sequentially over the hosts. "
                             "(ie. -n 3 with 2 hosts will spawn 2 workers on "
                             "the first host and 1 on the second.) (default: "
                             "Number of CPUs on current machine)",
                        type=int,
                        metavar="NumberOfWorkers")
    parser.add_argument('-b',
                        help="Total number of brokers to launch on the hosts. "
                             "Brokers are spawned sequentially over the hosts. "
                             "(ie. -b 3 with 2 hosts will spawn 2 brokers on "
                             "the first host and 1 on the second.) (default: "
                             "1)",
                        type=int,
                        default=1,
                        metavar="NumberOfBrokers")
    parser.add_argument('--rsh',
                        help="Use RSH instead of SSH for the launching process. "
                             "Not compatible with --tunnel flag.",
                        action='store_true')
    parser.add_argument('--ssh-executable',
                        help="Name of the ssh executable. (default: 'ssh')",
                        default="ssh",
                        metavar="SSHExecutable")
    parser.add_argument('--tunnel',
                        help="Activate ssh tunnels to route toward the broker "
                             "sockets over remote connections (may eliminate "
                             "routing problems and activate encryption but "
                             "slows down communications)",
                        action='store_true')
    parser.add_argument('--external-hostname',
                        nargs=1,
                        help="The externally routable hostname / ip of this "
                             "machine. (defaults to the local hostname)",
                        metavar="Address")
    parser.add_argument('--python-interpreter',
                        nargs=1,
                        help="The python interpreter executable with which to "
                             "execute the script",
                        default=[sys.executable],
                        metavar="Path")
    parser.add_argument('--pythonpath',
                        nargs=1,
                        help="The PYTHONPATH environment variable (default is "
                             "current PYTHONPATH)",
                        default=[os.environ.get('PYTHONPATH', '')])
    parser.add_argument('--prolog',
                        nargs=1,
                        help="Absolute Path to a shell script or executable "
                             "that will be executed at the launch of every "
                             "worker",
                        default=[None])
    parser.add_argument('--debug',
                        help=argparse.SUPPRESS,
                        action='store_true')
    parser.add_argument('--profile',
                        help=("Turn on the profiling. SCOOP will call "
                        "cProfile.run on the executable for every worker and"
                        " will produce files in directory profile/ named "
                        "workerX where X is the number of the worker."),
                        action='store_true')
    parser.add_argument('--backend',
                        help="Choice of communication backend",
                        choices=['ZMQ', 'TCP'],
                        default='ZMQ')
    parser.add_argument('executable',
                        nargs='?',
                        help='The executable to start with SCOOP')
    parser.add_argument('args',
                        nargs=argparse.REMAINDER,
                        help='The arguments to pass to the executable',
                        default=[],
                        metavar="args")
    return parser

Example 39

Project: qgisSpaceSyntaxToolkit
Source File: processes.py
View license
    def __init__(self, name=None, target=None, executable=None, copySysPath=True, debug=False, timeout=20, wrapStdout=None):
        """
        ==============  =============================================================
        **Arguments:**
        name            Optional name for this process used when printing messages
                        from the remote process.
        target          Optional function to call after starting remote process.
                        By default, this is startEventLoop(), which causes the remote
                        process to process requests from the parent process until it
                        is asked to quit. If you wish to specify a different target,
                        it must be picklable (bound methods are not).
        copySysPath     If True, copy the contents of sys.path to the remote process
        debug           If True, print detailed information about communication
                        with the child process.
        wrapStdout      If True (default on windows) then stdout and stderr from the
                        child process will be caught by the parent process and
                        forwarded to its stdout/stderr. This provides a workaround
                        for a python bug: http://bugs.python.org/issue3905
                        but has the side effect that child output is significantly
                        delayed relative to the parent output.
        ==============  =============================================================
        """
        if target is None:
            target = startEventLoop
        if name is None:
            name = str(self)
        if executable is None:
            executable = sys.executable
        self.debug = 7 if debug is True else False  # 7 causes printing in white
        
        ## random authentication key
        authkey = os.urandom(20)

        ## Windows seems to have a hard time with hmac 
        if sys.platform.startswith('win'):
            authkey = None

        #print "key:", ' '.join([str(ord(x)) for x in authkey])
        ## Listen for connection from remote process (and find free port number)
        l = multiprocessing.connection.Listener(('localhost', 0), authkey=authkey)
        port = l.address[1]

        ## start remote process, instruct it to run target function
        sysPath = sys.path if copySysPath else None
        bootstrap = os.path.abspath(os.path.join(os.path.dirname(__file__), 'bootstrap.py'))
        self.debugMsg('Starting child process (%s %s)' % (executable, bootstrap))

        # Decide on printing color for this process
        if debug:
            procDebug = (Process._process_count%6) + 1  # pick a color for this process to print in
            Process._process_count += 1
        else:
            procDebug = False
        
        if wrapStdout is None:
            wrapStdout = sys.platform.startswith('win')

        if wrapStdout:
            ## note: we need all three streams to have their own PIPE due to this bug:
            ## http://bugs.python.org/issue3905
            stdout = subprocess.PIPE
            stderr = subprocess.PIPE
            self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE, stdout=stdout, stderr=stderr)
            ## to circumvent the bug and still make the output visible, we use 
            ## background threads to pass data from pipes to stdout/stderr
            self._stdoutForwarder = FileForwarder(self.proc.stdout, "stdout", procDebug)
            self._stderrForwarder = FileForwarder(self.proc.stderr, "stderr", procDebug)
        else:
            self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE)

        targetStr = pickle.dumps(target)  ## double-pickle target so that child has a chance to 
                                          ## set its sys.path properly before unpickling the target
        pid = os.getpid() # we must send pid to child because windows does not have getppid
        
        ## Send everything the remote process needs to start correctly
        data = dict(
            name=name+'_child', 
            port=port, 
            authkey=authkey, 
            ppid=pid, 
            targetStr=targetStr, 
            path=sysPath, 
            pyside=USE_PYSIDE,
            debug=procDebug
            )
        pickle.dump(data, self.proc.stdin)
        self.proc.stdin.close()
        
        ## open connection for remote process
        self.debugMsg('Listening for child process on port %d, authkey=%s..' % (port, repr(authkey)))
        while True:
            try:
                conn = l.accept()
                break
            except IOError as err:
                if err.errno == 4:  # interrupted; try again
                    continue
                else:
                    raise

        RemoteEventHandler.__init__(self, conn, name+'_parent', pid=self.proc.pid, debug=self.debug)
        self.debugMsg('Connected to child process.')
        
        atexit.register(self.join)

Example 40

Project: st2
Source File: config.py
View license
def register_opts(ignore_errors=False):
    rbac_opts = [
        cfg.BoolOpt('enable', default=False, help='Enable RBAC.'),
    ]
    do_register_opts(rbac_opts, 'rbac', ignore_errors)

    system_user_opts = [
        cfg.StrOpt('user',
                   default='stanley',
                   help='Default system user.'),
        cfg.StrOpt('ssh_key_file',
                   default='/home/vagrant/.ssh/stanley_rsa',
                   help='SSH private key for the system user.')
    ]
    do_register_opts(system_user_opts, 'system_user', ignore_errors)

    schema_opts = [
        cfg.IntOpt('version', default=4, help='Version of JSON schema to use.'),
        cfg.StrOpt('draft', default='http://json-schema.org/draft-04/schema#',
                   help='URL to the JSON schema draft.')
    ]
    do_register_opts(schema_opts, 'schema', ignore_errors)

    system_opts = [
        cfg.BoolOpt('debug', help='Enable debug mode.', default=False),
        cfg.StrOpt('base_path', default='/opt/stackstorm',
                   help='Base path to all st2 artifacts.')
    ]
    do_register_opts(system_opts, 'system', ignore_errors)

    system_packs_base_path = os.path.join(cfg.CONF.system.base_path, 'packs')
    system_runners_base_path = os.path.join(cfg.CONF.system.base_path, 'runners')
    content_opts = [
        cfg.StrOpt('pack_group', default='st2packs',
                   help='User group that can write to packs directory.'),
        cfg.StrOpt('system_packs_base_path', default=system_packs_base_path,
                   help='Path to the directory which contains system packs.'),
        cfg.StrOpt('system_runners_base_path', default=system_runners_base_path,
                   help='Path to the directory which contains system runners.'),
        cfg.StrOpt('packs_base_paths', default=None,
                   help='Paths which will be searched for integration packs.'),
        cfg.StrOpt('runners_base_paths', default=None,
                   help='Paths which will be searched for runners.'),
        cfg.ListOpt('index_url', default=['https://index.stackstorm.org/v1/index.json'],
                    help=('A URL pointing to the pack index. StackStorm Exchange is used by '
                          'default. Use a comma-separated list for multiple indexes if you '
                          'want to get other packs discovered with "st2 pack search".')),
    ]
    do_register_opts(content_opts, 'content', ignore_errors)

    webui_opts = [
        cfg.StrOpt('webui_base_url', default='https://%s' % socket.getfqdn(),
                   help='Base https URL to access st2 Web UI. This is used to construct' +
                        'history URLs that are sent out when chatops is used to kick off ' +
                        'executions.')
    ]
    do_register_opts(webui_opts, 'webui', ignore_errors)

    db_opts = [
        cfg.StrOpt('host', default='0.0.0.0', help='host of db server'),
        cfg.IntOpt('port', default=27017, help='port of db server'),
        cfg.StrOpt('db_name', default='st2', help='name of database'),
        cfg.StrOpt('username', help='username for db login'),
        cfg.StrOpt('password', help='password for db login'),
        cfg.IntOpt('connection_retry_max_delay_m', help='Connection retry total time (minutes).',
                   default=3),
        cfg.IntOpt('connection_retry_backoff_max_s', help='Connection retry backoff max (seconds).',
                   default=10),
        cfg.IntOpt('connection_retry_backoff_mul', help='Backoff multiplier (seconds).',
                   default=1),
        cfg.BoolOpt('ssl', help='Create the connection to mongodb using SSL', default=False),
        cfg.StrOpt('ssl_keyfile',
                   help='Private keyfile used to identify the local connection against MongoDB.',
                   default=None),
        cfg.StrOpt('ssl_certfile', help='Certificate file used to identify the localconnection',
                   default=None),
        cfg.StrOpt('ssl_cert_reqs', choices='none, optional, required',
                   help='Specifies whether a certificate is required from the other side of the ' +
                        'connection, and whether it will be validated if provided',
                   default=None),
        cfg.StrOpt('ssl_ca_certs',
                   help='ca_certs file contains a set of concatenated CA certificates, which are' +
                        ' used to validate certificates passed from MongoDB.',
                   default=None),
        cfg.BoolOpt('ssl_match_hostname',
                    help='If True and `ssl_cert_reqs` is not None, enables hostname verification',
                    default=True)
    ]
    do_register_opts(db_opts, 'database', ignore_errors)

    messaging_opts = [
        # It would be nice to be able to deprecate url and completely switch to using
        # url. However, this will be a breaking change and will have impact so allowing both.
        cfg.StrOpt('url', default='amqp://guest:[email protected]:5672//',
                   help='URL of the messaging server.'),
        cfg.ListOpt('cluster_urls', default=[],
                    help='URL of all the nodes in a messaging service cluster.'),
        cfg.IntOpt('connection_retries', default=10,
                   help='How many times should we retry connection before failing.'),
        cfg.IntOpt('connection_retry_wait', default=10000,
                   help='How long should we wait between connection retries.')
    ]
    do_register_opts(messaging_opts, 'messaging', ignore_errors)

    syslog_opts = [
        cfg.StrOpt('host', default='127.0.0.1',
                   help='Host for the syslog server.'),
        cfg.IntOpt('port', default=514,
                   help='Port for the syslog server.'),
        cfg.StrOpt('facility', default='local7',
                   help='Syslog facility level.'),
        cfg.StrOpt('protocol', default='udp',
                   help='Transport protocol to use (udp / tcp).')
    ]
    do_register_opts(syslog_opts, 'syslog', ignore_errors)

    log_opts = [
        cfg.ListOpt('excludes', default='',
                    help='Exclusion list of loggers to omit.'),
        cfg.BoolOpt('redirect_stderr', default=False,
                    help='Controls if stderr should be redirected to the logs.'),
        cfg.BoolOpt('mask_secrets', default=True,
                    help='True to mask secrets in the log files.')
    ]
    do_register_opts(log_opts, 'log', ignore_errors)

    # Common API options
    api_opts = [
        cfg.StrOpt('host', default='0.0.0.0', help='StackStorm API server host'),
        cfg.IntOpt('port', default=9101, help='StackStorm API server port'),
        cfg.ListOpt('allow_origin', default=['http://127.0.0.1:3000'],
                    help='List of origins allowed for api, auth and stream'),
        cfg.BoolOpt('mask_secrets', default=True,
                    help='True to mask secrets in the API responses')
    ]
    do_register_opts(api_opts, 'api', ignore_errors)

    # Key Value store options
    keyvalue_opts = [
        cfg.BoolOpt('enable_encryption', default=True,
                    help='Allow encryption of values in key value stored qualified as "secret".'),
        cfg.StrOpt('encryption_key_path', default='',
                   help='Location of the symmetric encryption key for encrypting values in ' +
                        'kvstore. This key should be in JSON and should\'ve been ' +
                        'generated using keyczar.')
    ]
    do_register_opts(keyvalue_opts, group='keyvalue')

    # Common auth options
    auth_opts = [
        cfg.StrOpt('api_url', default=None,
                   help='Base URL to the API endpoint excluding the version'),
        cfg.BoolOpt('enable', default=True, help='Enable authentication middleware.'),
        cfg.IntOpt('token_ttl', default=86400, help='Access token ttl in seconds.')
    ]
    do_register_opts(auth_opts, 'auth', ignore_errors)

    # Common action runner options
    default_python_bin_path = sys.executable
    base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
    default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv')
    action_runner_opts = [
        cfg.StrOpt('logging', default='conf/logging.conf',
                   help='location of the logging.conf file'),
        cfg.StrOpt('python_binary', default=default_python_bin_path,
                   help='Python binary which will be used by Python actions.'),
        cfg.StrOpt('virtualenv_binary', default=default_virtualenv_bin_path,
                   help='Virtualenv binary which should be used to create pack virtualenvs.'),
        cfg.ListOpt('virtualenv_opts', default=['--system-site-packages'],
                    help='List of virtualenv options to be passsed to "virtualenv" command that ' +
                         'creates pack virtualenv.')
    ]
    do_register_opts(action_runner_opts, group='actionrunner')

    # Common options (used by action runner and sensor container)
    action_sensor_opts = [
        cfg.BoolOpt('enable', default=True,
                    help='Whether to enable or disable the ability to post a trigger on action.'),
    ]
    do_register_opts(action_sensor_opts, group='action_sensor')

    # Coordination options
    coord_opts = [
        cfg.StrOpt('url', default=None, help='Endpoint for the coordination server.'),
        cfg.IntOpt('lock_timeout', default=60, help='TTL for the lock if backend suports it.')
    ]
    do_register_opts(coord_opts, 'coordination', ignore_errors)

    # Mistral options
    mistral_opts = [
        cfg.StrOpt('v2_base_url', default='http://127.0.0.1:8989/v2', help='v2 API root endpoint.'),
        cfg.IntOpt('retry_exp_msec', default=1000, help='Multiplier for the exponential backoff.'),
        cfg.IntOpt('retry_exp_max_msec', default=300000, help='Max time for each set of backoff.'),
        cfg.IntOpt('retry_stop_max_msec', default=600000, help='Max time to stop retrying.'),
        cfg.StrOpt('keystone_username', default=None, help='Username for authentication.'),
        cfg.StrOpt('keystone_password', default=None, help='Password for authentication.'),
        cfg.StrOpt('keystone_project_name', default=None, help='OpenStack project scope.'),
        cfg.StrOpt('keystone_auth_url', default=None, help='Auth endpoint for Keystone.'),
        cfg.StrOpt('cacert', default=None, help='Optional certificate to validate endpoint.'),
        cfg.BoolOpt('insecure', default=False, help='Allow insecure communication with Mistral.'),

        cfg.StrOpt('api_url', default=None, help=('URL Mistral uses to talk back to the API.'
            'If not provided it defaults to public API URL. Note: This needs to be a base '
            'URL without API version (e.g. http://127.0.0.1:9101)'))
    ]
    do_register_opts(mistral_opts, group='mistral', ignore_errors=ignore_errors)

    # Common CLI options
    debug = cfg.BoolOpt('debug', default=False,
        help='Enable debug mode. By default this will set all log levels to DEBUG.')
    profile = cfg.BoolOpt('profile', default=False,
        help=('Enable profile mode. In the profile mode all the MongoDB queries and related '
              'profile data are logged.'))
    use_debugger = cfg.BoolOpt('use-debugger', default=True,
        help='Enables debugger. Note that using this option changes how the '
             'eventlet library is used to support async IO. This could result in '
             'failures that do not occur under normal operation.')

    cli_opts = [debug, profile, use_debugger]
    do_register_cli_opts(cli_opts, ignore_errors=ignore_errors)

Example 41

Project: gyp
Source File: win_tool.py
View license
  def ExecLinkWithManifests(self, arch, embed_manifest, out, ldcmd, resname,
                            mt, rc, intermediate_manifest, *manifests):
    """A wrapper for handling creating a manifest resource and then executing
    a link command."""
    # The 'normal' way to do manifests is to have link generate a manifest
    # based on gathering dependencies from the object files, then merge that
    # manifest with other manifests supplied as sources, convert the merged
    # manifest to a resource, and then *relink*, including the compiled
    # version of the manifest resource. This breaks incremental linking, and
    # is generally overly complicated. Instead, we merge all the manifests
    # provided (along with one that includes what would normally be in the
    # linker-generated one, see msvs_emulation.py), and include that into the
    # first and only link. We still tell link to generate a manifest, but we
    # only use that to assert that our simpler process did not miss anything.
    variables = {
      'python': sys.executable,
      'arch': arch,
      'out': out,
      'ldcmd': ldcmd,
      'resname': resname,
      'mt': mt,
      'rc': rc,
      'intermediate_manifest': intermediate_manifest,
      'manifests': ' '.join(manifests),
    }
    add_to_ld = ''
    if manifests:
      subprocess.check_call(
          '%(python)s gyp-win-tool manifest-wrapper %(arch)s %(mt)s -nologo '
          '-manifest %(manifests)s -out:%(out)s.manifest' % variables)
      if embed_manifest == 'True':
        subprocess.check_call(
            '%(python)s gyp-win-tool manifest-to-rc %(arch)s %(out)s.manifest'
          ' %(out)s.manifest.rc %(resname)s' % variables)
        subprocess.check_call(
            '%(python)s gyp-win-tool rc-wrapper %(arch)s %(rc)s '
            '%(out)s.manifest.rc' % variables)
        add_to_ld = ' %(out)s.manifest.res' % variables
    subprocess.check_call(ldcmd + add_to_ld)

    # Run mt.exe on the theoretically complete manifest we generated, merging
    # it with the one the linker generated to confirm that the linker
    # generated one does not add anything. This is strictly unnecessary for
    # correctness, it's only to verify that e.g. /MANIFESTDEPENDENCY was not
    # used in a #pragma comment.
    if manifests:
      # Merge the intermediate one with ours to .assert.manifest, then check
      # that .assert.manifest is identical to ours.
      subprocess.check_call(
          '%(python)s gyp-win-tool manifest-wrapper %(arch)s %(mt)s -nologo '
          '-manifest %(out)s.manifest %(intermediate_manifest)s '
          '-out:%(out)s.assert.manifest' % variables)
      assert_manifest = '%(out)s.assert.manifest' % variables
      our_manifest = '%(out)s.manifest' % variables
      # Load and normalize the manifests. mt.exe sometimes removes whitespace,
      # and sometimes doesn't unfortunately.
      with open(our_manifest, 'rb') as our_f:
        with open(assert_manifest, 'rb') as assert_f:
          our_data = our_f.read().translate(None, string.whitespace)
          assert_data = assert_f.read().translate(None, string.whitespace)
      if our_data != assert_data:
        os.unlink(out)
        def dump(filename):
          sys.stderr.write('%s\n-----\n' % filename)
          with open(filename, 'rb') as f:
            sys.stderr.write(f.read() + '\n-----\n')
        dump(intermediate_manifest)
        dump(our_manifest)
        dump(assert_manifest)
        sys.stderr.write(
            'Linker generated manifest "%s" added to final manifest "%s" '
            '(result in "%s"). '
            'Were /MANIFEST switches used in #pragma statements? ' % (
              intermediate_manifest, our_manifest, assert_manifest))
        return 1

Example 42

View license
def _stage_virtualenv(options, install_virtualenv=True):
    '''Creates staging virtual environment in order to help with the "real"
    installation. Returns path to staged virtual env. If install_virtualenv is
    False, then unpack the virtual env. package, but don't actually create
    a virtual environment.
    '''
    # Create a temporary directory to put the virtual env. in.
    temp_dir = tempfile.mkdtemp()
    try:
        # Was a fixed virtualenv version specified?  If not, we need to check
        # the PyPI server for the latest version.
        if options.virtualenv_version is None:
            options.virtualenv_version, virtualenv_url = (
                _get_newest_package_version_and_url(
                    options.pypi_pull_server,
                    options.virtualenv_package_name))
        else:
            virtualenv_url = '/'.join([
                options.pypi_pull_server,
                options.virtualenv_package_name,
                '{}-{}.tar.gz'.format(
                    options.virtualenv_package_name,
                    options.virtualenv_version)])

        virtualenv_tar_filename = os.path.basename(
            urlparse.urlparse(virtualenv_url).path)

        f_remote = urllib.urlopen(virtualenv_url)
        f_local = open(os.path.join(temp_dir, virtualenv_tar_filename), 'wb')
        f_local.write(f_remote.read())
        f_local.close()

        # If a download dir or download cache directory was specified,
        # copy the virtualenv package file to that directory if it is not
        # already there.
        for directory in [options.download_dir, options.download_cache_dir]:
            if directory is None:
                directory = ''

            virtualenv_tar_exists = os.path.isfile(
                os.path.join(directory, virtualenv_tar_filename))

            if directory and not virtualenv_tar_exists:
                shutil.copy2(
                    os.path.join(temp_dir, virtualenv_tar_filename),
                    directory)

        # Unpack the tarball to the temporary directory.
        tarf = tarfile.open(
            os.path.join(temp_dir, virtualenv_tar_filename),
            'r:gz')
        tarf.extractall(temp_dir)
        tarf.close()
        unpacked_tar_directory = os.path.join(
            temp_dir, virtualenv_tar_filename.replace('.tar.gz', ''))
        bootstrap_vm_directory = os.path.join(
            temp_dir, VIRTUALENV_BOOTSTRAP_NAME)
        # Create the bootstrap virtualenv in the temporary directory using the
        # current python executable we are using plus the virtualenv stuff we
        # unpacked.
        if install_virtualenv:
            arguments = [sys.executable,
                         os.path.join(unpacked_tar_directory, 'virtualenv.py'),
                         '--distribute',
                         bootstrap_vm_directory]
            try:
                if options.quiet:
                    subprocess.check_output(arguments, shell=True)
                else:
                    subprocess.check_call(arguments, shell=True)
            except subprocess.CalledProcessError as e:
                if options.quiet:
                    print e.output
                print 'Bootstrap VM create failed, return code', e.returncode
                raise

            # Get the right options to pass to pip to install virtualenv
            # to the bootstrap environment.  Again, this is necessary because
            # pip does not support file:// index urls.
            if urlparse.urlparse(options.pypi_pull_server).scheme == 'file':
                install_options = [
                    '--no-index',
                    '--find-links',
                    options.pypi_pull_server]

            else:
                install_options = [
                    '-i',
                    options.pypi_pull_server]

            # Install virtualenv into this bootstrap environment using pip,
            # pointing at the right server.
            subprocess.check_call(
                [
                    os.path.join(bootstrap_vm_directory, 'Scripts', 'pip'),
                    'install'
                ]
                + install_options
                + [
                    '{}=={}'.format(
                        options.virtualenv_package_name,
                        options.virtualenv_version)
                ])

    except Exception:
        # Even though the calling code is normally responsible for cleaning
        # up the temp dir, if an exception occurs, we do it here because we
        # won't be able to return the temp_dir to the caller
        _cleanup_virtualenv(temp_dir)
        raise

    # Return the bootstrap vm dir that was created
    return temp_dir

Example 43

Project: plastex
Source File: FunctionalTests.py
View license
    def runTest(self):
        if not self.filename:
            return
            
        src = self.filename
        root = os.path.dirname(os.path.dirname(src))

        # Create temp dir and files
        outdir = tempfile.mkdtemp()
        texfile = os.path.join(outdir, os.path.basename(src))
        shutil.copyfile(src, texfile)

        # Run preprocessing commands
        for line in open(src):
            if line.startswith('%*'):
                command = line[2:].strip()
                p = Process(cwd=outdir, *command.split())
                if p.returncode:
                    raise OSError, 'Preprocessing command exited abnormally with return code %s: %s' % (command, p.log) 
            elif line.startswith('%#'):
                filename = line[2:].strip()
                shutil.copyfile(os.path.join(root,'extras',filename),
                                os.path.join(outdir,filename))
            elif line.startswith('%'):
                continue
            elif not line.strip():
                continue
            else:
                break

        # Run plastex
        outfile = os.path.join(outdir, os.path.splitext(os.path.basename(src))[0]+'.html')
        plastex = which('plastex') or 'plastex'
        python = sys.executable 
        p = Process(python, plastex,'--split-level=0','--no-theme-extras',
                    '--dir=%s' % outdir,'--theme=minimal',
                    '--filename=%s' % os.path.basename(outfile), os.path.basename(src),
                    cwd=outdir)
        if p.returncode:
            shutil.rmtree(outdir, ignore_errors=True)
            raise OSError, 'plastex failed with code %s: %s' % (p.returncode, p.log)

        # Read output file
        output = open(outfile)

        # Get name of output file / benchmark file
        benchfile = os.path.join(root,'benchmarks',os.path.basename(outfile))
        if os.path.isfile(benchfile):
            bench = open(benchfile).readlines()
            output = output.readlines()
        else:
            try: os.makedirs(os.path.join(root,'new'))
            except: pass
            newfile = os.path.join(root,'new',os.path.basename(outfile))
            open(newfile,'w').write(output.read())
            shutil.rmtree(outdir, ignore_errors=True)
            raise OSError, 'No benchmark file: %s' % benchfile

        # Compare files
        diff = ''.join(list(difflib.unified_diff(bench, output))).strip()
        if diff:
            shutil.rmtree(outdir, ignore_errors=True)
            try: os.makedirs(os.path.join(root,'new'))
            except: pass
            newfile = os.path.join(root,'new',os.path.basename(outfile))
            open(newfile,'w').writelines(output)
            assert not(diff), 'Differences were found: %s' % diff

        # Clean up
        shutil.rmtree(outdir, ignore_errors=True)

Example 44

Project: pymt
Source File: build.py
View license
    def run(self):
        print "---------------------------------"
        print "Building PyMT Portable for OSX"
        print "---------------------------------"

        print "\nPreparing Build..."
        print "---------------------------------------"
        if os.path.exists(self.build_dir):
            print "*Cleaning old build dir"
            shutil.rmtree(self.build_dir, ignore_errors=True)
        print "*Creating build directory:"
        print " "+self.build_dir
        os.makedirs(self.build_dir)


        print "\nGetting binary dependencies..."
        print "---------------------------------------"
        print "*Downloading:", self.deps_url
        #report_hook is called every time a piece of teh file is downloaded to print progress
        def report_hook(block_count, block_size, total_size):
            p = block_count*block_size*100.0/total_size
            print "\b\b\b\b\b\b\b\b\b", "%06.2f"%p +"%",
        print " Progress: 000.00%",
        urlretrieve(self.deps_url, #location of binary dependencioes needed for portable pymt
                    os.path.join(self.build_dir,'deps.zip'), #tmp file to store teh archive
                    reporthook=report_hook)
        print " [Done]"


        print "*Extracting binary dependencies..."
        #using osx sysetm command, becasue python zipfile cant handle the hidden files in teh archive
        Popen(['unzip', os.path.join(self.build_dir,'deps.zip')], cwd=self.build_dir, stdout=PIPE).communicate()

        print "\nPutting pymt into portable environment"
        print "---------------------------------------"
        print "*Building pymt source distribution"
        sdist_cmd = [sys.executable, #path to python.exe
                     os.path.join(self.src_dir,'setup.py'), #path to setup.py
                     'sdist', #make setup.py create a src distribution
                     '--dist-dir=%s'%self.build_dir] #put it into build folder
        Popen(sdist_cmd, stdout=PIPE).communicate()


        print "*Placing pymt source distribution in portable context"
        src_dist = os.path.join(self.build_dir,self.dist_name)
        #using osx sysetm command, becasue python zipfile cant handle the hidden files in teh archive
        Popen(['tar', 'xfv', src_dist+'.tar.gz'], cwd=self.build_dir, stdout=PIPE, stderr=PIPE).communicate()
        if self.no_cext:
            print "*Skipping C Extension build (either --no_cext or --no_mingw option set)"
        else:
            print "*Compiling C Extensions inplace for portable distribution"
            cext_cmd = [sys.executable, #path to python.exe
                        'setup.py',
                        'build_ext', #make setup.py create a src distribution
                        '--inplace'] #do it inplace
            #this time it runs teh setup.py inside the source distribution
            #thats has been generated inside the build dir (to generate ext
            #for teh target, instead of the source were building from)
            Popen(cext_cmd, cwd=src_dist, stdout=PIPE, stderr=PIPE).communicate()



        print "\nFinalizing Application Bundle"
        print "---------------------------------------"
        print "*Copying launcher script into the app bundle"
        script_target = os.path.join(self.build_dir, 'portable-deps-osx', 'PyMT.app', 'Contents', 'Resources', 'script')
        script = os.path.join(src_dist,'pymt','tools','packaging','osx', 'pymt.sh')
        shutil.copy(script, script_target)

        print "*Moving examples out of app bundle to be included in disk image"
        examples_target = os.path.join(self.build_dir, 'portable-deps-osx', 'examples')
        examples = os.path.join(src_dist,'examples')
        shutil.move(examples, examples_target)

        print "*Moving newly build pymt distribution into app bundle"
        pymt_target = os.path.join(self.build_dir, 'portable-deps-osx', 'PyMT.app', 'Contents', 'Resources', 'pymt')
        shutil.move(src_dist, pymt_target)

        print "*Removing intermediate file"
        os.remove(os.path.join(self.build_dir,'deps.zip'))
        os.remove(os.path.join(self.build_dir,src_dist+'.tar.gz'))
        shutil.rmtree(os.path.join(self.build_dir,'__MACOSX'), ignore_errors=True)


        #contents of portable-deps-osx, are now ready to go into teh disk image
        dmg_dir = os.path.join(self.build_dir, 'portable-deps-osx')
        vol_name = "PyMT"

        print "\nCreating disk image for distribution"
        print "---------------------------------------"
        print "\nCreating intermediate DMG disk image: temp.dmg"
        print "*checking how much space is needed for disk image..."
        du_cmd = 'du -sh %s'%dmg_dir
        du_out = Popen(shlex.split(du_cmd), stdout=PIPE).communicate()[0]
        size, unit = re.search('(\d+)(.*)\s+/.*', du_out).group(1,2)
        print "  build needs at least %s%s." % (size, unit)

        size = int(size)+10
        print "*allocating %d%s for temp.dmg (volume name:%s)" % (size, unit, vol_name)
        create_dmg_cmd = 'hdiutil create -srcfolder %s -volname %s -fs HFS+ \
                         -fsargs "-c c=64,a=16,e=16" -format UDRW -size %d%s temp.dmg' \
                         % (dmg_dir, vol_name, size+10, unit)
        Popen(shlex.split(create_dmg_cmd), cwd=self.build_dir).communicate()

        print "*mounting intermediate disk image:"
        mount_cmd = 'hdiutil attach -readwrite -noverify -noautoopen "temp.dmg"'
        Popen(shlex.split(mount_cmd), cwd=self.build_dir, stdout=PIPE).communicate()

        print "*running Apple Script to configure DMG layout properties:"
        dmg_config_script = """
           tell application "Finder"
             tell disk "%s"
                   open

                   set current view of container window to icon view
                   set toolbar visible of container window to false
                   set statusbar visible of container window to false
                   set the bounds of container window to {300,100,942,582}
                   set theViewOptions to the icon view options of container window
                   set arrangement of theViewOptions to not arranged
                   set icon size of theViewOptions to 72
                   set background picture of theViewOptions to file ".background:pymtdmg.png"
                   make new alias file at container window to POSIX file "/Applications" with properties {name:"Applications"}
                   set position of item "PyMT" of container window to {150, 130}
                   set position of item "Applications" of container window to {500, 130}
                   set position of item "examples" of container window to {575, 400}
                   set position of item "Readme.txt" of container window to {475, 400}
                   set position of item "make-symlinks" of container window to {375, 400}
                   close
                   open
                   update without registering applications
                   delay 2
                   eject
             end tell
           end tell
        """ % vol_name
        print Popen(['osascript'], cwd=self.build_dir, stdin=PIPE, stdout=PIPE).communicate(dmg_config_script)[0]


        print "\nCreating final disk image"

        print "*unmounting intermediate disk image"
        umount_cmd = 'hdiutil detach /Volumes/%s' % vol_name
        Popen(shlex.split(umount_cmd), cwd=self.build_dir, stdout=PIPE).communicate()

        print "*compressing and finalizing disk image"
        convert_cmd = 'hdiutil convert "temp.dmg" -format UDZO -imagekey zlib-level=9 -o %s.dmg' % os.path.join(self.dist_dir,vol_name)
        Popen(shlex.split(convert_cmd), cwd=self.build_dir, stdout=PIPE).communicate()

        print "*Writing disk image, and cleaning build directory"
        shutil.rmtree(self.build_dir, ignore_errors=True)

Example 45

Project: pymt
Source File: build.py
View license
    def run(self):
        print "---------------------------------"
        print "Building PyMT Portable for Win 32"
        print "---------------------------------"


        print "\nPreparing Build..."
        print "---------------------------------------"
        if os.path.exists(self.build_dir):
            print "*Cleaning old build dir"
            shutil.rmtree(self.build_dir, ignore_errors=True)
        print "*Creating build directory:"
        print " "+self.build_dir
        os.makedirs(self.build_dir)


        print "\nGetting binary dependencies..."
        print "---------------------------------------"
        print "*Downloading:", self.deps_url
        #report_hook is called every time a piece of teh file is downloaded to print progress
        def report_hook(block_count, block_size, total_size):
            p = block_count*block_size*100.0/total_size
            print "\b\b\b\b\b\b\b\b\b", "%06.2f"%p +"%",
        print " Progress: 000.00%",
        urlretrieve(self.deps_url, #location of binary dependencioes needed for portable pymt
                    os.path.join(self.build_dir,'deps.zip'), #tmp file to store teh archive
                    reporthook=report_hook)
        print " [Done]"


        print "*Extracting binary dependencies..."
        zf = ZipFile(os.path.join(self.build_dir,'deps.zip'))
        zf.extractall(self.build_dir)
        zf.close()
        if self.no_mingw:
            print "*Excluding MinGW from portable distribution (--no-mingw option is set)"
            shutil.rmtree(os.path.join(self.build_dir, 'MinGW'), ignore_errors=True)


        print "\nPutting pymt into portable environment"
        print "---------------------------------------"
        print "*Building pymt source distribution"
        sdist_cmd = [sys.executable, #path to python.exe
                     os.path.join(self.src_dir,'setup.py'), #path to setup.py
                     'sdist', #make setup.py create a src distribution
                     '--dist-dir=%s'%self.build_dir] #put it into build folder
        Popen(sdist_cmd, stdout=PIPE, stderr=PIPE).communicate()


        print "*Placing pymt source distribution in portable context"
        src_dist = os.path.join(self.build_dir,self.dist_name)
        zf = ZipFile(src_dist+'.zip')
        zf.extractall(self.build_dir)
        zf.close()
        if self.no_mingw or self.no_cext:
            print "*Skipping C Extension build (either --no_cext or --no_mingw option set)"
        else:
            print "*Compiling C Extensions inplace for portable distribution"
            cext_cmd = [sys.executable, #path to python.exe
                        'setup.py',
                        'build_ext', #make setup.py create a src distribution
                        '--inplace'] #do it inplace
            #this time it runs teh setup.py inside the source distribution
            #thats has been generated inside the build dir (to generate ext
            #for teh target, instead of the source were building from)
            Popen(cext_cmd, cwd=src_dist, stdout=PIPE, stderr=PIPE).communicate()


        print "\nFinalizing pymt portable distribution..."
        print "---------------------------------------"
        print "*Copying scripts and resources"
        #copy launcher script and readme to portable root dir/build dir
        pymt_bat = os.path.join(src_dist,'pymt','tools','packaging','win32', 'pymt.bat')
        shutil.copy(pymt_bat, os.path.join(self.build_dir, 'pymt.bat'))
        readme = os.path.join(src_dist,'pymt','tools','packaging','win32', 'README.txt')
        shutil.copy(readme, os.path.join(self.build_dir, 'README.txt'))
        #rename pymt directory to "pymt"
        os.rename(src_dist, os.path.join(self.build_dir,'pymt'))

        print "*Removing intermediate file"
        os.remove(os.path.join(self.build_dir,'deps.zip'))
        os.remove(os.path.join(self.build_dir,src_dist+'.zip'))

        print "*Compressing portable distribution target"
        target = os.path.join(self.dist_dir, self.dist_name+"-w32.zip")
        zip_directory(self.build_dir, target)
        print "*Writing target:", target
        print "*Removing build dir"
        shutil.rmtree(self.build_dir, ignore_errors=True)

Example 46

Project: tox
Source File: config.py
View license
@hookimpl
def tox_addoption(parser):
    # formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--version", nargs=0, action=VersionAction,
                        dest="version",
                        help="report version information to stdout.")
    parser.add_argument("-h", "--help", action="store_true", dest="help",
                        help="show help about options")
    parser.add_argument("--help-ini", "--hi", action="store_true", dest="helpini",
                        help="show help about ini-names")
    parser.add_argument("-v", nargs=0, action=CountAction, default=0,
                        dest="verbosity",
                        help="increase verbosity of reporting output.")
    parser.add_argument("--showconfig", action="store_true",
                        help="show configuration information for all environments. ")
    parser.add_argument("-l", "--listenvs", action="store_true",
                        dest="listenvs", help="show list of test environments")
    parser.add_argument("-c", action="store", default="tox.ini",
                        dest="configfile",
                        help="use the specified config file name.")
    parser.add_argument("-e", action="append", dest="env",
                        metavar="envlist",
                        help="work against specified environments (ALL selects all).")
    parser.add_argument("--notest", action="store_true", dest="notest",
                        help="skip invoking test commands.")
    parser.add_argument("--sdistonly", action="store_true", dest="sdistonly",
                        help="only perform the sdist packaging activity.")
    parser.add_argument("--installpkg", action="store", default=None,
                        metavar="PATH",
                        help="use specified package for installation into venv, instead of "
                             "creating an sdist.")
    parser.add_argument("--develop", action="store_true", dest="develop",
                        help="install package in the venv using 'setup.py develop' via "
                             "'pip -e .'")
    parser.add_argument('-i', action="append",
                        dest="indexurl", metavar="URL",
                        help="set indexserver url (if URL is of form name=url set the "
                             "url for the 'name' indexserver, specifically)")
    parser.add_argument("--pre", action="store_true", dest="pre",
                        help="install pre-releases and development versions of dependencies. "
                             "This will pass the --pre option to install_command "
                             "(pip by default).")
    parser.add_argument("-r", "--recreate", action="store_true",
                        dest="recreate",
                        help="force recreation of virtual environments")
    parser.add_argument("--result-json", action="store",
                        dest="resultjson", metavar="PATH",
                        help="write a json file with detailed information "
                        "about all commands and results involved.")

    # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED.
    parser.add_argument("--hashseed", action="store",
                        metavar="SEED", default=None,
                        help="set PYTHONHASHSEED to SEED before running commands.  "
                             "Defaults to a random integer in the range [1, 4294967295] "
                             "([1, 1024] on Windows). "
                             "Passing 'noset' suppresses this behavior.")
    parser.add_argument("--force-dep", action="append",
                        metavar="REQ", default=None,
                        help="Forces a certain version of one of the dependencies "
                             "when configuring the virtual environment. REQ Examples "
                             "'pytest<2.7' or 'django>=1.6'.")
    parser.add_argument("--sitepackages", action="store_true",
                        help="override sitepackages setting to True in all envs")
    parser.add_argument("--skip-missing-interpreters", action="store_true",
                        help="don't fail tests for missing interpreters")
    parser.add_argument("--workdir", action="store",
                        dest="workdir", metavar="PATH", default=None,
                        help="tox working directory")

    parser.add_argument("args", nargs="*",
                        help="additional arguments available to command positional substitution")

    parser.add_testenv_attribute(
        name="envdir", type="path", default="{toxworkdir}/{envname}",
        help="set venv directory -- be very careful when changing this as tox "
             "will remove this directory when recreating an environment")

    # add various core venv interpreter attributes
    def setenv(testenv_config, value):
        setenv = value
        config = testenv_config.config
        if "PYTHONHASHSEED" not in setenv and config.hashseed is not None:
            setenv['PYTHONHASHSEED'] = config.hashseed
        return setenv

    parser.add_testenv_attribute(
        name="setenv", type="dict_setenv", postprocess=setenv,
        help="list of X=Y lines with environment variable settings")

    def basepython_default(testenv_config, value):
        if value is None:
            for f in testenv_config.factors:
                if f in default_factors:
                    return default_factors[f]
            return sys.executable
        return str(value)

    parser.add_testenv_attribute(
        name="basepython", type="string", default=None, postprocess=basepython_default,
        help="executable name or path of interpreter used to create a "
             "virtual test environment.")

    parser.add_testenv_attribute(
        name="envtmpdir", type="path", default="{envdir}/tmp",
        help="venv temporary directory")

    parser.add_testenv_attribute(
        name="envlogdir", type="path", default="{envdir}/log",
        help="venv log directory")

    def downloadcache(testenv_config, value):
        if value:
            # env var, if present, takes precedence
            downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", value)
            return py.path.local(downloadcache)

    parser.add_testenv_attribute(
        name="downloadcache", type="string", default=None, postprocess=downloadcache,
        help="(deprecated) set PIP_DOWNLOAD_CACHE.")

    parser.add_testenv_attribute(
        name="changedir", type="path", default="{toxinidir}",
        help="directory to change to when running commands")

    parser.add_testenv_attribute_obj(PosargsOption())

    parser.add_testenv_attribute(
        name="skip_install", type="bool", default=False,
        help="Do not install the current package. This can be used when "
             "you need the virtualenv management but do not want to install "
             "the current package")

    parser.add_testenv_attribute(
        name="ignore_errors", type="bool", default=False,
        help="if set to True all commands will be executed irrespective of their "
             "result error status.")

    def recreate(testenv_config, value):
        if testenv_config.config.option.recreate:
            return True
        return value

    parser.add_testenv_attribute(
        name="recreate", type="bool", default=False, postprocess=recreate,
        help="always recreate this test environment.")

    def passenv(testenv_config, value):
        # Flatten the list to deal with space-separated values.
        value = list(
            itertools.chain.from_iterable(
                [x.split(' ') for x in value]))

        passenv = set(["PATH", "PIP_INDEX_URL", "LANG", "LD_LIBRARY_PATH"])

        # read in global passenv settings
        p = os.environ.get("TOX_TESTENV_PASSENV", None)
        if p is not None:
            passenv.update(x for x in p.split() if x)

        # we ensure that tmp directory settings are passed on
        # we could also set it to the per-venv "envtmpdir"
        # but this leads to very long paths when run with jenkins
        # so we just pass it on by default for now.
        if sys.platform == "win32":
            passenv.add("SYSTEMDRIVE")  # needed for pip6
            passenv.add("SYSTEMROOT")   # needed for python's crypto module
            passenv.add("PATHEXT")      # needed for discovering executables
            passenv.add("COMSPEC")      # needed for distutils cygwincompiler
            passenv.add("TEMP")
            passenv.add("TMP")
        else:
            passenv.add("TMPDIR")
        for spec in value:
            for name in os.environ:
                if fnmatchcase(name.upper(), spec.upper()):
                    passenv.add(name)
        return passenv

    parser.add_testenv_attribute(
        name="passenv", type="line-list", postprocess=passenv,
        help="environment variables needed during executing test commands "
             "(taken from invocation environment). Note that tox always "
             "passes through some basic environment variables which are "
             "needed for basic functioning of the Python system. "
             "See --showconfig for the eventual passenv setting.")

    parser.add_testenv_attribute(
        name="whitelist_externals", type="line-list",
        help="each lines specifies a path or basename for which tox will not warn "
             "about it coming from outside the test environment.")

    parser.add_testenv_attribute(
        name="platform", type="string", default=".*",
        help="regular expression which must match against ``sys.platform``. "
             "otherwise testenv will be skipped.")

    def sitepackages(testenv_config, value):
        return testenv_config.config.option.sitepackages or value

    parser.add_testenv_attribute(
        name="sitepackages", type="bool", default=False, postprocess=sitepackages,
        help="Set to ``True`` if you want to create virtual environments that also "
             "have access to globally installed packages.")

    def pip_pre(testenv_config, value):
        return testenv_config.config.option.pre or value

    parser.add_testenv_attribute(
        name="pip_pre", type="bool", default=False, postprocess=pip_pre,
        help="If ``True``, adds ``--pre`` to the ``opts`` passed to "
             "the install command. ")

    def develop(testenv_config, value):
        option = testenv_config.config.option
        return not option.installpkg and (value or option.develop)

    parser.add_testenv_attribute(
        name="usedevelop", type="bool", postprocess=develop, default=False,
        help="install package in develop/editable mode")

    parser.add_testenv_attribute_obj(InstallcmdOption())

    parser.add_testenv_attribute(
        name="list_dependencies_command",
        type="argv",
        default="pip freeze",
        help="list dependencies for a virtual environment")

    parser.add_testenv_attribute_obj(DepOption())

    parser.add_testenv_attribute(
        name="commands", type="argvlist", default="",
        help="each line specifies a test command and can use substitution.")

    parser.add_testenv_attribute(
        "ignore_outcome", type="bool", default=False,
        help="if set to True a failing result of this testenv will not make "
             "tox fail, only a warning will be produced")

    parser.add_testenv_attribute(
        "extras", type="line-list",
        help="list of extras to install with the source distribution or "
             "develop install")

Example 47

Project: WAPT
Source File: CGIHTTPServer.py
View license
    def run_cgi(self):
        """Execute a CGI script."""
        dir, rest = self.cgi_info
        path = dir + '/' + rest
        i = path.find('/', len(dir)+1)
        while i >= 0:
            nextdir = path[:i]
            nextrest = path[i+1:]

            scriptdir = self.translate_path(nextdir)
            if os.path.isdir(scriptdir):
                dir, rest = nextdir, nextrest
                i = path.find('/', len(dir)+1)
            else:
                break

        # find an explicit query string, if present.
        i = rest.rfind('?')
        if i >= 0:
            rest, query = rest[:i], rest[i+1:]
        else:
            query = ''

        # dissect the part after the directory name into a script name &
        # a possible additional path, to be stored in PATH_INFO.
        i = rest.find('/')
        if i >= 0:
            script, rest = rest[:i], rest[i:]
        else:
            script, rest = rest, ''

        scriptname = dir + '/' + script
        scriptfile = self.translate_path(scriptname)
        if not os.path.exists(scriptfile):
            self.send_error(404, "No such CGI script (%r)" % scriptname)
            return
        if not os.path.isfile(scriptfile):
            self.send_error(403, "CGI script is not a plain file (%r)" %
                            scriptname)
            return
        ispy = self.is_python(scriptname)
        if not ispy:
            if not (self.have_fork or self.have_popen2 or self.have_popen3):
                self.send_error(403, "CGI script is not a Python script (%r)" %
                                scriptname)
                return
            if not self.is_executable(scriptfile):
                self.send_error(403, "CGI script is not executable (%r)" %
                                scriptname)
                return

        # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
        # XXX Much of the following could be prepared ahead of time!
        env = copy.deepcopy(os.environ)
        env['SERVER_SOFTWARE'] = self.version_string()
        env['SERVER_NAME'] = self.server.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PROTOCOL'] = self.protocol_version
        env['SERVER_PORT'] = str(self.server.server_port)
        env['REQUEST_METHOD'] = self.command
        uqrest = urllib.unquote(rest)
        env['PATH_INFO'] = uqrest
        env['PATH_TRANSLATED'] = self.translate_path(uqrest)
        env['SCRIPT_NAME'] = scriptname
        if query:
            env['QUERY_STRING'] = query
        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]
        authorization = self.headers.getheader("authorization")
        if authorization:
            authorization = authorization.split()
            if len(authorization) == 2:
                import base64, binascii
                env['AUTH_TYPE'] = authorization[0]
                if authorization[0].lower() == "basic":
                    try:
                        authorization = base64.decodestring(authorization[1])
                    except binascii.Error:
                        pass
                    else:
                        authorization = authorization.split(':')
                        if len(authorization) == 2:
                            env['REMOTE_USER'] = authorization[0]
        # XXX REMOTE_IDENT
        if self.headers.typeheader is None:
            env['CONTENT_TYPE'] = self.headers.type
        else:
            env['CONTENT_TYPE'] = self.headers.typeheader
        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length
        referer = self.headers.getheader('referer')
        if referer:
            env['HTTP_REFERER'] = referer
        accept = []
        for line in self.headers.getallmatchingheaders('accept'):
            if line[:1] in "\t\n\r ":
                accept.append(line.strip())
            else:
                accept = accept + line[7:].split(',')
        env['HTTP_ACCEPT'] = ','.join(accept)
        ua = self.headers.getheader('user-agent')
        if ua:
            env['HTTP_USER_AGENT'] = ua
        co = filter(None, self.headers.getheaders('cookie'))
        if co:
            env['HTTP_COOKIE'] = ', '.join(co)
        # XXX Other HTTP_* headers
        # Since we're setting the env in the parent, provide empty
        # values to override previously set values
        for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
                  'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
            env.setdefault(k, "")

        self.send_response(200, "Script output follows")

        decoded_query = query.replace('+', ' ')

        if self.have_fork:
            # Unix -- fork as we should
            args = [script]
            if '=' not in decoded_query:
                args.append(decoded_query)
            nobody = nobody_uid()
            self.wfile.flush() # Always flush before forking
            pid = os.fork()
            if pid != 0:
                # Parent
                pid, sts = os.waitpid(pid, 0)
                # throw away additional data [see bug #427345]
                while select.select([self.rfile], [], [], 0)[0]:
                    if not self.rfile.read(1):
                        break
                if sts:
                    self.log_error("CGI script exit status %#x", sts)
                return
            # Child
            try:
                try:
                    os.setuid(nobody)
                except os.error:
                    pass
                os.dup2(self.rfile.fileno(), 0)
                os.dup2(self.wfile.fileno(), 1)
                os.execve(scriptfile, args, env)
            except:
                self.server.handle_error(self.request, self.client_address)
                os._exit(127)

        else:
            # Non Unix - use subprocess
            import subprocess
            cmdline = [scriptfile]
            if self.is_python(scriptfile):
                interp = sys.executable
                if interp.lower().endswith("w.exe"):
                    # On Windows, use python.exe, not pythonw.exe
                    interp = interp[:-5] + interp[-4:]
                cmdline = [interp, '-u'] + cmdline
            if '=' not in query:
                cmdline.append(query)

            self.log_message("command: %s", subprocess.list2cmdline(cmdline))
            try:
                nbytes = int(length)
            except (TypeError, ValueError):
                nbytes = 0
            p = subprocess.Popen(cmdline,
                                 stdin = subprocess.PIPE,
                                 stdout = subprocess.PIPE,
                                 stderr = subprocess.PIPE,
                                 env = env
                                )
            if self.command.lower() == "post" and nbytes > 0:
                data = self.rfile.read(nbytes)
            else:
                data = None
            # throw away additional data [see bug #427345]
            while select.select([self.rfile._sock], [], [], 0)[0]:
                if not self.rfile._sock.recv(1):
                    break
            stdout, stderr = p.communicate(data)
            self.wfile.write(stdout)
            if stderr:
                self.log_error('%s', stderr)
            p.stderr.close()
            p.stdout.close()
            status = p.returncode
            if status:
                self.log_error("CGI script exit status %#x", status)
            else:
                self.log_message("CGI script exited OK")

Example 48

Project: treeio
Source File: installdb.py
View license
    def handle(self, *args, **options):

        initial_db = {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': './initial.db',
            'HOST': '',
            'USER': '',
            'PASSWORD': ''
        }

        db = {}

        dbengine = raw_input(
            'Enter database engine <mysql,postgresql,postgresql_psycopg2,oracle,sqlite3> (defaults to postgres): ')
        if not dbengine:
            dbengine = 'postgresql_psycopg2'
        if dbengine in ('mysql', 'postgresql', 'postgresql_psycopg2', 'oracle', 'sqlite3'):
            dbengine = 'django.db.backends.' + dbengine
        else:
            raise CommandError('Unknown database engine: %s' % dbengine)

        if dbengine.endswith('sqlite3'):
            dbname = raw_input(
                'Enter database name (defaults to treeio.db): ')
            if not dbname:
                dbname = 'treeio.db'
        else:
            dbname = raw_input(
                'Enter database name (defaults to treeio): ')
            if not dbname:
                dbname = 'treeio'

            dbuser = raw_input('Database user (defaults to treeio): ')
            if not dbuser:
                dbuser = 'treeio'

            dbpassword = raw_input('Database password: ')

            dbhost = raw_input('Hostname (defaults to 127.0.0.1): ')
            if not dbhost:
                dbhost = '127.0.0.1'
            dbport = raw_input('Port (empty for default): ')

        self.stdout.write('\n-- Saving database configuration...\n')
        self.stdout.flush()
        settings.CONF.set('db', 'ENGINE', dbengine)
        settings.CONF.set('db', 'NAME', dbname)
        if not dbengine.endswith('sqlite3'):
            settings.CONF.set('db', 'USER', dbuser)
            settings.CONF.set('db', 'PASSWORD', dbpassword)
            settings.CONF.set('db', 'HOST', dbhost)
            settings.CONF.set('db', 'PORT', dbport)

        with open(settings.USER_CONFIG_FILE, 'w') as f:
            settings.CONF.write(f)

        answer = raw_input(
            'Would you like to create the tables (say no to use an existing database) [y/n] (defaults to yes): ')
        if not len(answer):
            answer = True
        else:
            answer = strtobool(answer)

        if answer:
            exit_code = subprocess.call(
                [sys.executable, 'manage.py', 'syncdb', '--all', '--noinput'])
            if not exit_code == 0:
                self.stdout.flush()
                raise CommandError('Failed to install database.')

            exit_code = subprocess.call(
                [sys.executable, 'manage.py', 'migrate', '--all', '--fake', '--noinput', '--no-initial-data'])

            self.stdout.write(
                '\n-- Successfully installed database. \n-- You\'re ready to go!\n\n')

Example 49

Project: cmdln
Source File: test_cmdln.py
View license
    def generate(self):
        """Return executable "expect" code for this spawn-block."""
        # If the python we are running with isn't "python", then update `cmd`
        # accordingly (they all call Python as "python").
        pythonExeName = os.path.basename(sys.executable)
        expect = ["spawn " + self.cmd.replace("python", pythonExeName, 1)]
        interactive = self.options.get("INTERACTIVE", False)
        if interactive:
            prompt = self.options["PROMPT"]
        if sys.platform == "win32":
            eol_expect = r"\n"
            eol_expect_repr = r"\\n"
            eof_expect = r"\032\r" # Ctrl-Z + return
        else:
            eol_expect = r"\r\n"
            eol_expect_repr = r"\\r\\n"
            eof_expect = r"\004" # Ctrl-D

        for line in self.lines:
            if interactive and line.startswith(prompt):
                expect.append(r"""expect {
    -i $spawn_id
    -re "^%s" {}
    default {
        puts stderr {ERROR: expected "%s"}
        puts stderr "       got      \"$expect_out(buffer)\""
        exit 1
    }
}""" % (prompt, prompt))
                input = line[len(prompt):]
                if input in ("^D", "^Z"):
                    #XXX Around the post-10.4 (Tiger) OS X release
                    #    updates for 10.3 this 'expect' started failing.
                    #    Adding the "(....)?" helps. I don't know enough
                    #    Tcl to figure out exactly what those friggin'
                    #    chars are.
                    expect += [r'send "%s"' % eof_expect,
                               r'expect -re "^(....)?%s$"' % eol_expect]
                else:
                    expect += [r'send "%s\r"' % input,
                               r'expect -re "^%s%s"' % (input, eol_expect)]
            else:
                expected = tcl_escape(line)
                if line == "<BLANKLINE>":
                    expected = r"\s*" # a "blank line" can have whitespace
                expect.append(r"""expect {
    -i $spawn_id
    -re {^%s%s} {}
    default {
        puts stderr {ERROR: expected "%s%s"}
        puts stderr "       got      \"$expect_out(buffer)\""
        exit 1
    }
}""" % (expected,
        eol_expect,
        expected.replace('\\', ''),
        eol_expect_repr))

        # Trap EOF for current process and make sure there isn't
        # unexpected trailing output.
        expect.append(r"""expect {
    -i $spawn_id
    eof {
    } -re "^.+$" {
        puts stderr "error: unexpected trailing output: '$expect_out(buffer)'\n"
        exit 1
    } timeout {
        puts stderr {ERROR: timed out waiting for EOF from '%s'}
        exit 1
    }
}""" % self.cmd)

        return '\n'.join(expect)

Example 50

Project: TrustRouter
Source File: sysconfig.py
View license
def get_config_vars(*args):
    """With no arguments, return a dictionary of all configuration
    variables relevant for the current platform.  Generally this includes
    everything needed to build extensions and install both pure modules and
    extensions.  On Unix, this means every variable defined in Python's
    installed Makefile; on Windows and Mac OS it's a much smaller set.

    With arguments, return a list of values that result from looking up
    each argument in the configuration variable dictionary.
    """
    global _config_vars
    if _config_vars is None:
        func = globals().get("_init_" + os.name)
        if func:
            func()
        else:
            _config_vars = {}

        # Normalized versions of prefix and exec_prefix are handy to have;
        # in fact, these are the standard versions used most places in the
        # Distutils.
        _config_vars['prefix'] = PREFIX
        _config_vars['exec_prefix'] = EXEC_PREFIX

        # Convert srcdir into an absolute path if it appears necessary.
        # Normally it is relative to the build directory.  However, during
        # testing, for example, we might be running a non-installed python
        # from a different directory.
        if python_build and os.name == "posix":
            base = os.path.dirname(os.path.abspath(sys.executable))
            if (not os.path.isabs(_config_vars['srcdir']) and
                base != os.getcwd()):
                # srcdir is relative and we are not in the same directory
                # as the executable. Assume executable is in the build
                # directory and make srcdir absolute.
                srcdir = os.path.join(base, _config_vars['srcdir'])
                _config_vars['srcdir'] = os.path.normpath(srcdir)

        if sys.platform == 'darwin':
            kernel_version = os.uname()[2] # Kernel version (8.4.3)
            major_version = int(kernel_version.split('.')[0])

            if major_version < 8:
                # On Mac OS X before 10.4, check if -arch and -isysroot
                # are in CFLAGS or LDFLAGS and remove them if they are.
                # This is needed when building extensions on a 10.3 system
                # using a universal build of python.
                for key in ('LDFLAGS', 'BASECFLAGS',
                        # a number of derived variables. These need to be
                        # patched up as well.
                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
                    flags = _config_vars[key]
                    flags = re.sub('-arch\s+\w+\s', ' ', flags, re.ASCII)
                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
                    _config_vars[key] = flags

            else:

                # Allow the user to override the architecture flags using
                # an environment variable.
                # NOTE: This name was introduced by Apple in OSX 10.5 and
                # is used by several scripting languages distributed with
                # that OS release.

                if 'ARCHFLAGS' in os.environ:
                    arch = os.environ['ARCHFLAGS']
                    for key in ('LDFLAGS', 'BASECFLAGS',
                        # a number of derived variables. These need to be
                        # patched up as well.
                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):

                        flags = _config_vars[key]
                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
                        flags = flags + ' ' + arch
                        _config_vars[key] = flags

    if args:
        vals = []
        for name in args:
            vals.append(_config_vars.get(name))
        return vals
    else:
        return _config_vars