sys.modules

Here are the examples of the python api sys.modules taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: pyunicorn
Source File: mpi.py
View license
    def submit_call(name_to_call, args=(), kwargs={},
                    module="__main__", time_est=1, id=None, slave=None):
        """
        Submit a call for parallel execution.

        If called by the master and slaves are available, the call is submitted
        to a slave for asynchronous execution.

        If called by a slave or if no slaves are available, the call is instead
        executed synchronously on this MPI node.

        **Examples:**

            1. Provide ids and time estimate explicitly:

               .. code-block:: python

                  for n in range(0,10):
                      mpi.submit_call("doit", (n,A[n]), id=n, time_est=n**2)

                  for n in range(0,10):
                      result[n] = mpi.get_result(n)

            2. Use generated ids stored in a list:

               .. code-block:: python

                  for n in range(0,10):
                      ids.append(mpi.submit_call("doit", (n,A[n])))

                  for n in range(0,10):
                      results.append(mpi.get_result(ids.pop()))

            3. Ignore ids altogether:

               .. code-block:: python

                  for n in range(0,10):
                      mpi.submit_call("doit", (n,A[n]))

                  for n in range(0,10):
                      results.append(mpi.get_next_result())

            4. Call a module function and use keyword arguments:

               .. code-block:: python

                  mpi.submit_call("solve", (), {"a":a, "b":b},
                      module="numpy.linalg")

            5. Call a static class method from a package:

               .. code-block:: python

                  mpi.submit_call("Network._get_histogram", (values, n_bins),
                      module="pyunicorn")

              Note that it is module="pyunicorn" and not
              module="pyunicorn.network" here.

        :arg str name_to_call: name of callable object (usually a function or
            static method of a class) as contained in the namespace specified
            by module.
        :arg tuple args: the positional arguments to provide to the callable
            object.  Tuples of length 1 must be written (arg,).  Default: ()
        :arg dict kwargs: the keyword arguments to provide to the callable
            object.  Default: {}
        :arg str module: optional name of the imported module or submodule in
            whose namespace the callable object is contained. For objects
            defined on the script level, this is "__main__", for objects
            defined in an imported package, this is the package name. Must be a
            key of the dictionary sys.modules (check there after import if in
            doubt).  Default: "__main__"
        :arg float time_est: estimated relative completion time for this call;
            used to find a suitable slave. Default: 1
        :type id: object or None
        :arg  id: unique id for this call. Must be a possible dictionary key.
            If None, a random id is assigned and returned. Can be re-used after
            get_result() for this is. Default: None
        :type slave: int > 0 and < mpi.size, or None
        :arg  slave: optional no. of slave to assign the call to. If None, the
            call is assigned to the slave with the smallest current total time
            estimate. Default: None
        :return object: id of call, to be used in get_result().
        """
        if id is None:
            id = numpy.random.uniform()
        if id in assigned:
            raise MPIException("id ", str(id), " already in queue!")
        if slave is not None and am_slave:
            raise MPIException(
                "only the master can use slave= in submit_call()")
        if slave is None or slave < 1 or slave >= size:
            # find slave with least estimated total time:
            slave = numpy.argmin(total_time_est)
        if available:
            # send name to call, args, time_est to slave:
            if _verbose:
                print "MPI master : assigning call with id", id, "to slave", \
                      slave, ":", name_to_call, args, kwargs, "..."
            comm.send((name_to_call, args, kwargs, module, time_est),
                      dest=slave)
        else:
            # do it myself right now:
            slave = 0
            if _verbose:
                print "MPI master : calling", name_to_call, args, kwargs, "..."
            try:
                object_to_call = eval(name_to_call,
                                      sys.modules[module].__dict__)
            except NameError:
                sys.stderr.write(str(sys.modules[module].__dict__.keys()))
                raise
            call_time = time.time()
            results[id] = object_to_call(*args, **kwargs)
            this_time = time.time() - call_time
            n_processed[0] += 1
            total_time[0] = time.time() - start_time
            stats.append({"id": id, "rank": 0,
                          "this_time": this_time,
                          "time_over_est": this_time / time_est,
                          "n_processed": n_processed[0],
                          "total_time": total_time[0]})

        total_time_est[slave] += time_est
        queue.append(id)
        slave_queue[slave].append(id)
        assigned[id] = slave
        return id

Example 2

Project: PokemonGo-Bot-Desktop
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        # XXX (ncoghlan): The following code makes several bogus
        # assumptions regarding the relationship between __file__
        # and a module's real name. See PEP 302 and issue #10845
        # The problem is resolved properly in Python 3.4+, as
        # described in issue #19946

        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name == '__main__':
            # For directory and zipfile execution, we assume an implicit
            # "if __name__ == '__main__':" around the module, and don't
            # rerun the main module code in spawned processes
            main_module = sys.modules['__main__']
            main_module.__file__ = main_path
        elif main_name != 'ipython':
            # Main modules not actually called __main__.py may
            # contain additional code that should still be executed
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 3

Project: pymo
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name != 'ipython':
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 4

Project: python-future
Source File: __init__.py
View license
    def load_module(self, fullname):
        logger.debug('Running load_module for {0}...'.format(fullname))
        if fullname in sys.modules:
            mod = sys.modules[fullname]
        else:
            if self.kind in (imp.PY_COMPILED, imp.C_EXTENSION, imp.C_BUILTIN,
                             imp.PY_FROZEN):
                convert = False
            # elif (self.pathname.startswith(_stdlibprefix)
            #       and 'site-packages' not in self.pathname):
            #     # We assume it's a stdlib package in this case. Is this too brittle?
            #     # Please file a bug report at https://github.com/PythonCharmers/python-future
            #     # if so.
            #     convert = False
            # in theory, other paths could be configured to be excluded here too
            elif any([fullname.startswith(path) for path in self.exclude_paths]):
                convert = False
            elif any([fullname.startswith(path) for path in self.include_paths]):
                convert = True
            else:
                convert = False
            if not convert:
                logger.debug('Excluded {0} from translation'.format(fullname))
                mod = imp.load_module(fullname, *self.found)
            else:
                logger.debug('Autoconverting {0} ...'.format(fullname))
                mod = imp.new_module(fullname)
                sys.modules[fullname] = mod

                # required by PEP 302
                mod.__file__ = self.pathname
                mod.__name__ = fullname
                mod.__loader__ = self

                # This:
                #     mod.__package__ = '.'.join(fullname.split('.')[:-1])
                # seems to result in "SystemError: Parent module '' not loaded,
                # cannot perform relative import" for a package's __init__.py
                # file. We use the approach below. Another option to try is the
                # minimal load_module pattern from the PEP 302 text instead.

                # Is the test in the next line more or less robust than the
                # following one? Presumably less ...
                # ispkg = self.pathname.endswith('__init__.py')
                
                if self.kind == imp.PKG_DIRECTORY:
                    mod.__path__ = [ os.path.dirname(self.pathname) ]
                    mod.__package__ = fullname
                else:
                    #else, regular module
                    mod.__path__ = []
                    mod.__package__ = fullname.rpartition('.')[0]
                    
                try:
                    cachename = imp.cache_from_source(self.pathname)
                    if not os.path.exists(cachename):
                        update_cache = True
                    else:
                        sourcetime = os.stat(self.pathname).st_mtime
                        cachetime = os.stat(cachename).st_mtime
                        update_cache = cachetime < sourcetime
                    # # Force update_cache to work around a problem with it being treated as Py3 code???
                    # update_cache = True
                    if not update_cache:
                        with open(cachename, 'rb') as f:
                            data = f.read()
                            try:
                                code = marshal.loads(data)
                            except Exception:
                                # pyc could be corrupt. Regenerate it
                                update_cache = True
                    if update_cache:
                        if self.found[0]:
                            source = self.found[0].read()
                        elif self.kind == imp.PKG_DIRECTORY:
                            with open(self.pathname) as f:
                                source = f.read()

                        if detect_python2(source, self.pathname):
                            source = self.transform(source)
                            with open('/tmp/futurized_code.py', 'w') as f:
                                f.write('### Futurized code (from %s)\n%s' % 
                                        (self.pathname, source))

                        code = compile(source, self.pathname, 'exec')

                        dirname = os.path.dirname(cachename)
                        if not os.path.exists(dirname):
                            os.makedirs(dirname)
                        try:
                            with open(cachename, 'wb') as f:
                                data = marshal.dumps(code)
                                f.write(data)
                        except Exception:   # could be write-protected
                            pass
                    exec(code, mod.__dict__)
                except Exception as e:
                    # must remove module from sys.modules
                    del sys.modules[fullname]
                    raise # keep it simple

        if self.found[0]:
            self.found[0].close()
        return mod

Example 5

Project: saga-python
Source File: engine.py
View license
    def _load_adaptors (self, inject_registry=None):
        """ Try to load all adaptors that are registered in
            saga.engine.registry.py. This method is called from the
            constructor.  As Engine is a singleton, this method is
            called once after the module is first loaded in any python
            application.

            :param inject_registry: Inject a fake registry. *For unit tests only*.
        """

        # get the engine config options
        global_config     = ruc.getConfig('saga')
        engine_config     = global_config.get_category('saga.engine')
        saga_adaptor_path = engine_config['adaptor_path'].get_value()

        # get the list of adaptors to load
        registry = saga.engine.registry.adaptor_registry

        # add the list of modpaths found in the config options
        for path in saga_adaptor_path.split(':'):
            if path:
                self._logger.debug ("adding   adaptor path: '%s'" % path)
                registry.append(path)

        self._logger.debug ("listing  adaptor registry: %s" % registry)


        # check if some unit test wants to use a special registry.  If
        # so, we reset cpi infos from the earlier singleton creation.
        if inject_registry != None :
            self._adaptor_registry = {}
            registry = inject_registry


        # attempt to load all registered modules
        for module_name in registry:

            self._logger.info ("loading  adaptor %s" % module_name)


            # first, import the module
            adaptor_module = None
            try :
                adaptor_module = __import__ (module_name, fromlist=['Adaptor'])

            except Exception as e:
                self._logger.warn ("Skipping adaptor %s 1: module loading failed: %s" % (module_name, e))
                continue # skip to next adaptor


            # we expect the module to have an 'Adaptor' class
            # implemented, which, on calling 'register()', returns
            # a info dict for all implemented adaptor classes.
            adaptor_instance = None
            adaptor_info     = None

            try:
                adaptor_instance = adaptor_module.Adaptor ()
                adaptor_info     = adaptor_instance.register ()

            except se.SagaException as e:
                self._logger.warn ("Skipping adaptor %s: loading failed: '%s'" % (module_name, e))
                continue # skip to next adaptor

            except Exception as e:
                self._logger.warn ("Skipping adaptor %s: loading failed: '%s'" % (module_name, e))
                continue # skip to next adaptor


            # the adaptor must also provide a sanity_check() method, which sould
            # be used to confirm that the adaptor can function properly in the
            # current runtime environment (e.g., that all pre-requisites and
            # system dependencies are met).
            try:
                adaptor_instance.sanity_check ()

            except Exception as e:
                self._logger.warn ("Skipping adaptor %s: failed self test: %s" % (module_name, e))
                continue # skip to next adaptor


            # check if we have a valid adaptor_info
            if adaptor_info is None :
                self._logger.warning ("Skipping adaptor %s: adaptor meta data are invalid" \
                                   % module_name)
                continue  # skip to next adaptor


            if  not 'name'    in adaptor_info or \
                not 'cpis'    in adaptor_info or \
                not 'version' in adaptor_info or \
                not 'schemas' in adaptor_info    :
                self._logger.warning ("Skipping adaptor %s: adaptor meta data are incomplete" \
                                   % module_name)
                continue  # skip to next adaptor


            adaptor_name    = adaptor_info['name']
            adaptor_version = adaptor_info['version']
            adaptor_schemas = adaptor_info['schemas']
            adaptor_enabled = True   # default unless disabled by 'enabled' option or version filer

            # disable adaptors in 'alpha' or 'beta' versions -- unless
            # the 'load_beta_adaptors' config option is set to True
            if not self._cfg['load_beta_adaptors'].get_value () :

                if 'alpha' in adaptor_version.lower() or \
                   'beta'  in adaptor_version.lower()    :

                    self._logger.warn ("Skipping adaptor %s: beta versions are disabled (%s)" \
                                    % (module_name, adaptor_version))
                    continue  # skip to next adaptor


            # get the 'enabled' option in the adaptor's config
            # section (saga.cpi.base ensures that the option exists,
            # if it is initialized correctly in the adaptor class.
            adaptor_config  = None
            adaptor_enabled = False

            try :
                adaptor_config  = global_config.get_category (adaptor_name)
                adaptor_enabled = adaptor_config['enabled'].get_value ()

            except se.SagaException as e:
                self._logger.warn ("Skipping adaptor %s: initialization failed: %s" % (module_name, e))
                continue # skip to next adaptor
            except Exception as e:
                self._logger.warn ("Skipping adaptor %s: initialization failed: %s" % (module_name, e))
                continue # skip to next adaptor


            # only load adaptor if it is not disabled via config files
            if adaptor_enabled == False :
                self._logger.info ("Skipping adaptor %s: 'enabled' set to False" \
                                % (module_name))
                continue # skip to next adaptor


            # check if the adaptor has anything to register
            if 0 == len (adaptor_info['cpis']) :
                self._logger.warn ("Skipping adaptor %s: does not register any cpis" \
                                % (module_name))
                continue # skip to next adaptor


            # we got an enabled adaptor with valid info - yay!  We can
            # now register all adaptor classes (cpi implementations).
            for cpi_info in adaptor_info['cpis'] :

                # check cpi information details for completeness
                if  not 'type'    in cpi_info or \
                    not 'class'   in cpi_info    :
                    self._logger.info ("Skipping adaptor %s cpi: cpi info detail is incomplete" \
                                    % (module_name))
                    continue # skip to next cpi info


                # adaptor classes are registered for specific API types.
                cpi_type  = cpi_info['type']
                cpi_cname = cpi_info['class']
                cpi_class = None

                try :
                    cpi_class = getattr (adaptor_module, cpi_cname)

                except Exception as e:
                    # this exception likely means that the adaptor does
                    # not call the saga.adaptors.Base initializer (correctly)
                    self._logger.warning ("Skipping adaptor %s: adaptor class invalid %s: %s" \
                                       % (module_name, cpi_info['class'], str(e)))
                    continue # skip to next adaptor

                # make sure the cpi class is a valid cpi for the given type.
                # We walk through the list of known modules, and try to find
                # a modules which could have that class.  We do the following
                # tests:
                #
                #   cpi_class: ShellJobService
                #   cpi_type:  saga.job.Service
                #   modules:   saga.adaptors.cpi.job
                #   modules:   saga.adaptors.cpi.job.service
                #   classes:   saga.adaptors.cpi.job.Service
                #   classes:   saga.adaptors.cpi.job.service.Service
                #
                #   cpi_class: X509Context
                #   cpi_type:  saga.Context
                #   modules:   saga.adaptors.cpi.context
                #   classes:   saga.adaptors.cpi.context.Context
                #
                # So, we add a 'adaptors.cpi' after the 'saga' namespace
                # element, then append the rest of the given namespace.  If that
                # gives a module which has the requested class, fine -- if not,
                # we add a lower cased version of the class name as last
                # namespace element, and check again.

                # ->   saga .  job .  Service
                # <- ['saga', 'job', 'Service']
                cpi_type_nselems = cpi_type.split ('.')

                if  len(cpi_type_nselems) < 2 or \
                    len(cpi_type_nselems) > 3    :
                    self._logger.warn ("Skipping adaptor %s: cpi type not valid: '%s'" \
                                     % (module_name, cpi_type))
                    continue # skip to next cpi info

                if cpi_type_nselems[0] != 'saga' :
                    self._logger.warn ("Skipping adaptor %s: cpi namespace not valid: '%s'" \
                                     % (module_name, cpi_type))
                    continue # skip to next cpi info

                # -> ['saga',                    'job', 'Service']
                # <- ['saga', 'adaptors', 'cpi', 'job', 'Service']
                cpi_type_nselems.insert (1, 'adaptors')
                cpi_type_nselems.insert (2, 'cpi')

                # -> ['saga', 'adaptors', 'cpi', 'job',  'Service']
                # <- ['saga', 'adaptors', 'cpi', 'job'], 'Service'
                cpi_type_cname = cpi_type_nselems.pop ()

                # -> ['saga', 'adaptors', 'cpi', 'job'], 'Service'
                # <-  'saga.adaptors.cpi.job
                # <-  'saga.adaptors.cpi.job.service
                cpi_type_modname_1 = '.'.join (cpi_type_nselems)
                cpi_type_modname_2 = '.'.join (cpi_type_nselems + [cpi_type_cname.lower()])

                # does either module exist?
                cpi_type_modname = None
                if  cpi_type_modname_1 in sys.modules :
                    cpi_type_modname = cpi_type_modname_1

                if  cpi_type_modname_2 in sys.modules :
                    cpi_type_modname = cpi_type_modname_2

                if  not cpi_type_modname :
                    self._logger.warn ("Skipping adaptor %s: cpi type not known: '%s'" \
                                     % (module_name, cpi_type))
                    continue # skip to next cpi info

                # so, make sure the given cpi is actually
                # implemented by the adaptor class
                cpi_ok = False
                for name, cpi_obj in inspect.getmembers (sys.modules[cpi_type_modname]) :
                    if  name == cpi_type_cname      and \
                        inspect.isclass (cpi_obj)       :
                        if  issubclass (cpi_class, cpi_obj) :
                            cpi_ok = True

                if not cpi_ok :
                    self._logger.warn ("Skipping adaptor %s: doesn't implement cpi '%s (%s)'" \
                                     % (module_name, cpi_class, cpi_type))
                    continue # skip to next cpi info


                # finally, register the cpi for all its schemas!
                registered_schemas = list()
                for adaptor_schema in adaptor_schemas:

                    adaptor_schema = adaptor_schema.lower ()

                    # make sure we can register that cpi type
                    if not cpi_type in self._adaptor_registry :
                        self._adaptor_registry[cpi_type] = {}

                    # make sure we can register that schema
                    if not adaptor_schema in self._adaptor_registry[cpi_type] :
                        self._adaptor_registry[cpi_type][adaptor_schema] = []

                    # we register the cpi class, so that we can create
                    # instances as needed, and the adaptor instance,
                    # as that is passed to the cpi class c'tor later
                    # on (the adaptor instance is used to share state
                    # between cpi instances, amongst others)
                    info = {'cpi_cname'        : cpi_cname,
                            'cpi_class'        : cpi_class,
                            'adaptor_name'     : adaptor_name,
                            'adaptor_instance' : adaptor_instance}

                    # make sure this tuple was not registered, yet
                    if info in self._adaptor_registry[cpi_type][adaptor_schema] :

                        self._logger.warn ("Skipping adaptor %s: already registered '%s - %s'" \
                                         % (module_name, cpi_class, adaptor_instance))
                        continue  # skip to next cpi info

                    self._adaptor_registry[cpi_type][adaptor_schema].append(info)
                    registered_schemas.append(str("%s://" % adaptor_schema))

                self._logger.info("Register adaptor %s for %s API with URL scheme(s) %s" %
                                      (module_name,
                                       cpi_type,
                                       registered_schemas))

Example 6

Project: python3-trepan
Source File: cmdproc.py
View license
def print_location(proc_obj):
    """Show where we are. GUI's and front-end interfaces often
    use this to update displays. So it is helpful to make sure
    we give at least some place that's located in a file.
    """
    i_stack = proc_obj.curindex
    if i_stack is None or proc_obj.stack is None:
        return False
    core_obj = proc_obj.core
    dbgr_obj = proc_obj.debugger
    intf_obj = dbgr_obj.intf[-1]

    # Evaluation routines like "exec" don't show useful location
    # info. In these cases, we will use the position before that in
    # the stack.  Hence the looping below which in practices loops
    # once and sometimes twice.
    remapped_file = None
    while i_stack >= 0:
        frame_lineno = proc_obj.stack[i_stack]
        i_stack -= 1
        frame, lineno = frame_lineno

#         # Next check to see that local variable breadcrumb exists and
#         # has the magic dynamic value.
#         # If so, it's us and we don't normally show this.a
#         if 'breadcrumb' in frame.f_locals:
#             if self.run == frame.f_locals['breadcrumb']:
#                 break

        filename = Mstack.frame2file(core_obj, frame, canonic=False)
        if '<string>' == filename and dbgr_obj.eval_string:
            remapped_file = filename
            filename = pyficache.unmap_file(filename)
            if '<string>' == filename:
                remapped = cmdfns.source_tempfile_remap('eval_string',
                                                        dbgr_obj.eval_string)
                pyficache.remap_file(filename, remapped)
                filename = remapped
                pass
            pass
        else:
            m = re.search('^<frozen (.*)>', filename)
            if m and m.group(1) in pyficache.file2file_remap:
                remapped_file = pyficache.file2file_remap[m.group(1)]
                pass
            elif filename in pyficache.file2file_remap:
                remapped_file = pyficache.unmap_file(filename)
                # FIXME: a remapped_file shouldn't be the same as its unmapped version
                if remapped_file == filename:
                    remapped_file = None
                    pass
                pass
            elif m and m.group(1) in sys.modules:
                remapped_file = m.group(1)
                pyficache.remap_file(filename, remapped_file)
            pass

        code = frame.f_code
        fn_name = code.co_name
        last_i  = frame.f_lasti

        opts = {
            'reload_on_change' : proc_obj.settings('reload'),
            'output'           : proc_obj.settings('highlight')
            }

        if 'style' in proc_obj.debugger.settings:
            opts['style'] = proc_obj.settings('style')

        line = pyficache.getline(filename, lineno, opts)
        if not line:
            line = linecache.getline(filename, lineno,
                                     proc_obj.curframe.f_globals)
            if not line:
                m = re.search('^<frozen (.*)>', filename)
                if m and m.group(1):
                    remapped_file = m.group(1)
                    try_module = sys.modules.get(remapped_file)
                    if (try_module and inspect.ismodule(try_module) and
                        hasattr(try_module, '__file__')):
                        remapped_file = sys.modules[remapped_file].__file__
                        pyficache.remap_file(filename, remapped_file)
                        line = linecache.getline(remapped_file, lineno,
                                                 proc_obj.curframe.f_globals)
                    else:
                        remapped_file = m.group(1)
                        filename, line = cmdfns.deparse_getline(code, remapped_file,
                                                                lineno, opts)
                    pass
            pass

        print_source_location_info(intf_obj.msg, filename, lineno, fn_name,
                                   remapped_file = remapped_file,
                                   f_lasti = last_i)

        if line and len(line.strip()) != 0:
            if proc_obj.event:
                print_source_line(intf_obj.msg, lineno, line,
                                  proc_obj.event2short[proc_obj.event])
            pass
        if '<string>' != filename: break
        pass

    if proc_obj.event in ['return', 'exception']:
        val = proc_obj.event_arg
        intf_obj.msg('R=> %s' % proc_obj._saferepr(val))
        pass
    return True

Example 7

Project: rootpy
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 8

Project: root_numpy
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            if sys.version_info[0] >= 3:
                obj.__name__ = funcname
            else:
                obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 9

Project: saxo
Source File: irc.py
View license
    def load(self):
        # Update symlinks
        common.populate(saxo_path, self.base)

        # Load events
        first = not self.events
        self.events.clear()

        def module_exists(name):
            try: imp.find_module(name)
            except ImportError:
                return False
            else: return True

        if first and module_exists("plugins"):
            debug("Warning: a 'plugins' module already exists")

        if first and ("plugins" in sys.modules):
            raise ImportError("'plugins' duplicated")

        # This means we're using plugins as a namespace module
        # Might have to move saxo.path's plugins/ to something else
        # Otherwise it gets unionised into the namespace module
        # if self.base not in sys.path:
        # - not needed, because we clear up below
        sys.path[:0] = [self.base]

        plugins = os.path.join(self.base, "plugins")
        plugins_package = importlib.__import__("plugins")
        if next(iter(plugins_package.__path__)) != plugins:
            # This is very unlikely to happen, because we pushed self.base
            # to the front of sys.path, but perhaps some site configuration
            # or other import mechanism may affect this
            raise ImportError("non-saxo 'plugins' module")

        setups = {}
        for name in os.listdir(plugins):
            if ("_" in name) or (not name.endswith(".py")):
                continue

            name = "plugins." + name[:-3]
            if not name in sys.modules:
                try: module = importlib.import_module(name)
                except Exception as err:
                    debug("Error loading %s:" % name, err)
            elif first:
                raise ImportError("%r duplicated" % name)
            else:
                module = sys.modules[name]
                try: module = imp.reload(module)
                except Exception as err:
                    debug("Error reloading %s:" % name, err)

            for attr in dir(module):
                obj = getattr(module, attr)

                if hasattr(obj, "saxo_event"):
                    try: self.events[obj.saxo_event].append(obj)
                    except KeyError:
                        self.events[obj.saxo_event] = [obj]

                elif hasattr(obj, "saxo_setup"):
                    obj.saxo_name = module.__name__ + "." + obj.__name__
                    setups[obj.saxo_name] = obj

            # debug("Loaded module:", name)

        debug("%s setup functions" % len(setups))

        graph = {}
        for setup in setups.values():
            deps = ["plugins." + dep for dep in setup.saxo_deps]
            graph[setup.saxo_name] = deps

        database_filename = os.path.join(self.base, "database.sqlite3")
        with sqlite.Database(database_filename) as self.db:
            for name in common.tsort(graph):
                debug(name)
                if name in setups:
                    setups[name](self)
                else:
                    debug("Warning: Missing dependency:", name)

        sys.path[:1] = []

Example 10

Project: scikit-rf
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read

    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])

    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']

    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1

        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 11

Project: trackpy
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read

    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])

    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError:
                    break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']

    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0:
            return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0:
                return x
            if a.attrib['id'] in b_bases:
                return -1
            if b.attrib['id'] in a_bases:
                return 1

        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n":
            doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent:
                break
            if parent in object_cache:
                break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object):
                pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None:
            continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 12

Project: spectralDNS
Source File: MPI_knee.py
View license
def get_parent(globals, level):
    """
    parent, name = get_parent(globals, level)

    Return the package that an import is being performed in.  If globals comes
    from the module foo.bar.bat (not itself a package), this returns the
    sys.modules entry for foo.bar.  If globals is from a package's __init__.py,
    the package's entry in sys.modules is returned.

    If globals doesn't come from a package or a module in a package, or a
    corresponding entry is not found in sys.modules, None is returned.
    """
    orig_level = level

    if not level or not isinstance(globals, dict):
        return None, ''

    pkgname = globals.get('__package__', None)

    if pkgname is not None:
        # __package__ is set, so use it
        if not hasattr(pkgname, 'rindex'):
            raise ValueError('__package__ set to non-string')
        if len(pkgname) == 0:
            if level > 0:
                raise ValueError('Attempted relative import in non-package')
            return None, ''
        name = pkgname
    else:
        # __package__ not set, so figure it out and set it
        if '__name__' not in globals:
            return None, ''
        modname = globals['__name__']

        if '__path__' in globals:
            # __path__ is set, so modname is already the package name
            globals['__package__'] = name = modname
        else:
            # Normal module, so work out the package name if any
            lastdot = modname.rfind('.')
            if lastdot < 0 and level > 0:
                raise ValueError("Attempted relative import in non-package")
            if lastdot < 0:
                globals['__package__'] = None
                return None, ''
            globals['__package__'] = name = modname[:lastdot]

    dot = len(name)
    for x in xrange(level, 1, -1):
        try:
            dot = name.rindex('.', 0, dot)
        except ValueError:
            raise ValueError("attempted relative import beyond top-level "
                             "package")
    name = name[:dot]

    try:
        parent = sys.modules[name]
    except:
        if orig_level < 1:
            warn("Parent module '%.200s' not found while handling absolute "
                 "import" % name)
            parent = None
        else:
            raise SystemError("Parent module '%.200s' not loaded, cannot "
                              "perform relative import" % name)

    # We expect, but can't guarantee, if parent != None, that:
    # - parent.__name__ == name
    # - parent.__dict__ is globals
    # If this is violated...  Who cares?
    return parent, name

Example 13

Project: deblaze
Source File: imports.py
View license
def lazyModule(modname, relativePath=None):
    """
    Return module 'modname', but with its contents loaded "on demand"

    This function returns 'sys.modules[modname]', if present.  Otherwise
    it creates a 'LazyModule' object for the specified module, caches it
    in 'sys.modules', and returns it.

    'LazyModule' is a subclass of the standard Python module type, that
    remains empty until an attempt is made to access one of its
    attributes.  At that moment, the module is loaded into memory, and
    any hooks that were defined via 'whenImported()' are invoked.

    Note that calling 'lazyModule' with the name of a non-existent or
    unimportable module will delay the 'ImportError' until the moment
    access is attempted.  The 'ImportError' will occur every time an
    attribute access is attempted, until the problem is corrected.

    This function also takes an optional second parameter, 'relativePath',
    which will be interpreted as a '/'-separated path string relative to
    'modname'.  If a 'relativePath' is supplied, the module found by
    traversing the path will be loaded instead of 'modname'.  In the path,
    '.' refers to the current module, and '..' to the current module's
    parent.  For example::

        fooBaz = lazyModule('foo.bar','../baz')

    will return the module 'foo.baz'.  The main use of the 'relativePath'
    feature is to allow relative imports in modules that are intended for
    use with module inheritance.  Where an absolute import would be carried
    over as-is into the inheriting module, an import relative to '__name__'
    will be relative to the inheriting module, e.g.::

        something = lazyModule(__name__,'../path/to/something')

    The above code will have different results in each module that inherits
    it.

    (Note: 'relativePath' can also be an absolute path (starting with '/');
    this is mainly useful for module '__bases__' lists.)
    """
    if relativePath:
        modname = joinPath(modname, relativePath)

    if modname not in sys.modules:
        file_name = path = None

        if '.' in modname:
            splitpos = modname.rindex('.')

            parent = sys.modules[modname[:splitpos]]
            file_name = find_module(modname[splitpos + 1:], parent.__path__)[1]
        else:
            file_name = find_module(modname)[1]

        if os.path.isdir(file_name):
            path = [file_name]
            py = os.path.join(file_name, '__init__')

            for full in ('.pyo', '.pyc', '.py'):
                full = py + full

                if os.path.exists(full):
                    break
            else:
                raise ImportError('No module name %d' % modname)

            file_name = full

        getModuleHooks(modname) # force an empty hook list into existence
        sys.modules[modname] = LazyModule(modname, file_name, path)

        if '.' in modname:
            # ensure parent module/package is in sys.modules
            # and parent.modname=module, as soon as the parent is imported

            splitpos = modname.rindex('.')

            whenImported(
                modname[:splitpos],
                lambda m: setattr(m, modname[splitpos + 1:], sys.modules[modname])
            )

    return sys.modules[modname]

Example 14

Project: pystan
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            if sys.version_info[0] >= 3:
                obj.__name__ = funcname
            else:
                obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 15

Project: tensorly
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            if sys.version_info[0] >= 3:
                obj.__name__ = funcname
            else:
                obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 16

Project: rpyc
Source File: vinegar.py
View license
def load(val, import_custom_exceptions, instantiate_custom_exceptions, instantiate_oldstyle_exceptions):
    """
    Loads a dumped exception (the tuple returned by :func:`dump`) info a 
    throwable exception object. If the exception cannot be instantiated for any
    reason (i.e., the security parameters do not allow it, or the exception 
    class simply doesn't exist on the local machine), a :class:`GenericException`
    instance will be returned instead, containing all of the original exception's
    details.
    
    :param val: the dumped exception
    :param import_custom_exceptions: whether to allow this function to import custom modules 
                                     (imposes a security risk)
    :param instantiate_custom_exceptions: whether to allow this function to instantiate "custom 
                                          exceptions" (i.e., not one of the built-in exceptions,
                                          such as ``ValueError``, ``OSError``, etc.)
    :param instantiate_oldstyle_exceptions: whether to allow this function to instantiate exception 
                                            classes that do not derive from ``BaseException``.
                                            This is required to support old-style exceptions. 
                                            Not applicable for Python 3 and above.
    
    :returns: A throwable exception object
    """
    if val == consts.EXC_STOP_ITERATION:
        return StopIteration # optimization
    if type(val) is str:
        return val # deprecated string exceptions

    (modname, clsname), args, attrs, tbtext = val
    if import_custom_exceptions and modname not in sys.modules:
        try:
            __import__(modname, None, None, "*")
        except Exception:
            pass
    
    if instantiate_custom_exceptions:
        if modname in sys.modules:
            cls = getattr(sys.modules[modname], clsname, None)
        else:
            cls = None
    elif modname == exceptions_module.__name__:
        cls = getattr(exceptions_module, clsname, None)
    else:
        cls = None

    if is_py3k:
        if not isinstance(cls, type) or not issubclass(cls, BaseException):
            cls = None
    else:
        if not isinstance(cls, (type, ClassType)):
            cls = None
        elif issubclass(cls, ClassType) and not instantiate_oldstyle_exceptions:
            cls = None
        elif not issubclass(cls, BaseException):
            cls = None

    if cls is None:
        fullname = "%s.%s" % (modname, clsname)
        if fullname not in _generic_exceptions_cache:
            fakemodule = {"__module__" : "%s/%s" % (__name__, modname)}
            if isinstance(GenericException, ClassType):
                _generic_exceptions_cache[fullname] = ClassType(fullname, (GenericException,), fakemodule)
            else:
                _generic_exceptions_cache[fullname] = type(fullname, (GenericException,), fakemodule)
        cls = _generic_exceptions_cache[fullname]

    cls = _get_exception_class(cls)
    
    # support old-style exception classes
    if ClassType is not type and isinstance(cls, ClassType):
        exc = InstanceType(cls)
    else:
        exc = cls.__new__(cls)

    exc.args = args
    for name, attrval in attrs:
        setattr(exc, name, attrval)
    exc._remote_tb = tbtext
    return exc

Example 17

Project: WAPT
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name != 'ipython':
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 18

Project: python-haystack
Source File: types.py
View license
    def __set_pointer(self):
        # TODO: c_char_p ?
        # if host pointersize is same as target, keep ctypes pointer function.
        if self.sizeof(self.__real_ctypes.c_void_p) == self.__pointersize:
            # use the same pointer cache
            self._pointer_type_cache = self.__real_ctypes._pointer_type_cache
            # see __init__
            # pylint: disable=access-member-before-definition
            self.__ptrt = self.POINTER(self.c_byte).__bases__[0]
            return
        # get the replacement type.
        if self.__pointersize == 4:
            replacement_type = self.__real_ctypes.c_uint32
            replacement_type_char = self.__real_ctypes.c_uint32._type_
        elif self.__pointersize == 8:
            replacement_type = self.__real_ctypes.c_uint64
            replacement_type_char = self.__real_ctypes.c_uint64._type_
        else:
            raise NotImplementedError('pointer size of %d is not handled' % self.__pointersize)
        POINTERSIZE = self.__pointersize
        # required to access _ctypes
        import _ctypes
        # Emulate a pointer class using the approriate c_int32/c_int64 type
        # The new class should have :
        # ['__module__', 'from_param', '_type_', '__dict__', '__weakref__', '__doc__']
        my_ctypes = self
        # special class for c_void_p

        class _T_Simple(_ctypes._SimpleCData,):
            _type_ = replacement_type_char

            @property
            def _sub_addr_(myself):
                return myself.value

            def __init__(myself, value):
                myself.value = value

            def __repr__(myself):
                return '%s(%d)' % (type(myself).__name__, myself.value)
        self._T_Simple = _T_Simple

        def POINTER_T(pointee):
            if pointee in my_ctypes._pointer_type_cache:
                return my_ctypes._pointer_type_cache[pointee]
            # specific case for c_void_p
            subtype = pointee
            if pointee is None:  # VOID pointer type. c_void_p.
                clsname = 'LP_%d_c_void_p' % POINTERSIZE
                _class = type(clsname, (_T_Simple,), {})
                _class._subtype_ = type(None)
                my_ctypes._pointer_type_cache[pointee] = _class
                # additionnaly register this type in this module fo pickling
                setattr(sys.modules[__name__], clsname, _class)
                return _class

            clsname = pointee.__name__
            # template that creates a PointerType to pointee (clsname *)
            # we have to fake the size of the structure to
            # replacement_type_char's size.
            # so we replace _type_ with the fake type of the expected size.
            # and we had _subtype_ that will be queried by our helper
            # functions.

            class _T(_T_Simple,):
                _subtype_ = subtype  # could use _pointer_type_cache

                def __repr__(myself):
                    return '%s(%d)' % (type(myself).__name__, myself.value)

                @property
                def contents(myself):
                    return myself._subtype_.from_address(myself.value)
                    # raise TypeError('This is not a ctypes pointer.')

                def __init__(myself, _value=None):
                    if _value is None:
                        myself.value = 0
                        return
                    if not isinstance(_value, subtype):
                        raise TypeError('%s expected, not %s' % (subtype, type(_value)))
                    myself.value = my_ctypes.addressof(_value)
                    # raise TypeError('This is not a ctypes pointer.')

            _class = type('LP_%d_%s' % (POINTERSIZE, clsname), (_T,), {})
            my_ctypes._pointer_type_cache[pointee] = _class
            # additionally register this type in this module fo pickling
            setattr(sys.modules[__name__], clsname, _class)
            return _class
        # end of POINTER_T
        self.POINTER = POINTER_T
        self.__ptrt = self._T_Simple
        self._pointer_type_cache.clear()
        self.c_void_p = self.POINTER(None)
        # c_void_p is a simple type
        # self.c_void_p = type('c_void_p', (_T_Simple,),{})
        # other are different
        self.c_char_p = self.POINTER(self.c_char)
        self.c_wchar_p = self.POINTER(self.c_wchar)

        setattr(sys.modules[__name__], 'c_void_p', self.c_void_p)
        setattr(sys.modules[__name__], 'c_char_p', self.c_char_p)
        setattr(sys.modules[__name__], 'c_wchar_p', self.c_wchar_p)

        # set the casting function
        self.cast = self.__cast
        return

Example 19

Project: TrustRouter
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        # XXX (ncoghlan): The following code makes several bogus
        # assumptions regarding the relationship between __file__
        # and a module's real name. See PEP 302 and issue #10845
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name == '__main__':
            main_module = sys.modules['__main__']
            main_module.__file__ = main_path
        elif main_name != 'ipython':
            # Main modules not actually called __main__.py may
            # contain additional code that should still be executed
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in list(main_module.__dict__.values()):
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 20

Project: multiprocess
Source File: forking.py
View license
    def prepare(name, mainpath, sys_path, sys_argv, authkey,
                cur_dir, orig_dir, log_args):
        '''
        Try to get this process ready to unpickle process object
        '''
        global original_main_module

        original_main_module = sys.modules['__main__']
        processing.currentProcess().setName(name)
        processing.currentProcess().setAuthKey(authkey)

        if log_args is not None:
            from processing.logger import enableLogging
            enableLogging(*log_args)

        if orig_dir is not None:
            processing.ORIGINAL_DIR = orig_dir

        if cur_dir is not None:
            try:
                os.chdir(cur_dir)
            except OSError:
                raise

        if sys_path is not None:
            sys.path = sys_path

        if mainpath is not None:
            mainname = splitext(basename(mainpath))[0]
            if mainname == '__init__':
                mainname = basename(dirname(mainpath))

            if not mainpath.lower().endswith('.exe') and mainname != 'ipython':
                if mainpath is None:
                    dirs = None
                elif basename(mainpath).startswith('__init__.py'):
                    dirs = [dirname(dirname(mainpath))]
                else:
                    dirs = [dirname(mainpath)]

                assert mainname not in sys.modules, mainname
                file, pathname, etc = imp.find_module(mainname, dirs)
                try:
                    # We would like to do "imp.load_module('__main__', ...)"
                    # here.  However, that would cause 'if __name__ ==
                    # "__main__"' clauses to be executed.
                    main_module = imp.load_module(
                        '__parents_main__', file, pathname, etc
                        )
                finally:
                    if file:
                        file.close()

                sys.modules['__main__'] = main_module
                main_module.__name__ = '__main__'

                # XXX Try to make the potentially picklable objects in
                # sys.modules['__main__'] realize they are in the main
                # module -- ugly
                for obj in main_module.__dict__.values():
                    try:
                        if obj.__module__ == '__parents_main__':
                            obj.__module__ = '__main__'
                    except (KeyboardInterrupt, SystemExit):
                        raise
                    except:
                        pass

        if sys_argv is not None:            # this needs to come last 
            sys.argv = sys_argv

Example 21

Project: multiprocess
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        # XXX (ncoghlan): The following code makes several bogus
        # assumptions regarding the relationship between __file__
        # and a module's real name. See PEP 302 and issue #10845
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name == '__main__':
            main_module = sys.modules['__main__']
            main_module.__file__ = main_path
        elif main_name != 'ipython':
            # Main modules not actually called __main__.py may
            # contain additional code that should still be executed
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in list(main_module.__dict__.values()):
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 22

Project: CommPy
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 23

Project: compoundpi
Source File: binding_helper.py
View license
def _load_pyside(required_modules, optional_modules):
    os.environ['QT_API'] = 'pyside'

    for module_name in required_modules:
        _named_import('PySide.%s' % module_name)
    for module_name in optional_modules:
        _named_optional_import('PySide.%s' % module_name)

    # set some names for compatibility with PyQt4
    sys.modules['PySide.QtCore'].pyqtSignal   = sys.modules['PySide.QtCore'].Signal
    sys.modules['PySide.QtCore'].pyqtSlot     = sys.modules['PySide.QtCore'].Slot
    sys.modules['PySide.QtCore'].pyqtProperty = sys.modules['PySide.QtCore'].Property

    try:
        import PySideQwt
        _register_binding_module('Qwt', PySideQwt)
    except ImportError:
        pass

    global _loadUi

    def _loadUi(uifile, baseinstance=None, custom_widgets=None):
        from PySide.QtUiTools import QUiLoader
        from PySide.QtCore import QMetaObject
        from PySide.QtGui import QDialog

        class CustomUiLoader(QUiLoader):
            class_aliases = {
                'Line': 'QFrame',
            }

            def __init__(self, baseinstance=None, custom_widgets=None):
                super(CustomUiLoader, self).__init__(baseinstance)
                self._base_instance = baseinstance
                self._custom_widgets = custom_widgets or {}

            def createWidget(self, class_name, parent=None, name=''):
                # don't create the top-level widget, if a base instance is set
                if self._base_instance and not parent:
                    return self._base_instance

                if class_name in self._custom_widgets:
                    widget = self._custom_widgets[class_name](parent)
                else:
                    widget = QUiLoader.createWidget(self, class_name, parent, name)

                if str(type(widget)).find(self.class_aliases.get(class_name, class_name)) < 0:
                    sys.modules['QtCore'].qDebug(
                            str('PySide.loadUi(): could not find widget '
                            'class "%s", defaulting to "%s"' % (
                                class_name, type(widget)))
                            )

                if self._base_instance:
                    setattr(self._base_instance, name, widget)

                return widget

        loader = CustomUiLoader(baseinstance)
        custom_widgets = custom_widgets or {}
        for custom_widget in custom_widgets.values():
            loader.registerCustomWidget(custom_widget)

        ui = loader.load(uifile)
        QMetaObject.connectSlotsByName(ui)
        # Workaround: PySide doesn't automatically center dialogs on their
        # parent windows
        if isinstance(baseinstance, QDialog) and (baseinstance.parentWidget() is not None):
            r = baseinstance.frameGeometry()
            r.moveCenter(baseinstance.parentWidget().frameGeometry().center())
            baseinstance.move(r.topLeft())
        return ui

    import PySide
    return PySide.__version__

Example 24

Project: stash
Source File: pip.py
View license
def fake_setuptools_modules():
    """
    Created a bunch of stub setuptools modules
    """
    setuptools_modules = [
        'setuptools',
        'setuptools.command',
        'setuptools.command.alias',
        'setuptools.command.bdist_egg',
        'setuptools.command.bdist_rpm',
        'setuptools.command.bdist_wininst',
        'setuptools.command.build_ext',
        'setuptools.command.build_py',
        'setuptools.command.develop',
        'setuptools.command.easy_install',
        'setuptools.command.egg_info',
        'setuptools.command.install',
        'setuptools.depends.install_egg_info',
        'setuptools.command.install_lib',
        'setuptools.command.install_scripts',
        'setuptools.command.register',
        'setuptools.command.rotate',
        'setuptools.command.saveopts',
        'setuptools.command.sdist',
        'setuptools.command.setopt',
        'setuptools.command.test',
        'setuptools.command.upload',
        'setuptools.command.upload_docs',
        'setuptools.extern',
        'setuptools.dist',
        'setuptools.extension',
        'setuptools.launch',
        'setuptools.lib2to3_ex',
        'setuptools.msvc9_support',
        'setuptools.package_index',
        'setuptools.py26compat',
        'setuptools.py27compat',
        'setuptools.py31compat',
        'setuptools.sandbox',
        'setuptools.site-patch',
        'setuptools.ssl_support',
        'setuptools.unicode_utils',
        'setuptools.utils',
        'setuptools.version',
        'setuptools.windows_support',
        # 'pkg_resources',
        # 'pkg_resources.extern',
    ]

    for m in setuptools_modules:
        fake_module(m)

    # First import importable distutils
    import distutils.command
    import distutils.core
    import distutils.util
    distutils_command_modules = [
        'distutils.command.bdist'
        'distutils.command.bdist_dumb',
        'distutils.command.bdist_msi',
        'distutils.command.bdist_rpm',
        'distutils.command.bdist_wininst',
        'distutils.command.build',
        'distutils.command.build_clib',
        'distutils.command.build_ext',
        'distutils.command.build_py',
        'distutils.command.build_scripts',
    ]
    for m in distutils_command_modules:
        fake_module(m)
    sys.modules['distutils.util'].get_platform = OmniClass()
    # fix for new problem in issue 169 
    sys.modules['distutils.command.build_ext'].sub_commands = []
    sys.modules['setuptools.command.build_ext'].sub_commands = []

Example 25

Project: acoular
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            if sys.version_info[0] >= 3:
                obj.__name__ = funcname
            else:
                obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 26

Project: PythonScript
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name != 'ipython':
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 27

Project: brython
Source File: test_frozen.py
View license
    def test_frozen(self):
        with captured_stdout() as stdout:
            try:
                import __hello__
            except ImportError as x:
                self.fail("import __hello__ failed:" + str(x))
            self.assertEqual(__hello__.initialized, True)
            expect = set(self.module_attrs)
            expect.add('initialized')
            self.assertEqual(set(dir(__hello__)), expect)
            self.assertEqual(stdout.getvalue(), 'Hello world!\n')

        with captured_stdout() as stdout:
            try:
                import __phello__
            except ImportError as x:
                self.fail("import __phello__ failed:" + str(x))
            self.assertEqual(__phello__.initialized, True)
            expect = set(self.package_attrs)
            expect.add('initialized')
            if not "__phello__.spam" in sys.modules:
                self.assertEqual(set(dir(__phello__)), expect)
            else:
                expect.add('spam')
                self.assertEqual(set(dir(__phello__)), expect)
            self.assertEqual(__phello__.__path__, [__phello__.__name__])
            self.assertEqual(stdout.getvalue(), 'Hello world!\n')

        with captured_stdout() as stdout:
            try:
                import __phello__.spam
            except ImportError as x:
                self.fail("import __phello__.spam failed:" + str(x))
            self.assertEqual(__phello__.spam.initialized, True)
            spam_expect = set(self.module_attrs)
            spam_expect.add('initialized')
            self.assertEqual(set(dir(__phello__.spam)), spam_expect)
            phello_expect = set(self.package_attrs)
            phello_expect.add('initialized')
            phello_expect.add('spam')
            self.assertEqual(set(dir(__phello__)), phello_expect)
            self.assertEqual(stdout.getvalue(), 'Hello world!\n')

        try:
            import __phello__.foo
        except ImportError:
            pass
        else:
            self.fail("import __phello__.foo should have failed")

            try:
                import __phello__.foo
            except ImportError:
                pass
            else:
                self.fail("import __phello__.foo should have failed")

        del sys.modules['__hello__']
        del sys.modules['__phello__']
        del sys.modules['__phello__.spam']

Example 28

View license
def LazyLoadModule(name, contents):
    """
    :param name: name of the module
    :param contents: dictionary of initial module globals

    This function returns a special module type with one method `_addattr`.  The signature
    of this method is:

        _addattr(name, creator, *creatorArgs, **creatorKwargs)

    Attributes added with this method will not be created until the first time that
    they are accessed, at which point a callback function will be called to generate
    the attribute's value.

    :param name: name of the attribute to lazily add
    :param creator: a function that create the

    Example::

        import sys
        mod = LazyLoadModule(__name__, globals())
        mod._addattr( 'foo', str, 'bar' )
        sys.modules[__name__] = mod

    One caveat of this technique is that if a user imports everything from your
    lazy module ( .e.g from module import * ), it will cause all lazy attributes
    to be evaluated.

    Also, if any module-level expression needs to reference something that only
    exists in the LazyLoadModule, it will need to be stuck in after the creation of the
    LazyLoadModule.  Then, typically, after defining all functions/classes/etc
    which rely on the LazyLoadModule attributes, you will wish to update the
    LazyLoadModule with the newly-created functions - typically, this is done
    with the _updateLazyModule method.

    Finally, any functions which reference any LazyLoadModule-only attributes,
    whether they are defined after OR before the creation of the LazyLoadModule,
    will have to prefix it with a reference to the LazyLoadModule.

    Example::

        import sys

        def myFunc():
            # need to preface foo with 'lazyModule',
            # even though this function is defined before
            # the creation of the lazy module!
            print 'foo is:', lazyModule.foo

        mod = lazyLoadModule(__name__, globals())
        mod._addattr( 'foo', str, 'bar' )
        sys.modules[__name__] = mod

        # create a reference to the LazyLoadModule in this module's
        # global space
        lazyModule = sys.modules[__name__]

        # define something which relies on something in the lazy module
        fooExpanded = lazyModule.foo + '... now with MORE!'

        # update the lazyModule with our new additions (ie, fooExpanded)
        lazyModule._updateLazyModule(globals())
    """
    class _LazyLoadModule(types.ModuleType):
        class LazyLoader(object):
            """
            A data descriptor that delays instantiation of an object
            until it is first accessed.
            """
            def __init__(self, name, creator, *creatorArgs, **creatorKwargs):
                self.creator = creator
                self.args = creatorArgs
                self.kwargs = creatorKwargs
                self.name = name

            def __get__(self, obj, objtype):
                # In case the LazyLoader happens to get stored on more
                # than one object, cache the created object so the exact
                # same one will be returned
                if not hasattr(self, 'newobj'):
                    # use the callback to create the object that will replace us
                    self.newobj = self.creator(*self.args, **self.kwargs)
                    if isinstance(obj, types.ModuleType) and hasattr(self.newobj, '__module__'):
                        self.newobj.__module__ = obj.__name__
                #print "Lazy-loaded object:", self.name
                #delattr( obj.__class__, self.name) # should we overwrite with None?
                # overwrite ourselves with the newly created object
                setattr( obj, self.name, self.newobj)
                return self.newobj

        def __init__(self, name, contents):
            types.ModuleType.__init__(self, name)
            self.__dict__.update(contents)
            self._lazyGlobals = contents # globals of original module
            # add ourselves to sys.modules, overwriting the original module
            sys.modules[name] = self
            # the above line assigns a None value to all entries in the original globals.
            # luckily, we have a copy on this module we can use to restore it.
            self._lazyGlobals.update( self.__dict__ )
        @property
        def __all__(self):
            public = [ x for x in self.__dict__.keys() + self.__class__.__dict__.keys() if not x.startswith('_') ]
            return public

        @classmethod
        def _lazyModule_addAttr(cls, name, creator, *creatorArgs, **creatorKwargs):
            lazyObj = cls.LazyLoader(name, creator, *creatorArgs, **creatorKwargs)
            setattr( cls, name, lazyObj )
            return lazyObj

        def __setitem__(self, attr, args):
            """
            dynModule['attrName'] = ( callbackFunc, ( 'arg1', ), {} )
            """
            # args will either be a single callable, or will be a tuple of
            # ( callable, (args,), {kwargs} )
            if hasattr( args, '__call__'):
                callback = args
            elif isinstance( args, (tuple, list) ):
                if len(args) >= 1:
                    assert hasattr( args[0], '__call__' ), 'first argument must be callable'
                    callback = args[0]
                else:
                    raise ValueError, "must supply at least one argument"
                if len(args) >= 2:
                    assert hasattr( args[1], '__iter__'), 'second argument must be iterable'
                    cb_args = args[1]
                else:
                    cb_args = ()
                    cb_kwargs = {}
                if len(args) == 3:
                    assert operator.isMappingType(args[2]), 'third argument must be a mapping type'
                    cb_kwargs = args[2]
                else:
                    cb_kwargs = {}
                if len(args) > 3:
                    raise ValueError, "if args and kwargs are desired, they should be passed as a tuple and dictionary, respectively"
            else:
                raise ValueError, "the item must be set to a callable, or to a 3-tuple of (callable, (args,), {kwargs})"
            self._lazyModule_addAttr(attr, callback, *cb_args, **cb_kwargs)

        def __getitem__(self, attr):
            """
            return a LazyLoader without initializing it, or, if a LazyLoader does not exist with this name,
            a real object
            """
            try:
                return self.__class__.__dict__[attr]
            except KeyError:
                return self.__dict__[attr]

        # Sort of a cumbersome name, but we want to make sure it doesn't conflict with any
        # 'real' entries in the module
        def _lazyModule_update(self):
            """
            Used to update the contents of the LazyLoadModule with the contents of another dict.
            """
            # For debugging, print out a list of things in the _lazyGlobals that
            # AREN'T in __dict__
#            print "_lazyModule_update:"
#            print "only in dynamic module:", [x for x in
#                                              (set(self.__class__.__dict__) | set(self.__dict__))- set(self._lazyGlobals)
#                                              if not x.startswith('__')]
            self.__dict__.update(self._lazyGlobals)


    return _LazyLoadModule(name, contents)

Example 29

View license
def LazyLoadModule(name, contents):
    """
    :param name: name of the module
    :param contents: dictionary of initial module globals

    This function returns a special module type with one method `_addattr`.  The signature
    of this method is:

        _addattr(name, creator, *creatorArgs, **creatorKwargs)

    Attributes added with this method will not be created until the first time that
    they are accessed, at which point a callback function will be called to generate
    the attribute's value.

    :param name: name of the attribute to lazily add
    :param creator: a function that create the

    Example::

        import sys
        mod = LazyLoadModule(__name__, globals())
        mod._addattr( 'foo', str, 'bar' )
        sys.modules[__name__] = mod

    One caveat of this technique is that if a user imports everything from your
    lazy module ( .e.g from module import * ), it will cause all lazy attributes
    to be evaluated.

    Also, if any module-level expression needs to reference something that only
    exists in the LazyLoadModule, it will need to be stuck in after the creation of the
    LazyLoadModule.  Then, typically, after defining all functions/classes/etc
    which rely on the LazyLoadModule attributes, you will wish to update the
    LazyLoadModule with the newly-created functions - typically, this is done
    with the _updateLazyModule method.

    Finally, any functions which reference any LazyLoadModule-only attributes,
    whether they are defined after OR before the creation of the LazyLoadModule,
    will have to prefix it with a reference to the LazyLoadModule.

    Example::

        import sys

        def myFunc():
            # need to preface foo with 'lazyModule',
            # even though this function is defined before
            # the creation of the lazy module!
            print 'foo is:', lazyModule.foo

        mod = lazyLoadModule(__name__, globals())
        mod._addattr( 'foo', str, 'bar' )
        sys.modules[__name__] = mod

        # create a reference to the LazyLoadModule in this module's
        # global space
        lazyModule = sys.modules[__name__]

        # define something which relies on something in the lazy module
        fooExpanded = lazyModule.foo + '... now with MORE!'

        # update the lazyModule with our new additions (ie, fooExpanded)
        lazyModule._updateLazyModule(globals())
    """
    class _LazyLoadModule(types.ModuleType):
        class LazyLoader(object):
            """
            A data descriptor that delays instantiation of an object
            until it is first accessed.
            """
            def __init__(self, name, creator, *creatorArgs, **creatorKwargs):
                self.creator = creator
                self.args = creatorArgs
                self.kwargs = creatorKwargs
                self.name = name

            def __get__(self, obj, objtype):
                # In case the LazyLoader happens to get stored on more
                # than one object, cache the created object so the exact
                # same one will be returned
                if not hasattr(self, 'newobj'):
                    # use the callback to create the object that will replace us
                    self.newobj = self.creator(*self.args, **self.kwargs)
                    if isinstance(obj, types.ModuleType) and hasattr(self.newobj, '__module__'):
                        self.newobj.__module__ = obj.__name__
                #print "Lazy-loaded object:", self.name
                #delattr( obj.__class__, self.name) # should we overwrite with None?
                # overwrite ourselves with the newly created object
                setattr( obj, self.name, self.newobj)
                return self.newobj

        def __init__(self, name, contents):
            types.ModuleType.__init__(self, name)
            self.__dict__.update(contents)
            self._lazyGlobals = contents # globals of original module
            # add ourselves to sys.modules, overwriting the original module
            sys.modules[name] = self
            # the above line assigns a None value to all entries in the original globals.
            # luckily, we have a copy on this module we can use to restore it.
            self._lazyGlobals.update( self.__dict__ )
        @property
        def __all__(self):
            public = [ x for x in self.__dict__.keys() + self.__class__.__dict__.keys() if not x.startswith('_') ]
            return public

        @classmethod
        def _lazyModule_addAttr(cls, name, creator, *creatorArgs, **creatorKwargs):
            lazyObj = cls.LazyLoader(name, creator, *creatorArgs, **creatorKwargs)
            setattr( cls, name, lazyObj )
            return lazyObj

        def __setitem__(self, attr, args):
            """
            dynModule['attrName'] = ( callbackFunc, ( 'arg1', ), {} )
            """
            # args will either be a single callable, or will be a tuple of
            # ( callable, (args,), {kwargs} )
            if hasattr( args, '__call__'):
                callback = args
            elif isinstance( args, (tuple, list) ):
                if len(args) >= 1:
                    assert hasattr( args[0], '__call__' ), 'first argument must be callable'
                    callback = args[0]
                else:
                    raise ValueError, "must supply at least one argument"
                if len(args) >= 2:
                    assert hasattr( args[1], '__iter__'), 'second argument must be iterable'
                    cb_args = args[1]
                else:
                    cb_args = ()
                    cb_kwargs = {}
                if len(args) == 3:
                    assert operator.isMappingType(args[2]), 'third argument must be a mapping type'
                    cb_kwargs = args[2]
                else:
                    cb_kwargs = {}
                if len(args) > 3:
                    raise ValueError, "if args and kwargs are desired, they should be passed as a tuple and dictionary, respectively"
            else:
                raise ValueError, "the item must be set to a callable, or to a 3-tuple of (callable, (args,), {kwargs})"
            self._lazyModule_addAttr(attr, callback, *cb_args, **cb_kwargs)

        def __getitem__(self, attr):
            """
            return a LazyLoader without initializing it, or, if a LazyLoader does not exist with this name,
            a real object
            """
            try:
                return self.__class__.__dict__[attr]
            except KeyError:
                return self.__dict__[attr]

        # Sort of a cumbersome name, but we want to make sure it doesn't conflict with any
        # 'real' entries in the module
        def _lazyModule_update(self):
            """
            Used to update the contents of the LazyLoadModule with the contents of another dict.
            """
            # For debugging, print out a list of things in the _lazyGlobals that
            # AREN'T in __dict__
#            print "_lazyModule_update:"
#            print "only in dynamic module:", [x for x in
#                                              (set(self.__class__.__dict__) | set(self.__dict__))- set(self._lazyGlobals)
#                                              if not x.startswith('__')]
            self.__dict__.update(self._lazyGlobals)


    return _LazyLoadModule(name, contents)

Example 30

Project: cython
Source File: Dependencies.py
View license
def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=False, language=None,
                          exclude_failures=False):
    if language is not None:
        print('Please put "# distutils: language=%s" in your .pyx or .pxd file(s)' % language)
    if exclude is None:
        exclude = []
    if patterns is None:
        return [], {}
    elif isinstance(patterns, basestring) or not isinstance(patterns, collections.Iterable):
        patterns = [patterns]
    explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)])
    seen = set()
    deps = create_dependency_tree(ctx, quiet=quiet)
    to_exclude = set()
    if not isinstance(exclude, list):
        exclude = [exclude]
    for pattern in exclude:
        to_exclude.update(map(os.path.abspath, extended_iglob(pattern)))

    module_list = []
    module_metadata = {}

    # workaround for setuptools
    if 'setuptools' in sys.modules:
        Extension_distutils = sys.modules['setuptools.extension']._Extension
        Extension_setuptools = sys.modules['setuptools'].Extension
    else:
        # dummy class, in case we do not have setuptools
        Extension_distutils = Extension
        class Extension_setuptools(Extension): pass

    for pattern in patterns:
        if isinstance(pattern, str):
            filepattern = pattern
            template = None
            name = '*'
            base = None
            exn_type = Extension
            ext_language = language
        elif isinstance(pattern, (Extension_distutils, Extension_setuptools)):
            for filepattern in pattern.sources:
                if os.path.splitext(filepattern)[1] in ('.py', '.pyx'):
                    break
            else:
                # ignore non-cython modules
                module_list.append(pattern)
                continue
            template = pattern
            name = template.name
            base = DistutilsInfo(exn=template)
            exn_type = template.__class__
            ext_language = None  # do not override whatever the Extension says
        else:
            msg = str("pattern is not of type str nor subclass of Extension (%s)"
                      " but of type %s and class %s" % (repr(Extension),
                                                        type(pattern),
                                                        pattern.__class__))
            raise TypeError(msg)

        for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern):
            if os.path.abspath(file) in to_exclude:
                continue
            pkg = deps.package(file)
            module_name = deps.fully_qualified_name(file)
            if '*' in name:
                if module_name in explicit_modules:
                    continue
            elif name != module_name:
                print("Warning: Extension name '%s' does not match fully qualified name '%s' of '%s'" % (
                    name, module_name, file))
                module_name = name

            if module_name not in seen:
                try:
                    kwds = deps.distutils_info(file, aliases, base).values
                except Exception:
                    if exclude_failures:
                        continue
                    raise
                if base is not None:
                    for key, value in base.values.items():
                        if key not in kwds:
                            kwds[key] = value

                sources = [file]
                if template is not None:
                    sources += [m for m in template.sources if m != filepattern]
                if 'sources' in kwds:
                    # allow users to add .c files etc.
                    for source in kwds['sources']:
                        source = encode_filename_in_py2(source)
                        if source not in sources:
                            sources.append(source)
                    extra_sources = kwds['sources']
                    del kwds['sources']
                else:
                    extra_sources = None
                if 'depends' in kwds:
                    depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + ["."])
                    if template is not None:
                        # Always include everything from the template.
                        depends = set(template.depends).union(depends)
                    # Sort depends to make the metadata dump in the
                    # Cython-generated C code predictable.
                    kwds['depends'] = sorted(depends)

                if ext_language and 'language' not in kwds:
                    kwds['language'] = ext_language

                module_list.append(exn_type(
                        name=module_name,
                        sources=sources,
                        **kwds))
                if extra_sources:
                    kwds['sources'] = extra_sources
                module_metadata[module_name] = {'distutils': kwds, 'module_name': module_name}
                m = module_list[-1]
                if file not in m.sources:
                    # Old setuptools unconditionally replaces .pyx with .c
                    m.sources.remove(file.rsplit('.')[0] + '.c')
                    m.sources.insert(0, file)
                seen.add(name)
    return module_list, module_metadata

Example 31

Project: databus
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name != 'ipython':
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 32

Project: GitSavvy
Source File: reload.py
View license
def reload_modules(main, modules, perform_reload=True):
    """Implements the machinery for reloading a given plugin module."""
    #
    # Here's the approach in general:
    #
    #   - Hide GitSavvy modules from the sys.modules temporarily;
    #
    #   - Install a special import hook onto sys.meta_path;
    #
    #   - Call sublime_plugin.reload_plugin(), which imports the main
    #     "git_savvy" module under the hood, triggering the hook;
    #
    #   - The hook, instead of creating a new module object, peeks the saved
    #     one and reloads it. Once the module encounters an import statement
    #     requesting another module, not yet reloaded, the hook reenters and
    #     processes that new module recursively, then get back to the previous
    #     one, and so on.
    #
    # This makes the modules reload in the very same order as they were loaded
    # initially, as if they were imported from scratch.
    #
    if perform_reload:
        sublime_plugin.unload_module(main)

    # Insert the main "git_savvy" module at the beginning to make the reload
    # order be as close to the order of the "natural" import as possible.
    module_names = [main.__name__] + sorted(name for name in modules
                                            if name != main.__name__)

    # First, remove all the loaded modules from the sys.modules cache,
    # otherwise the reloading hook won't be called.
    loaded_modules = dict(sys.modules)
    for name in loaded_modules:
        if name in modules:
            del sys.modules[name]

    stack_meter = StackMeter()
    @FilteringImportHook.when(condition=lambda name: name in modules)
    def module_reloader(name):
        module = modules[name]
        sys.modules[name] = module  # restore the module back

        if perform_reload:
            with stack_meter as depth:
                dprint("reloading", ('╿ '*depth) + '┡━─', name)
                try:
                    return module.__loader__.load_module(name)
                except:
                    if name in sys.modules:
                        del sys.modules[name]  # to indicate an error
                    raise
        else:
            if name not in loaded_modules:
                dprint("NO RELOAD", '╺━─', name)
            return module

    with intercepting_imports(module_reloader), \
         importing_fromlist_aggresively(modules):
        # Now, import all the modules back, in order, starting with the main
        # module. This will reload all the modules directly or indirectly
        # referenced by the main one, i.e. usually most of our modules.
        sublime_plugin.reload_plugin(main.__name__)

        # Be sure to bring back *all* the modules that used to be loaded, not
        # only these imported through the main one. Otherwise, some of them
        # might end up being created from scratch as new module objects in
        # case of being imported after detaching the hook. In general, most of
        # the imports below (if not all) are no-ops though.
        for name in module_names:
            importlib.import_module(name)

Example 33

Project: django-mako-plus
Source File: router.py
View license
def route_request(request):
    '''The main router for all calls coming in to the system.'''
    # output the variables so the programmer can debug where this is routing
    if log.isEnabledFor(logging.INFO):
        log.info('processing: app=%s, page=%s, func=%s, urlparams=%s' % (request.dmp_router_app, request.dmp_router_page, request.dmp_router_function, request.urlparams))

    # set the full function location
    request.dmp_router_module = '.'.join([ request.dmp_router_app, 'views', request.dmp_router_page ])

    # first try going to the view function for this request
    # we look for a views/name.py file where name is the same name as the HTML file
    response = None

    while True: # enables the InternalRedirectExceptions to loop around
        full_module_filename = os.path.normpath(os.path.join(settings.BASE_DIR, request.dmp_router_module.replace('.', '/') + '.py'))
        try:
            # look for the module, and if not found go straight to template
            if not os.path.exists(full_module_filename):
                log.warning('module %s not found; sending processing directly to template %s.html' % (request.dmp_router_module, request.dmp_router_page_full))
                try:
                    dmp_loader = get_dmp_instance().get_template_loader(request.dmp_router_app)
                    return dmp_loader.get_template('%s.html' % request.dmp_router_page_full).render_to_response(request=request)
                except (TemplateDoesNotExist, TemplateSyntaxError, ImproperlyConfigured) as e:
                    log.error('%s' % (e))
                    raise Http404

            # find the function
            module_obj = import_module(request.dmp_router_module)
            if not hasattr(module_obj, request.dmp_router_function):
                log.error('view function/class %s not in module %s; returning 404 not found.' % (request.dmp_router_function, request.dmp_router_module))
                raise Http404
            func_obj = getattr(module_obj, request.dmp_router_function)

            # if the func_obj is a View, we're doing class-based views and it needs converting to a function
            if isclass(func_obj) and issubclass(func_obj, View):
                request.dmp_router_class = request.dmp_router_function
                request.dmp_router_function = request.method.lower()
                func_obj = func_obj.as_view()  # this Django method wraps the view class with a function, so now we can treat it like a regular dmp call
                # we don't need the @view_function security check because the class is already subclassed from "View", so we know the site means to expose this class as an endpoint.

            # if the func_obj is a regular function, so ensure it is decorated with @view_function - this is for security so only certain functions can be called
            elif getattr(func_obj, 'dmp_view_function', False) != True:
                log.error('view function %s found successfully, but it is not decorated with @view_function; returning 404 not found.  Note that if you have multiple decorators on a function, the @view_function decorator must be listed first.' % (request.dmp_router_function))
                raise Http404

            # send the pre-signal
            if DMP_OPTIONS.get('SIGNALS', False):
                for receiver, ret_response in dmp_signal_pre_process_request.send(sender=sys.modules[__name__], request=request):
                    if isinstance(ret_response, (HttpResponse, StreamingHttpResponse)):
                        return ret_response

            # call view function
            if request.dmp_router_class == None and log.isEnabledFor(logging.INFO):
                log.info('calling view function %s.%s' % (request.dmp_router_module, request.dmp_router_function))
            elif log.isEnabledFor(logging.INFO):
                log.info('calling class-based view function %s.%s.%s' % (request.dmp_router_module, request.dmp_router_class, request.dmp_router_function))
            response = func_obj(request)

            # send the post-signal
            if DMP_OPTIONS.get('SIGNALS', False):
                for receiver, ret_response in dmp_signal_post_process_request.send(sender=sys.modules[__name__], request=request, response=response):
                    if ret_response != None:
                        response = ret_response # sets it to the last non-None in the signal receiver chain

            # if we didn't get a correct response back, send a 404
            if not isinstance(response, (HttpResponse, StreamingHttpResponse)):
                if request.dmp_router_class == None:
                    log.error('view function %s.%s failed to return an HttpResponse (or the post-signal overwrote it).  Returning Http404.' % (request.dmp_router_module, request.dmp_router_function))
                else:
                    log.error('class-based view function %s.%s.%s failed to return an HttpResponse (or the post-signal overwrote it).  Returning Http404.' % (request.dmp_router_module, request.dmp_router_class, request.dmp_router_function))
                raise Http404

            # return the response
            return response

        except InternalRedirectException as ivr:
            # send the signal
            if DMP_OPTIONS.get('SIGNALS', False):
                dmp_signal_internal_redirect_exception.send(sender=sys.modules[__name__], request=request, exc=ivr)
            # do the internal redirect
            request.dmp_router_module = ivr.redirect_module
            request.dmp_router_function = ivr.redirect_function
            full_module_filename = os.path.normpath(os.path.join(settings.BASE_DIR, request.dmp_router_module.replace('.', '/') + '.py'))
            log.info('received an InternalViewRedirect to %s -> %s' % (full_module_filename, request.dmp_router_function))

        except RedirectException as e: # redirect to another page
            if request.dmp_router_class == None:
                log.info('view function %s.%s redirected processing to %s' % (request.dmp_router_module, request.dmp_router_function, e.redirect_to))
            else:
                log.info('class-based view function %s.%s.%s redirected processing to %s' % (request.dmp_router_module, request.dmp_router_class, request.dmp_router_function, e.redirect_to))
            # send the signal
            if DMP_OPTIONS.get('SIGNALS', False):
                dmp_signal_redirect_exception.send(sender=sys.modules[__name__], request=request, exc=e)
            # send the browser the redirect command
            return e.get_response(request)

    # the code should never get here
    raise Exception("Django-Mako-Plus router error: The route_request() function should not have been able to get to this point.  Please notify the owner of the DMP project.  Thanks.")

Example 34

Project: duecredit
Source File: injector.py
View license
    def activate(self, retrospect=True):
        """
        Parameters
        ----------
        retrospect : bool, optional
          Either consider already loaded modules
        """

        if not self._orig_import:
            # for paranoid Yarik so we have assurance we are not somehow
            # overriding our decorator
            if hasattr(__builtin__.__import__, '__duecredited__'):
                raise RuntimeError("__import__ is already duecredited")

            self._orig_import = __builtin__.__import__

            @wraps(__builtin__.__import__)
            def __import(name, *args, **kwargs):
                if self.__processing_queue or name in self._processed_modules or name in self.__queue_to_process:
                    lgr.debug("Performing undecorated import of %s", name)
                    # return right away without any decoration in such a case
                    if self._orig_import:
                        return _very_orig_import(name, *args, **kwargs)
                    else:
                        return self._mitigate_None_orig_import(name, *args, **kwargs)
                import_level_prefix = self._import_level_prefix
                lgr.log(1, "%sProcessing request to import %s", import_level_prefix, name)
                # importing submodule might result in importing a new one and
                # name here is not sufficient to determine which module would actually
                # get imported unless level=0 (absolute import), but that one rarely used

                # could be old-style or new style relative import!
                # args[0] -> globals, [1] -> locals(), [2] -> fromlist, [3] -> level
                level = args[3] if len(args) >= 4 else kwargs.get('level', -1)
                # fromlist = args[2] if len(args) >= 3 else kwargs.get('fromlist', [])

                if not retrospect and not self._processed_modules:
                    # we were asked to not consider those modules which were already loaded
                    # so let's assume that they were all processed already
                    self._processed_modules = set(sys.modules)

                mod = None
                try:
                    self.__import_level += 1
                    # TODO: safe-guard all our logic so
                    # if anything goes wrong post-import -- we still return imported module
                    if self._orig_import:
                        mod = self._orig_import(name, *args, **kwargs)
                    else:
                        mod = self._mitigate_None_orig_import(name, *args, **kwargs)

                    self._handle_fresh_imports(name, import_level_prefix, level)
                finally:
                    self.__import_level -= 1

                if self.__import_level == 0 and self.__queue_to_process:
                    self._process_queue()

                lgr.log(1, "%sReturning %s", import_level_prefix, mod)
                return mod
            __import.__duecredited__ = True

            self._populate_delayed_injections()

            if retrospect:
                lgr.debug("Considering previously loaded %d modules", len(sys.modules))
                # operate on keys() (not iterator) since we might end up importing delayed injection modules etc
                for mod_name in sys.modules.keys():
                    self.process(sys.modules[mod_name])

            lgr.debug("Assigning our importer")
            __builtin__.__import__ = __import
            self._active = True

        else:
            lgr.warning("Seems that we are calling duecredit_importer twice."
                        " No harm is done but shouldn't happen")

Example 35

Project: chaco
Source File: numpydoc.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read

    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])

    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']

    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1

        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 36

Project: C-PAC
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 37

Project: pyrise
Source File: pyrise.py
View license
    @classmethod
    def from_xml(cls, xml, parent=None):
        """Create a new object from XML data"""
        
        # instiantiate the object
        if cls == Party:
            cls = getattr(sys.modules[__name__], xml.get('type'))
        self = cls()
        
        for child in xml:
            
            # convert the key to underscore notation for Python
            key = child.tag.replace('-', '_')
            
            # if this key is not recognized by pyrise, ignore it
            if key not in cls.fields:
                continue
        
            # if there is no data, just set the default
            if child.text == None:
                self.__dict__[key] = self.fields[key].default
                continue

            # handle the contact-data key differently
            if key == 'contact_data':
                klass = getattr(sys.modules[__name__], 'ContactData')
                self.contact_data = klass.from_xml(child, parent=self)
                continue

            # if this an element with children, it's an object relationship
            if len(list(child)) > 0:
                
                # is this element an array of objects?
                if cls.fields[key].type == list:
                    items = []
                    for item in child:
                        if item.tag == 'party':
                            class_string = item.find('type').text
                        else:
                            class_string = Highrise.key_to_class(item.tag.replace('_', '-'))
                        klass = getattr(sys.modules[__name__], class_string)
                        items.append(klass.from_xml(item, parent=self))
                    self.__dict__[child.tag.replace('-', '_')] = items
                    continue
                
                # otherwise, let's treat it like a single object
                else:
                    if child.tag == 'party':
                        class_string = child.find('type').text
                    else:
                        class_string = Highrise.key_to_class(child.tag)
                    klass = getattr(sys.modules[__name__], class_string)
                    self.__dict__[child.tag.replace('-', '_')] = klass.from_xml(child, parent=self)
                    continue
                
            # get and convert attribute value based on type
            data_type = child.get('type')
            if data_type == 'integer':
                value = int(child.text)
            elif data_type == 'datetime':
                value = Highrise.from_utc(datetime.strptime(child.text, '%Y-%m-%dT%H:%M:%SZ'))
            else:
                try:
                    value = unicode(child.text)
                except:
                    value = str(child.text)

            # add value to object dictionary
            self.__dict__[key] = value
                
        return self

Example 38

Project: datafari
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name != 'ipython':
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 39

Project: freeipa
Source File: schema.py
View license
def get_package(server_info, client):
    NO_FINGERPRINT = object()

    fingerprint = NO_FINGERPRINT
    if server_info.is_valid():
        fingerprint = server_info.get('fingerprint', fingerprint)

    if fingerprint is not None:
        try:
            try:
                if fingerprint is NO_FINGERPRINT:
                    schema = Schema(client)
                else:
                    schema = Schema(client, fingerprint)
            except SchemaUpToDate as e:
                schema = Schema(client, e.fingerprint)
        except NotAvailable:
            fingerprint = None
            ttl = None
        except SchemaUpToDate as e:
            fingerprint = e.fingerprint
            ttl = e.ttl
        else:
            fingerprint = schema.fingerprint
            ttl = schema.ttl

        server_info['fingerprint'] = fingerprint
        server_info.update_validity(ttl)

    if fingerprint is None:
        raise NotAvailable()

    fingerprint = str(fingerprint)
    package_name = '{}${}'.format(__name__, fingerprint)
    package_dir = '{}${}'.format(os.path.splitext(__file__)[0], fingerprint)

    try:
        return sys.modules[package_name]
    except KeyError:
        pass

    package = types.ModuleType(package_name)
    package.__file__ = os.path.join(package_dir, '__init__.py')
    package.modules = ['plugins']
    sys.modules[package_name] = package

    module_name = '.'.join((package_name, 'plugins'))
    module = types.ModuleType(module_name)
    module.__file__ = os.path.join(package_dir, 'plugins.py')
    module.register = plugable.Registry()
    for plugin_cls in (_SchemaCommandPlugin, _SchemaObjectPlugin):
        for full_name in schema[plugin_cls.schema_key]:
            plugin = plugin_cls(schema, str(full_name))
            plugin = module.register()(plugin)
    sys.modules[module_name] = module

    for full_name, topic in six.iteritems(schema['topics']):
        name = str(topic['name'])
        module_name = '.'.join((package_name, name))
        try:
            module = sys.modules[module_name]
        except KeyError:
            module = sys.modules[module_name] = types.ModuleType(module_name)
            module.__file__ = os.path.join(package_dir, '{}.py'.format(name))
        module.__doc__ = topic.get('doc')
        if 'topic_topic' in topic:
            module.topic = str(topic['topic_topic']).partition('/')[0]
        else:
            module.topic = None

    return package

Example 40

Project: girder
Source File: assetstore_test.py
View license
    @moto.mock_s3bucket_path
    def testS3AssetstoreAdapter(self):
        # Delete the default assetstore
        self.model('assetstore').remove(self.assetstore)
        s3Regex = r'^https://s3.amazonaws.com(:443)?/bucketname/foo/bar'

        params = {
            'name': 'S3 Assetstore',
            'type': AssetstoreType.S3,
            'bucket': '',
            'accessKeyId': 'someKey',
            'secret': 'someSecret',
            'prefix': '/foo/bar/'
        }

        # Validation should fail with empty bucket name
        resp = self.request(path='/assetstore', method='POST', user=self.admin, params=params)
        self.assertStatus(resp, 400)
        self.assertEqual(resp.json, {
            'type': 'validation',
            'field': 'bucket',
            'message': 'Bucket must not be empty.'
        })

        params['bucket'] = 'bucketname'
        # Validation should fail with a missing bucket
        resp = self.request(path='/assetstore', method='POST', user=self.admin, params=params)
        self.assertStatus(resp, 400)
        self.assertEqual(resp.json, {
            'type': 'validation',
            'field': 'bucket',
            'message': 'Unable to write into bucket "bucketname".'
        })

        # Validation should fail with a bogus service name
        params['service'] = 'ftp://nowhere'
        resp = self.request(path='/assetstore', method='POST', user=self.admin, params=params)
        self.assertStatus(resp, 400)
        del params['service']

        # Create a bucket (mocked using moto), so that we can create an
        # assetstore in it
        botoParams = makeBotoConnectParams(params['accessKeyId'],
                                           params['secret'])
        bucket = mock_s3.createBucket(botoParams, 'bucketname')

        # Create an assetstore
        resp = self.request(path='/assetstore', method='POST', user=self.admin, params=params)
        self.assertStatusOk(resp)
        assetstore = self.model('assetstore').load(resp.json['_id'])

        # Set the assetstore to current.  This is really to test the edit
        # assetstore code.
        params['current'] = True
        resp = self.request(path='/assetstore/%s' % assetstore['_id'],
                            method='PUT', user=self.admin, params=params)
        self.assertStatusOk(resp)

        # Test init for a single-chunk upload
        folders = self.model('folder').childFolders(self.admin, 'user')
        parentFolder = six.next(folders)
        params = {
            'parentType': 'folder',
            'parentId': parentFolder['_id'],
            'name': 'My File.txt',
            'size': 1024,
            'mimeType': 'text/plain'
        }
        resp = self.request(path='/file', method='POST', user=self.admin, params=params)
        self.assertStatusOk(resp)
        self.assertEqual(resp.json['received'], 0)
        self.assertEqual(resp.json['size'], 1024)
        self.assertEqual(resp.json['behavior'], 's3')

        singleChunkUpload = resp.json
        s3Info = singleChunkUpload['s3']
        self.assertEqual(s3Info['chunked'], False)
        self.assertIsInstance(s3Info['chunkLength'], int)
        self.assertEqual(s3Info['request']['method'], 'PUT')
        six.assertRegex(self, s3Info['request']['url'], s3Regex)
        self.assertEqual(s3Info['request']['headers']['x-amz-acl'], 'private')

        # Test resume of a single-chunk upload
        resp = self.request(path='/file/offset', method='GET', user=self.admin,
                            params={'uploadId': resp.json['_id']})
        self.assertStatusOk(resp)
        self.assertEqual(resp.json['method'], 'PUT')
        self.assertTrue('headers' in resp.json)
        six.assertRegex(self, resp.json['url'], s3Regex)

        # Test finalize for a single-chunk upload
        resp = self.request(
            path='/file/completion', method='POST', user=self.admin,
            params={'uploadId': singleChunkUpload['_id']})
        self.assertStatusOk(resp)
        self.assertEqual(resp.json['size'], 1024)
        self.assertEqual(resp.json['assetstoreId'], str(assetstore['_id']))
        self.assertFalse('s3Key' in resp.json)
        self.assertFalse('relpath' in resp.json)

        file = self.model('file').load(resp.json['_id'], force=True)
        self.assertTrue('s3Key' in file)
        six.assertRegex(self, file['relpath'], '^/bucketname/foo/bar/')

        # Test init for a multi-chunk upload
        params['size'] = 1024 * 1024 * 1024 * 5
        resp = self.request(path='/file', method='POST', user=self.admin, params=params)
        self.assertStatusOk(resp)

        multiChunkUpload = resp.json
        s3Info = multiChunkUpload['s3']
        self.assertEqual(s3Info['chunked'], True)
        self.assertIsInstance(s3Info['chunkLength'], int)
        self.assertEqual(s3Info['request']['method'], 'POST')
        six.assertRegex(self, s3Info['request']['url'], s3Regex)

        # Test uploading a chunk
        resp = self.request(path='/file/chunk', method='POST',
                            user=self.admin, params={
                                'uploadId': multiChunkUpload['_id'],
                                'offset': 0,
                                'chunk': json.dumps({
                                    'partNumber': 1,
                                    's3UploadId': 'abcd'
                                })
                            })
        self.assertStatusOk(resp)
        six.assertRegex(self, resp.json['s3']['request']['url'], s3Regex)
        self.assertEqual(resp.json['s3']['request']['method'], 'PUT')

        # We should not be able to call file/offset with multi-chunk upload
        resp = self.request(path='/file/offset', method='GET', user=self.admin,
                            params={'uploadId': multiChunkUpload['_id']})
        self.assertStatus(resp, 400)
        self.assertEqual(resp.json, {
            'type': 'validation',
            'message': 'You should not call requestOffset on a chunked '
                       'direct-to-S3 upload.'
        })

        # Test finalize for a multi-chunk upload
        resp = self.request(
            path='/file/completion', method='POST', user=self.admin,
            params={'uploadId': multiChunkUpload['_id']})
        largeFile = resp.json
        self.assertStatusOk(resp)
        six.assertRegex(self, resp.json['s3FinalizeRequest']['url'], s3Regex)
        self.assertEqual(resp.json['s3FinalizeRequest']['method'], 'POST')

        # Test init for an empty file (should be no-op)
        params['size'] = 0
        resp = self.request(path='/file', method='POST', user=self.admin, params=params)
        emptyFile = resp.json
        self.assertStatusOk(resp)
        self.assertFalse('behavior' in resp.json)
        self.assertFalse('s3' in resp.json)

        # Test download for an empty file
        resp = self.request(path='/file/%s/download' % emptyFile['_id'],
                            user=self.admin, method='GET', isJson=False)
        self.assertStatusOk(resp)
        self.assertEqual(self.getBody(resp), '')
        self.assertEqual(resp.headers['Content-Length'], 0)
        self.assertEqual(resp.headers['Content-Disposition'],
                         'attachment; filename="My File.txt"')

        # Test download of a non-empty file
        resp = self.request(path='/file/%s/download' % largeFile['_id'],
                            user=self.admin, method='GET', isJson=False)
        self.assertStatus(resp, 303)
        six.assertRegex(self, resp.headers['Location'], s3Regex)

        # Test download of a non-empty file, with Content-Disposition=inline.
        # Expect the special S3 header response-content-disposition.
        params = {'contentDisposition': 'inline'}
        inlineRegex = r'response-content-disposition=' + \
                      'inline%3B\+filename%3D%22My\+File.txt%22'
        resp = self.request(
            path='/file/%s/download' % largeFile['_id'], user=self.admin, method='GET',
            isJson=False, params=params)
        self.assertStatus(resp, 303)
        six.assertRegex(self, resp.headers['Location'], s3Regex)
        six.assertRegex(self, resp.headers['Location'], inlineRegex)

        # Test download as part of a streaming zip
        @httmock.all_requests
        def s3_pipe_mock(url, request):
            if url.netloc.startswith('s3.amazonaws.com') and url.scheme == 'https':
                return 'dummy file contents'
            else:
                raise Exception('Unexpected url %s' % url)

        with httmock.HTTMock(s3_pipe_mock):
            resp = self.request(
                '/folder/%s/download' % parentFolder['_id'],
                method='GET', user=self.admin, isJson=False)
            self.assertStatusOk(resp)
            zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
            self.assertTrue(zip.testzip() is None)

            extracted = zip.read('Public/My File.txt')
            self.assertEqual(extracted, b'dummy file contents')

        # Attempt to import item directly into user; should fail
        resp = self.request(
            '/assetstore/%s/import' % assetstore['_id'], method='POST', params={
                'importPath': '/foo/bar',
                'destinationType': 'user',
                'destinationId': self.admin['_id']
            }, user=self.admin)
        self.assertStatus(resp, 400)
        self.assertEqual(
            resp.json['message'], 'Keys cannot be imported directly underneath a user.')

        # Import existing data from S3
        resp = self.request('/folder', method='POST', params={
            'parentType': 'folder',
            'parentId': parentFolder['_id'],
            'name': 'import destinaton'
        }, user=self.admin)
        self.assertStatusOk(resp)
        importFolder = resp.json

        resp = self.request(
            '/assetstore/%s/import' % assetstore['_id'], method='POST', params={
                'importPath': '',
                'destinationType': 'folder',
                'destinationId': importFolder['_id'],
            }, user=self.admin)
        self.assertStatusOk(resp)

        # Data should now appear in the tree
        resp = self.request('/folder', user=self.admin, params={
            'parentId': importFolder['_id'],
            'parentType': 'folder'
        })
        self.assertStatusOk(resp)
        children = resp.json
        self.assertEqual(len(children), 1)
        self.assertEqual(children[0]['name'], 'foo')

        resp = self.request('/folder', user=self.admin, params={
            'parentId': children[0]['_id'],
            'parentType': 'folder'
        })
        self.assertStatusOk(resp)
        children = resp.json
        self.assertEqual(len(children), 1)
        self.assertEqual(children[0]['name'], 'bar')

        resp = self.request('/item', user=self.admin, params={
            'folderId': children[0]['_id']
        })
        self.assertStatusOk(resp)
        self.assertEqual(len(resp.json), 1)
        item = resp.json[0]
        self.assertEqual(item['name'], 'test')
        self.assertEqual(item['size'], 0)

        resp = self.request('/item/%s/files' % str(item['_id']),
                            user=self.admin)
        self.assertStatusOk(resp)
        self.assertEqual(len(resp.json), 1)
        self.assertFalse('imported' in resp.json[0])
        self.assertFalse('relpath' in resp.json[0])
        file = self.model('file').load(resp.json[0]['_id'], force=True)
        self.assertTrue(file['imported'])
        self.assertFalse('relpath' in file)
        self.assertEqual(file['size'], 0)
        self.assertEqual(file['assetstoreId'], assetstore['_id'])
        self.assertTrue(bucket.get_key('/foo/bar/test') is not None)

        # Deleting an imported file should not delete it from S3
        with mock.patch('girder.events.daemon.trigger') as daemon:
            resp = self.request('/item/%s' % str(item['_id']), method='DELETE', user=self.admin)
            self.assertStatusOk(resp)
            self.assertEqual(len(daemon.mock_calls), 0)

        # Create the file key in the moto s3 store so that we can test that it
        # gets deleted.
        file = self.model('file').load(largeFile['_id'], user=self.admin)
        bucket.initiate_multipart_upload(file['s3Key'])
        key = bucket.new_key(file['s3Key'])
        key.set_contents_from_string("test")

        # Test delete for a non-empty file
        resp = self.request(path='/file/%s' % largeFile['_id'], user=self.admin, method='DELETE')
        self.assertStatusOk(resp)

        # The file should be gone now
        resp = self.request(path='/file/%s/download' % largeFile['_id'],
                            user=self.admin, method='GET', isJson=False)
        self.assertStatus(resp, 400)
        # The actual delete may still be in the event queue, so we want to
        # check the S3 bucket directly.
        startTime = time.time()
        while True:
            if bucket.get_key(file['s3Key']) is None:
                break
            if time.time()-startTime > 15:
                break  # give up and fail
            time.sleep(0.1)
        self.assertIsNone(bucket.get_key(file['s3Key']))

        resp = self.request(
            path='/folder/%s' % parentFolder['_id'], method='DELETE', user=self.admin)
        self.assertStatusOk(resp)

        # Set the assetstore to read only, attempt to delete it
        assetstore['readOnly'] = True
        assetstore = self.model('assetstore').save(assetstore)

        def fn(*args, **kwargs):
            raise Exception('get_all_multipart_uploads should not be called')

        # Must mock globally (too tricky to get a direct mock.patch)
        old = sys.modules['boto.s3.bucket'].Bucket.get_all_multipart_uploads
        sys.modules['boto.s3.bucket'].Bucket.get_all_multipart_uploads = fn

        try:
            resp = self.request(
                path='/assetstore/%s' % assetstore['_id'], method='DELETE', user=self.admin)
            self.assertStatusOk(resp)
        finally:
            sys.modules['boto.s3.bucket'].Bucket.get_all_multipart_uploads = old

Example 41

Project: pygobject
Source File: __init__.py
View license
def load_overrides(introspection_module):
    """Loads overrides for an introspection module.

    Either returns the same module again in case there are no overrides or a
    proxy module including overrides. Doesn't cache the result.
    """

    namespace = introspection_module.__name__.rsplit(".", 1)[-1]
    module_key = 'gi.repository.' + namespace

    # We use sys.modules so overrides can import from gi.repository
    # but restore everything at the end so this doesn't have any side effects
    has_old = module_key in sys.modules
    old_module = sys.modules.get(module_key)

    # Create a new sub type, so we can separate descriptors like
    # _DeprecatedAttribute for each namespace.
    proxy_type = type(namespace + "ProxyModule", (OverridesProxyModule, ), {})

    proxy = proxy_type(introspection_module)
    sys.modules[module_key] = proxy

    # backwards compat:
    # gedit uses gi.importer.modules['Gedit']._introspection_module
    from ..importer import modules
    assert hasattr(proxy, "_introspection_module")
    modules[namespace] = proxy

    try:
        override_package_name = 'gi.overrides.' + namespace

        # http://bugs.python.org/issue14710
        try:
            override_loader = get_loader(override_package_name)

        except AttributeError:
            override_loader = None

        # Avoid checking for an ImportError, an override might
        # depend on a missing module thus causing an ImportError
        if override_loader is None:
            return introspection_module

        override_mod = importlib.import_module(override_package_name)

    finally:
        del modules[namespace]
        del sys.modules[module_key]
        if has_old:
            sys.modules[module_key] = old_module

    # backwards compat: for gst-python/gstmodule.c,
    # which tries to access Gst.Fraction through
    # Gst._overrides_module.Fraction. We assign the proxy instead as that
    # contains all overridden classes like Fraction during import anyway and
    # there is no need to keep the real override module alive.
    proxy._overrides_module = proxy

    override_all = []
    if hasattr(override_mod, "__all__"):
        override_all = override_mod.__all__

    for var in override_all:
        try:
            item = getattr(override_mod, var)
        except (AttributeError, TypeError):
            # Gedit puts a non-string in __all__, so catch TypeError here
            continue
        setattr(proxy, var, item)

    # Replace deprecated module level attributes with a descriptor
    # which emits a warning when accessed.
    for attr, replacement in _deprecated_attrs.pop(namespace, []):
        try:
            value = getattr(proxy, attr)
        except AttributeError:
            raise AssertionError(
                "%s was set deprecated but wasn't added to __all__" % attr)
        delattr(proxy, attr)
        deprecated_attr = _DeprecatedAttribute(
            namespace, attr, value, replacement)
        setattr(proxy_type, attr, deprecated_attr)

    return proxy

Example 42

Project: simian
Source File: cgi.py
View license
def HandleRequest(unused_environ, handler_name, unused_url, post_data,
                  unused_error, application_root, python_lib,
                  import_hook=None):
  """Handle a single CGI request.

  Handles a request for handler_name in the form 'path/to/handler.py' with the
  environment contained in environ.

  Args:
    handler_name: A str containing the user-specified handler file to use for
        this request as specified in the script field of a handler in app.yaml.
    post_data: A stream containing the post data for this request.
    application_root: A str containing the root path of the application.
    python_lib: A str containing the root the Python App Engine library.
    import_hook: Optional import hook (PEP 302 style loader).

  Returns:
    A dict containing zero or more of the following:
      error: App Engine error code. 0 for OK, 1 for error. Defaults to OK if not
          set. If set, then the other fields may be missing.
      response_code: HTTP response code.
      headers: A list of tuples (key, value) of HTTP headers.
      body: A str of the body of the response.
  """
  body = cStringIO.StringIO()
  module_name = _FileToModuleName(handler_name)
  parent_module, _, submodule_name = module_name.rpartition('.')
  parent_module = _GetModuleOrNone(parent_module)
  main = None



  if module_name in sys.modules:
    module = sys.modules[module_name]
    main = _GetValidMain(module)
  if not main:
    module = imp.new_module('__main__')
    if import_hook is not None:
      module.__loader__ = import_hook
  saved_streams = sys.stdin, sys.stdout
  try:
    sys.modules['__main__'] = module
    module.__dict__['__name__'] = '__main__'
    sys.stdin = post_data
    sys.stdout = body
    if main:
      os.environ['PATH_TRANSLATED'] = module.__file__
      main()
    else:
      filename = _AbsolutePath(handler_name, application_root, python_lib)
      if filename.endswith(os.sep + '__init__.py'):
        module.__path__ = [os.path.dirname(filename)]
      if import_hook is None:
        code, filename = _LoadModuleCode(filename)
      else:
        code = import_hook.get_code(module_name)
      if not code:
        return {'error': 2}
      os.environ['PATH_TRANSLATED'] = filename
      module.__file__ = filename
      try:
        sys.modules[module_name] = module
        eval(code, module.__dict__)
      except:


        del sys.modules[module_name]
        if parent_module and submodule_name in parent_module.__dict__:
          del parent_module.__dict__[submodule_name]
        raise
      else:
        if parent_module:
          parent_module.__dict__[submodule_name] = module
    return _ParseResponse(body)
  except:
    exception = sys.exc_info()


    message = ''.join(traceback.format_exception(exception[0], exception[1],
                                                 exception[2].tb_next))
    logging.error(message)
    return {'error': 1}
  finally:
    sys.stdin, sys.stdout = saved_streams
    module.__name__ = module_name
    if '__main__' in sys.modules:
      del sys.modules['__main__']

Example 43

Project: python-compat-runtime
Source File: cgi.py
View license
def HandleRequest(unused_environ, handler_name, unused_url, post_data,
                  unused_error, application_root, python_lib,
                  import_hook=None):
  """Handle a single CGI request.

  Handles a request for handler_name in the form 'path/to/handler.py' with the
  environment contained in environ.

  Args:
    handler_name: A str containing the user-specified handler file to use for
        this request as specified in the script field of a handler in app.yaml.
    post_data: A stream containing the post data for this request.
    application_root: A str containing the root path of the application.
    python_lib: A str containing the root the Python App Engine library.
    import_hook: Optional import hook (PEP 302 style loader).

  Returns:
    A dict containing zero or more of the following:
      error: App Engine error code. 0 for OK, 1 for error. Defaults to OK if not
          set. If set, then the other fields may be missing.
      response_code: HTTP response code.
      headers: A list of tuples (key, value) of HTTP headers.
      body: A str of the body of the response.
  """
  body = cStringIO.StringIO()
  module_name = _FileToModuleName(handler_name)
  parent_module, _, submodule_name = module_name.rpartition('.')
  parent_module = _GetModuleOrNone(parent_module)
  main = None



  if module_name in sys.modules:
    module = sys.modules[module_name]
    main = _GetValidMain(module)
  if not main:
    module = imp.new_module('__main__')
    if import_hook is not None:
      module.__loader__ = import_hook
  saved_streams = sys.stdin, sys.stdout
  try:
    sys.modules['__main__'] = module
    module.__dict__['__name__'] = '__main__'
    sys.stdin = post_data
    sys.stdout = body
    if main:
      os.environ['PATH_TRANSLATED'] = module.__file__
      main()
    else:
      filename = _AbsolutePath(handler_name, application_root, python_lib)
      if filename.endswith(os.sep + '__init__.py'):
        module.__path__ = [os.path.dirname(filename)]
      if import_hook is None:
        code, filename = _LoadModuleCode(filename)
      else:
        code = import_hook.get_code(module_name)
      if not code:
        return {'error': 2}
      os.environ['PATH_TRANSLATED'] = filename
      module.__file__ = filename
      try:
        sys.modules[module_name] = module
        eval(code, module.__dict__)
      except:


        del sys.modules[module_name]
        if parent_module and submodule_name in parent_module.__dict__:
          del parent_module.__dict__[submodule_name]
        raise
      else:
        if parent_module:
          parent_module.__dict__[submodule_name] = module
    return _ParseResponse(body)
  except:
    exception = sys.exc_info()


    message = ''.join(traceback.format_exception(exception[0], exception[1],
                                                 exception[2].tb_next))
    logging.error(message)
    return {'error': 1}
  finally:
    sys.stdin, sys.stdout = saved_streams
    module.__name__ = module_name
    if '__main__' in sys.modules:
      del sys.modules['__main__']

Example 44

Project: ZeroNet
Source File: StatsPlugin.py
View license
    def actionStats(self):
        import gc
        import sys
        from Ui import UiRequest
        from Db import Db
        from Crypt import CryptConnection

        hpy = None
        if self.get.get("size") == "1":  # Calc obj size
            try:
                import guppy
                hpy = guppy.hpy()
            except:
                pass
        self.sendHeader()

        if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
            yield "This function is disabled on this proxy"
            raise StopIteration

        s = time.time()
        main = sys.modules["main"]

        # Style
        yield """
        <style>
         * { font-family: monospace }
         table td, table th { text-align: right; padding: 0px 10px }
         .serving-False { color: gray }
        </style>
        """

        # Memory
        try:
            yield "rev%s | " % config.rev
            yield "%s | " % config.ip_external
            yield "Opened: %s | " % main.file_server.port_opened
            yield "Crypt: %s | " % CryptConnection.manager.crypt_supported
            yield "In: %.2fMB, Out: %.2fMB  | " % (
                float(main.file_server.bytes_recv) / 1024 / 1024,
                float(main.file_server.bytes_sent) / 1024 / 1024
            )
            yield "Peerid: %s  | " % main.file_server.peer_id
            import psutil
            process = psutil.Process(os.getpid())
            mem = process.get_memory_info()[0] / float(2 ** 20)
            yield "Mem: %.2fMB | " % mem
            yield "Threads: %s | " % len(process.threads())
            yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
            yield "Files: %s | " % len(process.open_files())
            yield "Sockets: %s | " % len(process.connections())
            yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
        except Exception:
            pass
        yield "<br>"

        # Connections
        yield "<b>Connections</b> (%s, total made: %s):<br>" % (
            len(main.file_server.connections), main.file_server.last_connection_id
        )
        yield "<table><tr> <th>id</th> <th>proto</th>  <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>"
        yield "<th>buff</th> <th>bad</th> <th>idle</th> <th>open</th> <th>delay</th> <th>cpu</th> <th>out</th> <th>in</th> <th>last sent</th>"
        yield "<th>waiting</th> <th>version</th> <th>sites</th> </tr>"
        for connection in main.file_server.connections:
            if "cipher" in dir(connection.sock):
                cipher = connection.sock.cipher()[0]
            else:
                cipher = connection.crypt
            yield self.formatTableRow([
                ("%3d", connection.id),
                ("%s", connection.protocol),
                ("%s", connection.type),
                ("%s:%s", (connection.ip, connection.port)),
                ("%s", connection.handshake.get("port_opened")),
                ("<span title='%s'>%s</span>", (connection.crypt, cipher)),
                ("%6.3f", connection.last_ping_delay),
                ("%s", connection.incomplete_buff_recv),
                ("%s", connection.bad_actions),
                ("since", max(connection.last_send_time, connection.last_recv_time)),
                ("since", connection.start_time),
                ("%.3f", connection.last_sent_time - connection.last_send_time),
                ("%.3fs", connection.cpu_time),
                ("%.0fkB", connection.bytes_sent / 1024),
                ("%.0fkB", connection.bytes_recv / 1024),
                ("%s", connection.last_cmd),
                ("%s", connection.waiting_requests.keys()),
                ("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?"))),
                ("%s", connection.sites)
            ])
        yield "</table>"

        # Tor hidden services
        yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status
        for site_address, onion in main.file_server.tor_manager.site_onions.items():
            yield "- %-34s: %s<br>" % (site_address, onion)

        # Db
        yield "<br><br><b>Db</b>:<br>"
        for db in sys.modules["Db.Db"].opened_dbs:
            yield "- %.3fs: %s<br>" % (time.time() - db.last_query_time, db.db_path)

        # Sites
        yield "<br><br><b>Sites</b>:"
        yield "<table>"
        yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th>  </tr>"
        for site in sorted(self.server.sites.values(), lambda a, b: cmp(a.address,b.address)):
            yield self.formatTableRow([
                (
                    """<a href='#' class='serving-%s' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
                    (site.settings["serving"], site.address, site.address)
                ),
                ("%s", [peer.connection.id for peer in site.peers.values() if peer.connection and peer.connection.connected]),
                ("%s/%s/%s", (
                    len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected]),
                    len(site.getConnectablePeers(100)),
                    len(site.peers)
                )),
                ("%s (loaded: %s)", (
                    len(site.content_manager.contents),
                    len([key for key, val in dict(site.content_manager.contents).iteritems() if val])
                )),
                ("%.0fkB", site.settings.get("bytes_sent", 0) / 1024),
                ("%.0fkB", site.settings.get("bytes_recv", 0) / 1024),
            ])
            yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address
            for key, peer in site.peers.items():
                if peer.time_found:
                    time_found = int(time.time() - peer.time_found) / 60
                else:
                    time_found = "--"
                if peer.connection:
                    connection_id = peer.connection.id
                else:
                    connection_id = None
                if site.content_manager.hashfield:
                    yield "Optional files: %4s " % len(peer.hashfield)
                yield "(#%4s, err: %s, found: %5s min ago) %30s -<br>" % (connection_id, peer.connection_error, time_found, key)
            yield "<br></td></tr>"
        yield "</table>"

        # No more if not in debug mode
        if not config.debug:
            raise StopIteration

        # Object types

        obj_count = {}
        for obj in gc.get_objects():
            obj_type = str(type(obj))
            if obj_type not in obj_count:
                obj_count[obj_type] = [0, 0]
            obj_count[obj_type][0] += 1  # Count
            obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024  # Size

        yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
            len(obj_count),
            sum([stat[0] for stat in obj_count.values()]),
            sum([stat[1] for stat in obj_count.values()])
        )

        for obj, stat in sorted(obj_count.items(), key=lambda x: x[1][0], reverse=True):  # Sorted by count
            yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))

        # Classes

        class_count = {}
        for obj in gc.get_objects():
            obj_type = str(type(obj))
            if obj_type != "<type 'instance'>":
                continue
            class_name = obj.__class__.__name__
            if class_name not in class_count:
                class_count[class_name] = [0, 0]
            class_count[class_name][0] += 1  # Count
            class_count[class_name][1] += float(sys.getsizeof(obj)) / 1024  # Size

        yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
            len(class_count),
            sum([stat[0] for stat in class_count.values()]),
            sum([stat[1] for stat in class_count.values()])
        )

        for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True):  # Sorted by count
            yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))

        from greenlet import greenlet
        objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
        yield "<br>Greenlets (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Worker import Worker
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
        yield "<br>Workers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Connection import Connection
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
        yield "<br>Connections (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from socket import socket
        objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
        yield "<br>Sockets (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from msgpack import Unpacker
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
        yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Site import Site
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
        yield "<br>Sites (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
        yield "<br>Loggers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj.name)))

        objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
        yield "<br>UiRequests (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Peer import Peer
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
        yield "<br>Peers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        objs = [(key, val) for key, val in sys.modules.iteritems() if val is not None]
        objs.sort()
        yield "<br>Modules (%s):<br>" % len(objs)
        for module_name, module in objs:
            yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module)))

        gc.collect()  # Implicit grabage collection
        yield "Done in %.1f" % (time.time() - s)

Example 45

View license
    def load_module(self, fullname):
        logger.debug('Running load_module for {0}...'.format(fullname))
        if fullname in sys.modules:
            mod = sys.modules[fullname]
        else:
            if self.kind in (imp.PY_COMPILED, imp.C_EXTENSION, imp.C_BUILTIN,
                             imp.PY_FROZEN):
                convert = False
            # elif (self.pathname.startswith(_stdlibprefix)
            #       and 'site-packages' not in self.pathname):
            #     # We assume it's a stdlib package in this case. Is this too brittle?
            #     # Please file a bug report at https://github.com/PythonCharmers/python-future
            #     # if so.
            #     convert = False
            # in theory, other paths could be configured to be excluded here too
            elif any([fullname.startswith(path) for path in self.exclude_paths]):
                convert = False
            elif any([fullname.startswith(path) for path in self.include_paths]):
                convert = True
            else:
                convert = False
            if not convert:
                logger.debug('Excluded {0} from translation'.format(fullname))
                mod = imp.load_module(fullname, *self.found)
            else:
                logger.debug('Autoconverting {0} ...'.format(fullname))
                mod = imp.new_module(fullname)
                sys.modules[fullname] = mod

                # required by PEP 302
                mod.__file__ = self.pathname
                mod.__name__ = fullname
                mod.__loader__ = self

                # This:
                #     mod.__package__ = '.'.join(fullname.split('.')[:-1])
                # seems to result in "SystemError: Parent module '' not loaded,
                # cannot perform relative import" for a package's __init__.py
                # file. We use the approach below. Another option to try is the
                # minimal load_module pattern from the PEP 302 text instead.

                # Is the test in the next line more or less robust than the
                # following one? Presumably less ...
                # ispkg = self.pathname.endswith('__init__.py')
                
                if self.kind == imp.PKG_DIRECTORY:
                    mod.__path__ = [ os.path.dirname(self.pathname) ]
                    mod.__package__ = fullname
                else:
                    #else, regular module
                    mod.__path__ = []
                    mod.__package__ = fullname.rpartition('.')[0]
                    
                try:
                    cachename = imp.cache_from_source(self.pathname)
                    if not os.path.exists(cachename):
                        update_cache = True
                    else:
                        sourcetime = os.stat(self.pathname).st_mtime
                        cachetime = os.stat(cachename).st_mtime
                        update_cache = cachetime < sourcetime
                    # # Force update_cache to work around a problem with it being treated as Py3 code???
                    # update_cache = True
                    if not update_cache:
                        with open(cachename, 'rb') as f:
                            data = f.read()
                            try:
                                code = marshal.loads(data)
                            except Exception:
                                # pyc could be corrupt. Regenerate it
                                update_cache = True
                    if update_cache:
                        if self.found[0]:
                            source = self.found[0].read()
                        elif self.kind == imp.PKG_DIRECTORY:
                            with open(self.pathname) as f:
                                source = f.read()

                        if detect_python2(source, self.pathname):
                            source = self.transform(source)
                            with open('/tmp/futurized_code.py', 'w') as f:
                                f.write('### Futurized code (from %s)\n%s' % 
                                        (self.pathname, source))

                        code = compile(source, self.pathname, 'exec')

                        dirname = os.path.dirname(cachename)
                        if not os.path.exists(dirname):
                            os.makedirs(dirname)
                        try:
                            with open(cachename, 'wb') as f:
                                data = marshal.dumps(code)
                                f.write(data)
                        except Exception:   # could be write-protected
                            pass
                    exec(code, mod.__dict__)
                except Exception as e:
                    # must remove module from sys.modules
                    del sys.modules[fullname]
                    raise # keep it simple

        if self.found[0]:
            self.found[0].close()
        return mod

Example 46

Project: openwrt-mt7620
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name != 'ipython':
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 47

Project: openwrt-mt7620
Source File: forking.py
View license
def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    old_main_modules.append(sys.modules['__main__'])

    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process()._authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'main_path' in data:
        main_path = data['main_path']
        main_name = os.path.splitext(os.path.basename(main_path))[0]
        if main_name == '__init__':
            main_name = os.path.basename(os.path.dirname(main_path))

        if main_name != 'ipython':
            import imp

            if main_path is None:
                dirs = None
            elif os.path.basename(main_path).startswith('__init__.py'):
                dirs = [os.path.dirname(os.path.dirname(main_path))]
            else:
                dirs = [os.path.dirname(main_path)]

            assert main_name not in sys.modules, main_name
            file, path_name, etc = imp.find_module(main_name, dirs)
            try:
                # We would like to do "imp.load_module('__main__', ...)"
                # here.  However, that would cause 'if __name__ ==
                # "__main__"' clauses to be executed.
                main_module = imp.load_module(
                    '__parents_main__', file, path_name, etc
                    )
            finally:
                if file:
                    file.close()

            sys.modules['__main__'] = main_module
            main_module.__name__ = '__main__'

            # Try to make the potentially picklable objects in
            # sys.modules['__main__'] realize they are in the main
            # module -- somewhat ugly.
            for obj in main_module.__dict__.values():
                try:
                    if obj.__module__ == '__parents_main__':
                        obj.__module__ = '__main__'
                except Exception:
                    pass

Example 48

Project: supersmoother
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            if sys.version_info[0] >= 3:
                obj.__name__ = funcname
            else:
                obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 49

Project: nmrglue
Source File: phantom_import.py
View license
def import_phantom_module(xml_file):
    """
    Insert a fake Python module to sys.modules, based on a XML file.

    The XML file is expected to conform to Pydocweb DTD. The fake
    module will contain dummy objects, which guarantee the following:

    - Docstrings are correct.
    - Class inheritance relationships are correct (if present in XML).
    - Function argspec is *NOT* correct (even if present in XML).
      Instead, the function signature is prepended to the function docstring.
    - Class attributes are *NOT* correct; instead, they are dummy objects.

    Parameters
    ----------
    xml_file : str
        Name of an XML file to read
    
    """
    import lxml.etree as etree

    object_cache = {}

    tree = etree.parse(xml_file)
    root = tree.getroot()

    # Sort items so that
    # - Base classes come before classes inherited from them
    # - Modules come before their contents
    all_nodes = dict([(n.attrib['id'], n) for n in root])
    
    def _get_bases(node, recurse=False):
        bases = [x.attrib['ref'] for x in node.findall('base')]
        if recurse:
            j = 0
            while True:
                try:
                    b = bases[j]
                except IndexError: break
                if b in all_nodes:
                    bases.extend(_get_bases(all_nodes[b]))
                j += 1
        return bases

    type_index = ['module', 'class', 'callable', 'object']
    
    def base_cmp(a, b):
        x = cmp(type_index.index(a.tag), type_index.index(b.tag))
        if x != 0: return x

        if a.tag == 'class' and b.tag == 'class':
            a_bases = _get_bases(a, recurse=True)
            b_bases = _get_bases(b, recurse=True)
            x = cmp(len(a_bases), len(b_bases))
            if x != 0: return x
            if a.attrib['id'] in b_bases: return -1
            if b.attrib['id'] in a_bases: return 1
        
        return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))

    nodes = root.getchildren()
    nodes.sort(base_cmp)

    # Create phantom items
    for node in nodes:
        name = node.attrib['id']
        doc = (node.text or '').decode('string-escape') + "\n"
        if doc == "\n": doc = ""

        # create parent, if missing
        parent = name
        while True:
            parent = '.'.join(parent.split('.')[:-1])
            if not parent: break
            if parent in object_cache: break
            obj = imp.new_module(parent)
            object_cache[parent] = obj
            sys.modules[parent] = obj

        # create object
        if node.tag == 'module':
            obj = imp.new_module(name)
            obj.__doc__ = doc
            sys.modules[name] = obj
        elif node.tag == 'class':
            bases = [object_cache[b] for b in _get_bases(node)
                     if b in object_cache]
            bases.append(object)
            init = lambda self: None
            init.__doc__ = doc
            obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
            obj.__name__ = name.split('.')[-1]
        elif node.tag == 'callable':
            funcname = node.attrib['id'].split('.')[-1]
            argspec = node.attrib.get('argspec')
            if argspec:
                argspec = re.sub('^[^(]*', '', argspec)
                doc = "%s%s\n\n%s" % (funcname, argspec, doc)
            obj = lambda: 0
            obj.__argspec_is_invalid_ = True
            obj.func_name = funcname
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__objclass__ = object_cache[parent]
        else:
            class Dummy(object): pass
            obj = Dummy()
            obj.__name__ = name
            obj.__doc__ = doc
            if inspect.isclass(object_cache[parent]):
                obj.__get__ = lambda: None
        object_cache[name] = obj

        if parent:
            if inspect.ismodule(object_cache[parent]):
                obj.__module__ = parent
                setattr(object_cache[parent], name.split('.')[-1], obj)

    # Populate items
    for node in root:
        obj = object_cache.get(node.attrib['id'])
        if obj is None: continue
        for ref in node.findall('ref'):
            if node.tag == 'class':
                if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
                    setattr(obj, ref.attrib['name'],
                            object_cache.get(ref.attrib['ref']))
            else:
                setattr(obj, ref.attrib['name'],
                        object_cache.get(ref.attrib['ref']))

Example 50

Project: Pyrit
Source File: cpyrit.py
View license
    def __init__(self):
        """Create a new instance that blocks calls to .enqueue() when more than
           the given amount of passwords are currently waiting to be scheduled
           to the hardware.
        """
        self.inqueue = []
        self.outqueue = {}
        self.workunits = []
        self.slices = {}
        self.in_idx = self.out_idx = 0
        self.cores = []
        self.CUDAs = []
        self.OpCL = []
        self.all = []
        self.cv = threading.Condition()

        # CUDA
        if config.cfg['use_CUDA'] == 'true' and 'cpyrit._cpyrit_cuda' in sys.modules and config.cfg['use_OpenCL'] == 'false':

            CUDA = _cpyrit_cuda.listDevices()

            for dev_idx, device in enumerate(CUDA):
                self.CUDAs.append(CUDACore(queue=self, dev_idx=dev_idx))


        # OpenCL
        if config.cfg['use_OpenCL'] == 'true' and 'cpyrit._cpyrit_opencl' in sys.modules:

            for platform_idx in range(_cpyrit_opencl.numPlatforms):
                p = _cpyrit_opencl.OpenCLPlatform(platform_idx)
                for dev_idx in range(p.numDevices):
                    dev = _cpyrit_opencl.OpenCLDevice(platform_idx, dev_idx)
                    if dev.deviceType in ('GPU', 'ACCELERATOR'):
                        core = OpenCLCore(self, platform_idx, dev_idx)
                        self.OpCL.append(core)
        # CAL++
        if 'cpyrit._cpyrit_calpp' in sys.modules:
            for dev_idx, device in enumerate(_cpyrit_calpp.listDevices()):
                self.cores.append(CALCore(queue=self, dev_idx=dev_idx))


        # CPUs
        for i in xrange(util.ncpus):
            self.cores.append(CPUCore(queue=self))


        # Network

        if config.cfg['rpc_server'] == 'true':
            for port in xrange(17935, 18000):
                try:
                    ncore = NetworkCore(queue=self, port=port)
                except socket.error:
                    pass
                else:
                    self.ncore_uuid = ncore.uuid
                    self.cores.append(ncore)
                    if config.cfg['rpc_announce'] == 'true':
                        cl = config.cfg['rpc_knownclients'].split(' ')
                        cl = filter(lambda x: len(x) > 0, map(str.strip, cl))
                        bcst = config.cfg['rpc_announce_broadcast'] == 'true'
                        self.announcer = network.NetworkAnnouncer(port=port, \
                                                          clients=cl, \
                                                          broadcast=bcst)
                    break
            else:
                self.ncore_uuid = None
        else:
            self.ncore_uuid = None


        for core in self.cores:
            self.all.append(core)
        for OCL in self.OpCL:
            self.all.append(OCL)
        for CD in self.CUDAs:
            self.all.append(CD)