sys.getfilesystemencoding

Here are the examples of the python api sys.getfilesystemencoding taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: beets
Source File: test_library.py
View license
    def test_non_mbcs_characters_on_windows(self):
        oldfunc = sys.getfilesystemencoding
        sys.getfilesystemencoding = lambda: 'mbcs'
        try:
            self.i.title = u'h\u0259d'
            self.lib.path_formats = [(u'default', u'$title')]
            p = self.i.destination()
            self.assertFalse(b'?' in p)
            # We use UTF-8 to encode Windows paths now.
            self.assertTrue(u'h\u0259d'.encode('utf-8') in p)
        finally:
            sys.getfilesystemencoding = oldfunc

Example 2

Project: beets
Source File: test_util.py
View license
    def _windows_bytestring_path(self, path):
        old_gfse = sys.getfilesystemencoding
        sys.getfilesystemencoding = lambda: 'mbcs'
        try:
            with _common.platform_windows():
                return util.bytestring_path(path)
        finally:
            sys.getfilesystemencoding = old_gfse

Example 3

Project: databus
Source File: glob.py
View license
def glob1(dirname, pattern):
    if not dirname:
        dirname = os.curdir
    if isinstance(pattern, unicode) and not isinstance(dirname, unicode):
        dirname = unicode(dirname, sys.getfilesystemencoding() or
                                   sys.getdefaultencoding())
    try:
        names = os.listdir(dirname)
    except os.error:
        return []
    if pattern[0] != '.':
        names = filter(lambda x: x[0] != '.', names)
    return fnmatch.filter(names, pattern)

Example 4

Project: ZeroNet
Source File: SiteStorage.py
View license
    def __init__(self, site, allow_create=True):
        self.site = site
        self.directory = "%s/%s" % (config.data_dir, self.site.address)  # Site data diretory
        self.allowed_dir = os.path.abspath(self.directory.decode(sys.getfilesystemencoding()))  # Only serve file within this dir
        self.log = site.log
        self.db = None  # Db class
        self.db_checked = False  # Checked db tables since startup
        self.event_db_busy = None  # Gevent AsyncResult if db is working on rebuild
        self.has_db = self.isFile("dbschema.json")  # The site has schema

        if not os.path.isdir(self.directory):
            if allow_create:
                os.mkdir(self.directory)  # Create directory if not found
            else:
                raise Exception("Directory not exists: %s" % self.directory)

Example 5

Project: django-gears
Source File: finders.py
View license
    def get_app_assets_dirs(self):
        if not six.PY3:
            fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
        app_assets_dirs = []
        for app in settings.INSTALLED_APPS:
            try:
                mod = import_module(app)
            except ImportError as e:
                raise ImproperlyConfigured('ImportError %s: %s' % (app, e.args[0]))
            assets_dir = os.path.join(os.path.dirname(mod.__file__), 'assets')
            if os.path.isdir(assets_dir):
                if not six.PY3:
                    assets_dir = assets_dir.decode(fs_encoding)
                app_assets_dirs.append(assets_dir)
        app_assets_dirs = tuple(app_assets_dirs)
        return app_assets_dirs

Example 6

Project: CythonCTypesBackend
Source File: Utils.py
View license
def decode_filename(filename):
    if isinstance(filename, unicode):
        return filename
    try:
        filename_encoding = sys.getfilesystemencoding()
        if filename_encoding is None:
            filename_encoding = sys.getdefaultencoding()
        filename = filename.decode(filename_encoding)
    except UnicodeDecodeError:
        pass
    return filename

Example 7

Project: PlotSummary
Source File: interactive.py
View license
    def _fsencoding(self):
        """Get the system's filesystem encoding. On Windows, this is always
        UTF-8 (not MBCS).
        """
        encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
        if encoding == 'mbcs':
            # On Windows, a broken encoding known to Python as "MBCS" is
            # used for the filesystem. However, we only use the Unicode API
            # for Windows paths, so the encoding is actually immaterial so
            # we can avoid dealing with this nastiness. We arbitrarily
            # choose UTF-8.
            encoding = 'utf8'
        return encoding

Example 8

Project: entropy
Source File: misc.py
View license
    def _encode_path(self, path):
        """
        Encode path using proper encoding for use with os functions.
        """
        try:
            path = const_convert_to_rawstring(
                path, from_enctype=etpConst['conf_encoding'])
        except (UnicodeEncodeError,):
            path = const_convert_to_rawstring(
                path, from_enctype=sys.getfilesystemencoding())
        return path

Example 9

Project: moul-scripts
Source File: glob.py
View license
def glob1(dirname, pattern):
    if not dirname:
        dirname = os.curdir
    if isinstance(pattern, unicode) and not isinstance(dirname, unicode):
        dirname = unicode(dirname, sys.getfilesystemencoding() or
                                   sys.getdefaultencoding())
    try:
        names = os.listdir(dirname)
    except os.error:
        return []
    if pattern[0] != '.':
        names = filter(lambda x: x[0] != '.', names)
    return fnmatch.filter(names, pattern)

Example 10

Project: openmetadata
Source File: utils.py
View license
def make_str(value):
    """Converts a value into a valid string."""
    if isinstance(value, bytes):
        try:
            return value.decode(sys.getfilesystemencoding())
        except UnicodeError:
            return value.decode('utf-8', 'replace')
    return text_type(value)

Example 11

Project: vim-epub
Source File: path.py
View license
    @classmethod
    def _always_unicode(cls, path):
        """
        Ensure the path as retrieved from a Python API, such as os.listdir,
        is a proper Unicode string.
        """
        if PY3 or isinstance(path, unicode):
            return path
        return path.decode(sys.getfilesystemencoding(), 'surrogateescape')

Example 12

Project: django-icanhaz
Source File: finders.py
View license
def _get_app_template_dirs():
    fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
    ret = []
    for app in conf.INSTALLED_APPS:
        try:
            mod = import_module(app)
        except ImportError:
            e = sys.exc_info()[1]
            raise ImproperlyConfigured("ImportError %s: %s" % (app, e.args[0]))
        app_dir = os.path.dirname(mod.__file__)
        for dirname in conf.ICANHAZ_APP_DIRNAMES:
            template_dir = os.path.join(app_dir, dirname)
            if os.path.isdir(template_dir):
                # Only python 2.x needs decoding
                if isinstance(template_dir, bytes):
                    template_dir = template_dir.decode(fs_encoding)
                ret.append(template_dir)
    return ret

Example 13

Project: cython
Source File: Utils.py
View license
def decode_filename(filename):
    if isinstance(filename, bytes):
        try:
            filename_encoding = sys.getfilesystemencoding()
            if filename_encoding is None:
                filename_encoding = sys.getdefaultencoding()
            filename = filename.decode(filename_encoding)
        except UnicodeDecodeError:
            pass
    return filename

Example 14

Project: sumatra
Source File: _git.py
View license
    def content(self, digest, file=None):
        """Get the file content from repository."""
        repo = git.Repo(self.path)
        curtree = repo.commit(digest).tree
        if file is None:
            return curtree.blobs[0].data_stream.read() # Get the latest added file content.
        dirname, filename = os.path.split(file)
        if dirname != '':
            for dname in dirname.split(os.path.sep):
                for subtree in curtree.trees:
                    if subtree.name == dname:
                        curtree = subtree
                        break
        for blob in curtree.blobs:
            if blob.name == filename:
                expected_encoding = sys.getfilesystemencoding()
                file_content = blob.data_stream.read().decode(expected_encoding)
                return file_content
        return 'File content not found.'

Example 15

Project: scandir
Source File: test_scandir.py
View license
    def test_bytes(self):
        # Check that unicode filenames are returned correctly as bytes in output
        path = os.path.join(TEST_PATH, 'subdir').encode(sys.getfilesystemencoding(), 'replace')
        self.assertTrue(isinstance(path, bytes))
        if IS_PY3 and sys.platform == 'win32':
            self.assertRaises(TypeError, self.scandir_func, path)
            return

        entries = [e for e in self.scandir_func(path) if e.name.startswith(b'unicod')]
        self.assertEqual(len(entries), 1)
        entry = entries[0]

        self.assertTrue(isinstance(entry.name, bytes))
        self.assertTrue(isinstance(entry.path, bytes))

        # b'unicod?.txt' on Windows, b'unicod\xc6\x8f.txt' (UTF-8) or similar on POSIX
        entry_name = 'unicod\u018f.txt'.encode(sys.getfilesystemencoding(), 'replace')
        self.assertEqual(entry.name, entry_name)
        self.assertEqual(entry.path, os.path.join(path, entry_name))

Example 16

Project: be
Source File: utils.py
View license
def make_str(value):
    """Converts a value into a valid string."""
    if isinstance(value, bytes):
        try:
            return value.decode(sys.getfilesystemencoding())
        except UnicodeError:
            return value.decode('utf-8', 'replace')
    return text_type(value)

Example 17

Project: pmatic
Source File: glob.py
View license
def glob1(dirname, pattern):
    if not dirname:
        dirname = os.curdir
    if isinstance(pattern, _unicode) and not isinstance(dirname, unicode):
        dirname = unicode(dirname, sys.getfilesystemencoding() or
                                   sys.getdefaultencoding())
    try:
        names = os.listdir(dirname)
    except os.error:
        return []
    if pattern[0] != '.':
        names = filter(lambda x: x[0] != '.', names)
    return fnmatch.filter(names, pattern)

Example 18

Project: Tardis
Source File: RemoteDB.py
View license
def fs_encode(val):
    """ Turn filenames into str's (ie, series of bytes) rather than Unicode things """
    if not isinstance(val, bytes):
        #return val.encode(sys.getfilesystemencoding())
        return val.encode(sys.getfilesystemencoding())
    else:
        return val

Example 19

Project: PySyncObj
Source File: atomic_replace.py
View license
    def atomicReplace(oldPath, newPath):
        if not isinstance(oldPath, str):
            oldPath = str(oldPath, sys.getfilesystemencoding())
        if not isinstance(newPath, str):
            newPath = str(newPath, sys.getfilesystemencoding())
        ta = CreateTransaction(None, 0, 0, 0, 0, 1000, 'atomic_replace')
        if ta == -1:
            return False
        res = MoveFileTransacted(oldPath, newPath, None, None, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, ta)
        if not res:
            CloseHandle(ta)
            return False
        res = CommitTransaction(ta)
        CloseHandle(ta)
        return bool(res)

Example 20

Project: pandocfilters
Source File: pandocfilters.py
View license
def get_filename4code(module, content, ext=None):
    """Generate filename based on content

    The function ensures that the (temporary) directory exists, so that the
    file can be written.

    Example:
        filename = get_filename4code("myfilter", code)
    """
    imagedir = module + "-images"
    fn = hashlib.sha1(content.encode(sys.getfilesystemencoding())).hexdigest()
    try:
        os.mkdir(imagedir)
        sys.stderr.write('Created directory ' + imagedir + '\n')
    except OSError:
        pass
    if ext:
        fn += "." + ext
    return os.path.join(imagedir, fn)

Example 21

Project: ComicStreamer
Source File: folders.py
View license
    @staticmethod
    def userFolder():
        filename_encoding = sys.getfilesystemencoding()
        if platform.system() == "Windows":
            folder = os.path.join( AppFolders.windowsAppDataFolder(), 'ComicStreamer' )
        elif platform.system() == "Darwin":
            folder = os.path.join( os.path.expanduser('~') , 'Library/Application Support/ComicStreamer')
        else:
            folder = os.path.join( os.path.expanduser('~') , '.ComicStreamer')
        if folder is not None:
            folder = folder.decode(filename_encoding)
        return folder

Example 22

Project: knitpy
Source File: py3compat.py
View license
        def execfile(fname, glob=None, loc=None, compiler=None):
            if isinstance(fname, unicode):
                filename = fname.encode(sys.getfilesystemencoding())
            else:
                filename = fname
            where = [ns for ns in [glob, loc] if ns is not None]
            if compiler is None:
                builtin_mod.execfile(filename, *where)
            else:
                scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
                exec(compiler(scripttext, filename, 'exec'), glob, loc)

Example 23

Project: spotter
Source File: spotter.py
View license
    def process_default(self, event):
        """Run the commands that have a pattern matching the events path

        Stops running commands once one fails or is marked as final"""
        path = event.pathname.decode(sys.getfilesystemencoding())

        for watchlist in self.watchlists:
            for watch in watchlist:
                if watch.pattern_matches(os.path.relpath(path)):
                    success = self.run(
                        watch.command,
                        filename=path, **watchlist.definitions)
                    # Always stop if the watch was final
                    if watch.final:
                        break
                    # Stop if we failed without --continue-on-fail
                    if not (success or self.options.continue_on_fail):
                        break

Example 24

Project: ionyweb
Source File: __init__.py
View license
def get_ionyweb_path():
    """
        Return full patt of ionyweb
    """
    fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
    template_dir = os.path.dirname(__file__)
    return template_dir.decode(fs_encoding)

Example 25

Project: geofront-cli
Source File: utils.py
View license
def get_env_path(key, default):
    """Get a UTF-8 encoded path from an environment variable."""
    if key in os.environ:
        # on windows, environment variables are mbcs bytes
        # so we must turn them into utf-8 Syncdaemon paths
        try:
            path = os.environb.get(key.encode('utf-8'))
        except AttributeError:
            path = os.environ[key]
        return path.decode(sys.getfilesystemencoding()).encode('utf-8')
    else:
        if not isinstance(default, bytes):
            return default.encode('utf-8')
        return default

Example 26

Project: winpython
Source File: utils.py
View license
def decode_fs_string(string):
    """Convert string from file system charset to unicode"""
    charset = sys.getfilesystemencoding()
    if charset is None:
        charset = locale.getpreferredencoding()
    return string.decode(charset)

Example 27

Project: sublime-wakatime
Source File: stats.py
View license
def number_lines_in_file(file_name):
    lines = 0
    try:
        with open(file_name, 'r', encoding='utf-8') as fh:
            for line in fh:
                lines += 1
    except:  # pragma: nocover
        try:
            with open(file_name, 'r', encoding=sys.getfilesystemencoding()) as fh:
                for line in fh:
                    lines += 1
        except:
            return None
    return lines

Example 28

Project: Veil-Evasion
Source File: test_archive_util.py
View license
def can_fs_encode(filename):
    """
    Return True if the filename can be saved in the file system.
    """
    if os.path.supports_unicode_filenames:
        return True
    try:
        filename.encode(sys.getfilesystemencoding())
    except UnicodeEncodeError:
        return False
    return True

Example 29

Project: rez
Source File: misc.py
View license
    def fsencode(filename):
        if isinstance(filename, bytes):
            return filename
        elif isinstance(filename, str):
            return filename.encode(sys.getfilesystemencoding())
        else:
            raise TypeError("expect bytes or str, not %s" %
                            type(filename).__name__)

Example 30

Project: Gooey
Source File: processor.py
View license
  def run(self, command):
    env = os.environ.copy()
    env["GOOEY"] = "1"
    self._process = subprocess.Popen(
      command.encode(sys.getfilesystemencoding()),
      bufsize=1, stdout=subprocess.PIPE,
      stderr=subprocess.STDOUT, shell=True, env=env)
    Pool(1).apply_async(self._forward_stdout, (self._process,))

Example 31

Project: SublimeIPythonNotebook
Source File: py3compat.py
View license
        def execfile(fname, *where):
            if isinstance(fname, unicode):
                filename = fname.encode(sys.getfilesystemencoding())
            else:
                filename = fname
            __builtin__.execfile(filename, *where)

Example 32

Project: zine
Source File: __init__.py
View license
    def test_instance_folder(self, request):
        """Check if the instance folder exists."""
        _ = request.translations.gettext
        if not path.exists(self.instance_folder):
            folder = self.instance_folder
            if not isinstance(folder, unicode):
                folder = folder.decode(sys.getfilesystemencoding() or 'utf-8',
                                       'ignore')
            return {'error': _(u'Instance folder does not exist.  You have to '
                    u'create the folder “%s” before proceeding.') % folder}

Example 33

View license
    def get_openfiles_counters(self):
        fs_enc = sys.getfilesystemencoding()
        ret, ps = popen('lsof')
        lines = ret[0].decode(fs_enc).splitlines()
        open_files = len(lines) - 1
        inot_files = len([line for line in lines  if 'anon_inode' in line])
        return {'open_files': open_files,
                'inotify_openfiles': inot_files}

Example 34

Project: veusz
Source File: veusz_main.py
View license
def convertArgsUnicode(args):
    '''Convert set of arguments to unicode.
    Arguments in argv use current file system encoding
    '''
    enc = sys.getfilesystemencoding()
    # bail out if not supported
    if enc is None:
        return args
    out = []
    for a in args:
        if isinstance(a, cbytes):
            out.append( a.decode(enc) )
        else:
            out.append(a)
    return out

Example 35

Project: hae
Source File: process.py
View license
	def decode(self, bytes):
		if self.encoding:
			return str(bytes, self.encoding)
		try:
			return str(bytes, sys.getfilesystemencoding())
		except:
			pass
		try:
			return str(bytes, sys.getdefaultencoding())
		except:
			pass
		return str(bytes)[2:-1]

Example 36

Project: emailed-me
Source File: sessions.py
View license
    def __init__(self, path=None, filename_template='werkzeug_%s.sess',
                 session_class=None, renew_missing=False, mode=0644):
        SessionStore.__init__(self, session_class)
        if path is None:
            path = tempfile.gettempdir()
        self.path = path
        if isinstance(filename_template, unicode):
            filename_template = filename_template.encode(
                sys.getfilesystemencoding() or 'utf-8')
        assert not filename_template.endswith(_fs_transaction_suffix), \
            'filename templates may not end with %s' % _fs_transaction_suffix
        self.filename_template = filename_template
        self.renew_missing = renew_missing
        self.mode = mode

Example 37

Project: eavatar-me
Source File: agent.py
View license
def _mygetfilesystemencoding():
    old = sys.getfilesystemencoding

    def inner_func():
        ret = old()
        if ret is None:
            return 'utf-8'
        else:
            return ret
    return inner_func

Example 38

Project: mavelous
Source File: sessions.py
View license
    def __init__(self, path=None, filename_template='werkzeug_%s.sess',
                 session_class=None, renew_missing=False, mode=0644):
        SessionStore.__init__(self, session_class)
        if path is None:
            path = tempfile.gettempdir()
        self.path = path
        if isinstance(filename_template, unicode):
            filename_template = filename_template.encode(
                sys.getfilesystemencoding() or 'utf-8')
        assert not filename_template.endswith(_fs_transaction_suffix), \
            'filename templates may not end with %s' % _fs_transaction_suffix
        self.filename_template = filename_template
        self.renew_missing = renew_missing
        self.mode = mode

Example 39

Project: komodo-wakatime
Source File: git.py
View license
    def branch(self):
        base = self._project_base()
        if base:
            head = os.path.join(self._project_base(), '.git', 'HEAD')
            try:
                with open(head, 'r', encoding='utf-8') as fh:
                    return u(fh.readline().strip().rsplit('/', 1)[-1])
            except UnicodeDecodeError:  # pragma: nocover
                try:
                    with open(head, 'r', encoding=sys.getfilesystemencoding()) as fh:
                        return u(fh.readline().strip().rsplit('/', 1)[-1])
                except:
                    log.traceback('warn')
            except IOError:  # pragma: nocover
                log.traceback('warn')
        return u('master')

Example 40

Project: powerline
Source File: encoding.py
View license
def get_preferred_file_name_encoding():
	'''Get preferred file name encoding
	'''
	return (
		sys.getfilesystemencoding()
		or locale.getpreferredencoding()
		or 'utf-8'
	)

Example 41

Project: peppy
Source File: utils.py
View license
def normalize(ref, base=None):
    """Normalize a url string into a reference and fix windows shenanigans"""
    if not isinstance(ref, Reference):
        if ref.startswith('file:'):
            # URLs always use /, so change windows path separators to forward
            # slashes
            try:
                ref = unicode(ref)
            except UnicodeDecodeError:
                try:
                    ref = str(ref).decode(sys.getfilesystemencoding())
                except UnicodeDecodeError:
                    ref = str(ref).decode('utf-8')
            #dprint(repr(ref))
            if os.path.sep == '\\':
                ref = ref.replace(os.path.sep, '/')
        ref = get_reference(ref)
    # If the reference is absolute (i.e.  contains a scheme), we return;
    # otherwise we assume it's a file:// URI
    #dprint(str(ref))
    if ref.scheme:
        return ref

    # Default to the current working directory
    if base is None:
        try:
            base = os.getcwd().decode(sys.getfilesystemencoding())
        except UnicodeDecodeError:
            base = os.getcwd().decode('utf-8')

    # URLs always use /
    if os.path.sep == '\\':
        base = base.replace(os.path.sep, '/')
    #dprint(base)
    # Check windows drive letters and add extra slash for correct URL syntax
    if len(base) > 1 and base[1] == ':':
        base = "/%s:%s" % (base[0].lower(), base[2:])
    baseref = get_reference(u'file://%s/' % base)
    try:
        path = unicode(ref.path)
    except UnicodeDecodeError:
        try:
            path = str(ref.path).decode(sys.getfilesystemencoding())
        except UnicodeDecodeError:
            path = str(ref.path).decode('utf-8')
    #dprint(repr(path))
    
    # Add the query string and fragment if they exist
    newref = baseref.resolve(path)
    newref.query = ref.query
    newref.fragment = ref.fragment
    #dprint(newref)
    return newref

Example 42

Project: luci-py
Source File: arfile_test.py
View license
  def assertCLI(self, *initems, **kw):
    extra_args = kw.get('extra_args', [])

    indir = None
    ardir = None
    outdir = None
    try:
      indir = tempfile.mkdtemp().decode(sys.getfilesystemencoding())
      ardir = tempfile.mkdtemp().decode(sys.getfilesystemencoding())
      outdir = tempfile.mkdtemp().decode(sys.getfilesystemencoding())

      arp = os.path.join(ardir, 'out.ar')
      assert not os.path.exists(arp)

      # Write out a directory tree
      files = []
      for fp, contents in initems:
        fn = os.path.join(indir, fp)
        dn = os.path.dirname(fn)
        if not os.path.exists(dn):
          os.makedirs(dn)

        with file(fn, 'wb') as f:
          f.write(contents)

        files.append(fp)

      files.sort()
      fileslist = '\n'.join(files)

      # Create an archive from a directory
      self.runCLI(['create', '--filename', arp, indir] + extra_args)
      self.assertTrue(
          os.path.exists(arp), '%s file should exists' % arp)

      # List the archive contents
      output, _ = self.runCLI(['list', '--filename', arp])
      filesoutput = '\n'.join(sorted(output[:-1].split('\n')))
      self.assertMultiLineEqual(fileslist, filesoutput)

      # Extract the archive
      os.chdir(outdir)
      self.runCLI(['extract', '--filename', arp] + extra_args)

      # Walk the directory tree and collect the extracted output
      outitems = []
      for root, _, files in os.walk(outdir):
        for fn in files:
          fp = os.path.join(root, fn)
          outitems.append([fp[len(outdir)+1:], file(fp, 'rb').read()])

      # Check the two are equal
      self.assertSequenceEqual(sorted(initems), sorted(outitems))

    finally:
      if indir:
        shutil.rmtree(indir, ignore_errors=True)
      if ardir:
        shutil.rmtree(ardir, ignore_errors=True)
      if outdir:
        shutil.rmtree(outdir, ignore_errors=True)

Example 43

Project: luci-py
Source File: arfile_test.py
View license
  def assertCLI(self, *initems, **kw):
    extra_args = kw.get('extra_args', [])

    indir = None
    ardir = None
    outdir = None
    try:
      indir = tempfile.mkdtemp().decode(sys.getfilesystemencoding())
      ardir = tempfile.mkdtemp().decode(sys.getfilesystemencoding())
      outdir = tempfile.mkdtemp().decode(sys.getfilesystemencoding())

      arp = os.path.join(ardir, 'out.ar')
      assert not os.path.exists(arp)

      # Write out a directory tree
      files = []
      for fp, contents in initems:
        fn = os.path.join(indir, fp)
        dn = os.path.dirname(fn)
        if not os.path.exists(dn):
          os.makedirs(dn)

        with file(fn, 'wb') as f:
          f.write(contents)

        files.append(fp)

      files.sort()
      fileslist = '\n'.join(files)

      # Create an archive from a directory
      self.runCLI(['create', '--filename', arp, indir] + extra_args)
      self.assertTrue(
          os.path.exists(arp), '%s file should exists' % arp)

      # List the archive contents
      output, _ = self.runCLI(['list', '--filename', arp])
      filesoutput = '\n'.join(sorted(output[:-1].split('\n')))
      self.assertMultiLineEqual(fileslist, filesoutput)

      # Extract the archive
      os.chdir(outdir)
      self.runCLI(['extract', '--filename', arp] + extra_args)

      # Walk the directory tree and collect the extracted output
      outitems = []
      for root, _, files in os.walk(outdir):
        for fn in files:
          fp = os.path.join(root, fn)
          outitems.append([fp[len(outdir)+1:], file(fp, 'rb').read()])

      # Check the two are equal
      self.assertSequenceEqual(sorted(initems), sorted(outitems))

    finally:
      if indir:
        shutil.rmtree(indir, ignore_errors=True)
      if ardir:
        shutil.rmtree(ardir, ignore_errors=True)
      if outdir:
        shutil.rmtree(outdir, ignore_errors=True)

Example 44

Project: OpenSesame
Source File: plugins.py
View license
def load_mod(path, mod, pkg=None):

	"""
	Dynamically loads a module from a path.

	Arguments:
	path		--	A folder or file name. If a filename is specified, the
					file's directory will be used as path.
	mod			--	The name of a module, which corresponds to the name of the
					source file without the `.py` extension.
	pkg			--	The module's package. This is effectively a subfolder that
					is added to the path. (default=None)

	Returns:
	A Python module.
	"""

	import imp
	path = safe_decode(path, enc=sys.getfilesystemencoding())
	if not os.path.isdir(path):
		path = os.path.dirname(path)
	if pkg is not None:
		path = os.path.join(path, pkg)
	path = os.path.join(path, mod+u'.py')
	if not os.path.exists(path):
		raise osexception(u'%s does not exist' % path)
	debug.msg(u'loading module from %s' % path)
	if not py3:
		mod = safe_encode(mod, enc=sys.getfilesystemencoding())
		path = safe_encode(path, enc=sys.getfilesystemencoding())
	return imp.load_source(mod, path)

Example 45

Project: rbtools
Source File: process.py
View license
def execute(command,
            env=None,
            split_lines=False,
            ignore_errors=False,
            extra_ignore_errors=(),
            translate_newlines=True,
            with_errors=True,
            none_on_ignored_error=False,
            return_error_code=False,
            log_output_on_error=True,
            results_unicode=True,
            return_errors=False):
    """Utility function to execute a command and return the output.

    :param command:
        The command to execute as either a string or a list of strings.
    :param env:
        The environment variables to pass to the called executable.
        These variables will be added to the current set of environment
        variables.
    :param split_lines:
        Determines if the program's output will be split into multiple lines.
    :param ignore_errors:
        If ``False``, RBTools will exit if a command returns a non-zero status.
    :param extra_ignore_errors:
        The set of return status codes that should be treated as if the program
        exited with status 0.
    :param translate_newlines:
        If ``True``, all line endings will be translated to ``\n``.
    :param with_errors:
        If ``True``, the command's standard output and standard error streams
        will be combined. This parameter is mutually exclusive with the
        ``return_errors`` parameter.
    :param none_on_ignored_error:
        If ``True``, this function will return ``None`` if either
        ``ignore_errors`` is ``True` and the program returns a non-zero exit
        status or the program exits with a status code in
        ``extra_ignored_errors``.
    :param return_error_code:
        Determines if the exit status of the executed command will also be
        returned.
    :param log_output_on_error:
        Determines if the output of the executed command will be logged if it
        returns a non-zero status code.
    :param results_unicode:
        Determines if the output will be interpreted as UTF-8. If ``True``,
        the process's output will be returned as a ``six.text_type``.
        Otherwise, it will return a ``six.binary_type``.
    :param return_errors:
        Determines if the standard error stream will be returned. This
        parameter is mutually exclusive with the ``with_errors`` parameter.

    :returns:
        This function returns either a single value or a 2- or 3-tuple.

        If ``return_error_code`` is True, the error code of the process will be
        returned as the first element of the tuple.

        If ``return_errors`` is True, the process' standard error stream will
        be returned as the last element of the tuple.

        If both of ``return_error_code`` and ``return_errors`` are ``False``,
        then the process' output will be returned. If either or both of them
        are ``True``, then this is the other element of the returned tuple.
    """
    def post_process_output(output):
        """Post process the given output to convert it to the desired type."""
        # If Popen is called with universal_newlines=True, the resulting data
        # returned from stdout will be a text stream (and therefore a unicode
        # object). Otherwise, it will be a byte stream. Translate the results
        # into the desired type.
        encoding = sys.getfilesystemencoding()

        if split_lines and len(output) > 0:
            if results_unicode and isinstance(output[0], six.binary_type):
                return [line.decode(encoding) for line in output]
            elif not results_unicode and isinstance(output[0], six.text_type):
                return [line.encode('utf-8') for line in output]
        elif not split_lines:
            if results_unicode and isinstance(output, six.binary_type):
                return output.decode(encoding)
            elif not results_unicode and isinstance(output, six.text_type):
                return output.encode('utf-8')

        return output

    assert not (with_errors and return_errors)

    if isinstance(command, list):
        logging.debug(b'Running: ' + subprocess.list2cmdline(command))
    else:
        logging.debug(b'Running: ' + command)

    new_env = os.environ.copy()

    if env:
        new_env.update(env)

    # TODO: This can break on systems that don't have the en_US locale
    # installed (which isn't very many). Ideally in this case, we could
    # put something in the config file, but that's not plumbed through to here.
    new_env['LC_ALL'] = 'en_US.UTF-8'
    new_env['LANGUAGE'] = 'en_US.UTF-8'

    if with_errors:
        errors_output = subprocess.STDOUT
    else:
        errors_output = subprocess.PIPE

    if sys.platform.startswith('win'):
        # Convert all environment variables to byte strings, so that subprocess
        # doesn't blow up on Windows.
        new_env = dict(
            (six.binary_type(key), six.binary_type(value))
            for key, value in six.iteritems(new_env)
        )

        p = subprocess.Popen(command,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=errors_output,
                             shell=False,
                             universal_newlines=translate_newlines,
                             env=new_env)
    else:
        p = subprocess.Popen(command,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=errors_output,
                             shell=False,
                             close_fds=True,
                             universal_newlines=translate_newlines,
                             env=new_env)

    errors = None

    if split_lines:
        data = p.stdout.readlines()

        if return_errors:
            errors = p.stderr.readlines()
    else:
        data = p.stdout.read()

        if return_errors:
            errors = p.stderr.read()

    rc = p.wait()

    if rc and not ignore_errors and rc not in extra_ignore_errors:
        die('Failed to execute command: %s\n%s' % (command, data))
    elif rc:
        if log_output_on_error:
            logging.debug('Command exited with rc %s: %s\n%s---'
                          % (rc, command, data))
        else:
            logging.debug('Command exited with rc %s: %s'
                          % (rc, command))

    if rc and none_on_ignored_error:
        data = None

    if data is not None:
        data = post_process_output(data)

    if return_errors:
        errors = post_process_output(errors)

    if return_error_code or return_errors:
        if return_error_code and return_errors:
            return rc, data, errors
        elif return_error_code:
            return rc, data
        else:
            return data, errors
    else:
        return data

Example 46

Project: PySide
Source File: pysideinclude.py
View license
    def run(self):
        document = self.state.document
        filename = self.arguments[0]
        if not document.settings.file_insertion_enabled:
            return [document.reporter.warning('File insertion disabled',
                                              line=self.lineno)]
        env = document.settings.env
        if filename.startswith('/') or filename.startswith(os.sep):
            rel_fn = filename[1:]
        else:
            docdir = path.dirname(env.doc2path(env.docname, base=None))
            rel_fn = path.join(docdir, filename)
        try:
            fn = path.join(env.srcdir, rel_fn)
        except UnicodeDecodeError:
            # the source directory is a bytestring with non-ASCII characters;
            # let's try to encode the rel_fn in the file system encoding
            rel_fn = rel_fn.encode(sys.getfilesystemencoding())
            fn = path.join(env.srcdir, rel_fn)

        if 'pyobject' in self.options and 'lines' in self.options:
            return [document.reporter.warning(
                'Cannot use both "pyobject" and "lines" options',
                line=self.lineno)]

        encoding = self.options.get('encoding', env.config.source_encoding)
        codec_info = codecs.lookup(encoding)
        try:
            f = codecs.StreamReaderWriter(open(fn, 'U'),
                    codec_info[2], codec_info[3], 'strict')
            lines = f.readlines()
            f.close()
        except (IOError, OSError):
            return [document.reporter.warning(
                'Include file %r not found or reading it failed' % filename,
                line=self.lineno)]
        except UnicodeError:
            return [document.reporter.warning(
                'Encoding %r used for reading included file %r seems to '
                'be wrong, try giving an :encoding: option' %
                (encoding, filename))]

        objectname = self.options.get('pyobject')
        if objectname is not None:
            from sphinx.pycode import ModuleAnalyzer
            analyzer = ModuleAnalyzer.for_file(fn, '')
            tags = analyzer.find_tags()
            if objectname not in tags:
                return [document.reporter.warning(
                    'Object named %r not found in include file %r' %
                    (objectname, filename), line=self.lineno)]
            else:
                lines = lines[tags[objectname][1]-1 : tags[objectname][2]-1]

        linespec = self.options.get('lines')
        if linespec is not None:
            try:
                linelist = parselinenos(linespec, len(lines))
            except ValueError, err:
                return [document.reporter.warning(str(err), line=self.lineno)]
            lines = [lines[i] for i in linelist]

        startafter = self.options.get('start-after')
        endbefore  = self.options.get('end-before')
        prepend    = self.options.get('prepend')
        append     = self.options.get('append')
        snippet    = self.options.get('snippet')

        if snippet:
            startafter = "//![%s]" % snippet
            endbefore = "//![%s]" % snippet

        if startafter is not None or endbefore is not None:
            use = not startafter
            res = []
            for line in lines:
                if not use and startafter and startafter in line:
                    use = True
                elif use and endbefore and endbefore in line:
                    use = False
                    break
                elif use:
                    if not line.startswith("//!"):
                        res.append(line)
            lines = res

        if prepend:
           lines.insert(0, prepend + '\n')
        if append:
           lines.append(append + '\n')

        text = ''.join(lines)
        if self.options.get('tab-width'):
            text = text.expandtabs(self.options['tab-width'])
        retnode = nodes.literal_block(text, text, source=fn)
        retnode.line = 1
        retnode.attributes['line_number'] = self.lineno
        if self.options.get('language', ''):
            retnode['language'] = self.options['language']
        if 'linenos' in self.options:
            retnode['linenos'] = True
        document.settings.env.note_dependency(rel_fn)
        return [retnode]

Example 47

Project: qutebrowser
Source File: mhtml.py
View license
def start_download_checked(dest, tab):
    """First check if dest is already a file, then start the download.

    Args:
        dest: The filename where the resulting file should be saved.
        tab: Specify the tab whose page should be loaded.
    """
    # The default name is 'page title.mht'
    title = tab.title()
    default_name = utils.sanitize_filename(title + '.mht')

    # Remove characters which cannot be expressed in the file system encoding
    encoding = sys.getfilesystemencoding()
    default_name = utils.force_encoding(default_name, encoding)
    dest = utils.force_encoding(dest, encoding)

    dest = os.path.expanduser(dest)

    # See if we already have an absolute path
    path = downloads.create_full_filename(default_name, dest)
    if path is None:
        # We still only have a relative path, prepend download_dir and
        # try again.
        path = downloads.create_full_filename(
            default_name, os.path.join(downloads.download_dir(), dest))
    downloads.last_used_directory = os.path.dirname(path)

    # Avoid downloading files if we can't save the output anyway...
    # Yes, this is prone to race conditions, but we're checking again before
    # saving the file anyway.
    if not os.path.isdir(os.path.dirname(path)):
        folder = os.path.dirname(path)
        message.error("Directory {} does not exist.".format(folder))
        return

    if not os.path.isfile(path):
        _start_download(path, tab=tab)
        return

    q = usertypes.Question()
    q.mode = usertypes.PromptMode.yesno
    q.title = "Overwrite existing file?"
    q.text = "<b>{}</b> already exists. Overwrite?".format(
        html.escape(path))
    q.completed.connect(q.deleteLater)
    q.answered_yes.connect(functools.partial(
        _start_download, path, tab=tab))
    message.global_bridge.ask(q, blocking=False)

Example 48

Project: OmniMarkupPreviewer
Source File: images.py
View license
    def run(self):
        figwidth = self.options.pop('figwidth', None)
        figclasses = self.options.pop('figclass', None)
        align = self.options.pop('align', None)
        (image_node,) = Image.run(self)
        if isinstance(image_node, nodes.system_message):
            return [image_node]
        figure_node = nodes.figure('', image_node)
        if figwidth == 'image':
            if PIL and self.state.document.settings.file_insertion_enabled:
                imagepath = urllib.url2pathname(image_node['uri'])
                try:
                    img = PIL.Image.open(
                            imagepath.encode(sys.getfilesystemencoding()))
                except (IOError, UnicodeEncodeError):
                    pass # TODO: warn?
                else:
                    self.state.document.settings.record_dependencies.add(
                        imagepath.replace('\\', '/'))
                    figure_node['width'] = img.size[0]
                    del img
        elif figwidth is not None:
            figure_node['width'] = figwidth
        if figclasses:
            figure_node['classes'] += figclasses
        if align:
            figure_node['align'] = align
        if self.content:
            node = nodes.Element()          # anonymous container for parsing
            self.state.nested_parse(self.content, self.content_offset, node)
            first_node = node[0]
            if isinstance(first_node, nodes.paragraph):
                caption = nodes.caption(first_node.rawsource, '',
                                        *first_node.children)
                caption.source = first_node.source
                caption.line = first_node.line
                figure_node += caption
            elif not (isinstance(first_node, nodes.comment)
                      and len(first_node) == 0):
                error = self.state_machine.reporter.error(
                      'Figure caption must be a paragraph or empty comment.',
                      nodes.literal_block(self.block_text, self.block_text),
                      line=self.lineno)
                return [figure_node, error]
            if len(node) > 1:
                figure_node += nodes.legend('', *node[1:])
        return [figure_node]

Example 49

Project: rivulet
Source File: __init__.py
View license
    def torrents_status(self, detailed):
        torrents = self._session.get_torrents()
        torrent_statuses = []
        for th in torrents:
            ts = th.status()
            pieces = ts.pieces
            ti = get_torrent_info(th)
            torrent_status = {}
            torrent_status['state'] = str(lt.torrent_status.states.values[ts.state])
            torrent_status['download_rate'] = ts.download_rate
            torrent_status['upload_rate'] = ts.upload_rate
            torrent_status['progress'] = round(ts.progress * 100, 2)
            torrent_status['peers'] = ts.num_peers
            torrent_status['num_pieces'] = ts.num_pieces
            torrent_status['num_complete'] = ts.num_complete
            torrent_status['files'] = []
            if ti == None:
                continue
            info_hash = str(ti.info_hash())
            torrent_status['total_done'] = round(float(ts.total_done) / ti.total_size() * 100, 2)
            torrent_status['info_hash'] = info_hash
            torrent_status['name'] = ti.name()
            if not detailed:
                torrent_statuses.append(torrent_status)
                continue
            for index in range(ti.num_files()):
                file_status = {}
                file_ = ti.file_at(index)

                path = file_.path.decode(sys.getfilesystemencoding())
                file_status['path'] = path

                file_status['requested'] = th.file_priority(index) != 0

                file_size = file_.size
                start = ti.map_file(index, 0, 0)
                # I don't know why we need -1; we shouldn't need it.
                end = ti.map_file(index, file_size-1, 0)

                file_status['pieces'] = pieces[start.piece:end.piece+1]

                torrent_status['files'].append(file_status)
            torrent_statuses.append(torrent_status)
        return torrent_statuses

Example 50

Project: OmniMarkupPreviewer
Source File: images.py
View license
    def run(self):
        figwidth = self.options.pop('figwidth', None)
        figclasses = self.options.pop('figclass', None)
        align = self.options.pop('align', None)
        (image_node,) = Image.run(self)
        if isinstance(image_node, nodes.system_message):
            return [image_node]
        figure_node = nodes.figure('', image_node)
        if figwidth == 'image':
            if PIL and self.state.document.settings.file_insertion_enabled:
                imagepath = urllib.request.url2pathname(image_node['uri'])
                try:
                    img = PIL.Image.open(
                            imagepath.encode(sys.getfilesystemencoding()))
                except (IOError, UnicodeEncodeError):
                    pass # TODO: warn?
                else:
                    self.state.document.settings.record_dependencies.add(
                        imagepath.replace('\\', '/'))
                    figure_node['width'] = img.size[0]
                    del img
        elif figwidth is not None:
            figure_node['width'] = figwidth
        if figclasses:
            figure_node['classes'] += figclasses
        if align:
            figure_node['align'] = align
        if self.content:
            node = nodes.Element()          # anonymous container for parsing
            self.state.nested_parse(self.content, self.content_offset, node)
            first_node = node[0]
            if isinstance(first_node, nodes.paragraph):
                caption = nodes.caption(first_node.rawsource, '',
                                        *first_node.children)
                caption.source = first_node.source
                caption.line = first_node.line
                figure_node += caption
            elif not (isinstance(first_node, nodes.comment)
                      and len(first_node) == 0):
                error = self.state_machine.reporter.error(
                      'Figure caption must be a paragraph or empty comment.',
                      nodes.literal_block(self.block_text, self.block_text),
                      line=self.lineno)
                return [figure_node, error]
            if len(node) > 1:
                figure_node += nodes.legend('', *node[1:])
        return [figure_node]