Here are the examples of the python api sys.stdout.encoding taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
132 Examples
0
Example 101
Project: kupfer Source File: c_config.py
@conf
def get_cc_version(conf, cc, gcc=False, icc=False):
"""
Run the preprocessor to determine the compiler version
The variables CC_VERSION, DEST_OS, DEST_BINFMT and DEST_CPU will be set in *conf.env*
"""
cmd = cc + ['-dM', '-E', '-']
env = conf.env.env or None
try:
p = Utils.subprocess.Popen(cmd, stdin=Utils.subprocess.PIPE, stdout=Utils.subprocess.PIPE, stderr=Utils.subprocess.PIPE, env=env)
p.stdin.write('\n'.encode())
out = p.communicate()[0]
except:
conf.fatal('Could not determine the compiler version %r' % cmd)
if not isinstance(out, str):
out = out.decode(sys.stdout.encoding)
if gcc:
if out.find('__INTEL_COMPILER') >= 0:
conf.fatal('The intel compiler pretends to be gcc')
if out.find('__GNUC__') < 0:
conf.fatal('Could not determine the compiler type')
if icc and out.find('__INTEL_COMPILER') < 0:
conf.fatal('Not icc/icpc')
k = {}
if icc or gcc:
out = out.split('\n')
for line in out:
lst = shlex.split(line)
if len(lst)>2:
key = lst[1]
val = lst[2]
k[key] = val
def isD(var):
return var in k
def isT(var):
return var in k and k[var] != '0'
# Some docuementation is available at http://predef.sourceforge.net
# The names given to DEST_OS must match what Utils.unversioned_sys_platform() returns.
if not conf.env.DEST_OS:
conf.env.DEST_OS = ''
for i in MACRO_TO_DESTOS:
if isD(i):
conf.env.DEST_OS = MACRO_TO_DESTOS[i]
break
else:
if isD('__APPLE__') and isD('__MACH__'):
conf.env.DEST_OS = 'darwin'
elif isD('__unix__'): # unix must be tested last as it's a generic fallback
conf.env.DEST_OS = 'generic'
if isD('__ELF__'):
conf.env.DEST_BINFMT = 'elf'
elif isD('__WINNT__') or isD('__CYGWIN__'):
conf.env.DEST_BINFMT = 'pe'
conf.env.LIBDIR = conf.env['PREFIX'] + '/bin'
elif isD('__APPLE__'):
conf.env.DEST_BINFMT = 'mac-o'
if not conf.env.DEST_BINFMT:
# Infer the binary format from the os name.
conf.env.DEST_BINFMT = Utils.destos_to_binfmt(conf.env.DEST_OS)
for i in MACRO_TO_DEST_CPU:
if isD(i):
conf.env.DEST_CPU = MACRO_TO_DEST_CPU[i]
break
Logs.debug('ccroot: dest platform: ' + ' '.join([conf.env[x] or '?' for x in ('DEST_OS', 'DEST_BINFMT', 'DEST_CPU')]))
if icc:
ver = k['__INTEL_COMPILER']
conf.env['CC_VERSION'] = (ver[:-2], ver[-2], ver[-1])
else:
conf.env['CC_VERSION'] = (k['__GNUC__'], k['__GNUC_MINOR__'], k['__GNUC_PATCHLEVEL__'])
return k
0
Example 102
Project: estnltk Source File: conv_oldMrf_to_vabamrfJson.py
def getJSONanalysis(root, pos, form):
result = { "clitic":"", "ending":"", "form":form, "partofspeech":pos, "root":""}
# {
# "clitic": "string",
# "ending": "string",
# "form": "string",
# "partofspeech": "string",
# "root": "string"
# },
breakpoint = -1
for i in range(len(root)-1, -1, -1):
if root[i] == '+':
breakpoint = i
break
if breakpoint == -1:
result["root"] = root
result["ending"] = "0"
if not re.match("^\W+$", root):
try:
print( " No breakpoint found from: ", root, pos, form )
except UnicodeEncodeError:
print( " No breakpoint found from: ", (root+" "+pos+" "+form).encode('utf8').decode(sys.stdout.encoding) )
else:
result["root"] = root[0:breakpoint]
result["ending"] = root[breakpoint+1:]
return result
0
Example 103
def addstr(*args):
scr, args = args[0], list(args[1:])
x = 2 if len(args) > 2 else 0
args[x] = args[x].encode(sys.stdout.encoding)
return scr.addstr(*args)
0
Example 104
@retry_before_failing()
def test_pids(self):
# Note: this test might fail if the OS is starting/killing
# other processes in the meantime
if SUNOS:
cmd = ["ps", "-A", "-o", "pid"]
else:
cmd = ["ps", "ax", "-o", "pid"]
p = get_test_subprocess(cmd, stdout=subprocess.PIPE)
output = p.communicate()[0].strip()
assert p.poll() == 0
if PY3:
output = str(output, sys.stdout.encoding)
pids_ps = []
for line in output.split('\n')[1:]:
if line:
pid = int(line.split()[0].strip())
pids_ps.append(pid)
# remove ps subprocess pid which is supposed to be dead in meantime
pids_ps.remove(p.pid)
pids_psutil = psutil.pids()
pids_ps.sort()
pids_psutil.sort()
# on OSX ps doesn't show pid 0
if OSX and 0 not in pids_ps:
pids_ps.insert(0, 0)
if pids_ps != pids_psutil:
difference = [x for x in pids_psutil if x not in pids_ps] + \
[x for x in pids_ps if x not in pids_psutil]
self.fail("difference: " + str(difference))
0
Example 105
def stdout_encode(u, default='UTF8'):
encoding = sys.stdout.encoding or default
if sys.version_info > (3, 0):
return u.encode(encoding).decode(encoding)
return u.encode(encoding)
0
Example 106
def stdoutencode(data):
retVal = None
try:
# Reference: http://bugs.python.org/issue1602
if IS_WIN:
output = data.encode("ascii", "replace")
if output != data:
warnMsg = "cannot properly display Unicode characters "
warnMsg += "inside Windows OS command prompt "
warnMsg += "(http://bugs.python.org/issue1602). All "
warnMsg += "unhandled occurances will result in "
warnMsg += "replacement with '?' character. Please, find "
warnMsg += "proper character representation inside "
warnMsg += "corresponding output files. "
singleTimeWarnMessage(warnMsg)
retVal = output
else:
retVal = data.encode(sys.stdout.encoding)
except:
retVal = data.encode(UNICODE_ENCODING)
return retVal
0
Example 107
Project: apitools Source File: app2.py
@staticmethod
def EncodeForPrinting(s):
"""Safely encode a string as the encoding for sys.stdout."""
encoding = sys.stdout.encoding or 'ascii'
return six.text_type(s).encode(encoding, 'backslashreplace')
0
Example 108
Project: econ-project-templates Source File: c_emscripten.py
@conf
def get_emscripten_version(conf, cc):
"""
Emscripten doesn't support processing '-' like clang/gcc
"""
dummy = conf.cachedir.parent.make_node("waf-emscripten.c")
dummy.write("")
cmd = cc + ['-dM', '-E', '-x', 'c', dummy.abspath()]
env = conf.env.env or None
try:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
out = p.communicate()[0]
except Exception as e:
conf.fatal('Could not determine emscripten version %r: %s' % (cmd, e))
if not isinstance(out, str):
out = out.decode(sys.stdout.encoding or 'iso8859-1')
k = {}
out = out.splitlines()
for line in out:
lst = shlex.split(line)
if len(lst)>2:
key = lst[1]
val = lst[2]
k[key] = val
if not ('__clang__' in k and 'EMSCRIPTEN' in k):
conf.fatal('Could not determine the emscripten compiler version.')
conf.env.DEST_OS = 'generic'
conf.env.DEST_BINFMT = 'elf'
conf.env.DEST_CPU = 'asm-js'
conf.env.CC_VERSION = (k['__clang_major__'], k['__clang_minor__'], k['__clang_patchlevel__'])
return k
0
Example 109
Project: econ-project-templates Source File: print_commands.py
def exec_command(self, cmd, **kw):
subprocess = Utils.subprocess
kw['shell'] = isinstance(cmd, str)
if isinstance(cmd, str):
kw['shell'] = True
txt = cmd
else:
txt = ' '.join(repr(x) if ' ' in x else x for x in cmd)
Logs.debug('runner: %s', txt)
Logs.debug('runner_env: kw=%s', kw)
if self.logger:
self.logger.info(cmd)
if 'stdout' not in kw:
kw['stdout'] = subprocess.PIPE
if 'stderr' not in kw:
kw['stderr'] = subprocess.PIPE
if Logs.verbose and not kw['shell'] and not Utils.check_exe(cmd[0]):
raise Errors.WafError("Program %s not found!" % cmd[0])
wargs = {}
if 'timeout' in kw:
if kw['timeout'] is not None:
wargs['timeout'] = kw['timeout']
del kw['timeout']
if 'input' in kw:
if kw['input']:
wargs['input'] = kw['input']
kw['stdin'] = Utils.subprocess.PIPE
del kw['input']
if 'cwd' in kw:
if not isinstance(kw['cwd'], str):
kw['cwd'] = kw['cwd'].abspath()
try:
if kw['stdout'] or kw['stderr']:
p = subprocess.Popen(cmd, **kw)
(out, err) = p.communicate(**wargs)
ret = p.returncode
else:
out, err = (None, None)
ret = subprocess.Popen(cmd, **kw).wait(**wargs)
except Exception as e:
raise Errors.WafError('Execution failure: %s' % str(e), ex=e)
if out:
if not isinstance(out, str):
out = out.decode(sys.stdout.encoding or 'iso8859-1')
if self.logger:
self.logger.debug('out: %s' % out)
else:
Logs.info(out, extra={'stream':sys.stdout, 'c1': ''})
if err:
if not isinstance(err, str):
err = err.decode(sys.stdout.encoding or 'iso8859-1')
if self.logger:
self.logger.error('err: %s' % err)
else:
Logs.info(err, extra={'stream':sys.stderr, 'c1': ''})
return ret
0
Example 110
Project: kay Source File: frontend.py
def run(self, argv=sys.argv):
"""Main entry point of the command-line interface.
:param argv: list of arguments passed on the command-line
"""
self.parser = OptionParser(usage=self.usage % ('command', '[args]'),
version=self.version)
self.parser.disable_interspersed_args()
self.parser.print_help = self._help
self.parser.add_option('--list-locales', dest='list_locales',
action='store_true',
help="print all known locales and exit")
self.parser.add_option('-v', '--verbose', action='store_const',
dest='loglevel', const=logging.DEBUG,
help='print as much as possible')
self.parser.add_option('-q', '--quiet', action='store_const',
dest='loglevel', const=logging.ERROR,
help='print as little as possible')
self.parser.set_defaults(list_locales=False, loglevel=logging.INFO)
options, args = self.parser.parse_args(argv[1:])
# Configure logging
self.log = logging.getLogger('babel')
self.log.setLevel(options.loglevel)
handler = logging.StreamHandler()
handler.setLevel(options.loglevel)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
self.log.addHandler(handler)
if options.list_locales:
identifiers = localedata.list()
longest = max([len(identifier) for identifier in identifiers])
identifiers.sort()
format = u'%%-%ds %%s' % (longest + 1)
for identifier in identifiers:
locale = Locale.parse(identifier)
output = format % (identifier, locale.english_name)
print output.encode(sys.stdout.encoding or
getpreferredencoding() or
'ascii', 'replace')
return 0
if not args:
self.parser.error('no valid command or option passed. '
'Try the -h/--help option for more information.')
cmdname = args[0]
if cmdname not in self.commands:
self.parser.error('unknown command "%s"' % cmdname)
return getattr(self, cmdname)(args[1:])
0
Example 111
Project: topic-explorer Source File: prep.py
def get_high_filter(args, c, words=None):
import numpy as np
header = "FILTER HIGH FREQUENCY WORDS"
stars = (80 - len(header) - 2) / 2
print "\n\n{0} {1} {0}".format('*' * stars, header)
print " This will remove all words occurring more than N times."
print " The histogram below shows how many words will be removed"
print " by selecting each maximum frequency threshold.\n"
items, counts = get_items_counts(c.corpus)
items = items[get_mask(c, words)]
counts = counts[get_mask(c, words)]
high_filter = False
bins = np.array([0., 0.025, 0.05, 0.075, 0.1, 0.15, 0.20, 0.25, 0.3, 0.35, 0.4, 0.5, 1.0])
bins = 1. - bins
thresh = np.cuemsum(counts[counts.argsort()]) / float(counts.sum())
bins = [counts[counts.argsort()][np.searchsorted(thresh, bin)] for bin in bins]
bins = sorted(set(bins))
bins.append(max(counts))
while not high_filter:
bin_counts, bins = np.histogram(counts, bins=bins)
print "{0:>8s} {1:>8s} {2:<36s} {3:>14s} {4:>8s}".format("Rate", 'Top', '% of corpus',
"# words", "Rate")
for bin, count in zip(bins[-2::-1], np.cuemsum(bin_counts[::-1])):
if count:
percentage = 1. - (counts[counts < bin].sum() / float(counts.sum()))
print "{0:>5.0f}x".format(bin - 1).rjust(8),
print '{0:2.1f}%'.format(percentage * 100).rjust(8),
print (u'\u2588' * int(percentage * 36)).ljust(36),
print " {0:0.0f} words".format(count).rjust(14),
print "> {0:>5.0f}x".format(bin - 1).ljust(8)
print ' ' * 17, "{} total occurrences".format(counts.sum()).ljust(36),
print '{} words total'.format(get_mask(c, words).sum()).rjust(20)
print ''
input_filter = 0
accept = None
while not input_filter or input_filter <= 0:
try:
if high_filter:
input_filter = high_filter
else:
input_filter = int(raw_input("Enter the maximum rate: ").replace('x', ''))
candidates = get_candidate_words(c, input_filter, words=words)
places = np.in1d(c.words[get_mask(c, words)], candidates)
places = dict(zip(candidates, np.where(places)[0]))
candidates = sorted(candidates, key=lambda x: counts[places[x]], reverse=True)
print "Filter will remove", counts[counts > input_filter].sum(),
print "occurrences", "of these", len(counts[counts > input_filter]), "words:"
print u' '.join(candidates).encode(
sys.stdout.encoding, errors='replace')
print "\nFilter will remove", counts[counts > input_filter].sum(),
print "occurrences", "of these", len(counts[counts > input_filter]), "words.",
if len(candidates) == len(c.words):
print "\n\nChoice of", input_filter, "will remove ALL words from the corpus."
print "Please choose a different filter."
high_filter = 0
input_filter = 0
else:
accept = None
while accept not in ['y', 'n']:
accept = raw_input("\nAccept filter? [y/n/[different max number]] ")
if isint(accept):
high_filter = int(accept)
input_filter = 0
accept = 'n'
elif accept == 'y':
high_filter = input_filter
elif accept == 'n':
high_filter = 0
except ValueError:
input_filter = 0
return (high_filter, candidates)
0
Example 112
Project: topic-explorer Source File: prep.py
def get_low_filter(args, c, words=None):
import numpy as np
header = "FILTER LOW FREQUENCY WORDS"
stars = (80 - len(header) - 2) / 2
print "\n\n{0} {1} {0}".format('*' * stars, header)
print " This will remove all words occurring less than N times."
print " The histogram below shows how many words will be removed"
print " by selecting each minimum frequency threshold.\n"
items, counts = get_items_counts(c.corpus)
items = items[get_mask(c, words)]
counts = counts[get_mask(c, words)]
bins = np.linspace(0, 1.0, 11)
bins = 1. - np.array([0., 0.025, 0.05, 0.075, 0.1, 0.15, 0.20, 0.25, 0.3, 0.35, 0.4, 0.5, 1.0])
thresh = np.cuemsum(counts[counts.argsort()[::-1]]) / float(counts.sum())
bins = [counts[counts.argsort()[::-1]][np.searchsorted(thresh, bin)] for bin in bins]
bins = sorted(set(bins))
low_filter = False
while low_filter is False:
bin_counts, bins = np.histogram(counts[counts.argsort()[::-1]], bins=bins)
# print "{0:>10s} {1:>10s}".format("# Tokens", "# Words")
print "{0:>8s} {1:>8s} {2:<36s} {3:>14s} {4:>8s}".format("Rate", 'Bottom', '% of corpus',
"# words", "Rate")
for bin, count in zip(bins[1:], np.cuemsum(bin_counts)):
if count:
percentage = (counts[counts <= bin].sum() / float(counts.sum()))
print "{0:>5.0f}x".format(bin - 1).rjust(8),
print '{0:2.1f}%'.format(percentage * 100).rjust(8),
print (u'\u2588' * int(percentage * 36)).ljust(36),
print " {0:0.0f} words".format(count).rjust(14),
print "<= {0:>5.0f}x".format(bin - 1).ljust(8)
print ' ' * 17, "{} total occurrences".format(counts.sum()).ljust(36),
print '{} words total'.format(get_mask(c, words).sum()).rjust(20)
print ''
input_filter = 0
accept = None
while not input_filter or input_filter <= 0:
try:
if low_filter:
input_filter = low_filter
else:
input_filter = int(raw_input("Enter the minimum rate: ").replace('x', ''))
candidates = get_candidate_words(c, -input_filter, words=words)
places = np.in1d(c.words[get_mask(c, words)], candidates)
places = dict(zip(candidates, np.where(places)[0]))
candidates = sorted(candidates, key=lambda x: counts[places[x]])
print "Filter will remove", counts[counts <= input_filter].sum(), "tokens",
print "of these", len(counts[counts <= input_filter]), "words:"
print u' '.join(candidates).encode(
sys.stdout.encoding, errors='replace')
print "\nFilter will remove", counts[counts <= input_filter].sum(), "tokens",
print "of these", len(counts[counts <= input_filter]), "words.",
if len(candidates) == len(c.words):
print "\n\nChoice of", input_filter, "will remove ALL words from the corpus."
print "Please choose a different filter."
low_filter = 0
input_filter = 0
else:
accept = None
while accept not in ['y', 'n']:
accept = raw_input("\nAccept filter? [y/n/[different min. number] ")
if isint(accept):
low_filter = int(accept)
input_filter = 0
accept = 'n'
elif accept == 'y':
low_filter = input_filter
elif accept == 'n':
low_filter = False
except ValueError:
input_filter = 0
return (low_filter, candidates)
0
Example 113
Project: ironpython3 Source File: compileall.py
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
legacy=False, optimize=-1):
"""Byte-compile one file.
Arguments (only fullname is required):
fullname: the file to byte-compile
ddir: if given, the directory name compiled in to the
byte-code file.
force: if True, force compilation, even if timestamps are up-to-date
quiet: if True, be quiet during compilation
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
optimize: optimization level or -1 for level of the interpreter
"""
success = 1
name = os.path.basename(fullname)
if ddir is not None:
dfile = os.path.join(ddir, name)
else:
dfile = None
if rx is not None:
mo = rx.search(fullname)
if mo:
return success
if os.path.isfile(fullname):
if legacy:
cfile = fullname + ('c' if __debug__ else 'o')
else:
if optimize >= 0:
cfile = importlib.util.cache_from_source(
fullname, debug_override=not optimize)
else:
cfile = importlib.util.cache_from_source(fullname)
cache_dir = os.path.dirname(cfile)
head, tail = name[:-3], name[-3:]
if tail == '.py':
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
mtime)
with open(cfile, 'rb') as chandle:
actual = chandle.read(8)
if expect == actual:
return success
except OSError:
pass
if not quiet:
print('Compiling {!r}...'.format(fullname))
try:
ok = py_compile.compile(fullname, cfile, dfile, True,
optimize=optimize)
except py_compile.PyCompileError as err:
if quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
# escape non-printable characters in msg
msg = err.msg.encode(sys.stdout.encoding,
errors='backslashreplace')
msg = msg.decode(sys.stdout.encoding)
print(msg)
success = 0
except (SyntaxError, UnicodeError, OSError) as e:
if quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
print(e.__class__.__name__ + ':', e)
success = 0
else:
if ok == 0:
success = 0
return success
0
Example 114
Project: glow Source File: frontend.py
def run(self, argv=sys.argv):
"""Main entry point of the command-line interface.
:param argv: list of arguments passed on the command-line
"""
self.parser = OptionParser(usage=self.usage % ('command', '[args]'),
version=self.version)
self.parser.disable_interspersed_args()
self.parser.print_help = self._help
self.parser.add_option('--list-locales', dest='list_locales',
action='store_true',
help="print all known locales and exit")
self.parser.add_option('-v', '--verbose', action='store_const',
dest='loglevel', const=logging.DEBUG,
help='print as much as possible')
self.parser.add_option('-q', '--quiet', action='store_const',
dest='loglevel', const=logging.ERROR,
help='print as little as possible')
self.parser.set_defaults(list_locales=False, loglevel=logging.INFO)
options, args = self.parser.parse_args(argv[1:])
# Configure logging
self.log = logging.getLogger('babel')
self.log.setLevel(options.loglevel)
handler = logging.StreamHandler()
handler.setLevel(options.loglevel)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
self.log.addHandler(handler)
if options.list_locales:
identifiers = localedata.list()
longest = max([len(identifier) for identifier in identifiers])
format = u'%%-%ds %%s' % (longest + 1)
for identifier in localedata.list():
locale = Locale.parse(identifier)
output = format % (identifier, locale.english_name)
print output.encode(sys.stdout.encoding or
getpreferredencoding() or
'ascii', 'replace')
return 0
if not args:
self.parser.error('incorrect number of arguments')
cmdname = args[0]
if cmdname not in self.commands:
self.parser.error('unknown command "%s"' % cmdname)
return getattr(self, cmdname)(args[1:])
0
Example 115
Project: timegaps Source File: main.py
def read_items_from_stdin():
"""Read items from standard input.
Regarding stdin decoding: http://stackoverflow.com/a/16549381/145400
Reading a stream of chunks/records with a different separator than newline
is not easily possible with stdlib (http://bugs.python.org/issue1152248).
Take simplest approach for now: read all data (processing cannot start
before that anyway), then split data (byte string) at sep byte occurrences
(NUL or newline), then decode each record and return list of unicode
strings.
"""
log.debug("Read binary data from standard input, until EOF.")
try:
bytedata = stdin_read_bytes_until_eof()
except (OSError, IOError) as e:
err("Error reading from stdin: %s" % e)
log.debug("%s bytes have been read.", len(bytedata))
enc = sys.stdout.encoding
sep = "\0" if options.nullsep else "\n"
sep_bytes = sep.encode(enc)
log.debug("Split binary stdin data on byte separator %r.", sep_bytes)
chunks = bytedata.split(sep_bytes)
log.debug("Decode non-empty chunks using %s.", enc)
# `split()` is the inverse of `join()`, i.e. it introduces empty strings for
# leading and trailing separators, and for separator sequences. That is why
# the `if c` part below is essential. Also see
# http://stackoverflow.com/a/2197493/145400
items_unicode = [c.decode(enc) for c in chunks if c]
log.debug("Identified %s item(s).", len(items_unicode))
return items_unicode
0
Example 116
def prepare_input():
"""Return a list of objects that can be categorized by `TimeFilter.filter`.
"""
if not options.stdin:
itemstrings = options.items
# `itemstrings` can be either unicode or byte strings. On Unix, we
# want to keep cmdline arguments as raw binary data as long as possible.
# On Python 3 argv already comes in as sequence of unicode strings.
# In file system mode on Python 2, treat items (i.e. paths) as byte
# strings. In time-from-string mode, decode itemstrings (later).
if options.glob:
if WINDOWS:
import glob
expandeditems = []
for pattern in itemstrings:
expandeditems.extend(glob.glob(pattern))
itemstrings = expandeditems
else:
log.info("Item wildcard expansion only allowed on Windows.")
else:
itemstrings = read_items_from_stdin()
# `itemstrings` as returned by `read_items_from_stdin()` are unicode.
if options.time_from_string is not None:
log.info("--time-from-string set, don't interpret items as paths.")
fmt = options.time_from_string
# Decoding of each single item string.
# If items came from stdin, they are already unicode. If they came from
# argv and Python 2 on Unix, they are still byte strings.
if isinstance(itemstrings[0], binary_type):
# Again, use sys.stdout.encoding to decode item byte strings, which
# can be set/overridden via PYTHONIOENCODING.
itemstrings = [s.decode(sys.stdout.encoding) for s in itemstrings]
items = []
for s in itemstrings:
log.debug("Parsing seconds since epoch from item: %r", s)
mtime = seconds_since_epoch_from_localtime_string(s, fmt)
log.debug("Seconds since epoch: %s", mtime)
items.append(FilterItem(modtime=mtime, text=s))
return items
log.info("Interpret items as paths.")
log.info("Validate paths and extract modification time.")
fses = []
for path in itemstrings:
log.debug("Type of path string: %s.", type(path))
# On the one hand, a unicode-aware Python program should only use
# unicode type strings internally. On the other hand, when it comes
# to file system interaction, byte strings are the more portable choice
# on Unix-like systems.
# See https://wiki.python.org/moin/Python3UnicodeDecodeError:
# "A robust program will have to use only the bytes type to make sure
# that it can open / copy / remove any file or directory."
#
# On Python 3, which automatically populates argv with unicode objects,
# we could therefore re-encode towards byte strings. See:
# http://stackoverflow.com/a/7077803/145400
# http://bugs.python.org/issue8514
# https://github.com/oscarbenjamin/opster/commit/61f693a2c553944394ba286baed20abc31958f03
# On the other hand, there is http://www.python.org/dev/peps/pep-0383/
# which describes how surrogate encoding is used by Python 3 for
# auto-correcting argument decoding issues (unknown byte sequences are
# conserved and re-created upon encoding). Interesting in this regard:
# http://stackoverflow.com/a/846931/145400
# Definite choice for Python 2 and Unix: keep paths as byte strings.
modtime = None
if options.time_from_basename:
bn = os.path.basename(path)
fmt = options.time_from_basename
log.debug("Parsing modification time from basename: %r", bn)
modtime = seconds_since_epoch_from_localtime_string(bn, fmt)
log.debug("Modification time (seconds since epoch): %s", modtime)
try:
fses.append(FileSystemEntry(path, modtime))
except OSError:
err("Cannot access '%s'." % path)
log.debug("Created %s item(s) (type: file system entry).", len(fses))
return fses
0
Example 117
def addText(self, text, tag=None):
"""
Add text to the canvas with a specified tag.
"""
sys.stdout.write(text.encode(sys.stdout.encoding,'replace'))
0
Example 118
Project: kivy-ios Source File: compat.py
def read_response(prompt=''):
"""
Prompt the user for a response.
Prints the given prompt (which should be a Unicode string),
and returns the text entered by the user as a Unicode string.
:param prompt: A Unicode string that is presented to the user.
"""
# For Python 2, raw_input takes a byte string argument for the prompt.
# This must be encoded using the encoding used by sys.stdout.
# The result is a byte string encoding using sys.stdin.encoding.
# However, if the program is not being run interactively, sys.stdout
# and sys.stdin may not have encoding attributes.
# In that case we don't print a prompt (stdin/out isn't interactive,
# so prompting is pointless), and we assume the returned data is
# encoded using sys.getdefaultencoding(). This may not be right,
# but it's likely the best we can do.
# Isn't Python 2 encoding support wonderful? :-)
if sys.stdout.encoding:
prompt = prompt.encode(sys.stdout.encoding)
else:
prompt = ''
enc = sys.stdin.encoding or sys.getdefaultencoding()
return raw_input(prompt).decode(enc)
0
Example 119
Project: lino Source File: diag.py
def babel_message():
s = u"""
Some sentences in different languages:
Ännchen Müller isst gern süße Soßen.
Cède à César les pâturages reçues.
Tõesti, ma ütlen teile, see ei ole ükskõik.
Overview table with some accented characters:
A E I O U a e i o u
¨ Ä Ë Ï Ö Ü ä ë ï ö ü
~ Ã . . Õ . ã . . õ .
´ Á É Í Ó Ú á é í ó ú
` À È Ì Ò Ù à è ì ò ù
^ Â Ê Î Ô Û â ê î ô û
"""
try:
out_encoding=repr(sys.stdout.encoding)
except AttributeError:
out_encoding="(undefined)"
try:
in_encoding=repr(sys.stdin.encoding)
except AttributeError:
in_encoding="(undefined)"
try:
nl_langinfo=repr(locale.nl_langinfo(locale.CODESET))
except AttributeError:
nl_langinfo="(undefined)"
s+="""
Some system settings related to encodings:
locale.getdefaultlocale() : %r
sys.getdefaultencoding() : %r
sys.getfilesystemencoding() : %r
sys.stdout.encoding : %s
sys.stdin.encoding : %s
locale codeset : %s""" % (
locale.getdefaultlocale(),
sys.getdefaultencoding(),
sys.getfilesystemencoding(),
out_encoding,
in_encoding,
nl_langinfo)
func = getattr(locale,'nl_langinfo',None)
if func: # nl_langinfo not available on win32
s += """
locale codeset : """ + func(locale.CODESET)
return s
0
Example 120
def write(self, msg, scroll=True):
row, col = self.row, self.col
self.e('[%i;%iH' % (row, col))
for line in self.wrap(msg):
if row > self.height:
row -= 1
if scroll:
self.scroll()
row, col = self.row, self.col
self.flush()
self.move(row, 1)
else:
break
for char in line:
self.lines[row-1][col-1] = char
char = char.encode(sys.stdout.encoding, 'replace')
sys.stdout.write(char)
col += 1
col = 1
row += 1
sys.stdout.write('\n')
self.row, self.col = row, col
0
Example 121
Project: pulseaudio-dlna Source File: encoding.py
def decode_default(bytes):
if type(bytes) is not str:
raise NotBytesException(bytes)
guess = chardet.detect(bytes)
encodings = {
'sys.stdout.encoding': sys.stdout.encoding,
'locale.getpreferredencoding': locale.getpreferredencoding(),
'chardet.detect': guess['encoding'],
'utf-8': 'utf-8',
'latin1': 'latin1',
}
for encoding in encodings.values():
if encoding and encoding != 'ascii':
try:
return bytes.decode(encoding)
except UnicodeDecodeError:
continue
try:
return bytes.decode('ascii', errors='replace')
except UnicodeDecodeError:
logger.error(
'Decoding failed using the following encodings: "{}"'.format(
','.join(
['{}:{}'.format(f, e) for f, e in encodings.items()]
)))
return 'Unknown'
0
Example 122
def get_next_cost_dict(self,params_dict):
'''
Implementation of running a command with parameters on the command line and reading the result.
'''
self.command_count += 1
self.log.debug('Running command count' + repr(self.command_count))
self.last_params_dict = params_dict
params = params_dict['params']
curr_command = self.command
if self.params_args_type == 'direct':
for p in params:
curr_command += ' ' + str(p)
elif self.params_args_type == 'named':
for ind,p in enumerate(params):
curr_command += ' ' + '--param' + str(ind +1) + ' ' + str(p)
else:
self.log.error('THIS SHOULD NOT HAPPEN. params_args_type not recognized')
#execute command and look at output
cli_return = sp.check_output(curr_command.split()).decode(sys.stdout.encoding)
print(cli_return)
tdict_string = ''
take_flag = False
for line in cli_return.splitlines():
temp = (line.partition('#')[0]).strip('\n').strip()
if temp == 'M-LOOP_start' or temp == 'MLOOP_start':
take_flag = True
elif temp == 'M-LOOP_end' or temp == 'MLOOP_end':
take_flag = False
elif take_flag:
tdict_string += temp + ','
print(tdict_string)
#Setting up words for parsing a dict, ignore eclipse warnings
array = np.array #@UnusedVariable
inf = float('inf') #@UnusedVariable
nan = float('nan') #@UnusedVariable
tdict = eval('dict('+tdict_string+')')
return tdict
0
Example 123
@contextlib.contextmanager
def capture_stdout(and_stderr=False):
"""Captures stdout and yields the new bytes stream that backs it.
It also wraps it in a fake object that flushes on getting the
underlying value.
"""
ll_stream = io.BytesIO()
stream = _NonClosingTextIOWrapper(ll_stream, sys.stdout.encoding,
sys.stdout.errors)
old_stdout = sys.stdout
sys.stdout = stream
if and_stderr:
old_stderr = sys.stderr
sys.stderr = stream
try:
yield _CapturedStream(stream)
finally:
stream.flush()
sys.stdout = old_stdout
if and_stderr:
sys.stderr = old_stderr
0
Example 124
def addToLog(self, message, messageCode="", messageArgs=None, file="", refs=None, level=logging.INFO):
"""Add a simple info message to the default logger
:param message: Text of message to add to log.
:type message: str
: param messageArgs: optional dict of message format-string key-value pairs
:type messageArgs: dict
:param messageCode: Message code (e.g., a prefix:id of a standard error)
:param messageCode: str
:param file: File name (and optional line numbers) pertaining to message
:type file: str
"""
if self.logger is not None:
if messageArgs:
args = (message, messageArgs)
else:
args = (message,) # pass no args if none provided
if refs is None:
refs = []
if isinstance(file, (tuple,list,set)):
for _file in file:
refs.append( {"href": _file} )
elif isinstance(file, _STR_BASE):
refs.append( {"href": file} )
if isinstance(level, _STR_BASE):
level = logging._checkLevel(level)
self.logger.log(level, *args, extra={"messageCode":messageCode,"refs":refs})
else:
try:
print(message)
except UnicodeEncodeError:
# extra parentheses in print to allow for 3-to-2 conversion
print((message
.encode(sys.stdout.encoding, 'backslashreplace')
.decode(sys.stdout.encoding, 'strict')))
0
Example 125
def analyze(self, filename):
info={"dependencies":[]}
script=os.path.join(os.path.dirname(__file__),"blender","analyze.py")
dependencyPattern = re.compile("RenderChan dependency: (.*)$")
startFramePattern = re.compile("RenderChan start: (.*)$")
endFramePattern = re.compile("RenderChan end: (.*)$")
env=os.environ.copy()
env["PYTHONPATH"]=""
commandline=[self.conf['binary'], "-b",filename, "-P",script]
commandline.append("-y") # Enable automatic script execution
out = subprocess.Popen(commandline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
while True:
line = out.stdout.readline()
try:
line = line.decode(sys.stdout.encoding)
except:
# In some files we hit weird strings, not handled properly by utf-8. Here goes a faillback.
line = line.decode('latin-1')
if not line:
break
#print line,
sys.stdout.flush()
dep = dependencyPattern.search(line)
if dep:
info["dependencies"].append(dep.group(1).strip())
start=startFramePattern.search(line)
if start:
info["startFrame"]=start.group(1).strip()
end=endFramePattern.search(line)
if end:
info["endFrame"]=end.group(1).strip()
rc = out.poll()
if rc != 0:
print(' Blender command failed...')
return info
0
Example 126
Project: RenderChan Source File: blender.py
def render(self, filename, outputPath, startFrame, endFrame, format, updateCompletion, extraParams={}):
comp = 0.0
updateCompletion(comp)
totalFrames = endFrame - startFrame + 1
frameCompletionPattern = re.compile("Saved:(\d+) Time: .* \(Saving: .*\)")
frameCompletionPattern2 = re.compile("Append frame (\d+) Time: .* \(Saving: .*\)")
frameNumberPattern = re.compile("Fra:(\d+) Mem:.*")
stereo_camera = ""
if extraParams["stereo"]=="left":
stereo_camera = "left"
elif extraParams["stereo"]=="right":
stereo_camera = "right"
if (extraParams["disable_gpu"]!="False") or ('BLENDER_DISABLE_GPU' in os.environ):
print("================== FORCE DISABLE GPU =====================")
gpu_device='None'
else:
gpu_device='"'+self.conf["gpu_device"]+'"'
random_string = "%08d" % (random.randint(0,99999999))
renderscript=os.path.join(tempfile.gettempdir(),"renderchan-"+os.path.basename(filename)+"-"+random_string+".py")
with open(os.path.join(os.path.dirname(__file__),"blender","render.py")) as f:
script=f.read()
script=script.replace("params[UPDATE]","False")\
.replace("params[WIDTH]", str(int(extraParams["width"])))\
.replace("params[HEIGHT]", str(int(extraParams["height"])))\
.replace("params[STEREO_CAMERA]", '"'+stereo_camera+'"')\
.replace("params[AUDIOFILE]", '"'+os.path.splitext(outputPath)[0]+'.wav"')\
.replace("params[FORMAT]", '"'+format+'"')\
.replace("params[CYCLES_SAMPLES]",str(int(extraParams["cycles_samples"])))\
.replace("params[PRERENDER_COUNT]",str(int(extraParams["prerender_count"])))\
.replace("params[GPU_DEVICE]",gpu_device)
with open(renderscript,'w') as f:
f.write(script)
if format in RenderChanModule.imageExtensions:
if extraParams["single"]!="None":
outputPath=outputPath+"-######"
elif extraParams["projectVersion"]<1:
outputPath=os.path.join(outputPath, "file")+".####"
else:
outputPath=os.path.join(outputPath, "file")+".#####"
print('====================================================')
print(' Output Path: %s' % outputPath)
print('====================================================')
env=os.environ.copy()
env["PYTHONPATH"]=""
commandline=[self.conf['binary'], "-b",filename, "-S","Scene", "-P",renderscript, "-o",outputPath]
commandline.append("-y") # Enable automatic script execution
if extraParams["single"]=="None":
commandline.append("-x")
commandline.append("1")
commandline.append("-s")
commandline.append(str(startFrame))
commandline.append("-e")
commandline.append(str(endFrame))
commandline.append("-a")
else:
commandline.append("-x")
commandline.append("0")
commandline.append("-f")
commandline.append(extraParams["single"])
out = subprocess.Popen(commandline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
rc = None
currentFrame = None
while rc is None:
line = out.stdout.readline()
line = line.decode(sys.stdout.encoding)
if not line:
break
print(line, end=' ')
sys.stdout.flush()
if line.startswith("CUDA error: Out of memory"):
out.kill()
raise Exception(' Blender command failed...')
fn = frameNumberPattern.search(line)
if fn:
currentFrame = float(fn.group(1).strip())
elif currentFrame is not None:
fcp = frameCompletionPattern.search(line)
if fcp :
fc = float(currentFrame / 100) / float(totalFrames)
updateCompletion(comp + fc)
else:
fcp = frameCompletionPattern2.search(line)
if fcp:
fc = float(currentFrame / 100) / float(totalFrames)
updateCompletion(comp + fc)
rc = out.poll()
out.communicate()
rc = out.poll()
print('====================================================')
print(' Blender command returns with code %d' % rc)
print('====================================================')
if format in RenderChanModule.imageExtensions and extraParams["single"]!="None":
outputPath=outputPath[:-7]
tmp=outputPath+"-%06d" % int(extraParams["single"])
os.rename(tmp, outputPath)
if rc != 0:
print(' Blender command failed...')
raise Exception(' Blender command failed...')
os.remove(renderscript)
0
Example 127
Project: iot-utilities Source File: compileall.py
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
legacy=False, optimize=-1):
"""Byte-compile one file.
Arguments (only fullname is required):
fullname: the file to byte-compile
ddir: if given, the directory name compiled in to the
byte-code file.
force: if True, force compilation, even if timestamps are up-to-date
quiet: full output with False or 0, errors only with 1,
no output with 2
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
optimize: optimization level or -1 for level of the interpreter
"""
success = 1
name = os.path.basename(fullname)
if ddir is not None:
dfile = os.path.join(ddir, name)
else:
dfile = None
if rx is not None:
mo = rx.search(fullname)
if mo:
return success
if os.path.isfile(fullname):
if legacy:
cfile = fullname + 'c'
else:
if optimize >= 0:
opt = optimize if optimize >= 1 else ''
cfile = importlib.util.cache_from_source(
fullname, optimization=opt)
else:
cfile = importlib.util.cache_from_source(fullname)
cache_dir = os.path.dirname(cfile)
head, tail = name[:-3], name[-3:]
if tail == '.py':
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
mtime)
with open(cfile, 'rb') as chandle:
actual = chandle.read(8)
if expect == actual:
return success
except OSError:
pass
if not quiet:
print('Compiling {!r}...'.format(fullname))
try:
ok = py_compile.compile(fullname, cfile, dfile, True,
optimize=optimize)
except py_compile.PyCompileError as err:
success = 0
if quiet >= 2:
return success
elif quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
# escape non-printable characters in msg
msg = err.msg.encode(sys.stdout.encoding,
errors='backslashreplace')
msg = msg.decode(sys.stdout.encoding)
print(msg)
except (SyntaxError, UnicodeError, OSError) as e:
success = 0
if quiet >= 2:
return success
elif quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
print(e.__class__.__name__ + ':', e)
else:
if ok == 0:
success = 0
return success
0
Example 128
Project: shinken Source File: notify_by_email.py
def create_html_message(msg):
# Get url and add it in footer
url = get_shinken_url()
logging.debug('Grabbed Shinken URL : %s' % url)
# Header part
html_content = ['''
<html>\r
<head>\r
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\r
<style type="text/css">\r
body {text-align: center; font-family: Verdana, sans-serif; font-size: 10pt;}\r
img.logo {float: left; margin: 10px 10px 10px; vertical-align: middle}\r
span {font-family: Verdana, sans-serif; font-size: 12pt;}\r
table {text-align:center; margin-left: auto; margin-right: auto;}\r
th {white-space: nowrap;}\r
th.even {background-color: #D9D9D9;}\r
td.even {background-color: #F2F2F2;}\r
th.odd {background-color: #F2F2F2;}\r
td.odd {background-color: #FFFFFF;}\r
th,td {font-family: Verdana, sans-serif; font-size: 10pt; text-align:left;}\r
th.customer {width: 600px; background-color: #004488; color: #ffffff;}\r
</style>\r
</head>\r
<body>\r'''
]
full_logo_path = get_webui_logo()
if full_logo_path:
msg = add_image2mail(full_logo_path, msg)
html_content.append('<img src="cid:customer_logo">')
html_content.append('<table width="600px"><tr><th colspan="2"><span>%s</span></th></tr>' % mail_welcome)
else:
html_content.append('<table width="600px"><tr><th colspan="2"><span>%s</span></th></tr>' % mail_welcome)
# Update shinken_var dict with appropriate dict depending which is object notified
# then we can fill mail content.
odd=True
get_content_to_send()
logging.debug('Content to send: %s' % shinken_var)
for k,v in sorted(shinken_var.iteritems()):
logging.debug('type %s : %s' % (k, type(v)))
if odd:
html_content.append('<tr><th class="odd">' + k + '</th><td class="odd">' + v + '</td></tr>')
odd=False
else:
html_content.append('<tr><th class="even">' + k + '</th><td class="even">' + v + '</td></tr>')
odd=True
html_content.append('</table>')
if url != None:
html_content.append('More details on Shinken WebUI at : <a href="%s">%s</a></body></html>' % (url, url))
else:
html_content.append('</body></html>')
# Make final string var to send and encode it to stdout encoding
# avoiding decoding error.
html_content = '\r\n'.join(html_content)
try:
if sys.stdout.encoding is not None:
encoding = sys.stdout.encoding
else:
encoding = 'utf-8'
html_msg = html_content.encode(encoding)
except UnicodeDecodeError:
logging.debug('Content is Unicode encoded.')
html_msg = html_content.decode('utf-8').encode(encoding)
logging.debug('HTML string: %s' % html_msg)
msgText = MIMEText(html_msg, 'html', encoding)
logging.debug('MIMEText: %s' % msgText)
msg.attach(msgText)
logging.debug('Mail object: %s' % msg)
return msg
0
Example 129
def __str__(self):
return self.__repr__().encode(sys.stdout.encoding)
0
Example 130
def __str__(self):
return self.__repr__().encode(sys.stdout.encoding)
0
Example 131
def insstr(*args):
scr, args = args[0], list(args[1:])
x = 2 if len(args) > 2 else 0
args[x] = args[x].encode(sys.stdout.encoding)
return scr.insstr(*args)
0
Example 132
def __init__(self, *args, **kwargs):
# Fix for python curses resize bug:
# http://bugs.python.org/issue2675
os.unsetenv('LINES')
os.unsetenv('COLUMNS')
self.scr = args[0]
self.data = args[1]['data']
self.header_offset_orig = 3
self.header = args[1]['header']
self.header_offset = self.header_offset_orig
self.num_data_columns = len(self.header)
self._init_double_width(kwargs.get('double_width'))
self.column_width_mode = kwargs.get('column_width')
self.column_gap = kwargs.get('column_gap')
self._init_column_widths(kwargs.get('column_width'),
kwargs.get('column_widths'))
try:
kwargs.get('trunc_char').encode(sys.stdout.encoding or 'utf-8')
self.trunc_char = kwargs.get('trunc_char')
except (UnicodeDecodeError, UnicodeError):
self.trunc_char = '>'
self.x, self.y = 0, 0
self.win_x, self.win_y = 0, 0
self.max_y, self.max_x = 0, 0
self.num_columns = 0
self.vis_columns = 0
self.init_search = self.search_str = kwargs.get('search_str')
self._search_win_open = 0
self.modifier = str()
self.define_keys()
self.resize()
self.display()
# Handle goto initial position (either (y,x), [y] or y)
try:
self.goto_y(kwargs.get('start_pos')[0])
except TypeError:
self.goto_y(kwargs.get('start_pos'))
try:
self.goto_x(kwargs.get('start_pos')[1])
except (IndexError, TypeError):
pass