Here are the examples of the python api sys.getsizeof taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
116 Examples
2
Example 51
Project: joliebulle Source File: base.py
def __init__(self):
logger.debug("Import %s", database_file)
fichierBeerXML = database_file
self.arbre = ET.parse(fichierBeerXML)
presentation=self.arbre.find('.//RECIPE')
fermentables=self.arbre.findall('.//FERMENTABLE')
hops = self.arbre.findall('.//HOP')
levures = self.arbre.findall('.//YEAST')
misc = self.arbre.findall('.//MISC')
self.listeFermentables = list()
self.listeHops = list()
self.listeYeasts = list()
self.listeMiscs = list()
#Ingredient fermentescibles
for element in fermentables:
self.listeFermentables.append( Fermentable.parse(element) )
self.listeFermentables = sorted(self.listeFermentables, key=attrgetter('name'))
logger.debug( "%s fermentables in database, using %s bytes in memory", len(self.listeFermentables), sys.getsizeof(self.listeFermentables) )
#Houblons
for element in hops:
self.listeHops.append( Hop.parse(element) )
self.listeHops = sorted(self.listeHops, key=attrgetter('name'))
logger.debug( "%s hops in database, using %s bytes in memory", len(self.listeHops), sys.getsizeof(self.listeHops) )
#Levures
for element in levures:
self.listeYeasts.append( Yeast.parse(element) )
self.listeYeasts = sorted(self.listeYeasts, key=attrgetter('name'))
logger.debug( "%s yeasts in database, using %s bytes in memory", len(self.listeYeasts), sys.getsizeof(self.listeYeasts) )
#Ingredients divers
for element in misc:
self.listeMiscs.append( Misc.parse(element) )
self.listeMiscs = sorted(self.listeMiscs, key=attrgetter('name'))
logger.debug( "%s miscs in database, using %s bytes in memory", len(self.listeMiscs), sys.getsizeof(self.listeMiscs) )
logger.debug("Import %s terminé", database_file)
#Import Mash file
logger.debug("Import %s", mash_file)
arbre = ET.parse(mash_file)
mash = arbre.findall('.//MASH')
self.listeMashes = list()
for element in mash:
self.listeMashes.append( Mash.parse(element) )
logger.debug( "%s mash in database, using %s bytes in memory", len(self.listeMashes), sys.getsizeof(self.listeMashes) )
logger.debug("Import %s terminé", mash_file)
2
Example 52
def __init__(self, struct_or_mol, authors, projects=None, references='',
remarks=None, data=None, history=None, created_at=None):
# initialize root-level structure keys
self.structure = struct_or_mol
# turn authors into list of Author objects
authors = authors.split(',')\
if isinstance(authors, string_types) else authors
self.authors = [Author.parse_author(a) for a in authors]
# turn projects into list of Strings
projects = projects if projects else []
self.projects = [projects] \
if isinstance(projects, string_types) else projects
# check that references are valid BibTeX
if references and not is_valid_bibtex(references):
raise ValueError("Invalid format for SNL reference! Should be "
"BibTeX string.")
if len(references) > MAX_BIBTEX_CHARS:
raise ValueError("The BibTeX string must be fewer than {} chars "
", you have {}"
.format(MAX_BIBTEX_CHARS, len(references)))
self.references = references
# turn remarks into list of Strings
remarks = remarks if remarks else []
self.remarks = [remarks] if isinstance(remarks, string_types) \
else remarks
# check remarks limit
for r in self.remarks:
if len(r) > 140:
raise ValueError("The remark exceeds the maximum size of"
"140 characters: {}".format(r))
# check data limit
self.data = data if data else {}
if not sys.getsizeof(self.data) < MAX_DATA_SIZE:
raise ValueError("The data dict exceeds the maximum size limit of"
" {} bytes (you have {})"
.format(MAX_DATA_SIZE, sys.getsizeof(data)))
for k, v in self.data.items():
if not k.startswith("_"):
raise ValueError("data must contain properly namespaced data "
"with keys starting with an underscore. The "
"key {} does not start with an underscore.",
format(k))
# check for valid history nodes
history = history if history else [] # initialize null fields
if len(history) > MAX_HNODES:
raise ValueError("A maximum of {} History nodes are supported, "
"you have {}!".format(MAX_HNODES, len(history)))
self.history = [HistoryNode.parse_history_node(h) for h in history]
if not all([sys.getsizeof(h) < MAX_HNODE_SIZE for h in history]):
raise ValueError("One or more history nodes exceeds the maximum "
"size limit of {} bytes".format(MAX_HNODE_SIZE))
self.created_at = created_at if created_at \
else datetime.datetime.utcnow()
2
Example 53
def test_errors(self):
class BadSizeof(object):
def __sizeof__(self):
raise ValueError
self.assertRaises(ValueError, sys.getsizeof, BadSizeof())
class InvalidSizeof(object):
def __sizeof__(self):
return None
self.assertRaises(TypeError, sys.getsizeof, InvalidSizeof())
sentinel = ["sentinel"]
self.assertIs(sys.getsizeof(InvalidSizeof(), sentinel), sentinel)
class OverflowSizeof(long):
def __sizeof__(self):
return int(self)
self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)),
sys.maxsize + self.gc_headsize)
with self.assertRaises(OverflowError):
sys.getsizeof(OverflowSizeof(sys.maxsize + 1))
with self.assertRaises(ValueError):
sys.getsizeof(OverflowSizeof(-1))
with self.assertRaises((ValueError, OverflowError)):
sys.getsizeof(OverflowSizeof(-sys.maxsize - 1))
2
Example 54
def test_errors(self):
class BadSizeof:
def __sizeof__(self):
raise ValueError
self.assertRaises(ValueError, sys.getsizeof, BadSizeof())
class InvalidSizeof:
def __sizeof__(self):
return None
self.assertRaises(TypeError, sys.getsizeof, InvalidSizeof())
sentinel = ["sentinel"]
self.assertIs(sys.getsizeof(InvalidSizeof(), sentinel), sentinel)
class FloatSizeof:
def __sizeof__(self):
return 4.5
self.assertRaises(TypeError, sys.getsizeof, FloatSizeof())
self.assertIs(sys.getsizeof(FloatSizeof(), sentinel), sentinel)
class OverflowSizeof(int):
def __sizeof__(self):
return int(self)
self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)),
sys.maxsize + self.gc_headsize)
with self.assertRaises(OverflowError):
sys.getsizeof(OverflowSizeof(sys.maxsize + 1))
with self.assertRaises(ValueError):
sys.getsizeof(OverflowSizeof(-1))
with self.assertRaises((ValueError, OverflowError)):
sys.getsizeof(OverflowSizeof(-sys.maxsize - 1))
0
Example 55
Project: translator Source File: api.py
def __translate__(text, source, target, client='x',
user_agent=DEFAULT_USER_AGENT):
"""
text: text to be translated
source: source language
target: target language
"""
if source == target:
return text
if not re.match(r'Mozilla/\d+\.\d+ \(.*', user_agent):
user_agent = 'Mozilla/5.0 (%s)' % user_agent
headers = {
'Referer': 'http://translate.google.com',
'User-Agent': user_agent,
'Content-Length': str(sys.getsizeof(text))
}
payload = {
'client': client,
'sl': source,
'tl': target,
'text': text,
}
url = 'http://translate.google.com/translate_a/t'
req = requests.post(url, headers=headers, data=payload)
if req.status_code != 200:
raise HTTPException(
('Google Translate returned HTTP {}'.format(req.status_code)),
req.status_code)
if client == 'x':
data = json.loads(req.text)
# It appears in some cases the Google Translate returns a string
# rather than a dictionary
try:
if isinstance(data, unicode):
return data
except NameError:
if isinstance(data, str):
return data
try:
sentences = data['sentences']
except TypeError:
sentences = data['results'][0]['sentences']
result = ' '.join(map(lambda x: x['trans'], sentences))
# Remove unneccessary white spaces
return '\n'.join(map(lambda x: x.strip(), result.split('\n')))
elif client == 't':
return parse_javascript(req.text)
else:
raise Exception("Unsupported client '{}'".format(client))
0
Example 56
def __sizeof__(self):
return getsizeof(self.__value)
0
Example 57
def sizeof(obj, exclude=None):
"""Return the size in bytes of *obj*."""
if obj is None or obj is False or obj is True:
return 0
size = sys.getsizeof(obj)
if hasattr(obj, '__dict__'):
size += sys.getsizeof(obj.__dict__)
for key, value in obj.__dict__.items():
if exclude is not None and key in exclude:
continue
s = sizeof(key)
s += sizeof(value, exclude)
# print('{}.{}: {}'.format(type(obj).__name__, key, s))
size += s
elif hasattr(obj, '__slots__'):
for key in obj.__slots__:
if hasattr(obj, key):
if exclude is not None and key in exclude:
continue
s = sizeof(getattr(obj, key), exclude)
# print('{}.{}: {}'.format(type(obj).__name__, key, s))
size += s
return size
0
Example 58
Project: ZeroNet Source File: StatsPlugin.py
def actionListobj(self):
import gc
import sys
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
raise StopIteration
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
raise StopIteration
type_filter = self.get.get("type")
yield """
<style>
* { font-family: monospace; white-space: pre }
table * { text-align: right; padding: 0px 10px }
</style>
"""
yield "Listing all %s objects in memory...<br>" % cgi.escape(type_filter)
ref_count = {}
objs = gc.get_objects()
for obj in objs:
obj_type = str(type(obj))
if obj_type != type_filter:
continue
refs = [
ref for ref in gc.get_referrers(obj)
if hasattr(ref, "__class__") and
ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"]
]
if not refs:
continue
try:
yield "%.1fkb <span title=\"%s\">%s</span>... " % (
float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100))
)
except:
continue
for ref in refs:
yield " ["
if "object at" in str(ref) or len(str(ref)) > 100:
yield str(ref.__class__.__name__)
else:
yield str(ref.__class__.__name__) + ":" + cgi.escape(str(ref))
yield "] "
ref_type = ref.__class__.__name__
if ref_type not in ref_count:
ref_count[ref_type] = [0, 0]
ref_count[ref_type][0] += 1 # Count
ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "<br>"
yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (len(ref_count), sum([stat[1] for stat in ref_count.values()]))
for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count
yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], cgi.escape(str(obj)))
gc.collect() # Implicit grabage collection
0
Example 59
Project: rockstor-core Source File: data_collector.py
def on_readlog(self, reader, logfile):
logs_loader = {'slow' : {'lines' : 200, 'sleep' : 0.50},
'fast' : {'lines' : 1, 'sleep' : 0.05},
}
def valid_log(logfile):
#If file exist and size greater than 0 return true
#else false and avoid processing
if path.exists(logfile):
return (path.getsize(logfile) > 0)
else:
return False
def build_reader_command(reader):
command = []
command.append(self.readers[reader]['command'])
#If our reader has opt args we add them to popen command
if ('args' in self.readers[reader]):
command.append(self.readers[reader]['args'])
#Queue log file to popen command
command.append(log_path)
return command
def static_reader(reader, log_path):
if (valid_log(log_path)):#Log file exist and greater than 0, perform data collecting
#Build reader command
read_command = build_reader_command(reader)
#Define popen process and once completed split stdout by lines
reader_process = Popen(read_command, bufsize=1, stdout=PIPE)
log_content = reader_process.communicate()[0]
log_contentsize = getsizeof(log_content)
log_content = log_content.splitlines(True)
#Starting from content num of lines decide if serve it 1 line/time or in 200 lines chunks
reader_type = 'fast' if (len(log_content) <= 200) else 'slow'
chunk_size = logs_loader[reader_type]['lines']
reader_sleep = logs_loader[reader_type]['sleep']
log_content_chunks = [log_content[x:x+chunk_size] for x in xrange(0, len(log_content), chunk_size)]
total_rows = len(log_content)
else:#Log file missing or size 0, gently inform user
#Log not exist or empty so we send fake values
#for rows, chunks, etc to uniform data on existing functions
#and avoid client side extra checks
log_content = 'Selected log file is empty or doesn\'t exist'
log_content = log_content.splitlines(True)
total_rows = 1
log_contentsize = getsizeof(log_content)
log_content_chunks = []
log_content_chunks.append(log_content)
reader_sleep = 0
#Serve each chunk with emit and sleep before next one to avoid client side browser overload
current_rows = 0
for data_chunks in log_content_chunks:
chunk_content = ''.join(data_chunks)
current_rows += len(data_chunks)
self.emit('logManager:logcontent', {
'key': 'logManager:logcontent', 'data': {
'current_rows' : current_rows,
'total_rows' : total_rows,
'chunk_content' : chunk_content,
'content_size' : log_contentsize
}
})
gevent.sleep(reader_sleep)
def live_reader(log_path):
#Switch live reader state to True
self.livereading = True
#Build reader command from readers dict
read_command = build_reader_command('tailf')
self.livereader_process = Popen(read_command, bufsize=1, stdout=PIPE)
while self.livereading:
live_out = self.livereader_process.stdout.readline()
self.emit('logManager:logcontent', {
'key': 'logManager:logcontent', 'data': {
'current_rows' : 1,
'total_rows' : 1,
'chunk_content' : live_out,
'content_size' : 1
}
})
log_path = self.build_log_path(logfile)
if (reader == 'tailf'):
gevent.spawn(live_reader, log_path)
else:
gevent.spawn(static_reader, reader, log_path)
0
Example 60
Project: WeCase Source File: debug.py
def objectSize(obj, ignore=None, verbose=False, depth=0, recursive=False):
"""Guess how much memory an object is using"""
ignoreTypes = [types.MethodType, types.UnboundMethodType, types.BuiltinMethodType, types.FunctionType, types.BuiltinFunctionType]
ignoreRegex = re.compile('(method-wrapper|Flag|ItemChange|Option|Mode)')
if ignore is None:
ignore = {}
indent = ' '*depth
try:
hash(obj)
hsh = obj
except:
hsh = "%s:%d" % (str(type(obj)), id(obj))
if hsh in ignore:
return 0
ignore[hsh] = 1
try:
size = sys.getsizeof(obj)
except TypeError:
size = 0
if isinstance(obj, ndarray):
try:
size += len(obj.data)
except:
pass
if recursive:
if type(obj) in [list, tuple]:
if verbose:
print(indent+"list:")
for o in obj:
s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
if verbose:
print(indent+' +', s)
size += s
elif isinstance(obj, dict):
if verbose:
print(indent+"list:")
for k in obj:
s = objectSize(obj[k], ignore=ignore, verbose=verbose, depth=depth+1)
if verbose:
print(indent+' +', k, s)
size += s
#elif isinstance(obj, QtCore.QObject):
#try:
#childs = obj.children()
#if verbose:
#print indent+"Qt children:"
#for ch in childs:
#s = objectSize(obj, ignore=ignore, verbose=verbose, depth=depth+1)
#size += s
#if verbose:
#print indent + ' +', ch.objectName(), s
#except:
#pass
#if isinstance(obj, types.InstanceType):
gc.collect()
if verbose:
print(indent+'attrs:')
for k in dir(obj):
if k in ['__dict__']:
continue
o = getattr(obj, k)
if type(o) in ignoreTypes:
continue
strtyp = str(type(o))
if ignoreRegex.search(strtyp):
continue
#if isinstance(o, types.ObjectType) and strtyp == "<type 'method-wrapper'>":
#continue
#if verbose:
#print indent, k, '?'
refs = [r for r in gc.get_referrers(o) if type(r) != types.FrameType]
if len(refs) == 1:
s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
size += s
if verbose:
print(indent + " +", k, s)
#else:
#if verbose:
#print indent + ' -', k, len(refs)
return size
0
Example 61
Project: StarCluster Source File: utils.py
def size_in_kb(obj):
return sys.getsizeof(obj) / 1024.
0
Example 62
Project: python_zklib Source File: zkAtt.py
def zkAtt(self):
command = CMD_ATTLOG_RRQ
comand_string = ''
chksum = 0
session_id = self.session_id
reply_id = unpack('4H', self.data_recv[:8])[3]
buf = self.createHeader(command, chksum, session_id,
reply_id, comand_string)
self.zkclient.sendto(buf, self.address)
size = None
attendance = []
self.data_recv, addr = self.zkclient.recvfrom(1024)
# print unpack('24s1s4s11s',self.data_recv.ljust(40)[:40])
# print "unpack HcHc :7", unpack('HcHc',self.data_recv[:7])
# print "unpack HHHH :8", unpack('HHHH',self.data_recv[:8])
# print "unpack HHHH :8 [1]", unpack('4H',self.data_recv[:8])[0]
print "size", sys.getsizeof(self.data_recv)
print "size", len(self.data_recv)
lensi = len(self.data_recv) / 2
fstri = str(lensi) + "H"
print "unpack 4I ", unpack(fstri, self.data_recv)
# print "unpack 8H", unpack('8H', self.data_recv)
# print "unpack I 8:12 [0]", unpack ('I', self.data_recv[8:12])[0]
attendance = []
self.data_recv, addr = self.zkclient.recvfrom(1024)
if unpack('4H', self.data_recv[:8])[0] == CMD_PREPARE_DATA:
print "received CMD_PREPARE_DATA"
size = unpack('I', self.data_recv[8:12])[0]
wa = unpack('II', self.data_recv[:8])[0]
print "received ", wa
print 'Receiving %s %s' % (size, "bytes")
try:
data_recv, addr = self.zkclient.recvfrom(size)
print "size of data_recv", sys.getsizeof(data_recv)
self.attendancedata.append(data_recv)
acmOK(self)
print unpack('24s1s4s11s', self.data_recv.ljust(40)[:40])
except:
print "socket timeout 1 - no more data to receive"
while unpack('4H', self.data_recv[:8])[0] != 2000 or unpack('4H', self.data_recv[:8])[0] == 1501:
try:
data_recv, addr = self.zkclient.recvfrom(size)
acmOK(self)
print unpack('24s1s4s11s', self.data_recv.ljust(40)[:40])
except:
print "socket timeout 2 - no more data to receive"
self.attendancedata.append(data_recv)
acmOK(self)
break
# print "length of reiceived data packet", len(data_recv)
# print "unpack 126H", unpack('126H', data_recv)
data_recv, addr = self.zkclient.recvfrom(1024)
self.attendancedata.append(data_recv)
if unpack('4H', self.data_recv[:8])[0] == CMD_PREPARE_DATA:
print "received CMD_PREPARE_DATA"
size = unpack('I', self.data_recv[8:12])[0]
print 'Receiving %s %s' % (size, "bytes")
elif unpack('4H', data_recv[:8])[0] == 1501:
print "receiving Data packet"
elif unpack('4H', self.data_recv[:8])[0] == 2000:
print "received CMD_ACK_OK"
try:
self.data_recv, addr = self.zkclient.recvfrom(size)
acmOK(self)
# print len(self.data_recv)
except:
print "socket timeout 3 - no more data to receive"
#self.data_recv, addr = self.zkclient.recvfrom(1024)
# print "length of reiceived data packet", len(self.data_recv)
print "length", len(self.data_recv)
lens = len(self.data_recv) / 2
fstr = str(lens) + "H"
print unpack(fstr, self.data_recv)
if unpack('4H', self.data_recv[:8])[0] == 2000:
print "received CMD_ACK_OK"
try:
self.data_recv, addr = self.zkclient.recvfrom(4096)
# print len(self.data_recv)
except:
print "socket timeout 4 - no more data to receive"
print "length of att data", len(self.attendancedata)
#data_recv = self.zkclient.recvfrom(8)
for x in xrange(len(self.attendancedata)):
# print self.attendancedata[x][8:]
#self.attendancedata[x] = self.attendancedata[x][8:]
# print self.attendancedata[x][0:]
self.attendancedata[x] = self.attendancedata[x][0:]
self.data_recv, addr = self.zkclient.recvfrom(1024)
if unpack('4H', self.data_recv)[0] == 2000:
print "received CMD_ACK_OK"
try:
self.data_recv, addr = self.zkclient.recvfrom(1024)
except:
print "socket timeout - no more data to receive"
for x in xrange(len(self.attendancedata)):
self.attendancedata[x] = self.attendancedata[x][0:]
attendancedata = self.attendancedata
attendancedata = self.attendancedata
attendancedata = ''.join(self.attendancedata)
attendancedata = attendancedata[14:]
#attendancedata = attendancedata[14:]
print "len attendancedata", len(attendancedata)
while len(attendancedata):
#uidm = unpack('24s1s4s11s',attendancedata.ljust(40)[:40])
#uidt = unpack('38s',attendancedata[:38])
#uidmn = unpack('16s',attendancedata[:16])[0]
pls = unpack('c', attendancedata[29:30]) # [3]
#statev = unpack('=2c',attendancedata[21:23])
#datem = unpack('ii',attendancedata[:8])[1]
uid, state, timestamp, space = unpack(
'24s1s4s11s', attendancedata.ljust(40)[:40])
print "%s, %s, %s, %s" % (uid, 1, ord(space[0]), decode_time(int(reverseHex(timestamp.encode('hex')), 16)))
# print "%s, %s, %s, %s" % (uid, ord(pls[0]), ord(space[0]), decode_time( int( reverseHex( timestamp.encode('hex') ), 16 ) ) )
# print "%s, %s, %s, %s" % (uid, state, space, timestamp)
#attendance.append( ( uid, ord(pls[0]), decode_time( int( reverseHex( timestamp.encode('hex') ), 16 ) ) ) )
attendance.append((uid, ord(pls[0]), decode_time(
int(reverseHex(timestamp.encode('hex')), 16))))
attendancedata = attendancedata[40:]
while len(attendancedata):
pls = unpack('c', attendancedata[29:30]) # [3]
uid, state, timestamp, space = unpack(
'24s1s4s11s', attendancedata.ljust(40)[:40])
# print "%s, %s, %s, %s" % (uid, ord(pls[0]), ord(space[0]),
# decode_time( int( reverseHex( timestamp.encode('hex') ), 16 )
# ) )
attendance.append((uid, ord(pls[0]), decode_time(
int(reverseHex(timestamp.encode('hex')), 16))))
attendancedata = attendancedata[40:]
0
Example 63
def _send_chunks(self, to_put):
put_dump = pickle.dumps(to_put)
data_size = sys.getsizeof(put_dump)
nchunks = data_size / 20000000. # chunks with size 20 MB
chunksize = int(len(put_dump) / nchunks)
chunk_iterator = self._make_chunk_iterator(put_dump, chunksize)
for chunk in chunk_iterator:
# print('S: sending False')
self.conn.send(False)
# print('S: sent False')
# print('S: sending chunk')
self.conn.send_bytes(chunk)
# print('S: sent chunk %s' % chunk[0:10])
# print('S: recv signal')
self.conn.recv() # wait for signal that message was received
# print('S: read signal')
# print('S: sending True')
self.conn.send(True)
# print('S: sent True')
# print('S: recving last signal')
self.conn.recv() # wait for signal that message was received
0
Example 64
def __params__(text, source, target, client='at',
user_agent=DEFAULT_USER_AGENT):
"""Returns a dictionary containing all parameters to send a translation
request to Google Translate."""
headers = {
'User-Agent': user_agent,
'Content-Length': str(sys.getsizeof(text))
}
remote_addr = request.remote_addr if request.remote_addr else ''
payload = {
'client': client,
'sl': source,
'tl': target,
'q': text,
'dt': ['t', 'ld', 'qc', 'rm', 'bd'],
'dj': 1,
# Generate a UUID based on the remote client's IP address
'iid': str(uuid.uuid5(uuid.NAMESPACE_DNS, remote_addr)),
# 'itid': 'pk',
# 'otf': 1,
'ie': 'UTF-8',
}
url = 'https://translate.google.com/translate_a/single'
try:
# Python 3
from urllib.parse import quote, urlencode
except ImportError:
# Python 2
quote = urllib.quote
urlencode = urllib.urlencode
if len(quote(text)) > 1000:
method = 'post'
del payload['q']
else:
method = 'get'
del headers['Content-Length']
return {
'headers': headers,
'payload': payload,
'method': method,
'url': url,
'query': urlencode(list(__payload_as_tuples__(payload)))
}
0
Example 65
Project: python-beaver Source File: sqs_transport.py
def callback(self, filename, lines, **kwargs):
timestamp = self.get_timestamp(**kwargs)
if kwargs.get('timestamp', False):
del kwargs['timestamp']
if self._bulk_lines:
message_batch = ''
message_count = 0
else:
message_batch = []
message_batch_size = 0
message_batch_size_max = 250000 # Max 256KiB but leave some headroom
for line in lines:
if self._bulk_lines:
m = self.format(filename, line, timestamp, **kwargs)
message_size = getsizeof(m)
else:
m = Message()
m.set_body(self.format(filename, line, timestamp, **kwargs))
message_size = len(m)
if (message_size > message_batch_size_max):
self._logger.debug('Dropping the message as it is too large to send ({0} bytes)'.format(message_size))
continue
# Check the new total size before adding a new message and don't try to send an empty batch
if self._bulk_lines and (len(message_batch) > 0) and (((message_batch_size + message_size) >= message_batch_size_max)):
self._logger.debug('Flushing {0} messages to SQS queue {1} bytes'.format(message_count, message_batch_size))
self._send_message(message_batch)
message_batch = ''
message_count = 0
message_batch_size = 0
# SQS can only handle up to 10 messages in batch send and it can not exceed 256KiB (see above)
elif (len(message_batch) > 0) and (((message_batch_size + message_size) >= message_batch_size_max) or (len(message_batch) == 10)):
self._logger.debug('Flushing {0} messages to SQS queue {1} bytes'.format(len(message_batch), message_batch_size))
self._send_message_batch(message_batch)
message_batch = []
message_batch_size = 0
message_batch_size = message_batch_size + message_size
if self._bulk_lines:
message_batch += '{0},'.format(m)
message_count += 1
else:
message_batch.append((uuid.uuid4(), self.format(filename, line, timestamp, **kwargs), 0))
if len(message_batch) > 0:
if self._bulk_lines:
self._logger.debug('Flushing the last {0} messages to SQS queue {1} bytes'.format(message_count, message_batch_size))
self._send_message(message_batch)
else:
self._logger.debug('Flushing the last {0} messages to SQS queue {1} bytes'.format(len(message_batch), message_batch_size))
self._send_message_batch(message_batch)
return True
0
Example 66
Project: pygobject Source File: test_everything.py
@classmethod
def from_wrapped(cls, obj):
offset = sys.getsizeof(object()) # size of PyObject_HEAD
return ctypes.POINTER(cls).from_address(id(obj) + offset)
0
Example 67
Project: flask-mwoauth Source File: __init__.py
def request(self, api_query, url=None):
""" e.g. {'action': 'query', 'meta': 'userinfo'}. format=json not required
function returns a python dict that resembles the api's json response
"""
api_query['format'] = 'json'
url = url or self.base_url
size = sum([sys.getsizeof(v) for k, v in iteritems(api_query)])
if size > (1024 * 8):
# if request is bigger than 8 kB (the limit is somewhat arbitrary,
# see https://www.mediawiki.org/wiki/API:Edit#Large_texts) then
# transmit as multipart message
req = self._prepare_long_request(url=url + "/api.php?",
api_query=api_query
)
return self.mwoauth.post(url + "/api.php?",
data=req.body,
content_type=req.headers['Content-Type']
).data
else:
return self.mwoauth.post(url + "/api.php",
data=api_query
).data
0
Example 68
Project: odoo-saas-tools Source File: saas_client.py
@api.model
def _transport_backup(self, dump_db, filename=None):
'''
send db dump to S3
'''
with tempfile.TemporaryFile() as t:
dump_db(t)
t.seek(0)
db_dump = base64.b64decode(t.read().encode('base64'))
if parallel_upload and sys.getsizeof(db_dump) > 5242880:
ir_params = self.env['ir.config_parameter']
aws_key = ir_params.get_param('saas_s3.saas_s3_aws_accessid')
aws_secret = ir_params.get_param('saas_s3.saas_s3_aws_accesskey')
bucketname = ir_params.get_param('saas_s3.saas_s3_aws_bucket')
self._transport_backup_parallel(db_dump, filename, aws_key, aws_secret, bucketname)
else:
conn, bucket_name = _get_s3_conn(self.env)
self._transport_backup_simple(conn, bucket_name, db_dump, filename)
return True
0
Example 69
Project: pympler Source File: test_summary.py
def test_summary_diff(self):
"""Test summary diff. """
left = [[str(str), 3, 3*getsizeof('a')],\
[str(int), 2, 2*getsizeof(1)],\
[str(list), 1, getsizeof([])],\
[str(dict), 1, getsizeof({})]]
right = [[str(str), 2, 2*getsizeof('a')],\
[str(int), 3, 3*getsizeof(1)],\
[str(list), 1, getsizeof([1,2,3])],\
[str(dict), 1, getsizeof({})],
[str(tuple), 1, getsizeof((1,2))]]
expected = [[str(str), -1, -1*getsizeof('a')],\
[str(int), 1, +1*getsizeof(1)],\
[str(list), 0, getsizeof([1,2,3]) - getsizeof([])],\
[str(dict), 0, 0],
[str(tuple), 1, getsizeof((1,2))]]
res = summary.get_diff(left, right)
for row_e in res:
self.assertTrue(row_e in expected)
0
Example 70
def put(self, query, result):
### only if valid result (simple test) ###
if sys.getsizeof(result) > 21:
self.resultdict[query] = result
0
Example 71
@classmethod
def info(cls):
images = cls.__images
blacklist = cls.__blacklist
info = {
"blacklist": len(blacklist),
"blacklist_size": sys.getsizeof(blacklist, 0),
"images_per_site": {}
}
images_count = 0
images_size = 0
for crawler in images:
for site in images[crawler]:
site_image_count = len(images[crawler][site])
images_count += site_image_count
images_size += sys.getsizeof(images[crawler][site])
info["images_per_site"][crawler + "_" + site] = site_image_count
info["images_size"] = images_size
info["images"] = images_count
return info
0
Example 72
@support.bigmemtest(size=LARGE_SIZE,
memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
dry_run=False)
def test_frozenset(self, size):
self.check_unmarshallable(frozenset(range(size)))
0
Example 73
def test_default(self):
size = test.support.calcvobjsize
self.assertEqual(sys.getsizeof(True), size('') + self.longdigit)
self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit)
0
Example 74
Project: pympler Source File: test_summary.py
def test_subtract(self):
"""Test that a single object's data is correctly subtracted from a summary.
- result in correct total size and total number of objects
- if object was not listed before, it should be listed negative
afterwards
"""
objects = ['the', 'quick', 'brown', 'fox', 1298, 123, 234, [], {}]
summ = summary.summarize(objects)
summary._subtract(summ, 'the')
summary._subtract(summ, {})
summary._subtract(summ, (1,))
# to verify that these rows where actually included afterwards
checked_str = checked_dict = checked_tuple = False
for row in summ:
if row[0] == summary._repr(''):
totalsize = getsizeof('quick') + getsizeof('brown') +\
getsizeof('fox')
self.assert_(row[1] == 3, "%s != %s" % (row[1], 3))
self.assert_(row[2] == totalsize, totalsize)
checked_str = True
if row[0] == summary._repr({}):
self.assert_(row[1] == 0)
self.assert_(row[2] == 0)
checked_dict = True
if row[0] == summary._repr((1,)):
self.assert_(row[1] == -1)
self.assert_(row[2] == -getsizeof((1,)))
checked_tuple = True
self.assert_(checked_str, "no str found in summary")
self.assert_(checked_dict, "no dict found in summary")
self.assert_(checked_tuple, "no tuple found in summary")
summary._subtract(summ, 'quick')
summary._subtract(summ, 'brown')
checked_str = False
for row in summ:
if row[0] == summary._repr(''):
self.assert_(row[1] == 1)
self.assert_(row[2] == getsizeof('fox'))
checked_str = True
self.assert_(checked_str, "no str found in summ")
0
Example 75
def __sizeof__(self):
return sys.getsizeof(self._data)
0
Example 76
def get_size(self, obj):
return sys.getsizeof(obj.content)
0
Example 77
def body(self):
body = ''
content_type, char_set = _parse_content_type(self.response.get('Content-Type', ''))
content = getattr(self.response, 'content', '')
if content:
max_body_size = SilkyConfig().SILKY_MAX_RESPONSE_BODY_SIZE
if max_body_size > -1:
Logger.debug('Max size of response body defined so checking')
size = sys.getsizeof(content, None)
if not size:
Logger.error('Could not get size of response body. Ignoring')
content = ''
else:
if size > max_body_size:
content = ''
Logger.debug('Size of %d for %s is bigger than %d so ignoring response body' % (size, self.request.path, max_body_size))
else:
Logger.debug('Size of %d for %s is less than %d so saving response body' % (size, self.request.path, max_body_size))
if content and content_type in content_types_json:
# TODO: Perhaps theres a way to format the JSON without parsing it?
if not isinstance(content, str):
# byte string is not compatible with json.loads(...) and json.dumps(...) in python3
content = content.decode()
try:
body = json.dumps(json.loads(content), sort_keys=True, indent=4)
except (TypeError, ValueError):
Logger.warn('Response to request with pk %s has content type %s but was unable to parse it' % (self.request.pk, content_type))
return body, content
0
Example 78
Project: cyordereddict Source File: test_ordereddict.py
def test_sizeof(self):
# Wimpy test: Just verify the reported size is larger than a regular dict
d = dict(a=1)
od = OrderedDict(**d)
self.assertGreater(sys.getsizeof(od), sys.getsizeof(d))
0
Example 79
Project: Gnip-Python-Search-API-Utilities Source File: api.py
def parse_responses(self, count_bucket):
"""Parse returned responses.
When paged=True, manage paging using the API token mechanism
When output file is set, write output files for paged output."""
acs = []
repeat = True
page_count = 1
self.paged_file_list = []
while repeat:
doc = self.request()
tmp_response = json.loads(doc)
if "results" in tmp_response:
acs.extend(tmp_response["results"])
else:
raise ValueError("Invalid request\nQuery: %s\nResponse: %s"%(self.rule_payload, doc))
if len(acs) < self.hard_max:
repeat = False
if self.paged or (count_bucket and self.search_v2):
if len(acs) > 0:
if self.output_file_path is not None:
# writing to file
file_name = self.output_file_path + "/{0}_{1}.json".format(
str(datetime.datetime.utcnow().strftime(
"%Y%m%d%H%M%S"))
, str(self.file_name_prefix))
with codecs.open(file_name, "wb","utf-8") as out:
for item in tmp_response["results"]:
out.write(json.dumps(item)+"\n")
self.paged_file_list.append(file_name)
# if writing to file, don't keep track of all the data in memory
acs = []
else:
# storing in memory, so give some feedback as to size
sys.stderr.write("[%8d bytes] %5d total activities retrieved...\n"%(
sys.getsizeof(acs)
, len(acs)))
else:
sys.stderr.write( "No results returned for rule:{0}\n".format(str(self.rule_payload)) )
if "next" in tmp_response:
self.rule_payload["next"]=tmp_response["next"]
repeat = True
page_count += 1
sys.stderr.write( "Fetching page {}...\n".format(page_count) )
else:
if "next" in self.rule_payload:
del self.rule_payload["next"]
repeat = False
time.sleep(PAUSE)
else:
# stop iterating after reaching hard_max
repeat = False
return acs
0
Example 80
Project: pyqtgraph Source File: debug.py
def objectSize(obj, ignore=None, verbose=False, depth=0, recursive=False):
"""Guess how much memory an object is using"""
ignoreTypes = ['MethodType', 'UnboundMethodType', 'BuiltinMethodType', 'FunctionType', 'BuiltinFunctionType']
ignoreTypes = [getattr(types, key) for key in ignoreTypes if hasattr(types, key)]
ignoreRegex = re.compile('(method-wrapper|Flag|ItemChange|Option|Mode)')
if ignore is None:
ignore = {}
indent = ' '*depth
try:
hash(obj)
hsh = obj
except:
hsh = "%s:%d" % (str(type(obj)), id(obj))
if hsh in ignore:
return 0
ignore[hsh] = 1
try:
size = sys.getsizeof(obj)
except TypeError:
size = 0
if isinstance(obj, ndarray):
try:
size += len(obj.data)
except:
pass
if recursive:
if type(obj) in [list, tuple]:
if verbose:
print(indent+"list:")
for o in obj:
s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
if verbose:
print(indent+' +', s)
size += s
elif isinstance(obj, dict):
if verbose:
print(indent+"list:")
for k in obj:
s = objectSize(obj[k], ignore=ignore, verbose=verbose, depth=depth+1)
if verbose:
print(indent+' +', k, s)
size += s
#elif isinstance(obj, QtCore.QObject):
#try:
#childs = obj.children()
#if verbose:
#print indent+"Qt children:"
#for ch in childs:
#s = objectSize(obj, ignore=ignore, verbose=verbose, depth=depth+1)
#size += s
#if verbose:
#print indent + ' +', ch.objectName(), s
#except:
#pass
#if isinstance(obj, types.InstanceType):
gc.collect()
if verbose:
print(indent+'attrs:')
for k in dir(obj):
if k in ['__dict__']:
continue
o = getattr(obj, k)
if type(o) in ignoreTypes:
continue
strtyp = str(type(o))
if ignoreRegex.search(strtyp):
continue
#if isinstance(o, types.ObjectType) and strtyp == "<type 'method-wrapper'>":
#continue
#if verbose:
#print indent, k, '?'
refs = [r for r in gc.get_referrers(o) if type(r) != types.FrameType]
if len(refs) == 1:
s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
size += s
if verbose:
print(indent + " +", k, s)
#else:
#if verbose:
#print indent + ' -', k, len(refs)
return size
0
Example 81
def sizeof(object):
""" Returns the memory size of the given object (in bytes).
"""
return sys.getsizeof(object)
0
Example 82
def get_size(self):
if self._size is None:
self._size = sys.getsizeof(self)
return self._size
0
Example 83
Project: pattern Source File: metrics.py
def kb(object):
""" Returns the memory size of the given object (in kilobytes).
"""
return sys.getsizeof(object) * 0.01
0
Example 84
Project: datafari Source File: test_sys.py
def test_default(self):
size = test.test_support.calcobjsize
self.assertEqual(sys.getsizeof(True, -1), size('l'))
0
Example 85
def summarize(objects):
"""Summarize an objects list.
Return a list of lists, whereas each row consists of::
[str(type), number of objects of this type, total size of these objects].
No guarantee regarding the order is given.
"""
count = {}
total_size = {}
for o in objects:
otype = _repr(o)
if otype in count:
count[otype] += 1
total_size[otype] += getsizeof(o)
else:
count[otype] = 1
total_size[otype] = getsizeof(o)
rows = []
for otype in count:
rows.append([otype, count[otype], total_size[otype]])
return rows
0
Example 86
@classmethod
def from_iter(cls, iter):
offset = sys.getsizeof(object()) # size of PyObject_HEAD
return ctypes.POINTER(cls).from_address(id(iter) + offset)
0
Example 87
Project: tilequeue Source File: process.py
def _preprocess_data(feature_layers):
preproc_feature_layers = []
extra_data = dict(size={})
for feature_layer in feature_layers:
layer_datum = feature_layer['layer_datum']
geometry_types = layer_datum['geometry_types']
padded_bounds = feature_layer['padded_bounds']
features = []
features_size = 0
for row in feature_layer['features']:
wkb = row.pop('__geometry__')
shape = loads(wkb)
if shape.is_empty:
continue
if not shape.is_valid:
continue
if geometry_types is not None:
if shape.type not in geometry_types:
continue
# since a bounding box intersection is used, we
# perform a more accurate check here to filter out
# any extra features
# the formatter specific transformations will take
# care of any additional filtering
geom_type_bounds = padded_bounds[
normalize_geometry_type(shape.type)]
shape_padded_bounds = geometry.box(*geom_type_bounds)
if not shape_padded_bounds.intersects(shape):
continue
feature_id = row.pop('__id__')
props = dict()
feature_size = getsizeof(feature_id) + len(wkb)
for k, v in row.iteritems():
if k == 'mz_properties':
for output_key, output_val in v.items():
if output_val is not None:
# all other tags are utf8 encoded, encode
# these the same way to be consistent
if isinstance(output_key, unicode):
output_key = output_key.encode('utf-8')
if isinstance(output_val, unicode):
output_val = output_val.encode('utf-8')
props[output_key] = output_val
feature_size += len(output_key) + \
_sizeof(output_val)
else:
props[k] = v
feature_size += len(k) + _sizeof(v)
feature = shape, props, feature_id
features.append(feature)
features_size += feature_size
extra_data['size'][layer_datum['name']] = features_size
preproc_feature_layer = dict(
name=layer_datum['name'],
layer_datum=layer_datum,
features=features,
padded_bounds=padded_bounds,
)
preproc_feature_layers.append(preproc_feature_layer)
return preproc_feature_layers, extra_data
0
Example 88
Project: ZeroNet Source File: StatsPlugin.py
def actionDumpobj(self):
import gc
import sys
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
raise StopIteration
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
raise StopIteration
class_filter = self.get.get("class")
yield """
<style>
* { font-family: monospace; white-space: pre }
table * { text-align: right; padding: 0px 10px }
</style>
"""
objs = gc.get_objects()
for obj in objs:
obj_type = str(type(obj))
if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter:
continue
yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)))
for attr in dir(obj):
yield "- %s: %s<br>" % (attr, cgi.escape(str(getattr(obj, attr))))
yield "<br>"
gc.collect() # Implicit grabage collection
0
Example 89
Project: attention-lvcsr Source File: link.py
def raise_with_op(node, thunk=None, exc_info=None, storage_map=None):
"""
Re-raise an exception while annotating the exception object with
debug info.
Parameters
----------
node : Apply node
The Apply node object that resulted in the raised exception.
exc_info : tuple, optional
A tuple containing the exception type, exception object and
associated traceback, as would be returned by a call to
`sys.exc_info()` (which is done if `None` is passed).
storage_map: dict, optional
storage map of the theano function that resulted in the
raised exception.
Notes
-----
This re-raises the exception described by `exc_info` (or the last
one raised, if `exc_info` is omitted) and annotates the exception
object with several new members which may be helpful for debugging
Theano graphs. They are:
* __op_instance__: The Op that is responsible for the exception
being raised.
* __thunk_trace__: A traceback corresponding to the code that
actually generated the exception, if it is available.
* __applynode_index__: The index of the Apply node corresponding
to this op in `op.fgraph.toposort()`.
The exception is not annotated if it is of type `KeyboardInterrupt`.
"""
if exc_info is None:
exc_info = sys.exc_info()
exc_type, exc_value, exc_trace = exc_info
if exc_type == KeyboardInterrupt:
# print a simple traceback from KeyboardInterrupt
reraise(exc_type, exc_value, exc_trace)
try:
trace = node.outputs[0].tag.trace
except AttributeError:
try:
trace = node.op.tag.trace
except AttributeError:
trace = ()
exc_value.__thunk_trace__ = trace
exc_value.__op_instance__ = node
topo = node.fgraph.toposort()
if node in topo:
node_index = topo.index(node)
else:
node_index = None
exc_value.__applynode_index__ = node_index
hints = []
detailed_err_msg = "\nApply node that caused the error: " + str(node)
if exc_value.__applynode_index__ is not None:
detailed_err_msg += "\nToposort index: %d" % node_index
types = [getattr(ipt, 'type', 'No type') for ipt in node.inputs]
detailed_err_msg += "\nInputs types: %s\n" % types
if thunk is not None:
if hasattr(thunk, 'inputs'):
shapes = [getattr(ipt[0], 'shape', 'No shapes')
for ipt in thunk.inputs]
strides = [getattr(ipt[0], 'strides', 'No strides')
for ipt in thunk.inputs]
scalar_values = []
for ipt in thunk.inputs:
if getattr(ipt[0], "size", -1) <= 5:
scalar_values.append(ipt[0])
else:
scalar_values.append("not shown")
else:
shapes = "The thunk don't have an inputs attributes."
strides = "So we can't access the strides of inputs values"
scalar_values = "And can't print its inputs scalar value"
clients = [[c[0] for c in var.clients] for var in node.outputs]
detailed_err_msg += ("Inputs shapes: %s" % shapes +
"\nInputs strides: %s" % strides +
"\nInputs values: %s" % scalar_values)
if hasattr(node.op, '__input_name__'):
detailed_err_msg += "\nInputs name: %s\n" % str(node.op.__input_name__)
detailed_err_msg += "\nOutputs clients: %s\n" % clients
else:
hints.append(
"HINT: Use another linker then the c linker to"
" have the inputs shapes and strides printed.")
# Print node backtraces
tr = getattr(node.outputs[0].tag, 'trace', [])
if type(tr) is list and len(tr) > 0:
detailed_err_msg += "\nBacktrace when the node is created(use Theano flag traceback.limit=N to make it longer):\n"
# Print separate message for each element in the list of batcktraces
sio = StringIO()
for subtr in tr:
traceback.print_list(subtr, sio)
detailed_err_msg += str(sio.getvalue())
else:
hints.append(
"HINT: Re-running with most Theano optimization disabled could"
" give you a back-trace of when this node was created. This can"
" be done with by setting the Theano flag"
" 'optimizer=fast_compile'. If that does not work,"
" Theano optimizations can be disabled with 'optimizer=None'.")
if theano.config.exception_verbosity == 'high':
f = StringIO()
theano.printing.debugprint(node, file=f, stop_on_name=True,
print_type=True)
detailed_err_msg += "\nDebugprint of the apply node: \n"
detailed_err_msg += f.getvalue()
# Prints output_map
if theano.config.exception_verbosity == 'high' and storage_map is not None:
detailed_err_msg += "\nStorage map footprint:\n"
shared_input_list = [
item for item in node.fgraph.inputs
if isinstance(item, theano.compile.SharedVariable)]
nonshared_input_list = [
item for item in node.fgraph.inputs
if not isinstance(item, theano.compile.SharedVariable)]
storage_map_list = []
total_size = 0
total_size_inputs = 0
for k in storage_map:
storage_map_item = []
# storage_map_item[0]: the variable
storage_map_item.append(str(k))
# storage_map_item[1]: the shape
shapeinfo = None
if hasattr(storage_map[k][0], 'shape'):
shapeinfo = storage_map[k][0].shape
if len(shapeinfo) != 0:
storage_map_item.append(shapeinfo)
else:
storage_map_item.append(tuple())
else:
storage_map_item.append(None)
# storage_map_item[2]: itemsize
# storage_map_item[3]: bytes
if hasattr(storage_map[k][0], 'dtype'):
dtype = storage_map[k][0].dtype
storage_map_item.append(numpy.dtype(dtype).itemsize)
if shapeinfo is None:
storage_map_item.append(-1)
else:
sz = numpy.dtype(dtype).itemsize * numpy.prod(shapeinfo)
storage_map_item.append(sz)
total_size += sz
if not k.owner:
total_size_inputs += sz
else:
# If it is a view, don't count it twice.
if getattr(k.owner.op, 'view_map', None):
vmap = k.owner.op.view_map
out_idx = k.owner.outputs.index(k)
data = storage_map[k][0]
if out_idx in vmap:
assert len(vmap[out_idx]) == 1
input_data = storage_map[
k.owner.inputs[vmap[out_idx][0]]][0]
if k.type.may_share_memory(data, input_data):
total_size -= sz
# If it is a destroyed input, the input
# shouldn't be in the storage_map anymore
# except if there is a special flag used. So
# we still must check it.
if getattr(k.owner.op, 'destroy_map', None):
vmap = k.owner.op.destroy_map
out_idx = k.owner.outputs.index(k)
data = storage_map[k][0]
if out_idx in vmap:
assert len(vmap[out_idx]) == 1
input_data = storage_map[
k.owner.inputs[vmap[out_idx][0]]][0]
if k.type.may_share_memory(data, input_data):
total_size -= sz
else:
bytes = getsizeof(storage_map[k][0])
storage_map_item.append(bytes)
storage_map_item.append(-1)
# Flag of shared val
# storage_map_item[4]
if k in shared_input_list:
storage_map_item.append(True)
elif k in nonshared_input_list:
storage_map_item.append(False)
else:
storage_map_item.append(None)
storage_map_list.append(storage_map_item)
from operator import itemgetter
storage_map_list.sort(key=itemgetter(3), reverse=True)
for item in storage_map_list:
if item[3] == -1:
continue
detailed_err_msg += " - " + item[0] + ", "
if item[4] is True:
detailed_err_msg += "Shared Input, "
elif item[4] is False:
detailed_err_msg += "Input, "
if item[1] is not None:
detailed_err_msg += "Shape: %s, " % str(item[1])
detailed_err_msg += "ElemSize: %s Byte(s)" % item[2]
if item[3] is not None:
detailed_err_msg += ", TotalSize: %s Byte(s)\n" % item[3]
else:
detailed_err_msg += "\n"
detailed_err_msg += " TotalSize: %s Byte(s) %.3f GB\n" % (
total_size, total_size / 1024. / 1024 / 1024)
detailed_err_msg += " TotalSize inputs: %s Byte(s) %.3f GB\n" % (
total_size_inputs, total_size_inputs / 1024. / 1024 / 1024)
else:
hints.append(
"HINT: Use the Theano flag 'exception_verbosity=high'"
" for a debugprint and storage map footprint of this apply node.")
try:
exc_value = exc_type(str(exc_value) + detailed_err_msg +
'\n' + '\n'.join(hints))
except TypeError:
print("WARNING: %s error does not allow us to add extra error message" %
str(exc_type))
# Some exception need extra parameter in inputs. So forget the
# extra long error message in that case.
pass
reraise(exc_type, exc_value, exc_trace)
0
Example 90
Project: total-impact-core Source File: cache.py
def set_cache_entry(self, key, data):
""" Store a cache entry """
if sys.getsizeof(data["text"]) > MAX_PAYLOAD_SIZE_BYTES:
logger.debug(u"Not caching because payload is too large")
return None
mc = self._get_client()
if mc.info()["used_memory"] >= MAX_CACHE_SIZE_BYTES:
logger.debug(u"Not caching because redis cache is too full")
return None
hash_key = self._build_hash_key(key)
set_response = mc.set(hash_key, json.dumps(data))
mc.expire(hash_key, self.max_cache_age)
if not set_response:
logger.warning("Unable to store into Redis. Make sure redis server is running.")
raise CacheException("Unable to store into Redis. Make sure redis server is running.")
return set_response
0
Example 91
Project: intelhex Source File: getsizeof.py
def total_size(o, handlers={}, verbose=False):
""" Returns the approximate memory footprint an object and all of its contents.
Automatically finds the contents of the following builtin containers and
their subclasses: tuple, list, deque, dict, set and frozenset.
To search other containers, add handlers to iterate over their contents:
handlers = {SomeContainerClass: iter,
OtherContainerClass: OtherContainerClass.get_elements}
"""
dict_handler = lambda d: chain.from_iterable(d.items())
all_handlers = {tuple: iter,
list: iter,
deque: iter,
dict: dict_handler,
set: iter,
frozenset: iter,
}
all_handlers.update(handlers) # user handlers take precedence
seen = set() # track which object id's have already been seen
default_size = sys.getsizeof(0) # estimate sizeof object without __sizeof__
def sizeof(o):
if id(o) in seen: # do not double count the same object
return 0
seen.add(id(o))
s = sys.getsizeof(o, default_size)
if verbose:
print(s, type(o), repr(o))#, file=stderr)
for typ, handler in all_handlers.items():
if isinstance(o, typ):
s += sum(map(sizeof, handler(o)))
break
return s
return sizeof(o)
0
Example 92
def sizeof(self, key):
# XXX: not recursive
return sys.getsizeof(key)
0
Example 93
def test_default(self):
h = self.header
size = self.calcsize
self.assertEqual(sys.getsizeof(True, -1), size(h + 'l'))
0
Example 94
@support.bigmemtest(size=LARGE_SIZE,
memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
dry_run=False)
def test_set(self, size):
self.check_unmarshallable(set(range(size)))
0
Example 95
def nbytes(o):
""" Number of bytes of an object
>>> nbytes(123) # doctest: +SKIP
24
>>> nbytes('Hello, world!') # doctest: +SKIP
50
>>> import numpy as np
>>> nbytes(np.ones(1000, dtype='i4'))
4000
"""
name = type(o).__module__ + '.' + type(o).__name__
if name == 'pandas.core.series.Series':
return _array(o._data.blocks[0].values) + _array(o.index._data)
elif name == 'pandas.core.frame.DataFrame':
return _array(o.index) + sum([_array(blk.values)
for blk in o._data.blocks])
elif name == 'numpy.ndarray':
return _array(o)
elif hasattr(o, 'nbytes'):
return o.nbytes
else:
return sys.getsizeof(o)
0
Example 96
def callback(self, event):
headers(self)
if event.error:
self.response.write('ERROR:')
self.response.write('<br /><br />')
self.response.write('message: {0}<br />'.format(event.error.message))
for k, v in event.error.__dict__.iteritems():
if not k == 'message':
self.response.write('{0}: {1}<br />'.format(k, v))
elif event.user:
self.response.write('<br /><br />')
self.response.write('user = {0}<br /><br />'.format(event.user))
if event.user.credentials:
self.response.write('<br /><br />')
self.response.write('Credentials:<br /><br />')
for k, v in event.user.credentials.__dict__.items():
self.response.write('{0}: {1}<br />'.format(k, v))
serialized = event.user.credentials.serialize()
deserialized = authomatic.credentials(serialized)
self.response.write('<br /><br />')
self.response.write('Serialized:<br />{0}<br /><br />'.format(serialized))
self.response.write('Serialized size:<br />{0} B<br /><br />'.format(sys.getsizeof(serialized)))
# deserialized credentials
for k, v in deserialized.__dict__.items():
self.response.write('{0}: {1}<br />'.format(k, v))
self.response.write('<br /><br />')
self.response.write('User Info:<br /><br />')
self.response.write('<br /><br />')
event.user.update()
for k, v in event.user.__dict__.iteritems():
if k != 'data':
self.response.write('{0}: {1}<br />'.format(k, v))
if k == 'gae_user' and v:
for kk, vv in v.__dict__.iteritems():
self.response.write(' {0}: {1}<br />'.format(kk, vv))
self.response.write('<br /><br />')
self.response.write('Raw User Info:<br /><br />')
self.response.write(event.user.data)
0
Example 97
def filter(objects, Type=None, min=-1, max=-1):
"""Filter objects.
The filter can be by type, minimum size, and/or maximum size.
Keyword arguments:
Type -- object type to filter by
min -- minimum object size
max -- maximum object size
"""
res = []
if min > max:
raise ValueError("minimum must be smaller than maximum")
if Type is not None:
res = [o for o in objects if isinstance(o, Type)]
if min > -1:
res = [o for o in res if getsizeof(o) < min]
if max > -1:
res = [o for o in res if getsizeof(o) > max]
return res
0
Example 98
Project: pympler Source File: test_summary.py
def test_sweep(self):
"""Test that all and only empty entries are removed from a summary."""
objects = ['the', 'quick', 'brown', 'fox', 1298, 123, 234, [], {}]
summ = summary.summarize(objects)
# correct removal of rows when sizes are empty
summary._subtract(summ, {})
summary._subtract(summ, [])
summ = summary._sweep(summ)
found_dict = found_tuple = False
for row in summ:
if row[0] == "<type 'dict'>":
found_dict = True
if row[0] == "<type 'tuple'>":
found_tuple = True
self.assert_(found_dict == False)
self.assert_(found_tuple == False)
# do not remove row if one of the sizes is not empty
# e.g. if the number of objects of a type did not change, but the
# total size did
summ = summary._subtract(summ, 'the')
summ = summary._subtract(summ, 'quick')
summ = summary._subtract(summ, 'brown')
summ = summary._subtract(summ, '42')
summ = summary._sweep(summ)
found_string = False
for row in summ:
if row[0] == summary._repr(''):
found_string = True
self.assert_(row[1] == 0)
totalsize = getsizeof('fox') - getsizeof('42')
self.assert_(row[2] == totalsize)
self.assert_(found_string == True)
0
Example 99
Project: sfepy Source File: mem_usage.py
def get_mem_usage(obj, usage=None, name=None, traversal_order=None, level=0):
"""
Get lower bound of memory usage of an object.
Takes into account strings, numpy arrays and scipy CSR sparse matrices,
descends into sequences, mappings and objects.
Parameters
----------
obj : any object
The object to be measured.
usage : dict
The dict with memory usage records, serving also as a cache of already
traversed objects.
name : str
The name to be given to the object in its record.
traversal_order : list, internal
The traversal order of the object.
level : int, internal
The recurrence level.
Returns
-------
usage : int
The object's lower bound of memory usage.
"""
if usage is None:
usage = {}
if name is None:
name = getattr(obj, 'name', '-')
if traversal_order is None:
traversal_order = [0]
to = traversal_order
key = id(obj)
if key in usage:
usage[key].nrefs += 1
return 0
else:
record = usage.setdefault(key, Struct(name=name,
kind=type(obj).__name__,
usage=0, nrefs=1,
traversal_order=to[0],
level=level))
level += 1
if isinstance(obj, nm.ndarray):
record.usage = obj.nbytes
elif isinstance(obj, sp.csr_matrix):
record.usage = (get_mem_usage(obj.data, usage, name='data',
traversal_order=to, level=level)
+ get_mem_usage(obj.indices, usage, name='indices',
traversal_order=to, level=level)
+ get_mem_usage(obj.indptr, usage, name='indptr',
traversal_order=to, level=level))
elif isinstance(obj, basestr):
record.usage = len(obj)
elif isinstance(obj, Struct):
for subname, sub in six.iteritems(obj.__dict__):
to[0] += 1
record.usage += get_mem_usage(sub, usage,
name='attribute %s of %s'
% (subname, getattr(obj, 'name',
record.kind)),
traversal_order=to, level=level)
elif isinstance(obj, collections.Mapping):
try:
for subname, sub in six.iteritems(obj):
to[0] += 1
record.usage += get_mem_usage(sub, usage,
name='item %s of %s'
% (subname, record.kind),
traversal_order=to, level=level)
except:
pass
elif isinstance(obj, collections.Sequence):
for ii, sub in enumerate(obj):
to[0] += 1
record.usage += get_mem_usage(sub, usage,
name='item %d of %s'
% (ii, record.kind),
traversal_order=to, level=level)
else:
record.usage = sys.getsizeof(obj)
return record.usage
0
Example 100
def body(self):
content_type, char_set = self.content_type()
raw_body = self.request.body
if char_set:
try:
raw_body = raw_body.decode(char_set)
except AttributeError:
pass
except LookupError: # If no encoding exists, default to UTF-8
try:
raw_body = raw_body.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
raw_body = ''
except Exception as e:
Logger.error('Unable to decode request body using char_set %s due to error: %s. Will ignore. Stacktrace:' % (char_set, e))
traceback.print_exc()
else:
# Default to an attempt at UTF-8 decoding.
try:
raw_body = raw_body.decode('UTF-8')
except AttributeError:
pass
except UnicodeDecodeError:
raw_body = ''
max_size = SilkyConfig().SILKY_MAX_REQUEST_BODY_SIZE
body = ''
if raw_body:
if max_size > -1:
Logger.debug('A max request size is set so checking size')
size = sys.getsizeof(raw_body, default=None)
request_identifier = self.request.path
if not size:
Logger.error('No way in which to get size of request body for %s, will ignore it', request_identifier)
elif size <= max_size:
Logger.debug('Request %s has body of size %d which is less than %d so will save the body' % (request_identifier, size, max_size))
body = self._body(raw_body, content_type)
else:
Logger.debug('Request %s has body of size %d which is greater than %d, therefore ignoring' % (request_identifier, size, max_size))
raw_body = None
else:
Logger.debug('No maximum request body size is set, continuing.')
body = self._body(raw_body, content_type)
return body, raw_body