sys.getsizeof

Here are the examples of the python api sys.getsizeof taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

116 Examples 7

Example 51

Project: joliebulle Source File: base.py
    def __init__(self):
        logger.debug("Import %s", database_file)
        fichierBeerXML = database_file
        self.arbre = ET.parse(fichierBeerXML)

        presentation=self.arbre.find('.//RECIPE')
        fermentables=self.arbre.findall('.//FERMENTABLE')
        hops = self.arbre.findall('.//HOP')
        levures = self.arbre.findall('.//YEAST')
        misc = self.arbre.findall('.//MISC')
 
        
        self.listeFermentables = list()
        self.listeHops = list()
        self.listeYeasts = list()
        self.listeMiscs = list()

        #Ingredient fermentescibles
        for element in fermentables:
            self.listeFermentables.append( Fermentable.parse(element) )
        self.listeFermentables = sorted(self.listeFermentables, key=attrgetter('name'))
        logger.debug( "%s fermentables in database, using %s bytes in memory", len(self.listeFermentables), sys.getsizeof(self.listeFermentables) )

        #Houblons
        for element in hops:
            self.listeHops.append( Hop.parse(element) )
        self.listeHops = sorted(self.listeHops, key=attrgetter('name'))
        logger.debug( "%s hops in database, using %s bytes in memory", len(self.listeHops), sys.getsizeof(self.listeHops) )

        #Levures
        for element in levures:
            self.listeYeasts.append( Yeast.parse(element) )
        self.listeYeasts = sorted(self.listeYeasts, key=attrgetter('name'))
        logger.debug( "%s yeasts in database, using %s bytes in memory", len(self.listeYeasts), sys.getsizeof(self.listeYeasts) )

        #Ingredients divers
        for element in misc:
            self.listeMiscs.append( Misc.parse(element) )
        self.listeMiscs = sorted(self.listeMiscs, key=attrgetter('name'))
        logger.debug( "%s miscs in database, using %s bytes in memory", len(self.listeMiscs), sys.getsizeof(self.listeMiscs) )

        logger.debug("Import %s terminé", database_file)

        #Import Mash file
        logger.debug("Import %s", mash_file)
        arbre = ET.parse(mash_file)

        mash = arbre.findall('.//MASH')
        self.listeMashes = list()

        for element in mash:
            self.listeMashes.append( Mash.parse(element) )
        logger.debug( "%s mash in database, using %s bytes in memory", len(self.listeMashes), sys.getsizeof(self.listeMashes) )

        logger.debug("Import %s terminé", mash_file)

Example 52

Project: pymatgen Source File: snl.py
Function: init
    def __init__(self, struct_or_mol, authors, projects=None, references='',
                 remarks=None, data=None, history=None, created_at=None):
        # initialize root-level structure keys
        self.structure = struct_or_mol

        # turn authors into list of Author objects
        authors = authors.split(',')\
            if isinstance(authors, string_types) else authors
        self.authors = [Author.parse_author(a) for a in authors]

        # turn projects into list of Strings
        projects = projects if projects else []
        self.projects = [projects] \
            if isinstance(projects, string_types) else projects

        # check that references are valid BibTeX
        if references and not is_valid_bibtex(references):
            raise ValueError("Invalid format for SNL reference! Should be "
                             "BibTeX string.")
        if len(references) > MAX_BIBTEX_CHARS:
            raise ValueError("The BibTeX string must be fewer than {} chars "
                             ", you have {}"
                             .format(MAX_BIBTEX_CHARS, len(references)))

        self.references = references

        # turn remarks into list of Strings
        remarks = remarks if remarks else []
        self.remarks = [remarks] if isinstance(remarks, string_types) \
            else remarks

        # check remarks limit
        for r in self.remarks:
            if len(r) > 140:
                raise ValueError("The remark exceeds the maximum size of"
                                 "140 characters: {}".format(r))

        # check data limit
        self.data = data if data else {}
        if not sys.getsizeof(self.data) < MAX_DATA_SIZE:
            raise ValueError("The data dict exceeds the maximum size limit of"
                             " {} bytes (you have {})"
                             .format(MAX_DATA_SIZE, sys.getsizeof(data)))

        for k, v in self.data.items():
            if not k.startswith("_"):
                raise ValueError("data must contain properly namespaced data "
                                 "with keys starting with an underscore. The "
                                 "key {} does not start with an underscore.",
                                 format(k))

        # check for valid history nodes
        history = history if history else []  # initialize null fields
        if len(history) > MAX_HNODES:
            raise ValueError("A maximum of {} History nodes are supported, "
                             "you have {}!".format(MAX_HNODES, len(history)))
        self.history = [HistoryNode.parse_history_node(h) for h in history]
        if not all([sys.getsizeof(h) < MAX_HNODE_SIZE for h in history]):
            raise ValueError("One or more history nodes exceeds the maximum "
                             "size limit of {} bytes".format(MAX_HNODE_SIZE))

        self.created_at = created_at if created_at \
            else datetime.datetime.utcnow()

Example 53

Project: datafari Source File: test_sys.py
Function: test_errors
    def test_errors(self):
        class BadSizeof(object):
            def __sizeof__(self):
                raise ValueError
        self.assertRaises(ValueError, sys.getsizeof, BadSizeof())

        class InvalidSizeof(object):
            def __sizeof__(self):
                return None
        self.assertRaises(TypeError, sys.getsizeof, InvalidSizeof())
        sentinel = ["sentinel"]
        self.assertIs(sys.getsizeof(InvalidSizeof(), sentinel), sentinel)

        class OverflowSizeof(long):
            def __sizeof__(self):
                return int(self)
        self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)),
                         sys.maxsize + self.gc_headsize)
        with self.assertRaises(OverflowError):
            sys.getsizeof(OverflowSizeof(sys.maxsize + 1))
        with self.assertRaises(ValueError):
            sys.getsizeof(OverflowSizeof(-1))
        with self.assertRaises((ValueError, OverflowError)):
            sys.getsizeof(OverflowSizeof(-sys.maxsize - 1))

Example 54

Project: iot-utilities Source File: test_sys.py
Function: test_errors
    def test_errors(self):
        class BadSizeof:
            def __sizeof__(self):
                raise ValueError
        self.assertRaises(ValueError, sys.getsizeof, BadSizeof())

        class InvalidSizeof:
            def __sizeof__(self):
                return None
        self.assertRaises(TypeError, sys.getsizeof, InvalidSizeof())
        sentinel = ["sentinel"]
        self.assertIs(sys.getsizeof(InvalidSizeof(), sentinel), sentinel)

        class FloatSizeof:
            def __sizeof__(self):
                return 4.5
        self.assertRaises(TypeError, sys.getsizeof, FloatSizeof())
        self.assertIs(sys.getsizeof(FloatSizeof(), sentinel), sentinel)

        class OverflowSizeof(int):
            def __sizeof__(self):
                return int(self)
        self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)),
                         sys.maxsize + self.gc_headsize)
        with self.assertRaises(OverflowError):
            sys.getsizeof(OverflowSizeof(sys.maxsize + 1))
        with self.assertRaises(ValueError):
            sys.getsizeof(OverflowSizeof(-1))
        with self.assertRaises((ValueError, OverflowError)):
            sys.getsizeof(OverflowSizeof(-sys.maxsize - 1))

Example 55

Project: translator Source File: api.py
def __translate__(text, source, target, client='x',
                  user_agent=DEFAULT_USER_AGENT):
    """
    text: text to be translated
    source: source language
    target: target language
    """

    if source == target:
        return text

    if not re.match(r'Mozilla/\d+\.\d+ \(.*', user_agent):
        user_agent = 'Mozilla/5.0 (%s)' % user_agent

    headers = {
        'Referer': 'http://translate.google.com',
        'User-Agent': user_agent,
        'Content-Length': str(sys.getsizeof(text))
    }
    payload = {
        'client': client,
        'sl': source,
        'tl': target,
        'text': text,
    }
    url = 'http://translate.google.com/translate_a/t'

    req = requests.post(url, headers=headers, data=payload)

    if req.status_code != 200:
        raise HTTPException(
            ('Google Translate returned HTTP {}'.format(req.status_code)),
            req.status_code)

    if client == 'x':
        data = json.loads(req.text)

        # It appears in some cases the Google Translate returns a string
        # rather than a dictionary
        try:
            if isinstance(data, unicode):
                return data
        except NameError:
            if isinstance(data, str):
                return data

        try:
            sentences = data['sentences']
        except TypeError:
            sentences = data['results'][0]['sentences']

        result = ' '.join(map(lambda x: x['trans'], sentences))

        # Remove unneccessary white spaces
        return '\n'.join(map(lambda x: x.strip(), result.split('\n')))

    elif client == 't':
        return parse_javascript(req.text)

    else:
        raise Exception("Unsupported client '{}'".format(client))

Example 56

Project: pymaybe Source File: __init__.py
Function: sizeof
    def __sizeof__(self):
        return getsizeof(self.__value)

Example 57

Project: gruvi Source File: support.py
Function: sizeof
def sizeof(obj, exclude=None):
    """Return the size in bytes of *obj*."""
    if obj is None or obj is False or obj is True:
        return 0
    size = sys.getsizeof(obj)
    if hasattr(obj, '__dict__'):
        size += sys.getsizeof(obj.__dict__)
        for key, value in obj.__dict__.items():
            if exclude is not None and key in exclude:
                continue
            s = sizeof(key)
            s += sizeof(value, exclude)
            # print('{}.{}: {}'.format(type(obj).__name__, key, s))
            size += s
    elif hasattr(obj, '__slots__'):
        for key in obj.__slots__:
            if hasattr(obj, key):
                if exclude is not None and key in exclude:
                    continue
                s = sizeof(getattr(obj, key), exclude)
                # print('{}.{}: {}'.format(type(obj).__name__, key, s))
                size += s
    return size

Example 58

Project: ZeroNet Source File: StatsPlugin.py
    def actionListobj(self):

        import gc
        import sys

        self.sendHeader()

        if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
            yield "This function is disabled on this proxy"
            raise StopIteration

        # No more if not in debug mode
        if not config.debug:
            yield "Not in debug mode"
            raise StopIteration

        type_filter = self.get.get("type")

        yield """
        <style>
         * { font-family: monospace; white-space: pre }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        yield "Listing all %s objects in memory...<br>" % cgi.escape(type_filter)

        ref_count = {}
        objs = gc.get_objects()
        for obj in objs:
            obj_type = str(type(obj))
            if obj_type != type_filter:
                continue
            refs = [
                ref for ref in gc.get_referrers(obj)
                if hasattr(ref, "__class__") and
                ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"]
            ]
            if not refs:
                continue
            try:
                yield "%.1fkb <span title=\"%s\">%s</span>... " % (
                    float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100))
                )
            except:
                continue
            for ref in refs:
                yield " ["
                if "object at" in str(ref) or len(str(ref)) > 100:
                    yield str(ref.__class__.__name__)
                else:
                    yield str(ref.__class__.__name__) + ":" + cgi.escape(str(ref))
                yield "] "
                ref_type = ref.__class__.__name__
                if ref_type not in ref_count:
                    ref_count[ref_type] = [0, 0]
                ref_count[ref_type][0] += 1  # Count
                ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024  # Size
            yield "<br>"

        yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (len(ref_count), sum([stat[1] for stat in ref_count.values()]))

        for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]:  # Sorted by count
            yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], cgi.escape(str(obj)))

        gc.collect()  # Implicit grabage collection

Example 59

Project: rockstor-core Source File: data_collector.py
    def on_readlog(self, reader, logfile):

        logs_loader = {'slow' : {'lines' : 200, 'sleep' : 0.50},
                   'fast' : {'lines' : 1, 'sleep' : 0.05},
        }

        def valid_log(logfile):
            #If file exist and size greater than 0 return true
            #else false and avoid processing
            if path.exists(logfile):
                return (path.getsize(logfile) > 0)
            else:
                return False
                
        def build_reader_command(reader):

            command = []
            command.append(self.readers[reader]['command'])
            #If our reader has opt args we add them to popen command
            if ('args' in self.readers[reader]):
                command.append(self.readers[reader]['args'])
            #Queue log file to popen command
            command.append(log_path)
            return command
            
        def static_reader(reader, log_path):
            if (valid_log(log_path)):#Log file exist and greater than 0, perform data collecting

                #Build reader command
                read_command = build_reader_command(reader)

                #Define popen process and once completed split stdout by lines
                reader_process = Popen(read_command, bufsize=1, stdout=PIPE)
                log_content = reader_process.communicate()[0]
                log_contentsize = getsizeof(log_content)
                log_content = log_content.splitlines(True)
                
                #Starting from content num of lines decide if serve it 1 line/time or in 200 lines chunks
                reader_type = 'fast' if (len(log_content) <= 200) else 'slow'
                chunk_size = logs_loader[reader_type]['lines']
                reader_sleep = logs_loader[reader_type]['sleep']
                log_content_chunks = [log_content[x:x+chunk_size] for x in xrange(0, len(log_content), chunk_size)]
                total_rows = len(log_content)

            else:#Log file missing or size 0, gently inform user
            
                #Log not exist or empty so we send fake values
                #for rows, chunks, etc to uniform data on existing functions
                #and avoid client side extra checks
                log_content = 'Selected log file is empty or doesn\'t exist'
                log_content = log_content.splitlines(True)
                total_rows = 1
                log_contentsize = getsizeof(log_content)
                log_content_chunks = []
                log_content_chunks.append(log_content)
                reader_sleep = 0
            
            #Serve each chunk with emit and sleep before next one to avoid client side browser overload
            current_rows = 0
            
            for data_chunks in log_content_chunks:
                chunk_content = ''.join(data_chunks)
                current_rows += len(data_chunks)
                self.emit('logManager:logcontent', {
                    'key': 'logManager:logcontent', 'data': {
                    'current_rows' : current_rows,
                    'total_rows' : total_rows,
                    'chunk_content' : chunk_content,
                    'content_size' : log_contentsize
                    }
                })
                gevent.sleep(reader_sleep)
        
        def live_reader(log_path):

            #Switch live reader state to True
            self.livereading = True
            
            #Build reader command from readers dict
            read_command = build_reader_command('tailf')

            self.livereader_process = Popen(read_command, bufsize=1, stdout=PIPE)
            while self.livereading:
                live_out = self.livereader_process.stdout.readline()
                self.emit('logManager:logcontent', {
                    'key': 'logManager:logcontent', 'data': {
                    'current_rows' : 1,
                    'total_rows' : 1,
                    'chunk_content' : live_out,
                    'content_size' : 1
                    }
                })

        log_path = self.build_log_path(logfile)

        if (reader == 'tailf'):
            gevent.spawn(live_reader, log_path)
        else:
            gevent.spawn(static_reader, reader, log_path)

Example 60

Project: WeCase Source File: debug.py
def objectSize(obj, ignore=None, verbose=False, depth=0, recursive=False):
    """Guess how much memory an object is using"""
    ignoreTypes = [types.MethodType, types.UnboundMethodType, types.BuiltinMethodType, types.FunctionType, types.BuiltinFunctionType]
    ignoreRegex = re.compile('(method-wrapper|Flag|ItemChange|Option|Mode)')


    if ignore is None:
        ignore = {}

    indent = '  '*depth

    try:
        hash(obj)
        hsh = obj
    except:
        hsh = "%s:%d" % (str(type(obj)), id(obj))

    if hsh in ignore:
        return 0
    ignore[hsh] = 1

    try:
        size = sys.getsizeof(obj)
    except TypeError:
        size = 0

    if isinstance(obj, ndarray):
        try:
            size += len(obj.data)
        except:
            pass


    if recursive:
        if type(obj) in [list, tuple]:
            if verbose:
                print(indent+"list:")
            for o in obj:
                s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
                if verbose:
                    print(indent+'  +', s)
                size += s
        elif isinstance(obj, dict):
            if verbose:
                print(indent+"list:")
            for k in obj:
                s = objectSize(obj[k], ignore=ignore, verbose=verbose, depth=depth+1)
                if verbose:
                    print(indent+'  +', k, s)
                size += s
        #elif isinstance(obj, QtCore.QObject):
            #try:
                #childs = obj.children()
                #if verbose:
                    #print indent+"Qt children:"
                #for ch in childs:
                    #s = objectSize(obj, ignore=ignore, verbose=verbose, depth=depth+1)
                    #size += s
                    #if verbose:
                        #print indent + '  +', ch.objectName(), s

            #except:
                #pass
    #if isinstance(obj, types.InstanceType):
        gc.collect()
        if verbose:
            print(indent+'attrs:')
        for k in dir(obj):
            if k in ['__dict__']:
                continue
            o = getattr(obj, k)
            if type(o) in ignoreTypes:
                continue
            strtyp = str(type(o))
            if ignoreRegex.search(strtyp):
                continue
            #if isinstance(o, types.ObjectType) and strtyp == "<type 'method-wrapper'>":
                #continue

            #if verbose:
                #print indent, k, '?'
            refs = [r for r in gc.get_referrers(o) if type(r) != types.FrameType]
            if len(refs) == 1:
                s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
                size += s
                if verbose:
                    print(indent + "  +", k, s)
            #else:
                #if verbose:
                    #print indent + '  -', k, len(refs)
    return size

Example 61

Project: StarCluster Source File: utils.py
def size_in_kb(obj):
    return sys.getsizeof(obj) / 1024.

Example 62

Project: python_zklib Source File: zkAtt.py
def zkAtt(self):

    command = CMD_ATTLOG_RRQ
    comand_string = ''
    chksum = 0
    session_id = self.session_id
    reply_id = unpack('4H', self.data_recv[:8])[3]

    buf = self.createHeader(command, chksum, session_id,
                            reply_id, comand_string)
    self.zkclient.sendto(buf, self.address)

    size = None
    attendance = []
    self.data_recv, addr = self.zkclient.recvfrom(1024)
    # print unpack('24s1s4s11s',self.data_recv.ljust(40)[:40])
    # print "unpack HcHc :7", unpack('HcHc',self.data_recv[:7])
    # print "unpack HHHH :8", unpack('HHHH',self.data_recv[:8])
    # print "unpack HHHH :8 [1]", unpack('4H',self.data_recv[:8])[0]
    print "size", sys.getsizeof(self.data_recv)
    print "size", len(self.data_recv)
    lensi = len(self.data_recv) / 2
    fstri = str(lensi) + "H"
    print "unpack 4I  ", unpack(fstri, self.data_recv)
    # print "unpack 8H", unpack('8H', self.data_recv)
    # print "unpack I 8:12 [0]", unpack ('I', self.data_recv[8:12])[0]

    attendance = []
    self.data_recv, addr = self.zkclient.recvfrom(1024)

    if unpack('4H', self.data_recv[:8])[0] == CMD_PREPARE_DATA:
        print "received CMD_PREPARE_DATA"
        size = unpack('I', self.data_recv[8:12])[0]

        wa = unpack('II', self.data_recv[:8])[0]
        print "received ", wa
        print 'Receiving %s %s' % (size, "bytes")
        try:

            data_recv, addr = self.zkclient.recvfrom(size)
            print "size of data_recv", sys.getsizeof(data_recv)
            self.attendancedata.append(data_recv)
            acmOK(self)
            print unpack('24s1s4s11s', self.data_recv.ljust(40)[:40])

        except:

            print "socket timeout 1 - no more data to receive"

        while unpack('4H', self.data_recv[:8])[0] != 2000 or unpack('4H', self.data_recv[:8])[0] == 1501:

            try:

                data_recv, addr = self.zkclient.recvfrom(size)
                acmOK(self)
                print unpack('24s1s4s11s', self.data_recv.ljust(40)[:40])

            except:

                print "socket timeout 2 - no more data to receive"
                self.attendancedata.append(data_recv)
                acmOK(self)
                break
            # print "length of reiceived data packet", len(data_recv)
            # print "unpack 126H", unpack('126H', data_recv)

            data_recv, addr = self.zkclient.recvfrom(1024)

            self.attendancedata.append(data_recv)

            if unpack('4H', self.data_recv[:8])[0] == CMD_PREPARE_DATA:

                print "received CMD_PREPARE_DATA"
                size = unpack('I', self.data_recv[8:12])[0]
                print 'Receiving %s %s' % (size, "bytes")

            elif unpack('4H', data_recv[:8])[0] == 1501:
                print "receiving Data packet"

            elif unpack('4H', self.data_recv[:8])[0] == 2000:
                print "received CMD_ACK_OK"
                try:

                    self.data_recv, addr = self.zkclient.recvfrom(size)
                    acmOK(self)
                    # print len(self.data_recv)
                except:
                    print "socket timeout 3 - no more data to receive"

            #self.data_recv, addr = self.zkclient.recvfrom(1024)
            # print "length of reiceived data packet", len(self.data_recv)
            print "length", len(self.data_recv)
            lens = len(self.data_recv) / 2
            fstr = str(lens) + "H"

            print unpack(fstr, self.data_recv)
            if unpack('4H', self.data_recv[:8])[0] == 2000:
                print "received CMD_ACK_OK"
                try:

                    self.data_recv, addr = self.zkclient.recvfrom(4096)
                    # print len(self.data_recv)
                except:
                    print "socket timeout 4 - no more data to receive"
        print "length of att data", len(self.attendancedata)
        #data_recv = self.zkclient.recvfrom(8)

        for x in xrange(len(self.attendancedata)):

                                        # print self.attendancedata[x][8:]
                                        #self.attendancedata[x] = self.attendancedata[x][8:]
                                        # print self.attendancedata[x][0:]
            self.attendancedata[x] = self.attendancedata[x][0:]

            self.data_recv, addr = self.zkclient.recvfrom(1024)

            if unpack('4H', self.data_recv)[0] == 2000:
                print "received CMD_ACK_OK"
                try:

                    self.data_recv, addr = self.zkclient.recvfrom(1024)

                except:
                    print "socket timeout - no more data to receive"

                    for x in xrange(len(self.attendancedata)):

                        self.attendancedata[x] = self.attendancedata[x][0:]

                    attendancedata = self.attendancedata

        attendancedata = self.attendancedata

        attendancedata = ''.join(self.attendancedata)

        attendancedata = attendancedata[14:]
        #attendancedata = attendancedata[14:]

        print "len attendancedata", len(attendancedata)

    while len(attendancedata):

        #uidm = unpack('24s1s4s11s',attendancedata.ljust(40)[:40])

        #uidt = unpack('38s',attendancedata[:38])

        #uidmn = unpack('16s',attendancedata[:16])[0]

        pls = unpack('c', attendancedata[29:30])  # [3]

        #statev = unpack('=2c',attendancedata[21:23])
        #datem = unpack('ii',attendancedata[:8])[1]

        uid, state, timestamp, space = unpack(
            '24s1s4s11s', attendancedata.ljust(40)[:40])
        print "%s, %s, %s, %s" % (uid, 1, ord(space[0]), decode_time(int(reverseHex(timestamp.encode('hex')), 16)))
        # print "%s, %s, %s, %s" % (uid, ord(pls[0]), ord(space[0]), decode_time( int( reverseHex( timestamp.encode('hex') ), 16 ) ) )
        # print "%s, %s, %s, %s" % (uid, state, space, timestamp)
        #attendance.append( ( uid, ord(pls[0]), decode_time( int( reverseHex( timestamp.encode('hex') ), 16 ) ) ) )
        attendance.append((uid, ord(pls[0]), decode_time(
            int(reverseHex(timestamp.encode('hex')), 16))))
        attendancedata = attendancedata[40:]

            while len(attendancedata):

                pls = unpack('c', attendancedata[29:30])  # [3]

                uid, state, timestamp, space = unpack(
                    '24s1s4s11s', attendancedata.ljust(40)[:40])
                # print "%s, %s, %s, %s" % (uid, ord(pls[0]), ord(space[0]),
                # decode_time( int( reverseHex( timestamp.encode('hex') ), 16 )
                # ) )

                attendance.append((uid, ord(pls[0]), decode_time(
                    int(reverseHex(timestamp.encode('hex')), 16))))
                attendancedata = attendancedata[40:]

Example 63

Project: pypet Source File: mpwrappers.py
Function: send_chunks
    def _send_chunks(self, to_put):
        put_dump = pickle.dumps(to_put)
        data_size = sys.getsizeof(put_dump)
        nchunks = data_size / 20000000.   # chunks with size 20 MB
        chunksize = int(len(put_dump) / nchunks)
        chunk_iterator = self._make_chunk_iterator(put_dump, chunksize)
        for chunk in chunk_iterator:
            # print('S: sending False')
            self.conn.send(False)
            # print('S: sent False')
            # print('S: sending chunk')
            self.conn.send_bytes(chunk)
            # print('S: sent chunk %s' % chunk[0:10])
            # print('S: recv signal')
            self.conn.recv()  # wait for signal that message was received
            # print('S: read signal')
        # print('S: sending True')
        self.conn.send(True)
        # print('S: sent True')
        # print('S: recving last signal')
        self.conn.recv()  # wait for signal that message was received

Example 64

Project: translator Source File: api.py
Function: params
def __params__(text, source, target, client='at',
               user_agent=DEFAULT_USER_AGENT):
    """Returns a dictionary containing all parameters to send a translation
    request to Google Translate."""

    headers = {
        'User-Agent': user_agent,
        'Content-Length': str(sys.getsizeof(text))
    }
    remote_addr = request.remote_addr if request.remote_addr else ''
    payload = {
        'client': client,
        'sl': source,
        'tl': target,
        'q': text,
        'dt': ['t', 'ld', 'qc', 'rm', 'bd'],
        'dj': 1,
        # Generate a UUID based on the remote client's IP address
        'iid': str(uuid.uuid5(uuid.NAMESPACE_DNS, remote_addr)),
        # 'itid': 'pk',
        # 'otf': 1,
        'ie': 'UTF-8',
    }
    url = 'https://translate.google.com/translate_a/single'

    try:
        # Python 3
        from urllib.parse import quote, urlencode
    except ImportError:
        # Python 2
        quote = urllib.quote
        urlencode = urllib.urlencode

    if len(quote(text)) > 1000:
        method = 'post'
        del payload['q']
    else:
        method = 'get'
        del headers['Content-Length']

    return {
        'headers': headers,
        'payload': payload,
        'method': method,
        'url': url,
        'query': urlencode(list(__payload_as_tuples__(payload)))
    }

Example 65

Project: python-beaver Source File: sqs_transport.py
    def callback(self, filename, lines, **kwargs):
        timestamp = self.get_timestamp(**kwargs)
        if kwargs.get('timestamp', False):
            del kwargs['timestamp']

        if self._bulk_lines:
	    message_batch = ''
            message_count = 0
	else:
            message_batch = []

        message_batch_size = 0
        message_batch_size_max = 250000 # Max 256KiB but leave some headroom

        for line in lines:
            if self._bulk_lines:
               	m = self.format(filename, line, timestamp, **kwargs)
                message_size = getsizeof(m)
	    else:
                m = Message()
                m.set_body(self.format(filename, line, timestamp, **kwargs))
                message_size = len(m)

            if (message_size > message_batch_size_max):
                self._logger.debug('Dropping the message as it is too large to send ({0} bytes)'.format(message_size))
                continue

            # Check the new total size before adding a new message and don't try to send an empty batch
            if self._bulk_lines and (len(message_batch) > 0) and (((message_batch_size + message_size) >= message_batch_size_max)):
                self._logger.debug('Flushing {0} messages to SQS queue {1} bytes'.format(message_count, message_batch_size))
                self._send_message(message_batch)
                message_batch = ''
                message_count = 0
                message_batch_size = 0

            # SQS can only handle up to 10 messages in batch send and it can not exceed 256KiB (see above)
            elif (len(message_batch) > 0) and (((message_batch_size + message_size) >= message_batch_size_max) or (len(message_batch) == 10)):
                self._logger.debug('Flushing {0} messages to SQS queue {1} bytes'.format(len(message_batch), message_batch_size))
                self._send_message_batch(message_batch)
                message_batch = []
                message_batch_size = 0

            message_batch_size = message_batch_size + message_size

            if self._bulk_lines:
                message_batch += '{0},'.format(m)
                message_count += 1
            else:
                message_batch.append((uuid.uuid4(), self.format(filename, line, timestamp, **kwargs), 0))

        if len(message_batch) > 0:
            if self._bulk_lines:
                self._logger.debug('Flushing the last {0} messages to SQS queue {1} bytes'.format(message_count, message_batch_size))
                self._send_message(message_batch)
            else:
                self._logger.debug('Flushing the last {0} messages to SQS queue {1} bytes'.format(len(message_batch), message_batch_size))
                self._send_message_batch(message_batch)

        return True

Example 66

Project: pygobject Source File: test_everything.py
    @classmethod
    def from_wrapped(cls, obj):
        offset = sys.getsizeof(object())  # size of PyObject_HEAD
        return ctypes.POINTER(cls).from_address(id(obj) + offset)

Example 67

Project: flask-mwoauth Source File: __init__.py
    def request(self, api_query, url=None):
        """ e.g. {'action': 'query', 'meta': 'userinfo'}. format=json not required
            function returns a python dict that resembles the api's json response
        """
        api_query['format'] = 'json'
        url = url or self.base_url

        size = sum([sys.getsizeof(v) for k, v in iteritems(api_query)])

        if size > (1024 * 8):
            # if request is bigger than 8 kB (the limit is somewhat arbitrary,
            # see https://www.mediawiki.org/wiki/API:Edit#Large_texts) then
            # transmit as multipart message

            req = self._prepare_long_request(url=url + "/api.php?",
                                             api_query=api_query
                                             )
            return self.mwoauth.post(url + "/api.php?",
                                     data=req.body,
                                     content_type=req.headers['Content-Type']
                                     ).data
        else:
            return self.mwoauth.post(url + "/api.php",
                                     data=api_query
                                     ).data

Example 68

Project: odoo-saas-tools Source File: saas_client.py
    @api.model
    def _transport_backup(self, dump_db, filename=None):
        '''
        send db dump to S3
        '''
        with tempfile.TemporaryFile() as t:
            dump_db(t)
            t.seek(0)
            db_dump = base64.b64decode(t.read().encode('base64'))

        if parallel_upload and sys.getsizeof(db_dump) > 5242880:
            ir_params = self.env['ir.config_parameter']
            aws_key = ir_params.get_param('saas_s3.saas_s3_aws_accessid')
            aws_secret = ir_params.get_param('saas_s3.saas_s3_aws_accesskey')
            bucketname = ir_params.get_param('saas_s3.saas_s3_aws_bucket')
            self._transport_backup_parallel(db_dump, filename, aws_key, aws_secret, bucketname)
        else:
            conn, bucket_name = _get_s3_conn(self.env)
            self._transport_backup_simple(conn, bucket_name, db_dump, filename)

        return True

Example 69

Project: pympler Source File: test_summary.py
    def test_summary_diff(self):
        """Test summary diff. """
        left = [[str(str), 3, 3*getsizeof('a')],\
                [str(int), 2, 2*getsizeof(1)],\
                [str(list), 1, getsizeof([])],\
                [str(dict), 1, getsizeof({})]]
        right = [[str(str), 2, 2*getsizeof('a')],\
                 [str(int), 3, 3*getsizeof(1)],\
                 [str(list), 1, getsizeof([1,2,3])],\
                 [str(dict), 1, getsizeof({})],
                 [str(tuple), 1, getsizeof((1,2))]]

        expected = [[str(str), -1, -1*getsizeof('a')],\
                    [str(int), 1, +1*getsizeof(1)],\
                    [str(list), 0, getsizeof([1,2,3]) - getsizeof([])],\
                    [str(dict), 0, 0],
                    [str(tuple), 1, getsizeof((1,2))]]
        res = summary.get_diff(left, right)
        for row_e in res:
            self.assertTrue(row_e in expected)

Example 70

Project: fu Source File: cache.py
Function: put
    def put(self, query, result):
        ### only if valid result (simple test) ###
        if sys.getsizeof(result) > 21:
            self.resultdict[query] = result

Example 71

Project: nichtparasoup Source File: __init__.py
Function: info
    @classmethod
    def info(cls):
        images = cls.__images
        blacklist = cls.__blacklist

        info = {
            "blacklist": len(blacklist),
            "blacklist_size": sys.getsizeof(blacklist, 0),
            "images_per_site": {}
        }

        images_count = 0
        images_size = 0
        for crawler in images:
            for site in images[crawler]:
                site_image_count = len(images[crawler][site])
                images_count += site_image_count
                images_size += sys.getsizeof(images[crawler][site])
                info["images_per_site"][crawler + "_" + site] = site_image_count

        info["images_size"] = images_size
        info["images"] = images_count

        return info

Example 72

Project: brython Source File: test_marshal.py
Function: test_frozenset
    @support.bigmemtest(size=LARGE_SIZE,
            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
            dry_run=False)
    def test_frozenset(self, size):
        self.check_unmarshallable(frozenset(range(size)))

Example 73

Project: brython Source File: test_sys.py
Function: test_default
    def test_default(self):
        size = test.support.calcvobjsize
        self.assertEqual(sys.getsizeof(True), size('') + self.longdigit)
        self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit)

Example 74

Project: pympler Source File: test_summary.py
    def test_subtract(self):
        """Test that a single object's data is correctly subtracted from a summary.
        - result in correct total size and total number of objects
        - if object was not listed before, it should be listed negative
          afterwards
        """

        objects = ['the', 'quick', 'brown', 'fox', 1298, 123, 234, [], {}]
        summ = summary.summarize(objects)
        summary._subtract(summ, 'the')
        summary._subtract(summ, {})
        summary._subtract(summ, (1,))
        # to verify that these rows where actually included afterwards
        checked_str = checked_dict = checked_tuple = False
        for row in summ:
            if row[0] == summary._repr(''):
                totalsize = getsizeof('quick') + getsizeof('brown') +\
                            getsizeof('fox')
                self.assert_(row[1] == 3, "%s != %s" % (row[1], 3))
                self.assert_(row[2] == totalsize, totalsize)
                checked_str = True
            if row[0] == summary._repr({}):
                self.assert_(row[1] == 0)
                self.assert_(row[2] == 0)
                checked_dict = True
            if row[0] == summary._repr((1,)):
                self.assert_(row[1] == -1)
                self.assert_(row[2] == -getsizeof((1,)))
                checked_tuple = True

        self.assert_(checked_str, "no str found in summary")
        self.assert_(checked_dict, "no dict found in summary")
        self.assert_(checked_tuple, "no tuple found in summary")

        summary._subtract(summ, 'quick')
        summary._subtract(summ, 'brown')
        checked_str = False
        for row in summ:
            if row[0] == summary._repr(''):
                self.assert_(row[1] == 1)
                self.assert_(row[2] == getsizeof('fox'))
                checked_str = True
        self.assert_(checked_str, "no str found in summ")

Example 75

Project: sd-agent Source File: sdagent.py
Function: sizeof
    def __sizeof__(self):
        return sys.getsizeof(self._data)

Example 76

Project: osf.io Source File: serializers.py
Function: get_size
    def get_size(self, obj):
        return sys.getsizeof(obj.content)

Example 77

Project: silk Source File: model_factory.py
Function: body
    def body(self):
        body = ''
        content_type, char_set = _parse_content_type(self.response.get('Content-Type', ''))
        content = getattr(self.response, 'content', '')
        if content:
            max_body_size = SilkyConfig().SILKY_MAX_RESPONSE_BODY_SIZE
            if max_body_size > -1:
                Logger.debug('Max size of response body defined so checking')
                size = sys.getsizeof(content, None)
                if not size:
                    Logger.error('Could not get size of response body. Ignoring')
                    content = ''
                else:
                    if size > max_body_size:
                        content = ''
                        Logger.debug('Size of %d for %s is bigger than %d so ignoring response body' % (size, self.request.path, max_body_size))
                    else:
                        Logger.debug('Size of %d for %s is less than %d so saving response body' % (size, self.request.path, max_body_size))
            if content and content_type in content_types_json:
                # TODO: Perhaps theres a way to format the JSON without parsing it?
                if not isinstance(content, str):
                    # byte string is not compatible with json.loads(...) and json.dumps(...) in python3
                    content = content.decode()
                try:
                    body = json.dumps(json.loads(content), sort_keys=True, indent=4)
                except (TypeError, ValueError):
                    Logger.warn('Response to request with pk %s has content type %s but was unable to parse it' % (self.request.pk, content_type))
        return body, content

Example 78

Project: cyordereddict Source File: test_ordereddict.py
    def test_sizeof(self):
        # Wimpy test: Just verify the reported size is larger than a regular dict
        d = dict(a=1)
        od = OrderedDict(**d)
        self.assertGreater(sys.getsizeof(od), sys.getsizeof(d))

Example 79

Project: Gnip-Python-Search-API-Utilities Source File: api.py
    def parse_responses(self, count_bucket):
        """Parse returned responses.

           When paged=True, manage paging using the API token mechanism
           
           When output file is set, write output files for paged output."""
        acs = []
        repeat = True
        page_count = 1
        self.paged_file_list = []
        while repeat:
            doc = self.request()
            tmp_response = json.loads(doc)
            if "results" in tmp_response:
                acs.extend(tmp_response["results"])
            else:
                raise ValueError("Invalid request\nQuery: %s\nResponse: %s"%(self.rule_payload, doc))
            if len(acs) < self.hard_max:
                repeat = False
                if self.paged or (count_bucket and self.search_v2):
                    if len(acs) > 0:
                        if self.output_file_path is not None:
                            # writing to file
                            file_name = self.output_file_path + "/{0}_{1}.json".format(
                                    str(datetime.datetime.utcnow().strftime(
                                        "%Y%m%d%H%M%S"))
                                  , str(self.file_name_prefix))
                            with codecs.open(file_name, "wb","utf-8") as out:
                                for item in tmp_response["results"]:
                                    out.write(json.dumps(item)+"\n")
                            self.paged_file_list.append(file_name)
                            # if writing to file, don't keep track of all the data in memory
                            acs = []
                        else:
                            # storing in memory, so give some feedback as to size
                            sys.stderr.write("[%8d bytes] %5d total activities retrieved...\n"%(
                                                                sys.getsizeof(acs)
                                                              , len(acs)))
                    else:
                        sys.stderr.write( "No results returned for rule:{0}\n".format(str(self.rule_payload)) ) 
                    if "next" in tmp_response:
                        self.rule_payload["next"]=tmp_response["next"]
                        repeat = True
                        page_count += 1
                        sys.stderr.write( "Fetching page {}...\n".format(page_count) )
                    else:
                        if "next" in self.rule_payload:
                            del self.rule_payload["next"]
                        repeat = False
                    time.sleep(PAUSE)
            else:
                # stop iterating after reaching hard_max
                repeat = False
        return acs

Example 80

Project: pyqtgraph Source File: debug.py
def objectSize(obj, ignore=None, verbose=False, depth=0, recursive=False):
    """Guess how much memory an object is using"""
    ignoreTypes = ['MethodType', 'UnboundMethodType', 'BuiltinMethodType', 'FunctionType', 'BuiltinFunctionType']
    ignoreTypes = [getattr(types, key) for key in ignoreTypes if hasattr(types, key)]
    ignoreRegex = re.compile('(method-wrapper|Flag|ItemChange|Option|Mode)')
    
    
    if ignore is None:
        ignore = {}
        
    indent = '  '*depth
    
    try:
        hash(obj)
        hsh = obj
    except:
        hsh = "%s:%d" % (str(type(obj)), id(obj))
        
    if hsh in ignore:
        return 0
    ignore[hsh] = 1
    
    try:
        size = sys.getsizeof(obj)
    except TypeError:
        size = 0
        
    if isinstance(obj, ndarray):
        try:
            size += len(obj.data)
        except:
            pass
            
        
    if recursive:
        if type(obj) in [list, tuple]:
            if verbose:
                print(indent+"list:")
            for o in obj:
                s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
                if verbose:
                    print(indent+'  +', s)
                size += s
        elif isinstance(obj, dict):
            if verbose:
                print(indent+"list:")
            for k in obj:
                s = objectSize(obj[k], ignore=ignore, verbose=verbose, depth=depth+1)
                if verbose:
                    print(indent+'  +', k, s)
                size += s
        #elif isinstance(obj, QtCore.QObject):
            #try:
                #childs = obj.children()
                #if verbose:
                    #print indent+"Qt children:"
                #for ch in childs:
                    #s = objectSize(obj, ignore=ignore, verbose=verbose, depth=depth+1)
                    #size += s
                    #if verbose:
                        #print indent + '  +', ch.objectName(), s
                    
            #except:
                #pass
    #if isinstance(obj, types.InstanceType):
        gc.collect()
        if verbose:
            print(indent+'attrs:')
        for k in dir(obj):
            if k in ['__dict__']:
                continue
            o = getattr(obj, k)
            if type(o) in ignoreTypes:
                continue
            strtyp = str(type(o))
            if ignoreRegex.search(strtyp):
                continue
            #if isinstance(o, types.ObjectType) and strtyp == "<type 'method-wrapper'>":
                #continue
            
            #if verbose:
                #print indent, k, '?'
            refs = [r for r in gc.get_referrers(o) if type(r) != types.FrameType]
            if len(refs) == 1:
                s = objectSize(o, ignore=ignore, verbose=verbose, depth=depth+1)
                size += s
                if verbose:
                    print(indent + "  +", k, s)
            #else:
                #if verbose:
                    #print indent + '  -', k, len(refs)
    return size

Example 81

Project: pattern Source File: metrics.py
Function: sizeof
def sizeof(object):
    """ Returns the memory size of the given object (in bytes).
    """
    return sys.getsizeof(object)

Example 82

Project: sd-agent Source File: transaction.py
Function: get_size
    def get_size(self):
        if self._size is None:
            self._size = sys.getsizeof(self)

        return self._size

Example 83

Project: pattern Source File: metrics.py
def kb(object):
    """ Returns the memory size of the given object (in kilobytes).
    """
    return sys.getsizeof(object) * 0.01

Example 84

Project: datafari Source File: test_sys.py
    def test_default(self):
        size = test.test_support.calcobjsize
        self.assertEqual(sys.getsizeof(True, -1), size('l'))

Example 85

Project: pympler Source File: summary.py
Function: summarize
def summarize(objects):
    """Summarize an objects list.

    Return a list of lists, whereas each row consists of::
      [str(type), number of objects of this type, total size of these objects].

    No guarantee regarding the order is given.

    """
    count = {}
    total_size = {}
    for o in objects:
        otype = _repr(o)
        if otype in count:
            count[otype] += 1
            total_size[otype] += getsizeof(o)
        else:
            count[otype] = 1
            total_size[otype] = getsizeof(o)
    rows = []
    for otype in count:
        rows.append([otype, count[otype], total_size[otype]])
    return rows

Example 86

Project: pygobject Source File: generictreemodel.py
Function: from_iter
    @classmethod
    def from_iter(cls, iter):
        offset = sys.getsizeof(object())  # size of PyObject_HEAD
        return ctypes.POINTER(cls).from_address(id(iter) + offset)

Example 87

Project: tilequeue Source File: process.py
def _preprocess_data(feature_layers):
    preproc_feature_layers = []
    extra_data = dict(size={})

    for feature_layer in feature_layers:
        layer_datum = feature_layer['layer_datum']
        geometry_types = layer_datum['geometry_types']
        padded_bounds = feature_layer['padded_bounds']

        features = []
        features_size = 0
        for row in feature_layer['features']:
            wkb = row.pop('__geometry__')
            shape = loads(wkb)

            if shape.is_empty:
                continue

            if not shape.is_valid:
                continue

            if geometry_types is not None:
                if shape.type not in geometry_types:
                    continue

            # since a bounding box intersection is used, we
            # perform a more accurate check here to filter out
            # any extra features
            # the formatter specific transformations will take
            # care of any additional filtering
            geom_type_bounds = padded_bounds[
                normalize_geometry_type(shape.type)]
            shape_padded_bounds = geometry.box(*geom_type_bounds)
            if not shape_padded_bounds.intersects(shape):
                continue

            feature_id = row.pop('__id__')
            props = dict()
            feature_size = getsizeof(feature_id) + len(wkb)
            for k, v in row.iteritems():
                if k == 'mz_properties':
                    for output_key, output_val in v.items():
                        if output_val is not None:
                            # all other tags are utf8 encoded, encode
                            # these the same way to be consistent
                            if isinstance(output_key, unicode):
                                output_key = output_key.encode('utf-8')
                            if isinstance(output_val, unicode):
                                output_val = output_val.encode('utf-8')
                            props[output_key] = output_val
                            feature_size += len(output_key) + \
                                _sizeof(output_val)
                else:
                    props[k] = v
                    feature_size += len(k) + _sizeof(v)

            feature = shape, props, feature_id
            features.append(feature)
            features_size += feature_size

        extra_data['size'][layer_datum['name']] = features_size

        preproc_feature_layer = dict(
            name=layer_datum['name'],
            layer_datum=layer_datum,
            features=features,
            padded_bounds=padded_bounds,
        )
        preproc_feature_layers.append(preproc_feature_layer)

    return preproc_feature_layers, extra_data

Example 88

Project: ZeroNet Source File: StatsPlugin.py
    def actionDumpobj(self):

        import gc
        import sys

        self.sendHeader()

        if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
            yield "This function is disabled on this proxy"
            raise StopIteration

        # No more if not in debug mode
        if not config.debug:
            yield "Not in debug mode"
            raise StopIteration

        class_filter = self.get.get("class")

        yield """
        <style>
         * { font-family: monospace; white-space: pre }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        objs = gc.get_objects()
        for obj in objs:
            obj_type = str(type(obj))
            if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter:
                continue
            yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)))
            for attr in dir(obj):
                yield "- %s: %s<br>" % (attr, cgi.escape(str(getattr(obj, attr))))
            yield "<br>"

        gc.collect()  # Implicit grabage collection

Example 89

Project: attention-lvcsr Source File: link.py
def raise_with_op(node, thunk=None, exc_info=None, storage_map=None):
    """
    Re-raise an exception while annotating the exception object with
    debug info.

    Parameters
    ----------
    node : Apply node
        The Apply node object that resulted in the raised exception.
    exc_info : tuple, optional
        A tuple containing the exception type, exception object and
        associated traceback, as would be returned by a call to
        `sys.exc_info()` (which is done if `None` is passed).
    storage_map: dict, optional
        storage map of the theano function that resulted in the
        raised exception.

    Notes
    -----
    This re-raises the exception described by `exc_info` (or the last
    one raised, if `exc_info` is omitted) and annotates the exception
    object with several new members which may be helpful for debugging
    Theano graphs. They are:

     * __op_instance__: The Op that is responsible for the exception
       being raised.
     * __thunk_trace__: A traceback corresponding to the code that
       actually generated the exception, if it is available.
     * __applynode_index__: The index of the Apply node corresponding
       to this op in `op.fgraph.toposort()`.

    The exception is not annotated if it is of type `KeyboardInterrupt`.

    """
    if exc_info is None:
        exc_info = sys.exc_info()
    exc_type, exc_value, exc_trace = exc_info
    if exc_type == KeyboardInterrupt:
        # print a simple traceback from KeyboardInterrupt
        reraise(exc_type, exc_value, exc_trace)
    try:
        trace = node.outputs[0].tag.trace
    except AttributeError:
        try:
            trace = node.op.tag.trace
        except AttributeError:
            trace = ()
    exc_value.__thunk_trace__ = trace
    exc_value.__op_instance__ = node
    topo = node.fgraph.toposort()
    if node in topo:
        node_index = topo.index(node)
    else:
        node_index = None
    exc_value.__applynode_index__ = node_index

    hints = []
    detailed_err_msg = "\nApply node that caused the error: " + str(node)
    if exc_value.__applynode_index__ is not None:
        detailed_err_msg += "\nToposort index: %d" % node_index

    types = [getattr(ipt, 'type', 'No type') for ipt in node.inputs]
    detailed_err_msg += "\nInputs types: %s\n" % types

    if thunk is not None:
        if hasattr(thunk, 'inputs'):
            shapes = [getattr(ipt[0], 'shape', 'No shapes')
                      for ipt in thunk.inputs]
            strides = [getattr(ipt[0], 'strides', 'No strides')
                       for ipt in thunk.inputs]
            scalar_values = []
            for ipt in thunk.inputs:
                if getattr(ipt[0], "size", -1) <= 5:
                    scalar_values.append(ipt[0])
                else:
                    scalar_values.append("not shown")
        else:
            shapes = "The thunk don't have an inputs attributes."
            strides = "So we can't access the strides of inputs values"
            scalar_values = "And can't print its inputs scalar value"
        clients = [[c[0] for c in var.clients] for var in node.outputs]
        detailed_err_msg += ("Inputs shapes: %s" % shapes +
                             "\nInputs strides: %s" % strides +
                             "\nInputs values: %s" % scalar_values)
        if hasattr(node.op, '__input_name__'):
            detailed_err_msg += "\nInputs name: %s\n" % str(node.op.__input_name__)

        detailed_err_msg += "\nOutputs clients: %s\n" % clients
    else:
        hints.append(
            "HINT: Use another linker then the c linker to"
            " have the inputs shapes and strides printed.")

    # Print node backtraces
    tr = getattr(node.outputs[0].tag, 'trace', [])
    if type(tr) is list and len(tr) > 0:
        detailed_err_msg += "\nBacktrace when the node is created(use Theano flag traceback.limit=N to make it longer):\n"

        # Print separate message for each element in the list of batcktraces
        sio = StringIO()
        for subtr in tr:
            traceback.print_list(subtr, sio)
        detailed_err_msg += str(sio.getvalue())
    else:
        hints.append(
            "HINT: Re-running with most Theano optimization disabled could"
            " give you a back-trace of when this node was created. This can"
            " be done with by setting the Theano flag"
            " 'optimizer=fast_compile'. If that does not work,"
            " Theano optimizations can be disabled with 'optimizer=None'.")

    if theano.config.exception_verbosity == 'high':

        f = StringIO()
        theano.printing.debugprint(node, file=f, stop_on_name=True,
                                   print_type=True)
        detailed_err_msg += "\nDebugprint of the apply node: \n"
        detailed_err_msg += f.getvalue()

    # Prints output_map
    if theano.config.exception_verbosity == 'high' and storage_map is not None:
        detailed_err_msg += "\nStorage map footprint:\n"
        shared_input_list = [
            item for item in node.fgraph.inputs
            if isinstance(item, theano.compile.SharedVariable)]
        nonshared_input_list = [
            item for item in node.fgraph.inputs
            if not isinstance(item, theano.compile.SharedVariable)]
        storage_map_list = []
        total_size = 0
        total_size_inputs = 0
        for k in storage_map:
            storage_map_item = []

            # storage_map_item[0]: the variable
            storage_map_item.append(str(k))

            # storage_map_item[1]: the shape
            shapeinfo = None
            if hasattr(storage_map[k][0], 'shape'):
                shapeinfo = storage_map[k][0].shape
                if len(shapeinfo) != 0:
                    storage_map_item.append(shapeinfo)
                else:
                    storage_map_item.append(tuple())
            else:
                storage_map_item.append(None)

            # storage_map_item[2]: itemsize
            # storage_map_item[3]: bytes
            if hasattr(storage_map[k][0], 'dtype'):
                dtype = storage_map[k][0].dtype
                storage_map_item.append(numpy.dtype(dtype).itemsize)
                if shapeinfo is None:
                    storage_map_item.append(-1)
                else:
                    sz = numpy.dtype(dtype).itemsize * numpy.prod(shapeinfo)
                    storage_map_item.append(sz)
                    total_size += sz
                    if not k.owner:
                        total_size_inputs += sz
                    else:
                        # If it is a view, don't count it twice.
                        if getattr(k.owner.op, 'view_map', None):
                            vmap = k.owner.op.view_map
                            out_idx = k.owner.outputs.index(k)
                            data = storage_map[k][0]
                            if out_idx in vmap:
                                assert len(vmap[out_idx]) == 1
                                input_data = storage_map[
                                    k.owner.inputs[vmap[out_idx][0]]][0]
                                if k.type.may_share_memory(data, input_data):
                                    total_size -= sz
                        # If it is a destroyed input, the input
                        # shouldn't be in the storage_map anymore
                        # except if there is a special flag used. So
                        # we still must check it.
                        if getattr(k.owner.op, 'destroy_map', None):
                            vmap = k.owner.op.destroy_map
                            out_idx = k.owner.outputs.index(k)
                            data = storage_map[k][0]
                            if out_idx in vmap:
                                assert len(vmap[out_idx]) == 1
                                input_data = storage_map[
                                    k.owner.inputs[vmap[out_idx][0]]][0]
                                if k.type.may_share_memory(data, input_data):
                                    total_size -= sz
            else:
                bytes = getsizeof(storage_map[k][0])
                storage_map_item.append(bytes)
                storage_map_item.append(-1)

            # Flag of shared val
            # storage_map_item[4]
            if k in shared_input_list:
                storage_map_item.append(True)
            elif k in nonshared_input_list:
                storage_map_item.append(False)
            else:
                storage_map_item.append(None)
            storage_map_list.append(storage_map_item)

        from operator import itemgetter
        storage_map_list.sort(key=itemgetter(3), reverse=True)
        for item in storage_map_list:
            if item[3] == -1:
                continue
            detailed_err_msg += " - " + item[0] + ", "
            if item[4] is True:
                detailed_err_msg += "Shared Input, "
            elif item[4] is False:
                detailed_err_msg += "Input, "
            if item[1] is not None:
                detailed_err_msg += "Shape: %s, " % str(item[1])
            detailed_err_msg += "ElemSize: %s Byte(s)" % item[2]
            if item[3] is not None:
                detailed_err_msg += ", TotalSize: %s Byte(s)\n" % item[3]
            else:
                detailed_err_msg += "\n"
        detailed_err_msg += " TotalSize: %s Byte(s) %.3f GB\n" % (
            total_size, total_size / 1024. / 1024 / 1024)
        detailed_err_msg += " TotalSize inputs: %s Byte(s) %.3f GB\n" % (
            total_size_inputs, total_size_inputs / 1024. / 1024 / 1024)

    else:
        hints.append(
            "HINT: Use the Theano flag 'exception_verbosity=high'"
            " for a debugprint and storage map footprint of this apply node.")

    try:
        exc_value = exc_type(str(exc_value) + detailed_err_msg +
                             '\n' + '\n'.join(hints))
    except TypeError:
        print("WARNING: %s error does not allow us to add extra error message" %
              str(exc_type))
        # Some exception need extra parameter in inputs. So forget the
        # extra long error message in that case.
        pass
    reraise(exc_type, exc_value, exc_trace)

Example 90

Project: total-impact-core Source File: cache.py
    def set_cache_entry(self, key, data):
        """ Store a cache entry """

        if sys.getsizeof(data["text"]) > MAX_PAYLOAD_SIZE_BYTES:
            logger.debug(u"Not caching because payload is too large")
            return None

        mc = self._get_client()

        if mc.info()["used_memory"] >= MAX_CACHE_SIZE_BYTES:
            logger.debug(u"Not caching because redis cache is too full")
            return None

        hash_key = self._build_hash_key(key)
        set_response = mc.set(hash_key, json.dumps(data))
        mc.expire(hash_key, self.max_cache_age)

        if not set_response:
            logger.warning("Unable to store into Redis. Make sure redis server is running.")
            raise CacheException("Unable to store into Redis. Make sure redis server is running.")
        return set_response

Example 91

Project: intelhex Source File: getsizeof.py
def total_size(o, handlers={}, verbose=False):
    """ Returns the approximate memory footprint an object and all of its contents.

    Automatically finds the contents of the following builtin containers and
    their subclasses:  tuple, list, deque, dict, set and frozenset.
    To search other containers, add handlers to iterate over their contents:

        handlers = {SomeContainerClass: iter,
                    OtherContainerClass: OtherContainerClass.get_elements}

    """
    dict_handler = lambda d: chain.from_iterable(d.items())
    all_handlers = {tuple: iter,
                    list: iter,
                    deque: iter,
                    dict: dict_handler,
                    set: iter,
                    frozenset: iter,
                   }
    all_handlers.update(handlers)     # user handlers take precedence
    seen = set()                      # track which object id's have already been seen
    default_size = sys.getsizeof(0)       # estimate sizeof object without __sizeof__

    def sizeof(o):
        if id(o) in seen:       # do not double count the same object
            return 0
        seen.add(id(o))
        s = sys.getsizeof(o, default_size)

        if verbose:
            print(s, type(o), repr(o))#, file=stderr)

        for typ, handler in all_handlers.items():
            if isinstance(o, typ):
                s += sum(map(sizeof, handler(o)))
                break
        return s

    return sizeof(o)

Example 92

Project: compmake Source File: memorycache.py
Function: sizeof
    def sizeof(self, key):
        # XXX: not recursive
        return sys.getsizeof(key)

Example 93

Project: pymo Source File: test_sys.py
Function: test_default
    def test_default(self):
        h = self.header
        size = self.calcsize
        self.assertEqual(sys.getsizeof(True, -1), size(h + 'l'))

Example 94

Project: brython Source File: test_marshal.py
Function: test_set
    @support.bigmemtest(size=LARGE_SIZE,
            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
            dry_run=False)
    def test_set(self, size):
        self.check_unmarshallable(set(range(size)))

Example 95

Project: cachey Source File: nbytes.py
Function: nbytes
def nbytes(o):
    """ Number of bytes of an object

    >>> nbytes(123)  # doctest: +SKIP
    24

    >>> nbytes('Hello, world!')  # doctest: +SKIP
    50

    >>> import numpy as np
    >>> nbytes(np.ones(1000, dtype='i4'))
    4000
    """
    name = type(o).__module__ + '.' + type(o).__name__

    if name == 'pandas.core.series.Series':
        return _array(o._data.blocks[0].values) + _array(o.index._data)
    elif name == 'pandas.core.frame.DataFrame':
        return _array(o.index) + sum([_array(blk.values)
                                     for blk in o._data.blocks])
    elif name == 'numpy.ndarray':
        return _array(o)
    elif hasattr(o, 'nbytes'):
        return o.nbytes
    else:
        return sys.getsizeof(o)

Example 96

Project: authomatic Source File: main.py
Function: callback
    def callback(self, event):
        
        headers(self)
        
        if event.error:
            self.response.write('ERROR:')
            self.response.write('<br /><br />')
            
            self.response.write('message: {0}<br />'.format(event.error.message))
            for k, v in event.error.__dict__.iteritems():
                if not k == 'message':
                    self.response.write('{0}: {1}<br />'.format(k, v))
        
        elif event.user:
                
            self.response.write('<br /><br />')
            self.response.write('user = {0}<br /><br />'.format(event.user))
            
            if event.user.credentials:
                
                self.response.write('<br /><br />')
                self.response.write('Credentials:<br /><br />')
                
                for k, v in event.user.credentials.__dict__.items():
                    self.response.write('{0}: {1}<br />'.format(k, v))
                
                serialized = event.user.credentials.serialize()
                
                deserialized = authomatic.credentials(serialized)
                
                self.response.write('<br /><br />')
                self.response.write('Serialized:<br />{0}<br /><br />'.format(serialized))
                self.response.write('Serialized size:<br />{0} B<br /><br />'.format(sys.getsizeof(serialized)))
                
                # deserialized credentials
                for k, v in deserialized.__dict__.items():
                    self.response.write('{0}: {1}<br />'.format(k, v))
                
                self.response.write('<br /><br />')
                self.response.write('User Info:<br /><br />')
                self.response.write('<br /><br />')
                
                event.user.update()
                
            for k, v in event.user.__dict__.iteritems():
                if k != 'data':
                    self.response.write('{0}: {1}<br />'.format(k, v))
                    if k == 'gae_user' and v:
                        for kk, vv in v.__dict__.iteritems():
                            self.response.write('&nbsp;&nbsp;&nbsp;{0}: {1}<br />'.format(kk, vv))
                    
            
            self.response.write('<br /><br />')
            self.response.write('Raw User Info:<br /><br />')
            self.response.write(event.user.data)

Example 97

Project: pympler Source File: muppy.py
Function: filter
def filter(objects, Type=None, min=-1, max=-1):
    """Filter objects.

    The filter can be by type, minimum size, and/or maximum size.

    Keyword arguments:
    Type -- object type to filter by
    min -- minimum object size
    max -- maximum object size

    """
    res = []
    if min > max:
        raise ValueError("minimum must be smaller than maximum")
    if Type is not None:
        res = [o for o in objects if isinstance(o, Type)]
    if min > -1:
        res = [o for o in res if getsizeof(o) < min]
    if max > -1:
        res = [o for o in res if getsizeof(o) > max]
    return res

Example 98

Project: pympler Source File: test_summary.py
    def test_sweep(self):
        """Test that all and only empty entries are removed from a summary."""
        objects = ['the', 'quick', 'brown', 'fox', 1298, 123, 234, [], {}]
        summ = summary.summarize(objects)
        # correct removal of rows when sizes are empty
        summary._subtract(summ, {})
        summary._subtract(summ, [])
        summ = summary._sweep(summ)
        found_dict = found_tuple = False
        for row in summ:
            if row[0] == "<type 'dict'>":
                found_dict = True
            if row[0] == "<type 'tuple'>":
                found_tuple = True
        self.assert_(found_dict == False)
        self.assert_(found_tuple == False)
        # do not remove row if one of the sizes is not empty
        # e.g. if the number of objects of a type did not change, but the
        # total size did
        summ = summary._subtract(summ, 'the')
        summ = summary._subtract(summ, 'quick')
        summ = summary._subtract(summ, 'brown')
        summ = summary._subtract(summ, '42')
        summ = summary._sweep(summ)
        found_string = False
        for row in summ:
            if row[0] == summary._repr(''):
                found_string = True
                self.assert_(row[1] == 0)
                totalsize = getsizeof('fox') - getsizeof('42')
                self.assert_(row[2] == totalsize)
        self.assert_(found_string == True)

Example 99

Project: sfepy Source File: mem_usage.py
def get_mem_usage(obj, usage=None, name=None, traversal_order=None, level=0):
    """
    Get lower bound of memory usage of an object.

    Takes into account strings, numpy arrays and scipy CSR sparse matrices,
    descends into sequences, mappings and objects.

    Parameters
    ----------
    obj : any object
        The object to be measured.
    usage : dict
        The dict with memory usage records, serving also as a cache of already
        traversed objects.
    name : str
        The name to be given to the object in its record.
    traversal_order : list, internal
        The traversal order of the object.
    level : int, internal
        The recurrence level.

    Returns
    -------
    usage : int
        The object's lower bound of memory usage.
    """
    if usage is None:
        usage = {}

    if name is None:
        name = getattr(obj, 'name', '-')

    if traversal_order is None:
        traversal_order = [0]

    to = traversal_order

    key = id(obj)
    if key in usage:
        usage[key].nrefs += 1
        return 0

    else:
        record = usage.setdefault(key, Struct(name=name,
                                              kind=type(obj).__name__,
                                              usage=0, nrefs=1,
                                              traversal_order=to[0],
                                              level=level))
        level += 1

    if isinstance(obj, nm.ndarray):
        record.usage = obj.nbytes

    elif isinstance(obj, sp.csr_matrix):
        record.usage = (get_mem_usage(obj.data, usage, name='data',
                                      traversal_order=to, level=level)
                        + get_mem_usage(obj.indices, usage, name='indices',
                                        traversal_order=to, level=level)
                        + get_mem_usage(obj.indptr, usage, name='indptr',
                                        traversal_order=to, level=level))

    elif isinstance(obj, basestr):
        record.usage = len(obj)

    elif isinstance(obj, Struct):
        for subname, sub in six.iteritems(obj.__dict__):
            to[0] += 1
            record.usage += get_mem_usage(sub, usage,
                                          name='attribute %s of %s'
                                          % (subname, getattr(obj, 'name',
                                                              record.kind)),
                                          traversal_order=to, level=level)

    elif isinstance(obj, collections.Mapping):
        try:
            for subname, sub in six.iteritems(obj):
                to[0] += 1
                record.usage += get_mem_usage(sub, usage,
                                              name='item %s of %s'
                                              % (subname, record.kind),
                                              traversal_order=to, level=level)
        except:
            pass

    elif isinstance(obj, collections.Sequence):
        for ii, sub in enumerate(obj):
            to[0] += 1
            record.usage += get_mem_usage(sub, usage,
                                          name='item %d of %s'
                                          % (ii, record.kind),
                                          traversal_order=to, level=level)

    else:
        record.usage = sys.getsizeof(obj)

    return record.usage

Example 100

Project: silk Source File: model_factory.py
Function: body
    def body(self):
        content_type, char_set = self.content_type()
        raw_body = self.request.body
        if char_set:
            try:
                raw_body = raw_body.decode(char_set)
            except AttributeError:
                pass
            except LookupError:  # If no encoding exists, default to UTF-8
                try:
                    raw_body = raw_body.decode('UTF-8')
                except AttributeError:
                    pass
                except UnicodeDecodeError:
                    raw_body = ''
            except Exception as e:
                Logger.error('Unable to decode request body using char_set %s due to error: %s. Will ignore. Stacktrace:' % (char_set, e))
                traceback.print_exc()
        else:
            # Default to an attempt at UTF-8 decoding.
            try:
                raw_body = raw_body.decode('UTF-8')
            except AttributeError:
                pass
            except UnicodeDecodeError:
                raw_body = ''
        max_size = SilkyConfig().SILKY_MAX_REQUEST_BODY_SIZE
        body = ''
        if raw_body:
            if max_size > -1:
                Logger.debug('A max request size is set so checking size')
                size = sys.getsizeof(raw_body, default=None)
                request_identifier = self.request.path
                if not size:
                    Logger.error('No way in which to get size of request body for %s, will ignore it', request_identifier)
                elif size <= max_size:
                    Logger.debug('Request %s has body of size %d which is less than %d so will save the body' % (request_identifier, size, max_size))
                    body = self._body(raw_body, content_type)
                else:
                    Logger.debug('Request %s has body of size %d which is greater than %d, therefore ignoring' % (request_identifier, size, max_size))
                    raw_body = None
            else:
                Logger.debug('No maximum request body size is set, continuing.')
                body = self._body(raw_body, content_type)
        return body, raw_body
See More Examples - Go to Next Page
Page 1 Page 2 Selected Page 3