numpy.memmap

Here are the examples of the python api numpy.memmap taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

70 Examples 7

Page 1 Selected Page 2

Example 1

Project: invesalius3 Source File: surface_process.py
Function: run
    def run(self):
        if self.from_binary:
            self.mask = numpy.memmap(self.mask_filename, mode='r',
                                     dtype=self.mask_dtype,
                                     shape=self.mask_shape)
        else:
            self.image = numpy.memmap(self.filename, mode='r', dtype=self.dtype,
                                      shape=self.shape)
            self.mask = numpy.memmap(self.mask_filename, mode='r',
                                     dtype=self.mask_dtype,
                                     shape=self.mask_shape)

        while 1:
            roi = self.q_in.get()
            if roi is None:
                break
            self.CreateSurface(roi)

Example 2

Project: tvb-framework Source File: signals_importer.py
Function: read_data
    def read_data(self, mmap=False, dt='float32', mode='r'):
        """
        VHDR stores data in channel contiguous way such that reading disparate pieces
        in time is fast, when using memmap.
        """

        if mmap:
            ary = numpy.memmap(self.datafile, dt, mode)
        else:
            ary = numpy.fromfile(self.datafile, dt)
        self.data = ary.reshape((-1, self.nchan)).T
        self.nsamp = self.data.shape[1]

Example 3

Project: PyEMMA Source File: kmeans.py
    def _finish_estimate(self):
        fh = None
        if isinstance(self._in_memory_chunks, np.memmap):
            fh = self._in_memory_chunks.filename
        del self._in_memory_chunks
        if fh:
            os.unlink(fh)
        if self.init_strategy == 'uniform':
            del self._centers_iter_list
            del self._init_centers_indices
        if self.init_strategy == 'kmeans++':
            self._progress_force_finish(0)
        self._progress_force_finish(1)

Example 4

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
    def test_indexing_drops_references(self):
        fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        tmp = fp[[(1, 2), (2, 3)]]
        if isinstance(tmp, memmap):
            assert tmp._mmap is not fp._mmap

Example 5

Project: invesalius3 Source File: mask.py
Function: create_mask
    def create_mask(self, shape):
        """
        Creates a new mask object. This method do not append this new mask into the project.

        Parameters:
            shape(int, int, int): The shape of the new mask.
        """
        self.temp_file = tempfile.mktemp()
        shape = shape[0] + 1, shape[1] + 1, shape[2] + 1
        self.matrix = np.memmap(self.temp_file, mode='w+', dtype='uint8', shape=shape)

Example 6

Project: panns Source File: index.py
    def mmap_core_data(self):
        """
        Convert mtx and prj to mmap file to save memory space, very
        useful when dealing with large dataset and parallel mode is
        activated. Skip if the data is already mmaped.
        """
        if type(self.mtx) != numpy.memmap:
            shape_mtx = (len(self.mtx), self.dim)
            mmap_mtx = make_mmap(self.mtx, shape_mtx, self.typ)
            self.mtx = load_mmap(mmap_mtx, shape_mtx, self.typ)
        pass

Example 7

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
Function: test_file_name
    def test_filename(self):
        tmpname = mktemp('', 'mmap', dir=self.tempdir)
        fp = memmap(tmpname, dtype=self.dtype, mode='w+',
                       shape=self.shape)
        abspath = os.path.abspath(tmpname)
        fp[:] = self.data[:]
        self.assertEqual(abspath, fp.filename)
        b = fp[:1]
        self.assertEqual(abspath, b.filename)
        del b
        del fp

Example 8

Project: gensim Source File: test_ldamallet_wrapper.py
    def testLargeMmap(self):
        if not self.mallet_path:
            return
        fname = testfile()
        model = ldamallet.LdaMallet(self.mallet_path, self.corpus, num_topics=2, iterations=100)

        # simulate storing large arrays separately
        model.save(testfile(), sep_limit=0)

        # test loading the large model arrays with mmap
        model2 = ldamodel.LdaModel.load(testfile(), mmap='r')
        self.assertEqual(model.num_topics, model2.num_topics)
        self.assertTrue(isinstance(model2.word_topics, numpy.memmap))
        self.assertTrue(numpy.allclose(model.word_topics, model2.word_topics))
        tstvec = []
        self.assertTrue(numpy.allclose(model[tstvec], model2[tstvec])) # try projecting an empty vector

Example 9

Project: scikit-learn Source File: pool.py
def _strided_from_memmap(filename, dtype, mode, offset, order, shape, strides,
                         total_buffer_len):
    """Reconstruct an array view on a memory mapped file."""
    if mode == 'w+':
        # Do not zero the original data when unpickling
        mode = 'r+'

    if strides is None:
        # Simple, contiguous memmap
        return np.memmap(filename, dtype=dtype, shape=shape, mode=mode,
                         offset=offset, order=order)
    else:
        # For non-contiguous data, memmap the total enclosing buffer and then
        # extract the non-contiguous view with the stride-tricks API
        base = np.memmap(filename, dtype=dtype, shape=total_buffer_len,
                         mode=mode, offset=offset, order=order)
        return as_strided(base, shape=shape, strides=strides)

Example 10

Project: words2map Source File: words2map.py
Function: load_model
def load_model(directory="{}/vectors".format(getcwd())):
	# current model contains 100,000 vectors of 300 elements, each element containing a 16 bit floating point number, such that vectors total ~60 MB uncompressed; note that there is practically no loss of precision in saving vectors with 16 bits versus 32 bits, while data consumption is halved
	print "Loading 100,000 word vectors..."
	model = load(open(join(directory, 'model.pickle')))
	model.vocab = Loader(join(directory, 'word_to_index'))
	model.index2word = Loader(join(directory, 'index_to_word'))
	model.syn0norm = np.memmap(join(directory, 'syn0norm.dat'), dtype='float16', mode='r', shape=(len(model.vocab.keys()), model.layer1_size))
	model.syn0 = model.syn0norm
	return model

Example 11

Project: polar2grid Source File: plot_scene.py
def _parse_binary_info(parts):
    from polar2grid.core.dtype import str_to_dtype
    fn = parts[0]
    if not os.path.exists(fn):
        raise ValueError("File '%s' does not exist" % (fn,))

    dtype = str_to_dtype(parts[1])
    rows = int(parts[2])
    cols = int(parts[3])
    fill = float(parts[4])
    arr = numpy.memmap(fn, dtype=dtype, mode='r', shape=(rows, cols))
    return (fn, numpy.ma.masked_array(arr, numpy.isnan(arr) | (arr == fill)))

Example 12

Project: robothon Source File: test_memmap.py
Function: test_del
    def test_del(self):
        # Make sure a view does not delete the underlying mmap
        fp_base = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        fp_view = fp_base[:]
        class ViewCloseError(Exception):
            pass
        _close = memmap._close
        def replace_close(self):
            raise ViewCloseError('View should not call _close on memmap')
        try:
            memmap._close = replace_close
            del fp_view
        finally:
            memmap._close = _close

Example 13

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
    def test_arithmetic_drops_references(self):
        fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        tmp = (fp + 10)
        if isinstance(tmp, memmap):
            assert tmp._mmap is not fp._mmap

Example 14

Project: python-geoprobe Source File: volume.py
    def memmap_data(self):
        """Return an object similar to a memory-mapped numpy array."""
        header = self.read_header()
        nx, ny, nz = [header[item] for item in ('_nx', '_ny', '_nz')]
        dat = np.memmap(filename=self.filename, mode='r',
            offset=_headerLength, order='F',
            shape=(nx, ny, nz)
            )
        return dat

Example 15

Project: panns Source File: utils.py
Function: make_mmap
def make_mmap(mtx, shape, dtype, fname=None):
    m, n  = shape
    if fname is None:
        fname = tempfile.mkstemp()[1]
    logger.info('mmaping the data to %s ...' % fname)
    fpw = numpy.memmap(fname, dtype=dtype, mode='w+', shape=(m,n))
    for i in xrange(m):
        fpw[i] = mtx[i]
    del fpw
    return fname

Example 16

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
Function: test_open_with_filename
    def test_open_with_filename(self):
        tmpname = mktemp('', 'mmap', dir=self.tempdir)
        fp = memmap(tmpname, dtype=self.dtype, mode='w+',
                       shape=self.shape)
        fp[:] = self.data[:]
        del fp

Example 17

Project: spectral Source File: bsqfile.py
Function: open_memmap
    def _open_memmap(self, mode):
        import os
        import sys
        if (os.path.getsize(self.filename) < sys.maxsize):
            try:
                (R, C, B) = self.shape
                return np.memmap(self.filename, dtype=self.dtype, mode=mode,
                                 offset=self.offset, shape=(B, R, C))
            except:
                print('Unable to create memmap interface.')
                return None
        else:
            return None

Example 18

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
Function: test_del
    def test_del(self):
        # Make sure a view does not delete the underlying mmap
        fp_base = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        fp_base[0] = 5
        fp_view = fp_base[0:1]
        assert_equal(fp_view[0], 5)
        del fp_view
        # Should still be able to access and assign values after
        # deleting the view
        assert_equal(fp_base[0], 5)
        fp_base[0] = 6
        assert_equal(fp_base[0], 6)

Example 19

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
Function: test_attributes
    def test_attributes(self):
        offset = 1
        mode = "w+"
        fp = memmap(self.tmpfp, dtype=self.dtype, mode=mode,
                    shape=self.shape, offset=offset)
        self.assertEqual(offset, fp.offset)
        self.assertEqual(mode, fp.mode)
        del fp

Example 20

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: pool.py
Function: strided_from_memmap
def _strided_from_memmap(filename, dtype, mode, offset, order, shape, strides,
                         total_buffer_len):
    """Reconstruct an array view on a memmory mapped file"""
    if mode == 'w+':
        # Do not zero the original data when unpickling
        mode = 'r+'

    if strides is None:
        # Simple, contiguous memmap
        return np.memmap(filename, dtype=dtype, shape=shape, mode=mode,
                         offset=offset, order=order)
    else:
        # For non-contiguous data, memmap the total enclosing buffer and then
        # extract the non-contiguous view with the stride-tricks API
        base = np.memmap(filename, dtype=dtype, shape=total_buffer_len,
                         mode=mode, offset=offset, order=order)
        return as_strided(base, shape=shape, strides=strides)

Example 21

Project: gala Source File: test_pyintegrators.py
Function: test_memmap
@pytest.mark.parametrize("Integrator", integrator_list)
def test_memmap(tmpdir, Integrator):
    dt = 0.1
    n_steps = 1000
    nw0 = 10000
    mmap = np.memmap("/tmp/test_memmap.npy", mode='w+', shape=(2, n_steps+1, nw0))

    def sho(t,w,T):
        q,p = w
        return np.array([p, -(2*np.pi/T)**2*q])

    w0 = np.random.uniform(-1,1,size=(2,nw0))

    integrator = Integrator(sho, func_args=(1.,))

    orbit = integrator.run(w0, dt=dt, n_steps=n_steps, mmap=mmap)

Example 22

Project: distributed Source File: test_numpy.py
def test_memmap():
    with tmpfile('npy') as fn:
        with open(fn, 'wb') as f:  # touch file
            pass
        x = np.memmap(fn, shape=(5, 5), dtype='i4', mode='readwrite')
        x[:] = 5

        header, frames = serialize(x)
        if 'compression' in header:
            frames = decompress(header, frames)
        y = deserialize(header, frames)

        np.testing.assert_equal(x, y)

Example 23

Project: seisflows Source File: LBFGS.py
Function: restart
    def restart(self):
        """ Discards history and resets counters
        """
        self.iter = 1
        self.memory_used = 0

        unix.cd(self.path)
        S = np.memmap('LBFGS/S', mode='r+')
        Y = np.memmap('LBFGS/Y', mode='r+')
        S[:] = 0.
        Y[:] = 0.

Example 24

Project: ArrayServer Source File: client.py
Function: set_item
    def __setitem__(self, name, array):
        """Stores the array in a memmap file, then pings server."""
        fn = tempfile.mktemp()
        fp = np.memmap(fn, dtype=array.dtype, mode='w+', shape=array.shape)
        fp[:] = array[:]
        
        requests.put(self.url(name), json.dumps({
            'dtype': str(array.dtype),
            'shape': list(array.shape),
            'filename': fn,
            }))

Example 25

Project: polar2grid Source File: readers.py
Function: init
    def __init__(self, filename):
        self.filename = os.path.basename(filename)
        self.filepath = os.path.realpath(filename)

        with open(self.filepath, "rb") as fp_:
            header = numpy.memmap(fp_, dtype=_HEADERTYPE, mode="r", shape=(_HEADERTYPE.itemsize,))
            data = numpy.memmap(fp_, dtype=_SCANTYPE, offset=22016, mode="r")

        self._header = header
        self._data = data
        self.file_type = FT_AAPP

Example 26

Project: Twitch-plays-LSD-neural-net Source File: utils.py
def shared_mmp(data=None, file_name="shm", shape=(0,), dtype=floatX):
    """ Shared memory, only works for linux """
    if not data is None: shape = data.shape
    path = "/dev/shm/lio/"
    make_sure_path_exists(path)
    mmp = np.memmap(path+file_name+".mmp", dtype=dtype, mode='w+', shape=shape)
    if not data is None: mmp[:] = data
    return mmp

Example 27

Project: robothon Source File: test_memmap.py
Function: test_flush
    def test_flush(self):
        fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        fp[:] = self.data[:]
        fp.flush()

        warnings.simplefilter('ignore', DeprecationWarning)
        fp.sync()
        warnings.simplefilter('default', DeprecationWarning)

Example 28

Project: ArrayServer Source File: server.py
Function: put
    def put(self, name):
        """Store the metadata for this vector and open a handle to the file."""
        meta = json.loads(self.request.body)
        
        meta['obj'] = np.memmap(
            meta['filename'],
            dtype=meta['dtype'],
            mode='r',
            shape=tuple(meta['shape'])
            )

        arrays[name] = meta
        self.write(json.dumps({'success': 1}))

Example 29

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
Function: test_view
    def test_view(self):
        fp = memmap(self.tmpfp, dtype=self.dtype, shape=self.shape)
        new1 = fp.view()
        new2 = new1.view()
        assert(new1.base is fp)
        assert(new2.base is fp)
        new_array = asarray(fp)
        assert(new_array.base is fp)

Example 30

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
    @dec.knownfailureif(sys.platform == 'gnu0', "This test is known to fail on hurd")
    def test_flush(self):
        fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        fp[:] = self.data[:]
        assert_equal(fp[0], self.data[0])
        fp.flush()

Example 31

Project: robothon Source File: test_memmap.py
Function: test_round_trip
    def test_roundtrip(self):
        # Write data to file
        fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        fp[:] = self.data[:]
        del fp # Test __del__ machinery, which handles cleanup

        # Read data back from file
        newfp = memmap(self.tmpfp, dtype=self.dtype, mode='r',
                       shape=self.shape)
        assert allclose(self.data, newfp)
        assert_array_equal(self.data, newfp)

Example 32

Project: rdfspace Source File: space.py
Function: load
    @staticmethod
    def load(dirname):
        """Load an rdfspace instance from a directory"""
        if os.path.exists(dirname):
            f = open(os.path.join(dirname, 'space.dat'))
            space = pickle.load(f)
            f.close()
            space._ut = np.memmap(os.path.join(dirname, 'ut.dat'), dtype='float64', mode='r', shape=space._ut_shape)
            space._s = np.memmap(os.path.join(dirname, 's.dat'), dtype='float64', mode='r', shape=space._s_shape)
            space._vt = np.memmap(os.path.join(dirname, 'vt.dat'), dtype='float64', mode='r', shape=space._vt_shape)
            if os.path.exists(os.path.join(dirname, 'uri_index.db')) and os.path.exists(os.path.join(dirname, 'index_uri.db')):
                # If these files exist the index are stored through dbm
                # If not the indexes will be coming from the pickle
                space._uri_index = dbm.open(os.path.join(dirname, 'uri_index'), 'r')
                space._index_uri = dbm.open(os.path.join(dirname, 'index_uri'), 'r')
            return space
        else:
            raise Exception('No such directory')

Example 33

Project: spectral Source File: bilfile.py
Function: open_memmap
    def _open_memmap(self, mode):
        import os
        import sys
        if (os.path.getsize(self.filename) < sys.maxsize):
            try:
                (R, C, B) = self.shape
                return np.memmap(self.filename, dtype=self.dtype, mode=mode,
                                 offset=self.offset, shape=(R, B, C))
            except:
                print('Unable to create memmap interface.')
                return None
        else:
            return None

Example 34

Project: panns Source File: utils.py
Function: build_parallel
def build_parallel(mtx, shape_mtx, K, dtype, t):
    """
    The function for parallel building index. Implemented here because
    the default python serialization cannot pickle instance function.

    Parameters:
    mtx: a row-based data set, should be an numpy matrix.
    K:   max number of data points on a leaf.
    t:   index of binary trees.
    """
    logger.info('pass %i ...' % t)
    mtx = numpy.memmap(mtx, dtype=dtype, mode='r', shape=shape_mtx)
    numpy.random.seed(t**2 + numpy.random.randint(2**30))
    tree = NaiveTree()
    children = range(len(mtx))
    make_tree_parallel(tree.root, children, mtx, shape_mtx[1], dtype, K)
    return tree

Example 35

Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
Function: test_round_trip
    def test_roundtrip(self):
        # Write data to file
        fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        fp[:] = self.data[:]
        del fp  # Test __del__ machinery, which handles cleanup

        # Read data back from file
        newfp = memmap(self.tmpfp, dtype=self.dtype, mode='r',
                       shape=self.shape)
        assert_(allclose(self.data, newfp))
        assert_array_equal(self.data, newfp)

Example 36

Project: spectral Source File: bipfile.py
Function: open_memmap
    def _open_memmap(self, mode):
        import os
        import sys
        if (os.path.getsize(self.filename) < sys.maxsize):
            try:
                (R, C, B) = self.shape
                return np.memmap(self.filename, dtype=self.dtype, mode=mode,
                                 offset=self.offset, shape=self.shape)
            except:
                print('Unable to create memmap interface.')
                return None
        else:
            return None

Example 37

Project: gensim Source File: test_ldamodel.py
Function: testlargemmap
    def testLargeMmap(self):
        fname = testfile()
        model = self.model

        # simulate storing large arrays separately
        model.save(testfile(), sep_limit=0)

        # test loading the large model arrays with mmap
        model2 = self.class_.load(testfile(), mmap='r')
        self.assertEqual(model.num_topics, model2.num_topics)
        self.assertTrue(isinstance(model2.expElogbeta, numpy.memmap))
        self.assertTrue(numpy.allclose(model.expElogbeta, model2.expElogbeta))
        tstvec = []
        self.assertTrue(numpy.allclose(model[tstvec], model2[tstvec])) # try projecting an empty vector

Example 38

Project: gensim Source File: test_lsimodel.py
Function: testlargemmap
    def testLargeMmap(self):
        fname = testfile()
        model = self.model

        # test storing the internal arrays into separate files
        model.save(fname, sep_limit=0)

        # now load the external arrays via mmap
        model2 = lsimodel.LsiModel.load(fname, mmap='r')
        self.assertEqual(model.num_topics, model2.num_topics)
        self.assertTrue(isinstance(model2.projection.u, numpy.memmap))
        self.assertTrue(isinstance(model2.projection.s, numpy.memmap))
        self.assertTrue(numpy.allclose(model.projection.u, model2.projection.u))
        self.assertTrue(numpy.allclose(model.projection.s, model2.projection.s))
        tstvec = []
        self.assertTrue(numpy.allclose(model[tstvec], model2[tstvec]))  # try projecting an empty vector

Example 39

Project: ArrayServer Source File: client.py
Function: get_item
    def __getitem__(self, name):
        resp = requests.get(self.url(name))
        if resp.status_code == 404:
            raise KeyError

        meta = resp.json()
        
        return np.memmap(
            meta['filename'],
            dtype=meta['dtype'],
            mode='r',
            shape=tuple(meta['shape']))

Example 40

Project: robothon Source File: test_memmap.py
Function: test_open_with_filename
    def test_open_with_filename(self):
        tmpname = mktemp('','mmap')
        fp = memmap(tmpname, dtype=self.dtype, mode='w+',
                       shape=self.shape)
        fp[:] = self.data[:]
        del fp
        os.unlink(tmpname)

Example 41

Project: invesalius3 Source File: imagedata_utils.py
def analyze2mmap(analyze):
    data = analyze.get_data()
    header = analyze.get_header()
    temp_file = tempfile.mktemp()

    # Sagital
    if header['orient'] == 2:
        print "Orientation Sagital"
        shape = tuple([data.shape[i] for i in (1, 2, 0)])
        matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
        for n, slice in enumerate(data):
            matrix[:,:, n] = slice

    # Coronal
    elif header['orient'] == 1:
        print "Orientation coronal"
        shape = tuple([data.shape[i] for i in (1, 0, 2)])
        matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
        for n, slice in enumerate(data):
            matrix[:,n,:] = slice

    # AXIAL
    elif header['orient'] == 0:
        print "no orientation"
        shape = tuple([data.shape[i] for i in (0, 1, 2)])
        matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
        for n, slice in enumerate(data):
            matrix[n] = slice

    else:
        print "Orientation Sagital"
        shape = tuple([data.shape[i] for i in (1, 2, 0)])
        matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
        for n, slice in enumerate(data):
            matrix[:,:, n] = slice

    matrix.flush()
    return matrix, temp_file

Example 42

Project: pylearn2 Source File: new_norb.py
    def __getstate__(self):
        """
        Support method for pickling. Returns the complete state of this object
        as a dictionary, which is then pickled.

        This state does not include the memmaps' contents. Rather, it includes
        enough info to find the memmap and re-load it from disk in the same
        state.

        Note that pickling a NORB will set its memmaps (self.X and self.y) to
        be read-only. This is to prevent the memmaps from accidentally being
        edited after the save. To make them writeable again, the user must
        explicitly call setflags(write=True) on the memmaps.
        """
        _check_pickling_support()

        result = copy.copy(self.__dict__)

        assert isinstance(self.X, numpy.memmap), ("Expected X to be a memmap, "
                                                  "but it was a %s." %
                                                  str(type(self.X)))
        assert isinstance(self.y, numpy.memmap), ("Expected y to be a memmap, "
                                                  "but it was a %s." %
                                                  str(type(self.y)))

        # We don't want to pickle the memmaps; they're already on disk.
        del result['X']
        del result['y']

        # Replace memmaps with their constructor arguments
        def get_memmap_info(memmap):
            assert isinstance(memmap, numpy.memmap)

            if not isinstance(memmap.filename, str):
                raise ValueError("Expected memmap.filename to be a str; "
                                 "instead got a %s, %s" %
                                 (type(memmap.filename), str(memmap.filename)))

            result = {}

            def get_relative_path(full_path):
                """
                Returns the relative path to the PYLEARN2_DATA_PATH.
                """
                data_dir = string_utils.preprocess('${PYLEARN2_DATA_PATH}')

                if not memmap.filename.startswith(data_dir):
                    raise ValueError("Expected memmap.filename to start with "
                                     "the PYLEARN2_DATA_PATH (%s). Instead it "
                                     "was %s." % (data_dir, memmap.filename))

                return os.path.relpath(full_path, data_dir)

            return {'filename': get_relative_path(memmap.filename),
                    'dtype': memmap.dtype,
                    'shape': memmap.shape,
                    'offset': memmap.offset,
                    # We never want to set mode to w+, even if memmap.mode
                    # is w+. Otherwise we'll overwrite the memmap's contents
                    # when we open it.
                    'mode': 'r+' if memmap.mode in ('r+', 'w+') else 'r'}

        result['X_info'] = get_memmap_info(self.X)
        result['y_info'] = get_memmap_info(self.y)

        # This prevents self.X and self.y from being accidentally written to
        # after the save, thus unexpectedly changing the saved file. If the
        # user really wants to, they can make the memmaps writeable again
        # by calling setflags(write=True) on the memmaps.
        for memmap in (self.X, self.y):
            memmap.flush()
            memmap.setflags(write=False)

        return result

Example 43

Project: mpop Source File: aapp1b.py
Function: read
    def read(self):
        """Read the data.
        """
        tic = datetime.datetime.now()
        all_data = []
        all_headers = []
        for fname in self.filenames:
            with open(fname, "rb") as fp_:
                header = np.memmap(fp_, dtype=_HEADERTYPE, mode="r",
                                   shape=(_HEADERTYPE.itemsize, ))
                all_headers.append(header)
                data = np.memmap(fp_,
                                 dtype=_SCANTYPE,
                                 offset=22016, mode="r")
                all_data.append(data)

        LOGGER.debug("Reading time %s", str(datetime.datetime.now() - tic))

        self._header = all_headers
        self._data = all_data

Example 44

Project: attention-lvcsr Source File: var.py
Function: get_sum
    def _get_sum(self):
        """Compute sum of non NaN / Inf values in the array."""
        try:
            return self._sum
        except AttributeError:
            self._sum = self.no_nan.sum()
            # The following 2 lines are needede as in Python 3.3 with NumPy
            # 1.7.1, numpy.ndarray and numpy.memmap aren't hashable.
            if type(self._sum) is numpy.memmap:
                self._sum = numpy.asarray(self._sum).item()
            if self.has_nan and self.no_nan.mask.all():
                # In this case the sum is not properly computed by numpy.
                self._sum = 0
            if numpy.isinf(self._sum) or numpy.isnan(self._sum):
                # NaN may happen when there are both -inf and +inf values.
                if self.has_nan:
                    # Filter both NaN and Inf values.
                    mask = self.no_nan.mask + numpy.isinf(self[1])
                else:
                    # Filter only Inf values.
                    mask = numpy.isinf(self[1])
                if mask.all():
                    self._sum = 0
                else:
                    self._sum = numpy.ma.masked_array(self[1], mask).sum()
                # At this point there should be no more NaN.
                assert not numpy.isnan(self._sum)
        return self._sum

Example 45

Project: rdfspace Source File: space.py
Function: save
    def save(self, dirname = None):
        """Save the current rdfspace to a directory (by default the directory in which indexes are stored)"""
        if dirname is None and self._index_dir is not None:
            dirname = self._index_dir
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        # We memmap big matrices, as pickle eats the whole RAM
        # We don't save the full adjacency matrix
        ut_m = np.memmap(os.path.join(dirname, 'ut.dat'), dtype='float64', mode='w+', shape=self._ut_shape)
        ut_m[:] = self._ut[:]
        s_m = np.memmap(os.path.join(dirname, 's.dat'), dtype='float64', mode='w+', shape=self._s_shape)
        s_m[:] = self._s[:]
        vt_m = np.memmap(os.path.join(dirname, 'vt.dat'), dtype='float64', mode='w+', shape=self._vt_shape)
        vt_m[:] = self._vt[:]
        if self._index_dir is None:
            # The index is in memory, we'll pickle it with the rest
            (adjacency, ut, s, vt) = (self._adjacency, self._ut, self._s, self._vt)
            (self._adjacency, self._ut, self._s, self._vt) = (None, None, None, None)
            f = open(os.path.join(dirname, 'space.dat'), 'w')
            pickle.dump(self, f)
            f.close()
            (self._adjacency, self._ut, self._s, self._vt) = (adjacency, ut, s, vt)
        else:
            # Flushing indexes
            self._uri_index.close()
            self._index_uri.close()
            # The index is stored in dbm, we will exclude it from the pickle
            (adjacency, ut, s, vt) = (self._adjacency, self._ut, self._s, self._vt)
            (self._adjacency, self._ut, self._s, self._vt, self._uri_index, self._index_uri) = (None, None, None, None, None, None)
            f = open(os.path.join(dirname, 'space.dat'), 'w')
            pickle.dump(self, f)
            f.close()
            (self._adjacency, self._ut, self._s, self._vt) = (adjacency, ut, s, vt)
            self._uri_index = dbm.open(os.path.join(dirname, 'uri_index'), 'r')
            self._index_uri = dbm.open(os.path.join(dirname, 'index_uri'), 'r')

Example 46

Project: seisflows Source File: LBFGS.py
Function: apply
    def apply(self, q, S=[], Y=[]):
        """ Applies L-BFGS inverse Hessian to given vector
        """
        unix.cd(self.path)

        if S==[] or Y==[]:
            m = len(q)
            n = self.memory
            S = np.memmap('LBFGS/S', mode='w+', dtype='float32', shape=(m, n))
            Y = np.memmap('LBFGS/Y', mode='w+', dtype='float32', shape=(m, n))

        # first matrix product
        kk = self.memory_used
        rh = np.zeros(kk)
        al = np.zeros(kk)
        for ii in range(kk):
            rh[ii] = 1/np.dot(Y[:,ii], S[:,ii])
            al[ii] = rh[ii]*np.dot(S[:,ii], q)
            q = q - al[ii]*Y[:,ii]

        if self.precond:
            r = self.precond(q)
        else:
            r = q

        # use scaling M3 proposed by Liu and Nocedal 1989
        sty = np.dot(Y[:,0], S[:,0])
        yty = np.dot(Y[:,0], Y[:,0])
        r *= sty/yty

        # second matrix product
        for ii in range(kk-1, -1, -1):
            be = rh[ii]*np.dot(Y[:,ii], r)
            r = r + S[:,ii]*(al[ii] - be)

        return r

Example 47

Project: jcvi Source File: depth.py
    @property
    def mmarray(self):
        binfile = self.filename
        return np.memmap(binfile, dtype=self.dtype, mode="r")

Example 48

Project: seisflows Source File: LBFGS.py
Function: update
    def update(self):
        """ Updates L-BFGS algorithm history
        """
        unix.cd(self.path)

        s = self.load('m_new') - self.load('m_old')
        y = self.load('g_new') - self.load('g_old')

        m = len(s)
        n = self.memory

        if self.memory_used == 0:
            S = np.memmap('LBFGS/S', mode='w+', dtype='float32', shape=(m, n))
            Y = np.memmap('LBFGS/Y', mode='w+', dtype='float32', shape=(m, n))
            S[:, 0] = s
            Y[:, 0] = y
            self.memory_used = 1

        else:
            S = np.memmap('LBFGS/S', mode='r+', dtype='float32', shape=(m, n))
            Y = np.memmap('LBFGS/Y', mode='r+', dtype='float32', shape=(m, n))
            S[:, 1:] = S[:, :-1]
            Y[:, 1:] = Y[:, :-1]
            S[:, 0] = s
            Y[:, 0] = y

            if self.memory_used < self.memory:
                self.memory_used += 1

        return S, Y

Example 49

Project: panns Source File: utils.py
def load_mmap(fname, shape, dtype):
    mtx = numpy.memmap(fname, dtype=dtype, mode='r', shape=shape)
    return mtx

Example 50

Project: yolotf Source File: yolo.py
    def loadWeights(self, weight_path):
        self.startwith = np.array(
            np.memmap(weight_path, mode = 'r',
                offset = 0, shape = (),
                dtype = '(4)i4,'))
        #self.startwith = np.array(self.startwith)
        offset = 16
        chunkMB = 1000
        chunk = int(chunkMB * 2**18) 
        
        # Read byte arrays from file
        for i in range(self.layer_number):
            l = self.layers[i]
            if l.type == "CONVOLUTIONAL":
                weight_number = l.n * l.c * l.size * l.size
                l.biases = np.memmap(weight_path, mode = 'r',
                    offset = offset, shape = (),
                    dtype = '({})float32,'.format(l.n))
                offset += 4 * l.n
                l.weights = np.memmap(weight_path, mode = 'r',
                    offset = offset, shape = (),
                    dtype = '({})float32,'.format(weight_number))
                offset += 4 * weight_number

            elif l.type == "CONNECTED":
                bias_number = l.output_size
                weight_number = l.output_size * l.input_size
                l.biases = np.memmap(weight_path, mode = 'r',
                    offset = offset, shape = (),
                    dtype = '({})float32,'.format(bias_number))
                offset += bias_number * 4
            
                chunks  = [chunk] * (weight_number / chunk) 
                chunks += [weight_number % chunk]
                l.weights = np.array([], dtype = np.float32)
                for c in chunks:
                    l.weights = np.concatenate((l.weights,
                        np.memmap(weight_path, mode = 'r',
                        offset = offset, shape = (),
                        dtype = '({})float32,'.format(c))))
                    offset += c * 4
                    
        # Defensive python right here bietch.
        if offset == os.path.getsize(weight_path):
            print ('Successfully identified all {} bytes'.format(
                offset))
        else:
            print 'expect ', offset, ' bytes, found ', os.path.getsize(weight_path)
            exit()

        # Reshape
        for i in range(self.layer_number):
            l = self.layers[i]
            
            if l.type == 'CONVOLUTIONAL':
                weight_array = l.weights
                weight_array = np.reshape(weight_array,
                	[l.n, l.c, l.size, l.size])
                weight_array = weight_array.transpose([2,3,1,0])
                l.weights = weight_array

            if l.type == 'CONNECTED':
                weight_array = l.weights
                weight_array = np.reshape(weight_array,
                	[l.input_size, l.output_size])
                l.weights = weight_array
See More Examples - Go to Next Page
Page 1 Selected Page 2