Here are the examples of the python api numpy.memmap taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
70 Examples
5
Example 1
def run(self):
if self.from_binary:
self.mask = numpy.memmap(self.mask_filename, mode='r',
dtype=self.mask_dtype,
shape=self.mask_shape)
else:
self.image = numpy.memmap(self.filename, mode='r', dtype=self.dtype,
shape=self.shape)
self.mask = numpy.memmap(self.mask_filename, mode='r',
dtype=self.mask_dtype,
shape=self.mask_shape)
while 1:
roi = self.q_in.get()
if roi is None:
break
self.CreateSurface(roi)
3
Example 2
def read_data(self, mmap=False, dt='float32', mode='r'):
"""
VHDR stores data in channel contiguous way such that reading disparate pieces
in time is fast, when using memmap.
"""
if mmap:
ary = numpy.memmap(self.datafile, dt, mode)
else:
ary = numpy.fromfile(self.datafile, dt)
self.data = ary.reshape((-1, self.nchan)).T
self.nsamp = self.data.shape[1]
3
Example 3
Project: PyEMMA Source File: kmeans.py
def _finish_estimate(self):
fh = None
if isinstance(self._in_memory_chunks, np.memmap):
fh = self._in_memory_chunks.filename
del self._in_memory_chunks
if fh:
os.unlink(fh)
if self.init_strategy == 'uniform':
del self._centers_iter_list
del self._init_centers_indices
if self.init_strategy == 'kmeans++':
self._progress_force_finish(0)
self._progress_force_finish(1)
3
Example 4
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
def test_indexing_drops_references(self):
fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
tmp = fp[[(1, 2), (2, 3)]]
if isinstance(tmp, memmap):
assert tmp._mmap is not fp._mmap
3
Example 5
def create_mask(self, shape):
"""
Creates a new mask object. This method do not append this new mask into the project.
Parameters:
shape(int, int, int): The shape of the new mask.
"""
self.temp_file = tempfile.mktemp()
shape = shape[0] + 1, shape[1] + 1, shape[2] + 1
self.matrix = np.memmap(self.temp_file, mode='w+', dtype='uint8', shape=shape)
3
Example 6
Project: panns Source File: index.py
def mmap_core_data(self):
"""
Convert mtx and prj to mmap file to save memory space, very
useful when dealing with large dataset and parallel mode is
activated. Skip if the data is already mmaped.
"""
if type(self.mtx) != numpy.memmap:
shape_mtx = (len(self.mtx), self.dim)
mmap_mtx = make_mmap(self.mtx, shape_mtx, self.typ)
self.mtx = load_mmap(mmap_mtx, shape_mtx, self.typ)
pass
3
Example 7
def test_filename(self):
tmpname = mktemp('', 'mmap', dir=self.tempdir)
fp = memmap(tmpname, dtype=self.dtype, mode='w+',
shape=self.shape)
abspath = os.path.abspath(tmpname)
fp[:] = self.data[:]
self.assertEqual(abspath, fp.filename)
b = fp[:1]
self.assertEqual(abspath, b.filename)
del b
del fp
3
Example 8
Project: gensim Source File: test_ldamallet_wrapper.py
def testLargeMmap(self):
if not self.mallet_path:
return
fname = testfile()
model = ldamallet.LdaMallet(self.mallet_path, self.corpus, num_topics=2, iterations=100)
# simulate storing large arrays separately
model.save(testfile(), sep_limit=0)
# test loading the large model arrays with mmap
model2 = ldamodel.LdaModel.load(testfile(), mmap='r')
self.assertEqual(model.num_topics, model2.num_topics)
self.assertTrue(isinstance(model2.word_topics, numpy.memmap))
self.assertTrue(numpy.allclose(model.word_topics, model2.word_topics))
tstvec = []
self.assertTrue(numpy.allclose(model[tstvec], model2[tstvec])) # try projecting an empty vector
3
Example 9
Project: scikit-learn Source File: pool.py
def _strided_from_memmap(filename, dtype, mode, offset, order, shape, strides,
total_buffer_len):
"""Reconstruct an array view on a memory mapped file."""
if mode == 'w+':
# Do not zero the original data when unpickling
mode = 'r+'
if strides is None:
# Simple, contiguous memmap
return np.memmap(filename, dtype=dtype, shape=shape, mode=mode,
offset=offset, order=order)
else:
# For non-contiguous data, memmap the total enclosing buffer and then
# extract the non-contiguous view with the stride-tricks API
base = np.memmap(filename, dtype=dtype, shape=total_buffer_len,
mode=mode, offset=offset, order=order)
return as_strided(base, shape=shape, strides=strides)
3
Example 10
def load_model(directory="{}/vectors".format(getcwd())):
# current model contains 100,000 vectors of 300 elements, each element containing a 16 bit floating point number, such that vectors total ~60 MB uncompressed; note that there is practically no loss of precision in saving vectors with 16 bits versus 32 bits, while data consumption is halved
print "Loading 100,000 word vectors..."
model = load(open(join(directory, 'model.pickle')))
model.vocab = Loader(join(directory, 'word_to_index'))
model.index2word = Loader(join(directory, 'index_to_word'))
model.syn0norm = np.memmap(join(directory, 'syn0norm.dat'), dtype='float16', mode='r', shape=(len(model.vocab.keys()), model.layer1_size))
model.syn0 = model.syn0norm
return model
3
Example 11
Project: polar2grid Source File: plot_scene.py
def _parse_binary_info(parts):
from polar2grid.core.dtype import str_to_dtype
fn = parts[0]
if not os.path.exists(fn):
raise ValueError("File '%s' does not exist" % (fn,))
dtype = str_to_dtype(parts[1])
rows = int(parts[2])
cols = int(parts[3])
fill = float(parts[4])
arr = numpy.memmap(fn, dtype=dtype, mode='r', shape=(rows, cols))
return (fn, numpy.ma.masked_array(arr, numpy.isnan(arr) | (arr == fill)))
3
Example 12
def test_del(self):
# Make sure a view does not delete the underlying mmap
fp_base = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
fp_view = fp_base[:]
class ViewCloseError(Exception):
pass
_close = memmap._close
def replace_close(self):
raise ViewCloseError('View should not call _close on memmap')
try:
memmap._close = replace_close
del fp_view
finally:
memmap._close = _close
3
Example 13
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
def test_arithmetic_drops_references(self):
fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
tmp = (fp + 10)
if isinstance(tmp, memmap):
assert tmp._mmap is not fp._mmap
3
Example 14
Project: python-geoprobe Source File: volume.py
def memmap_data(self):
"""Return an object similar to a memory-mapped numpy array."""
header = self.read_header()
nx, ny, nz = [header[item] for item in ('_nx', '_ny', '_nz')]
dat = np.memmap(filename=self.filename, mode='r',
offset=_headerLength, order='F',
shape=(nx, ny, nz)
)
return dat
3
Example 15
def make_mmap(mtx, shape, dtype, fname=None):
m, n = shape
if fname is None:
fname = tempfile.mkstemp()[1]
logger.info('mmaping the data to %s ...' % fname)
fpw = numpy.memmap(fname, dtype=dtype, mode='w+', shape=(m,n))
for i in xrange(m):
fpw[i] = mtx[i]
del fpw
return fname
3
Example 16
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
Function: test_open_with_filename
Function: test_open_with_filename
def test_open_with_filename(self):
tmpname = mktemp('', 'mmap', dir=self.tempdir)
fp = memmap(tmpname, dtype=self.dtype, mode='w+',
shape=self.shape)
fp[:] = self.data[:]
del fp
3
Example 17
def _open_memmap(self, mode):
import os
import sys
if (os.path.getsize(self.filename) < sys.maxsize):
try:
(R, C, B) = self.shape
return np.memmap(self.filename, dtype=self.dtype, mode=mode,
offset=self.offset, shape=(B, R, C))
except:
print('Unable to create memmap interface.')
return None
else:
return None
3
Example 18
def test_del(self):
# Make sure a view does not delete the underlying mmap
fp_base = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
fp_base[0] = 5
fp_view = fp_base[0:1]
assert_equal(fp_view[0], 5)
del fp_view
# Should still be able to access and assign values after
# deleting the view
assert_equal(fp_base[0], 5)
fp_base[0] = 6
assert_equal(fp_base[0], 6)
3
Example 19
def test_attributes(self):
offset = 1
mode = "w+"
fp = memmap(self.tmpfp, dtype=self.dtype, mode=mode,
shape=self.shape, offset=offset)
self.assertEqual(offset, fp.offset)
self.assertEqual(mode, fp.mode)
del fp
3
Example 20
def _strided_from_memmap(filename, dtype, mode, offset, order, shape, strides,
total_buffer_len):
"""Reconstruct an array view on a memmory mapped file"""
if mode == 'w+':
# Do not zero the original data when unpickling
mode = 'r+'
if strides is None:
# Simple, contiguous memmap
return np.memmap(filename, dtype=dtype, shape=shape, mode=mode,
offset=offset, order=order)
else:
# For non-contiguous data, memmap the total enclosing buffer and then
# extract the non-contiguous view with the stride-tricks API
base = np.memmap(filename, dtype=dtype, shape=total_buffer_len,
mode=mode, offset=offset, order=order)
return as_strided(base, shape=shape, strides=strides)
3
Example 21
@pytest.mark.parametrize("Integrator", integrator_list)
def test_memmap(tmpdir, Integrator):
dt = 0.1
n_steps = 1000
nw0 = 10000
mmap = np.memmap("/tmp/test_memmap.npy", mode='w+', shape=(2, n_steps+1, nw0))
def sho(t,w,T):
q,p = w
return np.array([p, -(2*np.pi/T)**2*q])
w0 = np.random.uniform(-1,1,size=(2,nw0))
integrator = Integrator(sho, func_args=(1.,))
orbit = integrator.run(w0, dt=dt, n_steps=n_steps, mmap=mmap)
3
Example 22
Project: distributed Source File: test_numpy.py
def test_memmap():
with tmpfile('npy') as fn:
with open(fn, 'wb') as f: # touch file
pass
x = np.memmap(fn, shape=(5, 5), dtype='i4', mode='readwrite')
x[:] = 5
header, frames = serialize(x)
if 'compression' in header:
frames = decompress(header, frames)
y = deserialize(header, frames)
np.testing.assert_equal(x, y)
3
Example 23
def restart(self):
""" Discards history and resets counters
"""
self.iter = 1
self.memory_used = 0
unix.cd(self.path)
S = np.memmap('LBFGS/S', mode='r+')
Y = np.memmap('LBFGS/Y', mode='r+')
S[:] = 0.
Y[:] = 0.
3
Example 24
def __setitem__(self, name, array):
"""Stores the array in a memmap file, then pings server."""
fn = tempfile.mktemp()
fp = np.memmap(fn, dtype=array.dtype, mode='w+', shape=array.shape)
fp[:] = array[:]
requests.put(self.url(name), json.dumps({
'dtype': str(array.dtype),
'shape': list(array.shape),
'filename': fn,
}))
3
Example 25
def __init__(self, filename):
self.filename = os.path.basename(filename)
self.filepath = os.path.realpath(filename)
with open(self.filepath, "rb") as fp_:
header = numpy.memmap(fp_, dtype=_HEADERTYPE, mode="r", shape=(_HEADERTYPE.itemsize,))
data = numpy.memmap(fp_, dtype=_SCANTYPE, offset=22016, mode="r")
self._header = header
self._data = data
self.file_type = FT_AAPP
3
Example 26
Project: Twitch-plays-LSD-neural-net Source File: utils.py
def shared_mmp(data=None, file_name="shm", shape=(0,), dtype=floatX):
""" Shared memory, only works for linux """
if not data is None: shape = data.shape
path = "/dev/shm/lio/"
make_sure_path_exists(path)
mmp = np.memmap(path+file_name+".mmp", dtype=dtype, mode='w+', shape=shape)
if not data is None: mmp[:] = data
return mmp
3
Example 27
def test_flush(self):
fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
fp[:] = self.data[:]
fp.flush()
warnings.simplefilter('ignore', DeprecationWarning)
fp.sync()
warnings.simplefilter('default', DeprecationWarning)
3
Example 28
def put(self, name):
"""Store the metadata for this vector and open a handle to the file."""
meta = json.loads(self.request.body)
meta['obj'] = np.memmap(
meta['filename'],
dtype=meta['dtype'],
mode='r',
shape=tuple(meta['shape'])
)
arrays[name] = meta
self.write(json.dumps({'success': 1}))
3
Example 29
def test_view(self):
fp = memmap(self.tmpfp, dtype=self.dtype, shape=self.shape)
new1 = fp.view()
new2 = new1.view()
assert(new1.base is fp)
assert(new2.base is fp)
new_array = asarray(fp)
assert(new_array.base is fp)
3
Example 30
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: test_memmap.py
@dec.knownfailureif(sys.platform == 'gnu0', "This test is known to fail on hurd")
def test_flush(self):
fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
fp[:] = self.data[:]
assert_equal(fp[0], self.data[0])
fp.flush()
3
Example 31
def test_roundtrip(self):
# Write data to file
fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
fp[:] = self.data[:]
del fp # Test __del__ machinery, which handles cleanup
# Read data back from file
newfp = memmap(self.tmpfp, dtype=self.dtype, mode='r',
shape=self.shape)
assert allclose(self.data, newfp)
assert_array_equal(self.data, newfp)
3
Example 32
@staticmethod
def load(dirname):
"""Load an rdfspace instance from a directory"""
if os.path.exists(dirname):
f = open(os.path.join(dirname, 'space.dat'))
space = pickle.load(f)
f.close()
space._ut = np.memmap(os.path.join(dirname, 'ut.dat'), dtype='float64', mode='r', shape=space._ut_shape)
space._s = np.memmap(os.path.join(dirname, 's.dat'), dtype='float64', mode='r', shape=space._s_shape)
space._vt = np.memmap(os.path.join(dirname, 'vt.dat'), dtype='float64', mode='r', shape=space._vt_shape)
if os.path.exists(os.path.join(dirname, 'uri_index.db')) and os.path.exists(os.path.join(dirname, 'index_uri.db')):
# If these files exist the index are stored through dbm
# If not the indexes will be coming from the pickle
space._uri_index = dbm.open(os.path.join(dirname, 'uri_index'), 'r')
space._index_uri = dbm.open(os.path.join(dirname, 'index_uri'), 'r')
return space
else:
raise Exception('No such directory')
3
Example 33
def _open_memmap(self, mode):
import os
import sys
if (os.path.getsize(self.filename) < sys.maxsize):
try:
(R, C, B) = self.shape
return np.memmap(self.filename, dtype=self.dtype, mode=mode,
offset=self.offset, shape=(R, B, C))
except:
print('Unable to create memmap interface.')
return None
else:
return None
3
Example 34
def build_parallel(mtx, shape_mtx, K, dtype, t):
"""
The function for parallel building index. Implemented here because
the default python serialization cannot pickle instance function.
Parameters:
mtx: a row-based data set, should be an numpy matrix.
K: max number of data points on a leaf.
t: index of binary trees.
"""
logger.info('pass %i ...' % t)
mtx = numpy.memmap(mtx, dtype=dtype, mode='r', shape=shape_mtx)
numpy.random.seed(t**2 + numpy.random.randint(2**30))
tree = NaiveTree()
children = range(len(mtx))
make_tree_parallel(tree.root, children, mtx, shape_mtx[1], dtype, K)
return tree
3
Example 35
def test_roundtrip(self):
# Write data to file
fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+',
shape=self.shape)
fp[:] = self.data[:]
del fp # Test __del__ machinery, which handles cleanup
# Read data back from file
newfp = memmap(self.tmpfp, dtype=self.dtype, mode='r',
shape=self.shape)
assert_(allclose(self.data, newfp))
assert_array_equal(self.data, newfp)
3
Example 36
def _open_memmap(self, mode):
import os
import sys
if (os.path.getsize(self.filename) < sys.maxsize):
try:
(R, C, B) = self.shape
return np.memmap(self.filename, dtype=self.dtype, mode=mode,
offset=self.offset, shape=self.shape)
except:
print('Unable to create memmap interface.')
return None
else:
return None
3
Example 37
def testLargeMmap(self):
fname = testfile()
model = self.model
# simulate storing large arrays separately
model.save(testfile(), sep_limit=0)
# test loading the large model arrays with mmap
model2 = self.class_.load(testfile(), mmap='r')
self.assertEqual(model.num_topics, model2.num_topics)
self.assertTrue(isinstance(model2.expElogbeta, numpy.memmap))
self.assertTrue(numpy.allclose(model.expElogbeta, model2.expElogbeta))
tstvec = []
self.assertTrue(numpy.allclose(model[tstvec], model2[tstvec])) # try projecting an empty vector
3
Example 38
def testLargeMmap(self):
fname = testfile()
model = self.model
# test storing the internal arrays into separate files
model.save(fname, sep_limit=0)
# now load the external arrays via mmap
model2 = lsimodel.LsiModel.load(fname, mmap='r')
self.assertEqual(model.num_topics, model2.num_topics)
self.assertTrue(isinstance(model2.projection.u, numpy.memmap))
self.assertTrue(isinstance(model2.projection.s, numpy.memmap))
self.assertTrue(numpy.allclose(model.projection.u, model2.projection.u))
self.assertTrue(numpy.allclose(model.projection.s, model2.projection.s))
tstvec = []
self.assertTrue(numpy.allclose(model[tstvec], model2[tstvec])) # try projecting an empty vector
3
Example 39
def __getitem__(self, name):
resp = requests.get(self.url(name))
if resp.status_code == 404:
raise KeyError
meta = resp.json()
return np.memmap(
meta['filename'],
dtype=meta['dtype'],
mode='r',
shape=tuple(meta['shape']))
3
Example 40
def test_open_with_filename(self):
tmpname = mktemp('','mmap')
fp = memmap(tmpname, dtype=self.dtype, mode='w+',
shape=self.shape)
fp[:] = self.data[:]
del fp
os.unlink(tmpname)
2
Example 41
Project: invesalius3 Source File: imagedata_utils.py
def analyze2mmap(analyze):
data = analyze.get_data()
header = analyze.get_header()
temp_file = tempfile.mktemp()
# Sagital
if header['orient'] == 2:
print "Orientation Sagital"
shape = tuple([data.shape[i] for i in (1, 2, 0)])
matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
for n, slice in enumerate(data):
matrix[:,:, n] = slice
# Coronal
elif header['orient'] == 1:
print "Orientation coronal"
shape = tuple([data.shape[i] for i in (1, 0, 2)])
matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
for n, slice in enumerate(data):
matrix[:,n,:] = slice
# AXIAL
elif header['orient'] == 0:
print "no orientation"
shape = tuple([data.shape[i] for i in (0, 1, 2)])
matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
for n, slice in enumerate(data):
matrix[n] = slice
else:
print "Orientation Sagital"
shape = tuple([data.shape[i] for i in (1, 2, 0)])
matrix = numpy.memmap(temp_file, mode='w+', dtype=data.dtype, shape=shape)
for n, slice in enumerate(data):
matrix[:,:, n] = slice
matrix.flush()
return matrix, temp_file
2
Example 42
Project: pylearn2 Source File: new_norb.py
def __getstate__(self):
"""
Support method for pickling. Returns the complete state of this object
as a dictionary, which is then pickled.
This state does not include the memmaps' contents. Rather, it includes
enough info to find the memmap and re-load it from disk in the same
state.
Note that pickling a NORB will set its memmaps (self.X and self.y) to
be read-only. This is to prevent the memmaps from accidentally being
edited after the save. To make them writeable again, the user must
explicitly call setflags(write=True) on the memmaps.
"""
_check_pickling_support()
result = copy.copy(self.__dict__)
assert isinstance(self.X, numpy.memmap), ("Expected X to be a memmap, "
"but it was a %s." %
str(type(self.X)))
assert isinstance(self.y, numpy.memmap), ("Expected y to be a memmap, "
"but it was a %s." %
str(type(self.y)))
# We don't want to pickle the memmaps; they're already on disk.
del result['X']
del result['y']
# Replace memmaps with their constructor arguments
def get_memmap_info(memmap):
assert isinstance(memmap, numpy.memmap)
if not isinstance(memmap.filename, str):
raise ValueError("Expected memmap.filename to be a str; "
"instead got a %s, %s" %
(type(memmap.filename), str(memmap.filename)))
result = {}
def get_relative_path(full_path):
"""
Returns the relative path to the PYLEARN2_DATA_PATH.
"""
data_dir = string_utils.preprocess('${PYLEARN2_DATA_PATH}')
if not memmap.filename.startswith(data_dir):
raise ValueError("Expected memmap.filename to start with "
"the PYLEARN2_DATA_PATH (%s). Instead it "
"was %s." % (data_dir, memmap.filename))
return os.path.relpath(full_path, data_dir)
return {'filename': get_relative_path(memmap.filename),
'dtype': memmap.dtype,
'shape': memmap.shape,
'offset': memmap.offset,
# We never want to set mode to w+, even if memmap.mode
# is w+. Otherwise we'll overwrite the memmap's contents
# when we open it.
'mode': 'r+' if memmap.mode in ('r+', 'w+') else 'r'}
result['X_info'] = get_memmap_info(self.X)
result['y_info'] = get_memmap_info(self.y)
# This prevents self.X and self.y from being accidentally written to
# after the save, thus unexpectedly changing the saved file. If the
# user really wants to, they can make the memmaps writeable again
# by calling setflags(write=True) on the memmaps.
for memmap in (self.X, self.y):
memmap.flush()
memmap.setflags(write=False)
return result
0
Example 43
def read(self):
"""Read the data.
"""
tic = datetime.datetime.now()
all_data = []
all_headers = []
for fname in self.filenames:
with open(fname, "rb") as fp_:
header = np.memmap(fp_, dtype=_HEADERTYPE, mode="r",
shape=(_HEADERTYPE.itemsize, ))
all_headers.append(header)
data = np.memmap(fp_,
dtype=_SCANTYPE,
offset=22016, mode="r")
all_data.append(data)
LOGGER.debug("Reading time %s", str(datetime.datetime.now() - tic))
self._header = all_headers
self._data = all_data
0
Example 44
def _get_sum(self):
"""Compute sum of non NaN / Inf values in the array."""
try:
return self._sum
except AttributeError:
self._sum = self.no_nan.sum()
# The following 2 lines are needede as in Python 3.3 with NumPy
# 1.7.1, numpy.ndarray and numpy.memmap aren't hashable.
if type(self._sum) is numpy.memmap:
self._sum = numpy.asarray(self._sum).item()
if self.has_nan and self.no_nan.mask.all():
# In this case the sum is not properly computed by numpy.
self._sum = 0
if numpy.isinf(self._sum) or numpy.isnan(self._sum):
# NaN may happen when there are both -inf and +inf values.
if self.has_nan:
# Filter both NaN and Inf values.
mask = self.no_nan.mask + numpy.isinf(self[1])
else:
# Filter only Inf values.
mask = numpy.isinf(self[1])
if mask.all():
self._sum = 0
else:
self._sum = numpy.ma.masked_array(self[1], mask).sum()
# At this point there should be no more NaN.
assert not numpy.isnan(self._sum)
return self._sum
0
Example 45
def save(self, dirname = None):
"""Save the current rdfspace to a directory (by default the directory in which indexes are stored)"""
if dirname is None and self._index_dir is not None:
dirname = self._index_dir
if not os.path.exists(dirname):
os.makedirs(dirname)
# We memmap big matrices, as pickle eats the whole RAM
# We don't save the full adjacency matrix
ut_m = np.memmap(os.path.join(dirname, 'ut.dat'), dtype='float64', mode='w+', shape=self._ut_shape)
ut_m[:] = self._ut[:]
s_m = np.memmap(os.path.join(dirname, 's.dat'), dtype='float64', mode='w+', shape=self._s_shape)
s_m[:] = self._s[:]
vt_m = np.memmap(os.path.join(dirname, 'vt.dat'), dtype='float64', mode='w+', shape=self._vt_shape)
vt_m[:] = self._vt[:]
if self._index_dir is None:
# The index is in memory, we'll pickle it with the rest
(adjacency, ut, s, vt) = (self._adjacency, self._ut, self._s, self._vt)
(self._adjacency, self._ut, self._s, self._vt) = (None, None, None, None)
f = open(os.path.join(dirname, 'space.dat'), 'w')
pickle.dump(self, f)
f.close()
(self._adjacency, self._ut, self._s, self._vt) = (adjacency, ut, s, vt)
else:
# Flushing indexes
self._uri_index.close()
self._index_uri.close()
# The index is stored in dbm, we will exclude it from the pickle
(adjacency, ut, s, vt) = (self._adjacency, self._ut, self._s, self._vt)
(self._adjacency, self._ut, self._s, self._vt, self._uri_index, self._index_uri) = (None, None, None, None, None, None)
f = open(os.path.join(dirname, 'space.dat'), 'w')
pickle.dump(self, f)
f.close()
(self._adjacency, self._ut, self._s, self._vt) = (adjacency, ut, s, vt)
self._uri_index = dbm.open(os.path.join(dirname, 'uri_index'), 'r')
self._index_uri = dbm.open(os.path.join(dirname, 'index_uri'), 'r')
0
Example 46
def apply(self, q, S=[], Y=[]):
""" Applies L-BFGS inverse Hessian to given vector
"""
unix.cd(self.path)
if S==[] or Y==[]:
m = len(q)
n = self.memory
S = np.memmap('LBFGS/S', mode='w+', dtype='float32', shape=(m, n))
Y = np.memmap('LBFGS/Y', mode='w+', dtype='float32', shape=(m, n))
# first matrix product
kk = self.memory_used
rh = np.zeros(kk)
al = np.zeros(kk)
for ii in range(kk):
rh[ii] = 1/np.dot(Y[:,ii], S[:,ii])
al[ii] = rh[ii]*np.dot(S[:,ii], q)
q = q - al[ii]*Y[:,ii]
if self.precond:
r = self.precond(q)
else:
r = q
# use scaling M3 proposed by Liu and Nocedal 1989
sty = np.dot(Y[:,0], S[:,0])
yty = np.dot(Y[:,0], Y[:,0])
r *= sty/yty
# second matrix product
for ii in range(kk-1, -1, -1):
be = rh[ii]*np.dot(Y[:,ii], r)
r = r + S[:,ii]*(al[ii] - be)
return r
0
Example 47
Project: jcvi Source File: depth.py
@property
def mmarray(self):
binfile = self.filename
return np.memmap(binfile, dtype=self.dtype, mode="r")
0
Example 48
def update(self):
""" Updates L-BFGS algorithm history
"""
unix.cd(self.path)
s = self.load('m_new') - self.load('m_old')
y = self.load('g_new') - self.load('g_old')
m = len(s)
n = self.memory
if self.memory_used == 0:
S = np.memmap('LBFGS/S', mode='w+', dtype='float32', shape=(m, n))
Y = np.memmap('LBFGS/Y', mode='w+', dtype='float32', shape=(m, n))
S[:, 0] = s
Y[:, 0] = y
self.memory_used = 1
else:
S = np.memmap('LBFGS/S', mode='r+', dtype='float32', shape=(m, n))
Y = np.memmap('LBFGS/Y', mode='r+', dtype='float32', shape=(m, n))
S[:, 1:] = S[:, :-1]
Y[:, 1:] = Y[:, :-1]
S[:, 0] = s
Y[:, 0] = y
if self.memory_used < self.memory:
self.memory_used += 1
return S, Y
0
Example 49
Project: panns Source File: utils.py
def load_mmap(fname, shape, dtype):
mtx = numpy.memmap(fname, dtype=dtype, mode='r', shape=shape)
return mtx
0
Example 50
Project: yolotf Source File: yolo.py
def loadWeights(self, weight_path):
self.startwith = np.array(
np.memmap(weight_path, mode = 'r',
offset = 0, shape = (),
dtype = '(4)i4,'))
#self.startwith = np.array(self.startwith)
offset = 16
chunkMB = 1000
chunk = int(chunkMB * 2**18)
# Read byte arrays from file
for i in range(self.layer_number):
l = self.layers[i]
if l.type == "CONVOLUTIONAL":
weight_number = l.n * l.c * l.size * l.size
l.biases = np.memmap(weight_path, mode = 'r',
offset = offset, shape = (),
dtype = '({})float32,'.format(l.n))
offset += 4 * l.n
l.weights = np.memmap(weight_path, mode = 'r',
offset = offset, shape = (),
dtype = '({})float32,'.format(weight_number))
offset += 4 * weight_number
elif l.type == "CONNECTED":
bias_number = l.output_size
weight_number = l.output_size * l.input_size
l.biases = np.memmap(weight_path, mode = 'r',
offset = offset, shape = (),
dtype = '({})float32,'.format(bias_number))
offset += bias_number * 4
chunks = [chunk] * (weight_number / chunk)
chunks += [weight_number % chunk]
l.weights = np.array([], dtype = np.float32)
for c in chunks:
l.weights = np.concatenate((l.weights,
np.memmap(weight_path, mode = 'r',
offset = offset, shape = (),
dtype = '({})float32,'.format(c))))
offset += c * 4
# Defensive python right here bietch.
if offset == os.path.getsize(weight_path):
print ('Successfully identified all {} bytes'.format(
offset))
else:
print 'expect ', offset, ' bytes, found ', os.path.getsize(weight_path)
exit()
# Reshape
for i in range(self.layer_number):
l = self.layers[i]
if l.type == 'CONVOLUTIONAL':
weight_array = l.weights
weight_array = np.reshape(weight_array,
[l.n, l.c, l.size, l.size])
weight_array = weight_array.transpose([2,3,1,0])
l.weights = weight_array
if l.type == 'CONNECTED':
weight_array = l.weights
weight_array = np.reshape(weight_array,
[l.input_size, l.output_size])
l.weights = weight_array