Here are the examples of the python api numpy.save taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
143 Examples
3
Example 51
def DumpBuffer(self, i):
"""Write the contents of buffer i to disk."""
buf_index = self.buffer_index[i]
if buf_index == 0:
return
buf = self.buffers[i]
output_prefix = os.path.join(self.output_dir, self.names[i])
output_filename = '%s-%.5d-of-%.5d' % (
output_prefix, (self.dump_count[i]+1), self.max_dumps[i])
self.dump_count[i] += 1
np.save(output_filename, buf[:buf_index])
self.buffer_index[i] = 0
self.data_written[i] += buf_index
3
Example 52
def postNPZ (p, post_data):
"""Post data using the npz interface"""
# Build the url and then create a npz object
url = 'http://{}/{}/{}/npz/{}/{},{}/{},{}/{},{}/'.format(SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
fileobj = cStringIO.StringIO ()
np.save (fileobj, post_data)
cdz = zlib.compress (fileobj.getvalue())
return (url, cdz)
3
Example 53
Project: deepnet Source File: split_reps.py
def DumpLabelSplit(data, output_dir, name, dataset_pb):
data_pb = dataset_pb.data.add()
output_file_name = os.path.join(output_dir, name)
np.save(output_file_name, data)
data_pb.name = name
data_pb.file_pattern = '%s.npy' % output_file_name
data_pb.size = data.shape[0]
data_pb.dimensions.append(data.shape[1])
3
Example 54
Project: modl Source File: hcp.py
def _single_mask(masker, img, dest_data_dir, data_dir):
dest_file = img.replace('HCP', 'HCP_unmasked')
dest_file = dest_file.replace('.nii.gz', '')
dest_dir = os.path.abspath(os.path.join(dest_file, os.pardir))
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
print('Unmasking %s' % img)
data = masker.transform(img)
np.save(dest_file, data)
origin = dict(array=dest_file + '.npy', img=img)
with open(join(dest_dir, 'origin.json'), 'w+') as f:
json.dump(origin, f)
3
Example 55
Project: pylearn-parsimony Source File: test_svd.py
def get_err_fast_sparse_svd(self, nrow, ncol, density):
X = generate_sparse_matrix(shape=(nrow, ncol),
density=density)
# For debug
# np.save("/tmp/X_%d_%d.npy" % (nrow, ncol), X)
fd = None
try:
fd, tmpfilename = tempfile.mkstemp(suffix=".npy",
prefix="X_%d_%d" % (nrow, ncol))
np.save(tmpfilename, X)
finally:
if fd is not None:
os.close(fd)
# svd from parsimony
fast_sparse_svd = RankOneSparseSVD(max_iter=1000)
parsimony_v = fast_sparse_svd.run(X)
# return self.get_err_by_np_linalg_svd(parsimony_v, X)
return self.get_err_by_sp_sparse_linalg_svds(parsimony_v, X)
3
Example 56
def toNPZ ( self ):
"""Pickle and zip the object"""
try:
# Create the compressed cube
fileobj = cStringIO.StringIO ()
np.save ( fileobj, self.data )
return zlib.compress (fileobj.getvalue())
except:
logger.error ("Failed to compress database cube. Data integrity concern.")
raise
3
Example 57
Project: NeuroVault Source File: tasks.py
@shared_task
def save_resampled_transformation_single(pk1, resample_dim=[4, 4, 4]):
from neurovault.apps.statmaps.models import Image
from six import BytesIO
import numpy as np
img = get_object_or_404(Image, pk=pk1)
nii_obj = nib.load(img.file.path) # standard_mask=True is default
image_vector = make_resampled_transformation_vector(nii_obj,resample_dim)
f = BytesIO()
np.save(f, image_vector)
f.seek(0)
content_file = ContentFile(f.read())
img.reduced_representation.save("transform_%smm_%s.npy" %(resample_dim[0],img.pk), content_file)
return img
3
Example 58
Project: deepnet Source File: split_reps.py
def DumpDataSplit(data, output_dir, name, dataset_pb, stats_file):
data_pb = dataset_pb.data.add()
output_file_name = os.path.join(output_dir, name)
np.save(output_file_name, data)
data_pb.name = name
data_pb.file_pattern = '%s.npy' % output_file_name
data_pb.size = data.shape[0]
if stats_file:
data_pb.stats_file = stats_file
data_pb.dimensions.append(data.shape[1])
3
Example 59
Project: bluesky Source File: examples.py
def trigger(self):
# save file stash file name
self._result.clear()
for idx, (name, reading) in enumerate(super().read().items()):
# Save the actual reading['value'] to disk and create a record
# in FileStore.
np.save('{}_{}.npy'.format(self._path_stem, idx), reading['value'])
datum_id = new_uid()
self.fs.insert_datum(self._resource_id, datum_id,
dict(index=idx))
# And now change the reading in place, replacing the value with
# a reference to FileStore.
reading['value'] = datum_id
self._result[name] = reading
return NullStatus()
2
Example 60
Project: pyAudioAnalysis Source File: audioFeatureExtraction.py
def mtFeatureExtractionToFile(fileName, midTermSize, midTermStep, shortTermSize, shortTermStep, outPutFile,
storeStFeatures=False, storeToCSV=False, PLOT=False):
"""
This function is used as a wrapper to:
a) read the content of a WAV file
b) perform mid-term feature extraction on that signal
c) write the mid-term feature sequences to a numpy file
"""
[Fs, x] = audioBasicIO.readAudioFile(fileName) # read the wav file
x = audioBasicIO.stereo2mono(x) # convert to MONO if required
if storeStFeatures:
[mtF, stF] = mtFeatureExtraction(x, Fs, round(Fs * midTermSize), round(Fs * midTermStep), round(Fs * shortTermSize), round(Fs * shortTermStep))
else:
[mtF, _] = mtFeatureExtraction(x, Fs, round(Fs*midTermSize), round(Fs * midTermStep), round(Fs * shortTermSize), round(Fs * shortTermStep))
numpy.save(outPutFile, mtF) # save mt features to numpy file
if PLOT:
print "Mid-term numpy file: " + outPutFile + ".npy saved"
if storeToCSV:
numpy.savetxt(outPutFile+".csv", mtF.T, delimiter=",")
if PLOT:
print "Mid-term CSV file: " + outPutFile + ".csv saved"
if storeStFeatures:
numpy.save(outPutFile+"_st", stF) # save st features to numpy file
if PLOT:
print "Short-term numpy file: " + outPutFile + "_st.npy saved"
if storeToCSV:
numpy.savetxt(outPutFile+"_st.csv", stF.T, delimiter=",") # store st features to CSV file
if PLOT:
print "Short-term CSV file: " + outPutFile + "_st.csv saved"
0
Example 61
def save_clusters(self,Clusters,Nonzeros):
for i in range(len(Clusters)):
c = [Nonzeros[x] for x in Clusters[i]]
np.save(self.output_path+str(i)+'.cluster.npy',c)
np.save(self.output_path+'kmer_cluster_sizes.npy',[len(c) for c in Clusters])
0
Example 62
Project: datafeed Source File: server.py
def get_5minute(self, symbol, date, format='npy'):
"""Get 5min historical quotes.
Arguments:
symbol: String of security.
date: Which day data to get.
format: npy or json
"""
try:
if isinstance(date, str):
date = datetime.datetime.strptime(date, '%Y%m%d').date()
y = self.dbm.fiveminstore.get(symbol, date)
if format == 'npy':
memfile = StringIO()
np.save(memfile, y)
data = memfile.getvalue()
del(y)
else:
data = json_encode(y.tolist())
self._write_response(data)
except KeyError:
self.request.write("-ERR No data.\r\n")
0
Example 63
Project: tree-hmm Source File: vb_gmtkExact_continuous.py
def run_gmtk_lineagehmm(args):
try:
os.mkdir('gmtk_images')
except:
pass
args.iteration = 'initial'
args.observe = 'all'
args.run_name = 'gmtk'
args.out_params = 'gmtk_images/gmtk_{param}_{observe}'
args.out_dir = '.'
args.free_energy = []
plot_params(args)
# convert .npy parameters and data into gmtk format
args.observe_txt = []
for f in args.observe_matrix:
obs, obs_txt = npy_observations_to_txt(f)
args.observe_txt.append(obs_txt)
# prepare and triangulate the graphical model
strfile = 'lineagehmm.str'
gmtk_master = 'dummy.master'
with open(gmtk_master, 'w') as outfile:
outfile.write('''
MEAN_IN_FILE inline 2
0 gauss_emit_1_mean 1 1.0
1 gauss_emit_2_mean 1 50.0
COVAR_IN_FILE inline 2
0 gauss_emit_1_covar 1 1.0
1 gauss_emit_2_covar 1 50.0
MC_IN_FILE inline 2
0 1 0 gauss_emit_probs_1 gauss_emit_1_mean gauss_emit_1_covar
1 1 0 gauss_emit_probs_2 gauss_emit_2_mean gauss_emit_2_covar
DT_IN_FILE inline 1
0 parent_val 1
-1 { p0 }
''')
make_lineage_model(strfile, obs.shape[0], args.K, obs.shape[2], vert_parent=args.vert_parent, mark_avail=mark_avail, separate_theta=args.separate_theta)
cmd = 'gmtkTriangulate -strFile %s -inputMasterFile %s -rePart T -findBest T -triangulation completed ' % (strfile, gmtk_master)
#cmd = 'gmtkTriangulate -strFile %s' % strfile
subprocess.check_call(cmd, shell=True)
# populate theta parts if they don't exist
if args.separate_theta and any(not hasattr(args, 'theta_%s' % i) for i in range(2,args.I+1)):
for i in range(2,args.I+1):
if len(args.theta.shape) == 3:
setattr(args, 'theta_%s' % i, args.theta)
else:
setattr(args, 'theta_%s' % i, args.theta[i-2,:,:,:])
args.params_to_save.append('theta_%s' % i)
#del args.theta
#args.params_to_save.remove('theta')
# for each iteration...
for args.iteration in range(1, args.max_iterations+1):
# save args to disk
w = npy_params_to_workspace(args)
em_args = []
for index in range(len(args.observe_txt)):
gmtk_master = '%s.master' % args.observe_txt[index]
gmtk_obs = '%s.observations' % args.observe_txt[index]
write_workspace_simple_master(w, gmtk_master)
em_args.append((index, args.iteration, args.observe_txt[index], args.I, args.L, gmtk_master, gmtk_obs))
# run a gmtk em iteration on each file input, accuemulating results
if args.run_local:
try:
pool = multiprocessing.Pool(maxtasksperchild=1)
pool.map_async(do_em_from_args, em_args).get(99999999)
#map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, terminating workers"
pool.terminate()
pool.join()
else:
pool.close()
pool.join()
else:
pool = sge.SGEPool()
#jobs_handle = pool.map_async(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))], chunksize=10)
#jobs_handle = pool.map_async(do_em_from_args, [(i, i, i) for i in range(len(args.observe_txt))], chunksize=10)
#jobs_handle = pool.imap_unordered(do_em_from_args, em_args, chunksize=1)
jobs_handle = pool.map_async(do_em_from_args, em_args, chunksize=1)
# wait for all jobs to finish
for j in jobs_handle:
#pass
j.wait()
# run one final accuemulator to get params
accuemulate_em_runs(args, gmtk_obs, gmtk_master)
plot_params(args)
plot_energy(args)
#check convergence
f = args.free_energy[-1]
try:
print 'free energy is', f, 'percent change ll:', abs(args.last_free_energy - f) / args.last_free_energy
except AttributeError:
print 'first iteration. free energy is', f
else:
if abs(abs(args.last_free_energy - f) / args.last_free_energy) < args.epsilon_e:
print 'converged! free energy is', f
break
finally:
args.last_free_energy = f
for p in args.params_to_save:
numpy.save(os.path.join(args.out_dir, args.out_params.format(param=p, **args.__dict__)),
args.__dict__[p])
if args.run_local:
try:
pool = multiprocessing.Pool()
pool.map_async(do_viterbi_from_args, em_args).get(99999999)
#map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, terminating workers"
pool.terminate()
pool.join()
else:
pool.close()
pool.join()
else:
pool = sge.SGEPool()
jobs_handle = pool.map_async(do_viterbi_from_args, em_args, chunksize=1)
# wait for all jobs to finish
for j in jobs_handle:
j.wait()
for a in em_args:
numpy.save('viterbi_Q_' + os.path.split(a[2])[1], parse_viterbi_states_to_Q(args, a[-1]))
0
Example 64
Project: kaggle_diabetic Source File: config.py
def save_features(self, X, n_iter, skip=0, test=False):
np.save(open(self.get_features_fname(n_iter, skip=skip,
test=test), 'wb'), X)
0
Example 65
Project: windml Source File: aemo.py
def convert(self):
def time_to_unix(datestr):
t = datetime.datetime.strptime(datestr, "%Y-%m-%d %H:%M:%S")
return time.mktime(t.timetuple())
if not os.path.exists(self.data_home_npy):
os.makedirs(self.data_home_npy)
# convert meta csv to meta numpy array
csvf = open(self.data_home_raw + "meta.csv", "r")
buf = csvf.readlines()
reader = csv.reader(buf, delimiter=',')
reader.next()
data = []
for row in reader:
point = []
point.append(self.park_id[row[0]])
point.append(row[3])
point.append(row[4])
point.append(row[5])
data.append(point)
data_arr = array([(a,b,c,d) for (a,b,c,d) in data], dtype = self.AEMO_META_DTYPE)
save(self.data_home_npy + "meta.npy", data_arr)
# convert data to windml format
turbine_arrays, turbine_npy_arrays = {}, {}
for k in self.park_id.keys():
turbine_arrays[k] = []
print "The following procedures are only necessary for the first time."
print "Converting AEMO data to lists and filtering missing data."
for year in self.years:
for month in self.months_in_year[year]:
location = self.data_home_raw + self.filename(year, month)
current = open(location, "r")
buf = current.readlines()
reader = csv.reader(buf, delimiter=',')
keys = reader.next()
for row in reader:
for i in range(1, len(row)):
# filter corrupt data
if(row[0] == ""):
break
if(row[i] == ""):
continue
timestamp = time_to_unix(row[0])
power = row[i]
turbine_arrays[keys[i]].append([timestamp, power])
current.close()
print "Converting to numpy arrays"
for k in turbine_arrays.keys():
data = turbine_arrays[k]
a = array([(a,b,nan) for (a,b) in data], dtype = self.AEMO_DATA_DTYPE)
turbine_npy_arrays[k] = a
save(self.data_home_npy + "%i.npy" % self.park_id[k], a)
0
Example 66
def save_weight(self, dir, name):
print 'weight saved: ' + name
np.save(dir + name + '.npy', self.val.get_value())
0
Example 67
def save_data(self, data, name):
fn = self.get_fn(name)
np.save(fn, data)
0
Example 68
Project: rf_helicopter Source File: pytests.py
def test_saving_tracks():
routes = Obstacle_Tracks(MAX_OBS_HEIGHT=MAX_OBS_HEIGHT,
MAX_OBS_WIDTH=MAX_OBS_WIDTH,
WINDOW_HEIGHT=WINDOW_HEIGHT,
WINDOW_WIDTH=WINDOW_WIDTH,
N_OBSTABLE_GEN=N_OBSTABLE_GEN,
MIN_GAP=MIN_GAP,
N_TRACKS_GEN=N_TRACKS_GEN)
# Generate Obstacles
output5 = routes.generate_tracks
# Get the first obstacle
saved = output5[0]
# Save the obstacle
np.save(os.path.join(os.getcwd(),
'Tests',
'Test_Track'), saved)
# Load obstacle
loaded = np.load(os.path.join(os.getcwd(),
'Tests',
'Test_Track.npy'))
assert saved.shape == loaded.shape, 'Dimensions Incorrect'
0
Example 69
Project: kaggle_diabetic Source File: config.py
def save_std(self, X, n_iter, skip=0, test=False):
np.save(open(self.get_std_fname(n_iter, skip=skip,
test=test), 'wb'), X)
0
Example 70
Project: cat-fancier Source File: train_model.py
def report(clf, testdata, testlabel, traindata_all, trainlabel_all, labels,
reportdir, modeltype, istrain=False):
print('# ----- Classification report -----')
if hasattr(clf, 'best_estimator_'):
print('## --- best estimator:')
print(clf.best_estimator_)
else:
print('## --- estimator:')
print(clf)
print('## test data shape: %s' % (testdata.shape,))
predlabel = clf.predict(testdata)
print('## accuracy: %s' % (accuracy_score(testlabel, predlabel),)) ## == clf.score
cm = confusion_matrix(testlabel, predlabel)
print('## confusion matrix')
print(cm)
cr = classification_report(testlabel, predlabel, target_names=labels)
print(cr)
if istrain:
print('save classification report files')
np.save(reportdir + 'cm_' + modeltype, cm)
np.save(reportdir + 'cr_' + modeltype, cr)
0
Example 71
Project: westpa Source File: gen_bstate.py
def run(platform_name, deviceid, two):
not_obs = [True, True]
system, coordinates = wcadimer.WCADimer()
print "Time step: ", (wcadimer.stable_timestep * 2.0).in_units_of(units.femtoseconds)
# Minimization
platform = openmm.Platform.getPlatformByName('Reference')
print 'Minimizing energy...'
coordinates = minimize(platform, system, coordinates)
print 'Separation distance: {}'.format(norm(coordinates[1,:] - coordinates[0,:]) / units.angstroms)
print 'Equilibrating...'
platform = openmm.Platform.getPlatformByName(platform_name)
if platform_name == 'CUDA':
platform.setPropertyDefaultValue('CudaDeviceIndex', '%d' % deviceid)
platform.setPropertyDefaultValue('CudaPrecision', 'mixed')
elif platform_name == 'OpenCL':
platform.setPropertyDefaultValue('OpenCLDeviceIndex', '%d' % deviceid)
platform.setPropertyDefaultValue('OpenCLPrecision', 'mixed')
integrator = openmm.LangevinIntegrator(wcadimer.temperature,
wcadimer.collision_rate,
2.0 * wcadimer.stable_timestep)
context = openmm.Context(system, integrator, platform)
context.setPositions(coordinates)
context.setVelocitiesToTemperature(wcadimer.temperature)
if two:
while not_obs[0] or not_obs[1]:
integrator.step(5000)
state = context.getState(getPositions=True)
coordinates = state.getPositions(asNumpy=True)
sep_dist = norm(coordinates[1,:] - coordinates[0,:]) / units.angstroms
print 'Separation distance: {}'.format(sep_dist)
if sep_dist < 5.7:
not_obs[0] = False
tag = '_a'
sep_dist_a = sep_dist
else:
not_obs[1] = False
tag = '_b'
sep_dist_b = sep_dist
if not os.path.isdir('bstates'):
os.makedirs('bstates')
np.save('bstates/init_coords{}.npy'.format(tag), coordinates / units.nanometers)
print sep_dist_a, sep_dist_b
else:
integrator.step(5000)
state = context.getState(getPositions=True)
coordinates = state.getPositions(asNumpy=True)
print 'Separation distance: {}'.format(norm(coordinates[1,:] - coordinates[0,:]) / units.angstroms)
if not os.path.isdir('bstates'):
os.makedirs('bstates')
np.save('bstates/init_coords.npy', coordinates / units.nanometers)
0
Example 72
def load_niftis(source_dir, out_dir, name='mri', patterns=None):
'''Loads niftis from a directory.
Saves the data, paths, mask, and `sites`.
Args:
source_dir (str): Directory of nifti files.
out_dir (str): Output directory for saving arrays, etc.
name (str): Name of dataset.
patterns (Optional[list]): list of glob for filtering files.
'''
if patterns is not None:
file_lists = []
for i, pattern in enumerate(patterns):
file_list = glob(path.join(source_dir, pattern))
file_lists.append(file_list)
else:
file_lists = [find_niftis(source_dir)]
base_file = file_lists[0][0]
paths_file = path.join(out_dir, name + '_file_paths.npy')
sites_file = path.join(out_dir, name + '_sites.npy')
mask_file = path.join(out_dir, name + '_mask.npy')
yaml_file = path.join(out_dir, name + '.yaml')
tmp_dir = path.join(out_dir, name + '_tmp')
if not path.isdir(tmp_dir):
os.mkdir(tmp_dir)
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind('tab: complete')
readline.set_completer(complete_path)
print ('The MRI dataset requires an anatomical nifti file to visualize'
' properly. Enter the path for the anatomical file or leave blank'
' if you plan not to use visualization or will enter into the yaml'
' file later.')
anat_file = raw_input('Anat file: ')
if anat_file == '': yaml_file = None
datas = []
new_file_lists = []
data_paths = []
for i, file_list in enumerate(file_lists):
data, new_file_list = read_niftis(file_list)
new_file_lists.append(new_file_list)
datas.append(data)
data_path = path.join(out_dir, name + '_%d.npy' % i)
data_paths.append(data_path)
np.save(data_path, data)
sites = [[0 if 'st' in f else 1 for f in fl] for fl in file_lists]
sites = sites[0] + sites[1]
save_mask(np.concatenate(datas, axis=0), mask_file)
np.save(paths_file, new_file_lists)
np.save(sites_file, sites)
with open(yaml_file, 'w') as yf:
yf.write(
yaml.dump(
dict(name=name,
data=data_paths,
mask=mask_file,
nifti=base_file,
sites=sites_file,
tmp_path=tmp_dir,
anat_file=anat_file
)
)
)
0
Example 73
def main(input_file):
with open(input_file) as f:
graph = pickle.load(f)
node_map = {int(node_id): i for i, node_id in enumerate(graph.nodes())}
outcomes = []
for fn, name in [
('data/sfnodesdtINTOXICATIONCRIME.csv', 'intoxication'),
('data/sfnodesdtPROPERTYCRIME.csv', 'property'),
('data/sfnodesdtVIOLENTCRIME.csv', 'violent'),
]:
df = pd.read_csv(fn)
df['crime_type'] = name
outcomes.append(df)
df = pd.concat(outcomes)
df['id'] = df['id'].apply(node_map.get)
df = df[df['id'].notnull()]
for (tod, dow), time_df in df.groupby(['daytime', 'superday']):
mat = time_df.set_index(['id', 'crime_type'])['preds'].unstack()
outfile = 'data/sf_crime_risks_{}_{}.npy'.format(
tod.lower().replace('-','_'),
dow.lower()
)
np.save(outfile, mat.values)
0
Example 74
Project: models Source File: train_student.py
def prepare_student_data(dataset, nb_teachers, save=False):
"""
Takes a dataset name and the size of the teacher ensemble and prepares
training data for the student model, according to parameters indicated
in flags above.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param save: if set to True, will dump student training labels predicted by
the ensemble of teachers (with Laplacian noise) as npy files.
It also dumps the clean votes for each class (without noise) and
the labels assigned by teachers
:return: pairs of (data, labels) to be used for student training and testing
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Load the dataset
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=True)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=True)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
else:
print("Check value of dataset flag")
return False
# Make sure there is data leftover to be used as a test set
assert FLAGS.stdnt_share < len(test_data)
# Prepare [unlabeled] student training data (subset of test set)
stdnt_data = test_data[:FLAGS.stdnt_share]
# Compute teacher predictions for student training data
teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)
# Aggregate teacher predictions to get student training labels
if not save:
stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
else:
# Request clean votes and clean labels as well
stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line)
# Prepare filepath for numpy dump of clean votes
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_clean_votes_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Prepare filepath for numpy dump of clean labels
filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_teachers_labels_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Dump clean_votes array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, clean_votes)
# Dump labels_for_dump array
with tf.gfile.Open(filepath_labels, mode='w') as file_obj:
np.save(file_obj, labels_for_dump)
# Print accuracy of aggregated labels
ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share])
print("Accuracy of the aggregated labels: " + str(ac_ag_labels))
# Store unused part of test set for use as a test set after student training
stdnt_test_data = test_data[FLAGS.stdnt_share:]
stdnt_test_labels = test_labels[FLAGS.stdnt_share:]
if save:
# Prepare filepath for numpy dump of labels produced by noisy aggregation
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_labels_lap_' + str(FLAGS.lap_scale) + '.npy' #NOLINT(long-line)
# Dump student noisy labels array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, stdnt_labels)
return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
0
Example 75
Project: histwords Source File: pmi2svd.py
def main():
args = docopt("""
Usage:
pmi2svd.py [options] <pmi_path> <output_path>
Options:
--dim NUM Dimensionality of eigenvectors [default: 500]
--neg NUM Number of negative samples; subtracts its log from PMI [default: 1]
""")
pmi_path = args['<pmi_path>']
output_path = args['<output_path>']
dim = int(args['--dim'])
neg = int(args['--neg'])
explicit = PositiveExplicit(pmi_path, normalize=False, neg=neg)
ut, s, vt = sparsesvd(explicit.m.tocsc(), dim)
np.save(output_path + '.ut.npy', ut)
np.save(output_path + '.s.npy', s)
np.save(output_path + '.vt.npy', vt)
save_vocabulary(output_path + '.words.vocab', explicit.iw)
save_vocabulary(output_path + '.contexts.vocab', explicit.ic)
0
Example 76
def worker(proc_num, queue, out_dir, in_dir, count_dir, words, dim, num_words, min_count=100):
while True:
if queue.empty():
break
year = queue.get()
print "Loading embeddings for year", year
time.sleep(random.random() * 120)
valid_words = set(words_above_count(count_dir, year, min_count))
print len(valid_words)
words = list(valid_words.intersection(words[year][:num_words]))
print len(words)
base_embed = Explicit.load((in_dir + INPUT_FORMAT).format(year=year), normalize=False)
base_embed = base_embed.get_subembed(words, restrict_context=True)
print "SVD for year", year
u, s, v = randomized_svd(base_embed.m, n_components=dim, n_iter=5)
print "Saving year", year
np.save((out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-u.npy", u)
np.save((out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-v.npy", v)
np.save((out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-s.npy", s)
write_pickle(base_embed.iw, (out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-vocab.pkl")
0
Example 77
Project: datafeed Source File: server.py
def get_1minute(self, symbol, date, format='npy'):
"""Get 5min historical quotes.
Arguments:
symbol: String of security.
date: Which day data to get.
format: npy or json
"""
try:
if isinstance(date, str):
date = datetime.datetime.strptime(date, '%Y%m%d').date()
y = self.dbm.oneminstore.get(symbol, date)
if format == 'npy':
memfile = StringIO()
np.save(memfile, y)
data = memfile.getvalue()
del(y)
else:
data = json_encode(y.tolist())
self._write_response(data)
except KeyError:
self.request.write("-ERR No data.\r\n")
0
Example 78
Project: tree-hmm Source File: vb_gmtkExact.py
def run_gmtk_lineagehmm(args):
try:
os.mkdir('gmtk_images')
except:
pass
args.iteration = 'initial'
args.observe = 'all'
args.run_name = 'gmtk'
args.out_params = 'gmtk_images/gmtk_{param}_{observe}'
args.out_dir = '.'
args.free_energy = []
plot_params(args)
# convert .npy parameters and data into gmtk format
args.observe_txt = []
for f in args.observe_matrix:
obs, obs_txt = npy_observations_to_txt(f)
args.observe_txt.append(obs_txt)
# prepare and triangulate the graphical model
strfile = 'lineagehmm.str'
make_lineage_model(strfile, obs.shape[0], args.K, obs.shape[2], vert_parent=args.vert_parent, mark_avail=mark_avail, separate_theta=args.separate_theta)
cmd = 'gmtkTriangulate -strFile %s -rePart T -findBest T -triangulation completed ' % strfile
#cmd = 'gmtkTriangulate -strFile %s' % strfile
subprocess.check_call(cmd, shell=True)
# populate theta parts if they don't exist
if args.separate_theta and any(not hasattr(args, 'theta_%s' % i) for i in range(2,args.I+1)):
for i in range(2,args.I+1):
if len(args.theta.shape) == 3:
setattr(args, 'theta_%s' % i, args.theta)
else:
setattr(args, 'theta_%s' % i, args.theta[i-2,:,:,:])
args.params_to_save.append('theta_%s' % i)
#del args.theta
#args.params_to_save.remove('theta')
# for each iteration...
for args.iteration in range(1, args.max_iterations+1):
# save args to disk
w = npy_params_to_workspace(args)
em_args = []
for index in range(len(args.observe_txt)):
gmtk_master = '%s.master' % args.observe_txt[index]
gmtk_obs = '%s.observations' % args.observe_txt[index]
write_workspace_simple_master(w, gmtk_master)
em_args.append((index, args.iteration, args.observe_txt[index], args.I, args.L, gmtk_master, gmtk_obs))
# run a gmtk em iteration on each file input, accuemulating results
if args.run_local:
try:
pool = multiprocessing.Pool(maxtasksperchild=1)
pool.map_async(do_em_from_args, em_args).get(99999999)
#map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, terminating workers"
pool.terminate()
pool.join()
else:
pool.close()
pool.join()
else:
pool = sge.SGEPool()
#jobs_handle = pool.map_async(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))], chunksize=10)
#jobs_handle = pool.map_async(do_em_from_args, [(i, i, i) for i in range(len(args.observe_txt))], chunksize=10)
#jobs_handle = pool.imap_unordered(do_em_from_args, em_args, chunksize=1)
jobs_handle = pool.map_async(do_em_from_args, em_args, chunksize=1)
# wait for all jobs to finish
for j in jobs_handle:
#pass
j.wait()
# run one final accuemulator to get params
accuemulate_em_runs(args, gmtk_obs, gmtk_master)
plot_params(args)
plot_energy(args)
#check convergence
f = args.free_energy[-1]
try:
print 'free energy is', f, 'percent change ll:', abs(args.last_free_energy - f) / args.last_free_energy
except AttributeError:
print 'first iteration. free energy is', f
else:
if abs(abs(args.last_free_energy - f) / args.last_free_energy) < args.epsilon_e:
print 'converged! free energy is', f
break
finally:
args.last_free_energy = f
for p in args.params_to_save:
numpy.save(os.path.join(args.out_dir, args.out_params.format(param=p, **args.__dict__)),
args.__dict__[p])
if args.run_local:
try:
pool = multiprocessing.Pool()
pool.map_async(do_viterbi_from_args, em_args).get(99999999)
#map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, terminating workers"
pool.terminate()
pool.join()
else:
pool.close()
pool.join()
else:
pool = sge.SGEPool()
jobs_handle = pool.map_async(do_viterbi_from_args, em_args, chunksize=1)
# wait for all jobs to finish
for j in jobs_handle:
j.wait()
for a in em_args:
numpy.save('viterbi_Q_' + os.path.split(a[2])[1], parse_viterbi_states_to_Q(args, a[-1]))
0
Example 79
def get_day(self, symbol, length_or_date, format='npy'):
"""Get OHLCs quotes.
Return chronicle ordered quotes.
"""
try:
if len(length_or_date) == 8: # eg: 20101209
date = datetime.datetime.strptime(length_or_date, '%Y%m%d').date()
y = self.dbm.daystore.get_by_date(symbol, date)
else:
length = length_or_date
y = self.dbm.daystore.get(symbol, int(length))
if length == 1:
y = y[0]
if format == 'npy':
memfile = StringIO()
np.save(memfile, y)
data = memfile.getvalue()
del(y)
else:
data = json_encode(y.tolist())
self._write_response(data)
except KeyError:
self.request.write("-ERR Symbol %s not exists.\r\n" % symbol)
0
Example 80
def save_weight(self, dir, epoch = 0):
print '- Saved weight: ' + self.name + '_ep'+str(epoch)
numpy.save(dir + self.name + '_ep'+str(epoch), self.val.get_value())
0
Example 81
def save_npy(path, obj):
np.save(path, obj)
print(" [*] save %s" % path)
0
Example 82
def save_weight(self, dir, name):
#print 'weight saved: ' + name
np.save(dir + name + '.npy', self.val.get_value())
0
Example 83
def _memmap_array(x, memmap_dir=None, use_h5py=False, unique_id=''):
if memmap_dir is None:
memmap_dir = tempfile.gettempdir()
# generate the base filename
filename = os.path.join(memmap_dir,
unique_id + '_' + str(id(x)))
if use_h5py:
import h5py
filename += '.h5'
h = h5py.File(filename)
mmap_dat = h.create_dataset('mdat', data=x,
compression='gzip')
h.flush()
else:
# use normal memmap
# filename = os.path.join(memmap_dir, str(id(x)) + '.npy')
filename += '.npy'
np.save(filename, x)
mmap_dat = np.load(filename, 'r+')
return mmap_dat
0
Example 84
def save_npy(self, filename):
np.save(filename, self.get_params())
0
Example 85
Project: rf_helicopter Source File: pytests.py
def test_saving_obstacles():
routes = Obstacle_Tracks(MAX_OBS_HEIGHT=MAX_OBS_HEIGHT,
MAX_OBS_WIDTH=MAX_OBS_WIDTH,
WINDOW_HEIGHT=WINDOW_HEIGHT,
WINDOW_WIDTH=WINDOW_WIDTH,
N_OBSTABLE_GEN=N_OBSTABLE_GEN,
MIN_GAP=MIN_GAP,
N_TRACKS_GEN=N_TRACKS_GEN)
# Generate Obstacles
output4 = routes.generate_obstacles
# Get the first obstacle
saved = output4[0]
# Save the obstacle
np.save(os.path.join(os.getcwd(),
'Tests',
'Test_Obstacle'), saved)
# Load obstacle
loaded = np.load(os.path.join(os.getcwd(),
'Tests',
'Test_Obstacle.npy'))
assert saved.shape == loaded.shape, 'Dimensions Incorrect'
0
Example 86
def save_momentums(vels, weights_dir, epoch):
for ind in range(len(vels)):
np.save(os.path.join(weights_dir, 'mom_' + str(ind) + '_' + str(epoch)),
vels[ind].get_value())
0
Example 87
Project: models Source File: train_student.py
def prepare_student_data(dataset, nb_teachers, save=False):
"""
Takes a dataset name and the size of the teacher ensemble and prepares
training data for the student model, according to parameters indicated
in flags above.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param save: if set to True, will dump student training labels predicted by
the ensemble of teachers (with Laplacian noise) as npy files.
It also dumps the clean votes for each class (without noise) and
the labels assigned by teachers
:return: pairs of (data, labels) to be used for student training and testing
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Load the dataset
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=True)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=True)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
else:
print("Check value of dataset flag")
return False
# Make sure there is data leftover to be used as a test set
assert FLAGS.stdnt_share < len(test_data)
# Prepare [unlabeled] student training data (subset of test set)
stdnt_data = test_data[:FLAGS.stdnt_share]
# Compute teacher predictions for student training data
teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)
# Aggregate teacher predictions to get student training labels
if not save:
stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
else:
# Request clean votes and clean labels as well
stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line)
# Prepare filepath for numpy dump of clean votes
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_clean_votes_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Prepare filepath for numpy dump of clean labels
filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_teachers_labels_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Dump clean_votes array
with gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, clean_votes)
# Dump labels_for_dump array
with gfile.Open(filepath_labels, mode='w') as file_obj:
np.save(file_obj, labels_for_dump)
# Print accuracy of aggregated labels
ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share])
print("Accuracy of the aggregated labels: " + str(ac_ag_labels))
# Store unused part of test set for use as a test set after student training
stdnt_test_data = test_data[FLAGS.stdnt_share:]
stdnt_test_labels = test_labels[FLAGS.stdnt_share:]
if save:
# Prepare filepath for numpy dump of labels produced by noisy aggregation
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_labels_lap_' + str(FLAGS.lap_scale) + '.npy' #NOLINT(long-line)
# Dump student noisy labels array
with gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, stdnt_labels)
return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
0
Example 88
def main():
fname = os.path.join(DATA_DIR, CSV_FNAME)
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
# run this at first to make sure everything looks good
if RUN_CHECKS:
print 'Running checks'
checks()
sys.stdout.flush()
# then run to to generate the vectorized data from the raw dump (already done)
if REGEN_DATA:
print 'Re-generating data'
sys.stdout.flush()
data, labels, ranges = subsample_and_vectorize_data(fname, LABEL, PRETTY_PRINT_LABEL)
with open(VECTOR_DATA_PATH, 'wb') as file:
np.save(file, data)
data = None
with open(VECTOR_LABELS_PATH, 'wb') as file:
np.save(file, labels)
labels = None
with open(VALUE_RANGES_PATH, 'wb') as file:
json.dump(ranges, file, indent=4)
ranges = None
# finally train the model
if TRAIN == 'deep_model':
print 'Training deep model'
sys.stdout.flush()
train_deep_model()
test_deep_model()
# you can also train a logreg
if TRAIN == 'logistic_regression':
print 'Training logistic regression'
sys.stdout.flush()
train_logistic_regression()
# or train a random forest
if TRAIN == 'random_forest':
print 'Training random forest'
sys.stdout.flush()
top_features = train_random_forest()
with open(IMPORTANCES_CSV, 'wb') as f:
f.write('feature,importance\n')
for feature, importance in top_features:
f.write('%s,%.4f\n' % (feature, importance))
0
Example 89
def main():
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
# then run to to generate the vectorized data from the raw dump (already done)
if REGEN_DATA or \
not os.path.exists(VECTOR_DATA_PATH) or \
not os.path.exists(VECTOR_LABELS_PATH) or \
not os.path.exists(VALUE_RANGES_PATH):
print 'Re-generating data'
csv_fname = os.path.join(DATA_DIR, CSV_FNAME)
sys.stdout.flush()
data, labels, ranges = subsample_and_vectorize_data(csv_fname, LABEL, PRETTY_PRINT_LABEL)
with open(VECTOR_DATA_PATH, 'wb') as file:
np.save(file, data)
data = None
with open(VECTOR_LABELS_PATH, 'wb') as file:
np.save(file, labels)
labels = None
with open(VALUE_RANGES_PATH, 'wb') as file:
json.dump(ranges, file, indent=4)
ranges = None
features = load_feature_names()
(x_train_full, y_train), (x_val_full, y_val) = prepare_data()
train_rows, train_cols = x_train_full.shape
val_rows, val_cols = x_val_full.shape
# Figure out how many columns we need for the known starting features
fname = os.path.join(RESULTS_DIR, PRETTY_PRINT_LABEL + '_accuracy_test')
base_columns = 0;
for name in starting_features:
if name in features:
fname += "." + name
base_columns += features[name]['end'] - features[name]['start'] + 1
fname += ".csv"
# Try training each feature against the data set individually
feature_count = len(features)
feature_num = 0
with open(fname, 'wb', 1) as out:
for name, feature in features.iteritems():
if not len(test_features) or name in test_features:
feature_num += 1
#Build an input data set with just the columns we care about
count = feature['end'] - feature['start'] + 1
x_train = np.zeros((train_rows, base_columns + count))
x_val = np.zeros((val_rows, base_columns + count))
col = 0
# Populate the starting features
for n in starting_features:
if n == name:
continue
if n in features:
for column in xrange(features[n]['start'], features[n]['end'] + 1):
x_train[:, col] = x_train_full[:, column]
x_val[:, col] = x_val_full[:, column]
col += 1
# Populate the features we are testing
for column in xrange(feature['start'], feature['end'] + 1):
x_train[:,col] = x_train_full[:,column]
x_val[:, col] = x_val_full[:, column]
col += 1
# normalize the data
scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
x_val = scaler.transform(x_val)
# Run the actual training
print '[{0:d}/{1:d}] Training deep model on {2} ({3:d} columns)'.format(feature_num, feature_count, name, col)
sys.stdout.flush()
acc, model = train_deep_model(x_train, y_train, x_val, y_val)
print '{0} Accuracy: {1:0.4f}'.format(name, acc)
sys.stdout.flush()
out.write('{0},{1:0.4f}\n'.format(name,acc))
# Test the varous values for the feature
if len(test_features):
max_val = 100000
min_val = 100
step_size = 100
count = (max_val - min_val) / step_size
original_values = np.array([[0.0]] * count)
row = 0
for value in xrange(100, 100000, 100):
original_values[row] = value
row += 1
data = scaler.transform(original_values)
prob = model.predict_proba(data, verbose=0)
with open(os.path.join(RESULTS_DIR, PRETTY_PRINT_LABEL + '_values_' + name), 'wb') as v:
for row in xrange(0, count):
value = original_values[row][0]
probability = prob[row][0]
v.write('{0:d},{1:f}\n'.format(int(value), probability))
0
Example 90
def save(self, folder):
for param, name in zip(self.params, self.names):
np.save(os.path.join(folder, name + '.npy'), param.get_value())
0
Example 91
def main(min_portals=15,max_portals=60,d_portals=5,
min_agents=1,max_agents=5,d_agents=1,
trials=10,
outfile='runtimes.csv',
timeout=600):
"""
Run a series of maxfields to estimate the runtime for a
given number of portals and a given number of agents
Run each test trials times
Saves runtimes to outfile
"""
args = []
# Set up array of jobs
# loop over portal number
for nportals in range(min_portals,max_portals+1,d_portals):
# loop over number of agents
for nagents in range(min_agents,max_agents+1,d_agents):
# Loop over each trial
for trial in range(trials):
# Once with and once without skipping plots
for skipplot in [True,False]:
args.append((nportals,nagents,trial,skipplot))
with process.Pool(4) as p:
jobs = [p.schedule(worker,args=arg,timeout=timeout) for arg in args]
runtimes = []
for arg,job in zip(args,jobs):
try:
runtimes.append(job.get())
except TimeoutError:
runtimes.append((arg[0],arg[1],arg[2],arg[3],'nan'))
cleanup(arg[0],arg[1],arg[2],arg[3])
np.save('runtimes.npy',runtimes)
with open(outfile,'w') as f:
f.write('nportals, nagents, trial, skipplot, runtime\n')
for foo in runtimes:
nportals, nagents, trial, skipplot, runtime = foo
f.write('{0}, {1}, {2}, {3}, {4}\n'.format(nportals,nagents,trial,skipplot,runtime))
0
Example 92
Project: datafeed Source File: client.py
def put_minute(self, symbol, rawdata):
memfile = StringIO()
np.save(memfile, rawdata)
return self.execute_command('PUT_MINUTE', symbol, memfile.getvalue(), 'npy')
0
Example 93
Project: datafeed Source File: client.py
def put_1minute(self, symbol, rawdata):
memfile = StringIO()
np.save(memfile, rawdata)
return self.execute_command('PUT_1MINUTE', symbol, memfile.getvalue(), 'npy')
0
Example 94
Project: datafeed Source File: client.py
def put_5minute(self, symbol, rawdata):
memfile = StringIO()
np.save(memfile, rawdata)
return self.execute_command('PUT_5MINUTE', symbol, memfile.getvalue(), 'npy')
0
Example 95
Project: datafeed Source File: client.py
def put_day(self, symbol, rawdata):
memfile = StringIO()
np.save(memfile, rawdata)
return self.execute_command('PUT_DAY', symbol, memfile.getvalue(), 'npy')
0
Example 96
def savenpy(filename, v):
"""Saves numpy binary file."""
np.save(filename, v)
os.rename(filename + '.npy', filename)
0
Example 97
def load_niftis(source_dir, out_dir, name='fmri', patterns=None):
'''Loads niftis from a directory.
Saves the data, paths, mask, and `sites`.
Args:
source_dir (str): Directory of nifti files.
out_dir (str): Output directory for saving arrays, etc.
name (str): Name of dataset.
patterns (Optional[list]): list of glob for filtering files.
'''
if patterns is not None:
file_lists = []
for i, pattern in enumerate(patterns):
file_list = glob(path.join(source_dir, pattern))
file_lists.append(file_list)
else:
file_lists = [find_niftis(source_dir)]
base_file = file_lists[0][0]
paths_file = path.join(out_dir, name + '_file_paths.npy')
sites_file = path.join(out_dir, name + '_sites.npy')
mask_file = path.join(out_dir, name + '_mask.npy')
yaml_file = path.join(out_dir, name + '.yaml')
tmp_dir = path.join(out_dir, name + '_tmp')
if not path.isdir(tmp_dir):
os.mkdir(tmp_dir)
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind('tab: complete')
readline.set_completer(complete_path)
print ('The fMRI dataset requires an anatomical nifti file to visualize'
' properly. Enter the path for the anatomical file or leave blank'
' if you plan not to use visualization or will enter into the yaml'
' file later.')
anat_file = raw_input('Anat file: ')
if anat_file == '': yaml_file = None
datas = []
new_file_lists = []
data_paths = []
for i, file_list in enumerate(file_lists):
data, new_file_list = read_niftis(file_list)
new_file_lists.append(new_file_list)
datas.append(data)
data_path = path.join(out_dir, name + '_%d.npy' % i)
data_paths.append(data_path)
np.save(data_path, data)
sites = [[0 if 'st' in f else 1 for f in fl] for fl in file_lists]
sites = sites[0] + sites[1]
save_mask(np.concatenate(datas, axis=0), mask_file)
np.save(paths_file, new_file_lists)
np.save(sites_file, sites)
with open(yaml_file, 'w') as yf:
yf.write(
yaml.dump(
dict(name=name,
data=data_paths,
mask=mask_file,
nifti=base_file,
sites=sites_file,
tmp_path=tmp_dir,
anat_file=anat_file
)
)
)
0
Example 98
def endIter(self):
np.save('{0:03d}-{1!s}'.format(self.opt.iter, self.fileName), self.opt.xc)
0
Example 99
def main():
GTEx_gctobj = gct.GCT(GTEx_GCTX)
GTEx_gctobj.read()
GTEx_genes = map(lambda x:x.split('.')[0], GTEx_gctobj.get_rids())
lm_id = []
infile = open(BGEDV2_LM_ID)
for line in infile:
ID = line.strip('\n').split('\t')[0]
lm_id.append(ID)
infile.close()
lm_idx = map(GTEx_genes.index, lm_id)
tg_id = []
infile = open(BGEDV2_TG_ID)
for line in infile:
ID = line.strip('\n').split('\t')[0]
tg_id.append(ID)
infile.close()
tg_idx = map(GTEx_genes.index, tg_id)
genes_idx = lm_idx + tg_idx
data = GTEx_gctobj.matrix[genes_idx, :].astype('float64')
np.save('GTEx_float64.npy', data)
0
Example 100
def get_minute(self, symbol, timestamp, format='npy'):
"""Get daily minutes history.
Arguments:
symbol: String of security.
timestamp: Which day data to get.
format: npy or json
"""
try:
ts = int(timestamp)
if ts > 0:
store = self.dbm.get_minutestore_at(ts)
else:
store = self.dbm.minutestore
y = store.get(symbol)
if format == 'npy':
memfile = StringIO()
np.save(memfile, y)
data = memfile.getvalue()
del(y)
else:
data = json_encode(y.tolist())
self._write_response(data)
except KeyError:
self.request.write("-ERR Symbol %s not exists.\r\n" % symbol)