Here are the examples of the python api numpy.genfromtxt taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
136 Examples
4
Example 1
def load_file(self, fname):
cur_dir = os.path.dirname(os.path.abspath(__file__))
data = np.genfromtxt(os.path.join(cur_dir, 'results', fname),
delimiter=" ")
time = data[:,0]
status = data[:,1]
entry = data[:,2]
exog = data[:,3:]
return time, status, entry, exog
3
Example 2
Project: gala Source File: test_sgr.py
def test_against_David_Law():
""" Test my code against an output file from using David Law's cpp code. Do:
g++ SgrCoord.cpp; ./a.out
to generate the data file, SgrCoord_data.
"""
filename = get_pkg_data_filename('SgrCoord_data')
law_data = np.genfromtxt(filename, names=True, delimiter=',')
c = coord.Galactic(law_data["l"]*u.deg, law_data["b"]*u.deg)
sgr_coords = c.transform_to(Sagittarius)
law_sgr_coords = Sagittarius(Lambda=law_data["lambda"]*u.deg, Beta=law_data["beta"]*u.deg)
sep = sgr_coords.separation(law_sgr_coords).arcsec*u.arcsec
assert np.all(sep < 1.*u.arcsec)
3
Example 3
def train(classifier, train_data_filename, save_classifier_filename=None):
"""Trains and saves classifier so that it could be easily loaded later."""
file_data = genfromtxt(train_data_filename, delimiter=",")
train_data, labels = file_data[:, :-1], file_data[:, -1]
classifier.fit(train_data, labels)
if save_classifier_filename:
joblib.dump(classifier, save_classifier_filename)
return classifier
3
Example 4
Project: solpy Source File: epw_thermal.py
def read_EPW(self):
EPW_vars = ('year','month','day','hour','minute','datasource','t_amb','t_dp',
'rh_amb','press','g_o','g_on','ir_sky','g','g_n','g_d','ill','ill_n','ill_d',
'zen_lum','winddir','windspeed','sky_t','sky_o','visibility','ceiling',
'presweathobs','presweathcodes','precipwtr','aerosoloptdepth','snowdepth',
'dayslastsnow','albedo','rain','rain_hr')
out_vars = ['time','t_amb','t_dp','rh_amb','press',
'g_o','g_on','ir_sky','g','g_n','g_d','winddir','windspeed'] # Add as needed
out_types = tuple(['object'] + ['float']*12)
self.units = ('-','C','C','%','Pa','Wh/m2','Wh/m2','Wh/m2','Wh/m2',
'Wh/m2','Wh/m2','deg','m/s')
raw = np.genfromtxt(self.filename,dtype=None,delimiter=',',skiprows=8)
raw.dtype.names = EPW_vars
data = raw.astype(zip(out_vars,out_types))
data['time'] = [datetime(y,m,d,h-1,30) for y,m,d,h in zip(raw['year'],raw['month'],raw['day'],raw['hour'])]
print('\n Read from file %s: data for %s' % (self.filename, data['time'][0].strftime("%b %d, %Y (%A)") ))
return data
3
Example 5
Project: vsm Source File: FileReadWrite.py
def file_to_mat(filename):
"""
Data to an array. works for theta, phi.
Removes automatically added 'missing values' at the end of the rows.
"""
arr = np.genfromtxt(filename, delimiter=',')
return arr[:,:-1]
3
Example 6
Project: siphon Source File: ncss.py
def parse_csv_dataset(data, handle_units):
fobj = BytesIO(data)
names, units = parse_csv_header(fobj.readline().decode('utf-8'))
arrs = np.genfromtxt(fobj, dtype=None, names=names, delimiter=',', unpack=True,
converters={'date': lambda s: parse_iso_date(s.decode('utf-8'))})
d = dict()
for f in arrs.dtype.fields:
dat = arrs[f]
if dat.dtype == np.object:
dat = dat.tolist()
d[f] = handle_units(dat, units.get(f, None))
return d
3
Example 7
def setup(self):
with tempfile.NamedTemporaryFile(mode='w+b') as temp:
temp.write(_txt.encode('utf-8'))
temp.flush()
temp.seek(0)
self.data = np.genfromtxt(temp, names=True, skip_header=1)
3
Example 8
def loader(self, fn):
return np.genfromtxt(fn, delimiter=self.delimiter,
skip_header=self.skip_header,
skip_footer=self.skip_footer,
filling_values=self.filling_values,
usecols=self.usecols)
3
Example 9
@classmethod
def setupClass(cls):
cls.decimal_density = 2 # low accuracy because binning is different
res1 = KDE(Xi)
res1.fit(kernel="gau", fft=True, bw="silverman")
cls.res1 = res1
rfname2 = os.path.join(curdir,'results','results_kde_fft.csv')
cls.res_density = np.genfromtxt(open(rfname2, 'rb'))
3
Example 10
Project: SciDB-Py Source File: _py3k_compat.py
def genfromstr(s, **kwargs):
"""Utility routine to create an array from a string.
This uses either StringIO or BytesIO, depending on whether we're on
Python 2 or Python 3.
"""
return np.genfromtxt(stringio(s), **kwargs)
3
Example 11
@plac.annotations(features_fpath=plac.Annotation('Partial Features',
type=str),
tag_categ_fpath=plac.Annotation('Tags file', type=str),
tseries_fpath=plac.Annotation('Time series file', type=str),
num_days_to_use=plac.Annotation('Num Days Series', type=int),
assign_fpath=plac.Annotation('Series assignment file',
type=str),
out_foldpath=plac.Annotation('Output folder', type=str))
def main(features_fpath, tag_categ_fpath, tseries_fpath, num_days_to_use,
assign_fpath, out_foldpath):
X, feature_ids, _ = \
create_input_table(features_fpath, tseries_fpath, tag_categ_fpath,
num_days_to_use)
X = scale(X)
y_clf = np.genfromtxt(assign_fpath)
y_regr = scale(np.genfromtxt(tseries_fpath)[:,1:].sum(axis=1))
run_experiment(X, y_clf, y_regr, feature_ids, out_foldpath)
3
Example 12
def __init__(self, path):
file_path = "{}labels.csv".format(path)
try:
data = np.genfromtxt(file_path, delimiter=',', skip_header=1,
names=['timestamp', 'label'], dtype=[("timestamp", long), ('label', int)])
self.has_label = True
except IOError:
self.has_label = False
return
self.timestamp = data['timestamp']
label = data['label']
self.label = []
for i in range(0, len(label)):
self.label.append(chr(int(label[i])))
self.diff = np.diff(self.timestamp)
3
Example 13
Project: Deep-Spying Source File: View.py
def plot_sensor_data_from_file(self, file_path):
if not self.to_save and not self.to_show:
return
data = np.genfromtxt(file_path, delimiter=',', skip_header=1, names=['timestamp', 'x', 'y', 'z'])
file_name = basename(file_path)
file_name = file_name[:file_name.find(".")]
View.plot_sensor_data(basename(file_name), data['timestamp'], data['x'], data['y'], data['z'])
3
Example 14
def test_recover(self):
"""Test recovery from outliers.
"""
# run recovery
lr, sp, _ = ealm.recover(self._data, None)
# load baseline (no outliers)
file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "im_baseline.dat")
baseline = np.genfromtxt(file_path)
# Frobenius norm between recovered mat. and baseline
d = np.linalg.norm(np.round(lr)-baseline, ord='fro')
self.assertTrue(np.allclose(d,0.0))
3
Example 15
def get_testing_data(file='data/GAMETES-test.csv.gz', target=-1,
return_header=False, head=0):
"""Loads the desired data for testing ReliefF's functions"""
data = np.genfromtxt(file, delimiter=',')
header, data = data[head], data[head+1:] # Separate header (first) row
y = data[:, target] # labels of the data
X = np.delete(data, target, axis=1) # Remove labels from data
X_train, X_test, y_train, y_test = train_test_split(X, y,
random_state=34895)
# The user can decide if they want the headers returned as first value
if return_header:
return header, X_train, X_test, y_train, y_test
return X_train, X_test, y_train, y_test
3
Example 16
def getArray(buf):
"""Fill an intermediate ``numpy`` array with data read from the `CSV` file.
The dtypes are determined by the contents of each column.
Multidimensional columns will have string datatype.
:Parameter buf: the data buffer
"""
# The dtypes are determined by the contents of each column
# Multidimensional columns will have string datatype
temp_file = tempfile.TemporaryFile()
temp_file.writelines(buf)
temp_file.seek(0)
data = numpy.genfromtxt(temp_file, delimiter=',', dtype=None)
temp_file.close()
return data
3
Example 17
def _readcsv(self, path_to_csv):
"""reads a csv column"""
return np.genfromtxt(path_to_csv,
dtype=None,
delimiter=',',
names=True)
3
Example 18
def get_data( fn ):
data_file = open( fn, 'r' )
lines = [ l for l in data_file.readlines() if not l[0] in ['#','@'] ]
dat = np.genfromtxt(lines)
return dat[:,1]
3
Example 19
def __fromfile(self, f):
# d = np.fromfile(f,dtype=self.dtype,count=count)
try:
d = np.genfromtxt(f, dtype=self.dtype)
except Exception as e:
raise Exception("MfList.__fromfile() error reading recarray " + \
"from file " + str(e))
return d
3
Example 20
Project: OpenBCI_Python Source File: mne_openbci.py
def _get_data_dims(self, input_fname):
"""Briefly scan the data file for info"""
# raw data formatting is nsamps by nchans + counter
data = np.genfromtxt(input_fname, delimiter=',', comments='%',
skip_footer=1)
diff = np.abs(np.diff(data[:, 0]))
diff = np.mod(diff, 254) - 1
missing_idx = np.where(diff != 0)[0]
missing_samps = diff[missing_idx].astype(int)
nsamps, nchan = data.shape
# add the missing samples
nsamps += sum(missing_samps)
# remove the tracker column
nchan -= 1
del data
return nsamps, nchan
3
Example 21
def load_referrers(referrers_fpath, X = None, column_ids=None):
X_ref = np.genfromtxt(referrers_fpath)[:,1:].copy()
new_col_ids = {}
with open(referrers_fpath) as referrers_file:
for line in referrers_file:
if '#' in line:
spl = line.split()[1:]
new_col_ids = dict((k, v) for k, v in enumerate(spl))
return hstack_if_possible(X, X_ref), \
update_col_ids(new_col_ids, column_ids)
3
Example 22
def load_time_series(tseries_fpath, num_pts = -1, X = None, column_ids=None):
X_series = np.genfromtxt(tseries_fpath)[:,1:][:,range(num_pts)]
new_col_ids = dict((i, 'POINT_%d'%pnt) \
for i, pnt in enumerate(range(num_pts)))
return hstack_if_possible(X, X_series), \
update_col_ids(new_col_ids, column_ids)
3
Example 23
Project: pele Source File: _pointgrouporder.py
def testlj75(): # pragma: no cover
import numpy as np
coords = np.genfromtxt("tests/coords.lj75.gmin")
from pele.mindist import ExactMatchAtomicCluster
permlist = [range(75)]
match = ExactMatchAtomicCluster(permlist=permlist, can_invert=True)
calculator = PointGroupOrderCluster(match)
pgorder = calculator(coords)
print pgorder
3
Example 24
def check_col(self, pert, fit, redshift, col):
""" Able to check all columns"""
data = np.genfromtxt(LOCATION + "/tests/data/" + fit + '_' + str(int(redshift)))[::-1][400:1201]
# We have to do funky stuff to the data if its been cut by genmf
if col is "sigma":
assert max_diff_rel(pert.sigma, data[:, 5], 0.004)
elif col is "lnsigma": # We just do diff on this one because it passes through 0
assert max_diff(pert.lnsigma, data[:, 3], 0.001)
elif col is "n_eff":
assert max_diff_rel(pert.n_eff, data[:, 6], 0.001)
elif col is "dndlog10m":
assert rms_diff(pert.dndlog10m, 10 ** data[:, 1], 0.004)
elif col is "fsigma":
assert rms_diff(pert.fsigma, data[:, 4], 0.004)
elif col is "ngtm":
# # The reason this is only good to 5% is GENMF's problem -- it uses
# # poor integration.
assert rms_diff(pert.ngtm, 10 ** data[:, 2], 0.047)
3
Example 25
def main(tseries_fpath, centroids_fpath, test_fpath, assign_fpath, out_folder,
gamma_max):
gamma_max = int(gamma_max)
C = np.genfromtxt(centroids_fpath)
Xtest = ioutil.load_series(tseries_fpath, test_fpath)
y_train = np.arange(C.shape[0])
max_pts = gamma_max
for num_pts in range(1, max_pts + 1):
#for num_pts in [1, 25, 50, 75]:
probs = fit(C, y_train, Xtest, num_pts)
probs_fpath = os.path.join(out_folder, 'probs-%d-pts.dat' % num_pts)
np.savetxt(probs_fpath, probs)
3
Example 26
Project: Loan_Default_Prediction Source File: predict.py
def load_train_fs():
# In the validation process, the training data was randomly shuffled firstly.
# For the prediction process, there is no need to shuffle the dataset.
# Owing to out of memory problem, Gaussian process only use part of training data, the prediction of gaussian process
# may be a little different from the model,which the training data was shuffled.
train_fs = np.genfromtxt(open(dir + '/train_v2.csv','rb'), delimiter=',', skip_header=1)
col_mean = stats.nanmean(train_fs, axis=0)
inds = np.where(np.isnan(train_fs))
train_fs[inds] = np.take(col_mean, inds[1])
train_fs[np.isinf(train_fs)] = 0
return train_fs
3
Example 27
def _read_data(self, header_line_number):
names = [name.replace(' ', '_') for name in self.column_name]
data = np.genfromtxt(self.filename, delimiter=',', dtype=None,
names=names,
skip_header=header_line_number,
unpack=True)
self._data_dictionary = dict()
for i, name, name_dtype in zip(range(len(names)), self.column_name,
names):
if name == 'Notes':
self.notes = data[name_dtype].astype(str)
elif name == 'Time':
self.time_axis = data[name_dtype]
else:
self._data_dictionary[name] = data[name_dtype]
3
Example 28
Project: mlxtend Source File: three_blobs.py
def three_blobs_data():
"""A random dataset of 3 2D blobs for clustering.
Number of samples : 150
Suggested labels : {0, 1, 2}, distribution: [50, 50, 50]
Returns
--------
X, y : [n_samples, n_features], [n_cluster_labels]
X is the feature matrix with 159 samples as rows
and 2 feature columns.
y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2
"""
tmp = np.genfromtxt(fname=DATA_PATH, delimiter=',')
X, y = tmp[:, :-1], tmp[:, -1]
y = y.astype(int)
return X, y
3
Example 29
Project: python-rl Source File: compareParameters.py
def loadParameterData(filename, param_index):
data = numpy.genfromtxt(filename, delimiter=',')[:,(0,1,param_index)] # Grab the mean, std, and parameter
data = data[numpy.lexsort((data[:,2],)),:]
if data[:,2].std() <= 1.e-10:
xs = numpy.linspace(0, 1.0)
ys = xs.copy()
ys.fill(data[:,0].mean())
stdv = xs.copy()
xs.fill(0.0)
return numpy.array([ys, stdv, xs]).T
else:
return data
3
Example 30
Project: battery-status Source File: battery-status-graph.py
def parse_csv_np():
logging.debug('loading CSV file %s with NumPy', args.logfile)
data = np.genfromtxt(args.logfile,
delimiter=',', names=True,
filling_values = 0.0)
# convert timestamp to datetime, but also select only relevant
# fields to avoid having to specify all
# XXX: it seems that datetime is not what plot expects, so stick
# with float and we convert later
#return data.astype([('timestamp', 'datetime64[s]'),
# ('energy_full', 'f'),
# ('energy_full_design', 'f'),
# ('energy_now', 'f')])
return data
3
Example 31
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
prog=argv[0],
description=__doc__
)
parser.add_argument("-i", "--dat", required=True, help="data file")
parser.add_argument("-g", "--gam", required=True, help="gamma value", type=float)
options = parser.parse_args(argv[1:])
X = np.genfromtxt(options.dat)
t0 = time.clock()
low_rank, sparse, term, n_iter = opursuit(X, None, options.gam)
t1 = time.clock()
print ("%d iterations (%.2g [sec]) to convergence (val=%.10f) on X=(%d x %d) " %
(n_iter, (t1-t0), term, X.shape[0], X.shape[1]))
3
Example 32
def read_array(filename):
''' Read array and convert to 2d np arrays '''
array = np.genfromtxt(filename, dtype=float)
if len(array.shape)==1:
array = array.reshape( -1, 1 )
return array
3
Example 33
def test_recover(self):
"""Test recovery from outliers.
"""
# run recovery
res = ialm.recover(self._data, None)
lr = res[0]
sp = res[1]
# load baseline (no outliers)
file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "im_baseline.dat")
baseline = np.genfromtxt(file_path)
# Frobenius norm between recovered mat. and baseline
d = np.linalg.norm(np.round(lr)-baseline, ord='fro')
self.assertTrue(np.allclose(d,0.0))
3
Example 34
def setup(self):
with tempfile.NamedTemporaryFile() as temp:
temp.write(_txt.encode('utf-8'))
temp.flush()
temp.seek(0)
self.data = np.genfromtxt(temp, names=True, skip_header=1)
3
Example 35
Project: talon Source File: dataset_test.py
def test_build_extraction_dataset():
if os.path.exists(os.path.join(TMP_DIR, 'extraction.data')):
os.remove(os.path.join(TMP_DIR, 'extraction.data'))
d.build_extraction_dataset(os.path.join(EMAILS_DIR, 'P'),
os.path.join(TMP_DIR,
'extraction.data'), 1)
filename = os.path.join(TMP_DIR, 'extraction.data')
file_data = genfromtxt(filename, delimiter=",")
test_data = file_data[:, :-1]
# the result is a loadable signature extraction dataset
# 32 comes from 3 emails in emails/P folder, 11 lines checked to be
# a signature, one email has only 10 lines
eq_(test_data.shape[0], 32)
eq_(len(features('')), test_data.shape[1])
3
Example 36
def __init__(self, placename):
self.name = placename
self.fname = '/Users/daniel/work/solar/weather_data/EPW_data/BGD_%s/BGD_%s.epw' % \
(self.name,self.name)
self.d = np.genfromtxt(self.fname,dtype=None,delimiter=',',skiprows=8,
usecols=(1,2,3,4,6,7,8,9,10,11,12,13,14,15,20,21))
self.d.dtype.names = ('month','day','hour','minute','t_amb','t_dp','rh_amb','press',
'g_o','g_on','ir_sky','g','g_n','g_d','winddir','windspeed')
self.units = ('-','-','-','-','C','C','%','Pa','Wh/m2','Wh/m2','Wh/m2','Wh/m2',
'Wh/m2','Wh/m2','deg','m/s')
self.time = datetime(2010,1,1,1)
self.step = timedelta(hours=1)
self.curr = 0
3
Example 37
Project: SHARPpy Source File: pecan_decoder.py
def _parseSection(self, section):
parts = section.split('\n')
dt_obj = datetime.strptime(parts[1], 'TIME = %y%m%d/%H%M')
member = parts[0].split('=')[-1].strip()
location = parts[2].split('SLAT')[0].split('=')[-1].strip()
data = '\n'.join(parts[5:])
sound_data = StringIO( data )
p, h, t, td, wdir, wspd, omeg = np.genfromtxt( sound_data, delimiter=',', unpack=True)
prof = profile.create_profile(profile='raw', pres=p[1:], hght=h[1:], tmpc=t[1:], dwpc=td[1:], wspd=wspd[1:],\
wdir=wdir[1:], omeg=omeg[1:], location=location, date=dt_obj, missing=-999.0)
return prof, dt_obj, member
3
Example 38
Project: sncosmo Source File: builtins.py
def load_csp(**kwargs):
# this file contains the csp zeropoints and standards
fname = get_pkg_data_filename('data/bandpasses/csp/csp_filter_info.dat')
data = np.genfromtxt(fname, names=True, dtype=None, skip_header=3)
bands = data['name']
refsystems = data['reference_sed']
offsets = data['natural_mag']
# In Python 3, convert to native strings (Unicode)
if six.PY3:
bands = np.char.decode(bands)
refsystems = np.char.decode(refsystems)
return CompositeMagSystem(bands, refsystems, offsets, name='csp')
3
Example 39
Project: Loan_Default_Prediction Source File: predict.py
def load_test_fs():
test_fs = np.genfromtxt(open(dir + '/test_v2.csv','rb'), delimiter=',', skip_header = 1)
col_mean = stats.nanmean(test_fs, axis=0)
inds = np.where(np.isnan(test_fs))
test_fs[inds] = np.take(col_mean, inds[1])
test_fs[np.isinf(test_fs)] = 0
return test_fs
3
Example 40
Project: mlxtend Source File: mnist.py
def mnist_data():
"""5000 samples from the MNIST handwritten digits dataset.
Data Source : http://yann.lecun.com/exdb/mnist/
Returns
--------
X, y : [n_samples, n_features], [n_class_labels]
X is the feature matrix with 5000 image samples as rows,
each row consists of 28x28 pixels that were unrolled into
784 pixel feature vectors.
y contains the 10 unique class labels 0-9.
"""
tmp = np.genfromtxt(fname=DATA_PATH, delimiter=',')
X, y = tmp[:, :-1], tmp[:, -1]
y = y.astype(int)
return X, y
3
Example 41
Project: spotpy Source File: analyser.py
def load_csv_results(filename, usecols=None):
"""
Get an array of your results in the given file, without the first and the
last column. The first line may have a different objectivefunction and the last
line may be incomplete, which would result in an error.
:filename: Expects an available filename, without the csv, in your working directory
:type: str
:return: Result array
:rtype: array
"""
if usecols == None:
#return np.genfromtxt(filename+'.csv',delimiter=',',names=True,skip_footer=1,invalid_raise=False)[1:]
return np.genfromtxt(filename+'.csv',delimiter=',',names=True,invalid_raise=False)
else:
return np.genfromtxt(filename+'.csv',delimiter=',',names=True,skip_footer=1,invalid_raise=False,usecols=usecols)[1:]
3
Example 42
Project: oq-hazardlib Source File: rupture_test.py
def test_get_dppvalue(self):
rupture = self.make_rupture_fordpp(
ParametricProbabilisticRupture, occurrence_rate=0.01,
temporal_occurrence_model=PoissonTOM(50))
# Load the testing site.
data_path = os.path.dirname(__file__)
filename = os.path.join(
data_path, "./data/geo_cycs_ss3_testing_site.csv")
data = numpy.genfromtxt(filename,
dtype=float, delimiter=',', names=True,
skip_header=6675, skip_footer=6673)
for loc in range(len(data)):
lon = data[loc][0]
lat = data[loc][1]
ref_dpp = data[loc][2]
dpp = rupture.get_dppvalue(Point(lon, lat))
self.assertAlmostEqual(dpp, ref_dpp, delta=0.1)
2
Example 43
def __init__(self, filename=os.path.join(os.path.dirname(__file__),'configs', 'neurostim', 'params.dat'),
noise=0.00001, stim_penalty=-1.0, seizure_penalty=-40.0):
self.noise = noise
self.stim_penalty = stim_penalty
self.seizure_penalty = seizure_penalty
self.embed = -8
# Usefull utility function
def samedir(file):
return os.path.join(os.path.split(filename)[0], file)
# Load model parameters from input file
with open(filename, "rb") as f:
lines = [line.rstrip() for line in f]
self.features_filename = samedir(lines[0])
self.labels_filename = samedir(lines[1])
self.stimulation_filename = samedir(lines[2])
self.N, self.E, self.L, self.seiz_label = map(int, lines[3:7])
self.stim_magnitude = float(lines[7])
self.Ndt = int(lines[8])
# Load Data from Files...
# Model features
self.features = numpy.genfromtxt(self.features_filename)
# Model labels
self.labels = numpy.genfromtxt(self.labels_filename)
# Prototype stimulation
self.dstim = numpy.genfromtxt(self.stimulation_filename)
# Initialize model state to zero
self.state = numpy.zeros((self.E,))
# Construct the Noise Model
self.noise_mags = numpy.abs(self.features[1:,:] - self.features[:-1,:]).mean(0)
# Construct K-d Tree (for Nearest Neighbor computation)
self.knn = NearestNeighbors(n_neighbors=1, algorithm='kd_tree')
self.knn.fit(self.features)
# Initialize state
self.reset()
2
Example 44
Project: GParML Source File: gd-example.py
def main():
global N, Y, X_mu, X_S, flat_global_statistics_bounds, fix_beta, global_statistics_names
iterations = 100
# Load data
Y = numpy.concatenate((
numpy.genfromtxt('./easydata/inputs/easy_1', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_2', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_3', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_4', delimiter=',')))
N = Y.shape[0]
# We have several differet possible initialisations for the embeddings
#X_mu = numpy.load(X_file)
#X_mu = PCA(Y_file, Q)
#X_mu = scipy.randn(N, Q)
X_S = numpy.clip(numpy.ones((N, Q)) * 0.5
+ 0.01 * scipy.randn(N, Q),
0.001, 1)
#X_S = numpy.zeros((N, Q))
X_mu = numpy.concatenate((
scipy.load('./easydata/embeddings/easy_1.embedding.npy'),
scipy.load('./easydata/embeddings/easy_2.embedding.npy'),
scipy.load('./easydata/embeddings/easy_3.embedding.npy'),
scipy.load('./easydata/embeddings/easy_4.embedding.npy')))
'''
X_S = numpy.concatenate((
scipy.load('./easydata/embeddings/easy_1.variance.npy'),
scipy.load('./easydata/embeddings/easy_2.variance.npy'),
scipy.load('./easydata/embeddings/easy_3.variance.npy'),
scipy.load('./easydata/embeddings/easy_4.variance.npy')))
'''
# Initialise the inducing points
Z = X_mu[numpy.random.permutation(N)[:M],:]
#Z = X_mu[:M,:]
Z += scipy.randn(M, Q) * 0.1
global_statistics_names = {
'Z' : (M, Q), 'sf2' : (1,1), 'alpha' : (1, Q), 'beta' : (1,1), 'X_mu' : (N, Q), 'X_S' : (N, Q)
}
# Initialise the global statistics
global_statistics = {
'Z' : Z, # see GPy models/bayesian_gplvm.py
'sf2' : numpy.array([[1.0]]), # see GPy kern/rbf.py
'alpha' : scipy.ones((1, Q)), # see GPy kern/rbf.py
'beta' : numpy.array([[1.0]]), # see GPy likelihood/gaussian.py
'X_Zmu' : X_mu,
'X_S' : X_S,
}
# Initialise bounds for optimisation
global_statistics_bounds = {
'Z' : [(None, None) for i in range(M * Q)],
'sf2' : [(0, None)],
'alpha' : [(0, None) for i in range(Q)],
'beta' : [(0, None)],
'X_mu' : [(None, None) for i in range(N * Q)],
'X_S' : [(0, None) for i in range(N * Q)],
}
flat_global_statistics_bounds = []
for key, statistic in global_statistics_bounds.items():
flat_global_statistics_bounds = flat_global_statistics_bounds+statistic
'''
Run the optimiser
'''
x0 = flatten_global_statistics(global_statistics)
# Transform the positiv parameters to be in the range (-Inf, Inf)
x0 = numpy.array([transform_back(b, x) for b, x in zip(flat_global_statistics_bounds, x0)])
'''
SCG optimisation (adapted from GPy implementation to reduce function calls)
The number of iterations might be greater than max_f_eval
'''
#fix_beta = True
# We set the flag fixed_embeddings to true because we just need the globals (which now include the embeddings) optimised
x = GD(likelihood_and_gradient, x0, './easydata/tmp/', fixed_embeddings=True, display=True, maxiters=iterations)
#fix_beta = False
#x = SC(likelihood_and_gradient, x[0], display=True, maxiters=iterations)
flat_array = x[0]
# Transform the parameters that have to be positive to be positive
flat_array_transformed = numpy.array([transform(b, x) for b, x in zip(flat_global_statistics_bounds, flat_array)])
global_statistics = rebuild_global_statistics(global_statistics_names, flat_array_transformed)
print 'Final global_statistics'
print global_statistics
2
Example 45
Project: GParML Source File: scg-example.py
def main():
global N, Y, X_mu, X_S, flat_global_statistics_bounds, fix_beta, global_statistics_names
iterations = 20
# Load data
Y = numpy.concatenate((
numpy.genfromtxt('./easydata/inputs/easy_1', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_2', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_3', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_4', delimiter=',')))
N = Y.shape[0]
# We have several differet possible initialisations for the embeddings
#X_mu = numpy.load(X_file)
#X_mu = PCA(Y_file, Q)
#X_mu = scipy.randn(N, Q)
X_S = numpy.clip(numpy.ones((N, Q)) * 0.5
+ 0.01 * scipy.randn(N, Q),
0.001, 1)
#X_S = numpy.zeros((N, Q))
X_mu = numpy.concatenate((
scipy.load('./easydata/embeddings/easy_1.embedding.npy'),
scipy.load('./easydata/embeddings/easy_2.embedding.npy'),
scipy.load('./easydata/embeddings/easy_3.embedding.npy'),
scipy.load('./easydata/embeddings/easy_4.embedding.npy')))
'''
X_S = numpy.concatenate((
scipy.load('./easydata/embeddings/easy_1.variance.npy'),
scipy.load('./easydata/embeddings/easy_2.variance.npy'),
scipy.load('./easydata/embeddings/easy_3.variance.npy'),
scipy.load('./easydata/embeddings/easy_4.variance.npy')))
'''
# Initialise the inducing points
Z = X_mu[numpy.random.permutation(N)[:M],:]
#Z = X_mu[:M,:]
Z += scipy.randn(M, Q) * 0.1
global_statistics_names = {
'Z' : (M, Q), 'sf2' : (1,1), 'alpha' : (1, Q), 'beta' : (1,1), 'X_mu' : (N, Q), 'X_S' : (N, Q)
}
# Initialise the global statistics
global_statistics = {
'Z' : Z, # see GPy models/bayesian_gplvm.py
'sf2' : numpy.array([[1.0]]), # see GPy kern/rbf.py
'alpha' : scipy.ones((1, Q)), # see GPy kern/rbf.py
'beta' : numpy.array([[1.0]]), # see GPy likelihood/gaussian.py
'X_Zmu' : X_mu,
'X_S' : X_S
}
# Initialise bounds for optimisation
global_statistics_bounds = {
'Z' : [(None, None) for i in range(M * Q)],
'sf2' : [(0, None)],
'alpha' : [(0, None) for i in range(Q)],
'beta' : [(0, None)],
'X_mu' : [(None, None) for i in range(N * Q)],
'X_S' : [(0, None) for i in range(N * Q)]
}
flat_global_statistics_bounds = []
for key, statistic in global_statistics_bounds.items():
flat_global_statistics_bounds = flat_global_statistics_bounds+statistic
'''
Run the optimiser
'''
x0 = flatten_global_statistics(global_statistics)
# Transform the positiv parameters to be in the range (-Inf, Inf)
x0 = numpy.array([transform_back(b, x) for b, x in zip(flat_global_statistics_bounds, x0)])
'''
SCG optimisation (adapted from GPy implementation to reduce function calls)
The number of iterations might be greater than max_f_eval
'''
#fix_beta = True
x = SCG(likelihood, gradient, x0, display=True, maxiters=iterations)
#fix_beta = False
#x = SC(likelihood_and_gradient, x[0], display=True, maxiters=iterations)
flat_array = x[0]
# Transform the parameters that have to be positive to be positive
flat_array_transformed = numpy.array([transform(b, x) for b, x in zip(flat_global_statistics_bounds, flat_array)])
global_statistics = rebuild_global_statistics(global_statistics_names, flat_array_transformed)
print 'Final global_statistics'
print global_statistics
2
Example 46
Project: GParML Source File: scg_adapted-example.py
def main():
global N, Y, X_mu, X_S, flat_global_statistics_bounds, fix_beta, global_statistics_names
iterations = 20
# Load data
Y = numpy.concatenate((
numpy.genfromtxt('./easydata/inputs/easy_1', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_2', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_3', delimiter=','),
numpy.genfromtxt('./easydata/inputs/easy_4', delimiter=',')))
N = Y.shape[0]
# We have several differet possible initialisations for the embeddings
#X_mu = numpy.load(X_file)
#X_mu = PCA(Y_file, Q)
#X_mu = scipy.randn(N, Q)
X_S = numpy.clip(numpy.ones((N, Q)) * 0.5
+ 0.01 * scipy.randn(N, Q),
0.001, 1)
#X_S = numpy.zeros((N, Q))
X_mu = numpy.concatenate((
scipy.load('./easydata/embeddings/easy_1.embedding.npy'),
scipy.load('./easydata/embeddings/easy_2.embedding.npy'),
scipy.load('./easydata/embeddings/easy_3.embedding.npy'),
scipy.load('./easydata/embeddings/easy_4.embedding.npy')))
'''
X_S = numpy.concatenate((
scipy.load('./easydata/embeddings/easy_1.variance.npy'),
scipy.load('./easydata/embeddings/easy_2.variance.npy'),
scipy.load('./easydata/embeddings/easy_3.variance.npy'),
scipy.load('./easydata/embeddings/easy_4.variance.npy')))
'''
# Initialise the inducing points
Z = X_mu[numpy.random.permutation(N)[:M],:]
#Z = X_mu[:M,:]
Z += scipy.randn(M, Q) * 0.1
global_statistics_names = {
'Z' : (M, Q), 'sf2' : (1,1), 'alpha' : (1, Q), 'beta' : (1,1), 'X_mu' : (N, Q), 'X_S' : (N, Q)
}
# Initialise the global statistics
global_statistics = {
'Z' : Z, # see GPy models/bayesian_gplvm.py
'sf2' : numpy.array([[1.0]]), # see GPy kern/rbf.py
'alpha' : scipy.ones((1, Q)), # see GPy kern/rbf.py
'beta' : numpy.array([[1.0]]), # see GPy likelihood/gaussian.py
'X_Zmu' : X_mu,
'X_S' : X_S
}
# Initialise bounds for optimisation
global_statistics_bounds = {
'Z' : [(None, None) for i in range(M * Q)],
'sf2' : [(0, None)],
'alpha' : [(0, None) for i in range(Q)],
'beta' : [(0, None)],
'X_mu' : [(None, None) for i in range(N * Q)],
'X_S' : [(0, None) for i in range(N * Q)]
}
flat_global_statistics_bounds = []
for key, statistic in global_statistics_bounds.items():
flat_global_statistics_bounds = flat_global_statistics_bounds+statistic
'''
Run the optimiser
'''
x0 = flatten_global_statistics(global_statistics)
# Transform the positiv parameters to be in the range (-Inf, Inf)
x0 = numpy.array([transform_back(b, x) for b, x in zip(flat_global_statistics_bounds, x0)])
'''
SCG optimisation (adapted from GPy implementation to reduce function calls)
The number of iterations might be greater than max_f_eval
'''
#fix_beta = True
# We set the flag fixed_embeddings to true because we just need the globals (which now include the embeddings) optimised
x = SCG_adapted(likelihood_and_gradient, x0, './easydata/tmp/', fixed_embeddings=True, display=True, maxiters=iterations)
#fix_beta = False
#x = SC(likelihood_and_gradient, x[0], display=True, maxiters=iterations)
flat_array = x[0]
# Transform the parameters that have to be positive to be positive
flat_array_transformed = numpy.array([transform(b, x) for b, x in zip(flat_global_statistics_bounds, flat_array)])
global_statistics = rebuild_global_statistics(global_statistics_names, flat_array_transformed)
print 'Final global_statistics'
print global_statistics
0
Example 47
Project: pele Source File: test_optim_compatibility.py
def test1(self):
current_dir = os.path.dirname(__file__)
db = Database()
converter = OptimDBConverter(db,
mindata=os.path.join(current_dir, "min.data"),
tsdata=os.path.join(current_dir, "ts.data"),
pointsmin=os.path.join(current_dir, "points.min"),
pointsts=os.path.join(current_dir, "points.ts"),
endianness="<")
converter.convert()
mdata = tempfile.NamedTemporaryFile(delete=True)
tsdata = tempfile.NamedTemporaryFile(delete=True)
pm = tempfile.NamedTemporaryFile(delete=True)
pts = tempfile.NamedTemporaryFile(delete=True)
print mdata.name, tsdata.name
writer = WritePathsampleDB(db,
mindata=mdata.name,
tsdata=tsdata.name,
pointsmin=pm.name,
pointsts=pts.name,
endianness="<")
writer.write_db()
d1 = np.genfromtxt(os.path.join(current_dir, "min.data"))[:,:3]
d2 = np.genfromtxt(mdata.name)[:,:3]
_base_test.assert_arrays_almost_equal(self, d1, d2)
d1 = np.genfromtxt(os.path.join(current_dir, "ts.data")).reshape(-1,8)[:,:5]
d2 = np.genfromtxt(tsdata.name).reshape(-1,8)[:,:5]
print d1, d2
_base_test.assert_arrays_almost_equal(self, d1, d2)
self.assertEqual(sha1_of_file(os.path.join(current_dir, "points.min")),
sha1_of_file(pm.name))
self.assertEqual(sha1_of_file(os.path.join(current_dir, "points.ts")),
sha1_of_file(pts.name))
0
Example 48
def load_txt(path):
"""Load data from a text file.
Parameters
----------
path : str
Path to file.
Returns
-------
data : array
Loaded data.
mdata : dict
Metadata.
"""
# normalize path
path = utils.normpath(path)
with open(path, 'r') as fid:
lines = fid.readlines()
# extract header
mdata_tmp = {}
fields = ['Sampling Rate', 'Resolution', 'Date', 'Data Type', 'Labels']
values = []
for item in lines:
if '#' in item:
# parse comment
for f in fields:
if f in item:
mdata_tmp[f] = item.split(':= ')[1].strip()
fields.remove(f)
break
else:
values.append(item)
# convert mdata
mdata = {}
df = '%Y-%m-%dT%H:%M:%S.%f'
try:
mdata['sampling_rate'] = float(mdata_tmp['Sampling Rate'])
except KeyError:
pass
try:
mdata['resolution'] = int(mdata_tmp['Resolution'])
except KeyError:
pass
try:
dtype = mdata_tmp['Data Type']
except KeyError:
dtype = None
try:
d = datetime.datetime.strptime(mdata_tmp['Date'], df)
mdata['date'] = d
except (KeyError, ValueError):
pass
try:
labels = mdata_tmp['Labels'].split('\t')
mdata['labels'] = labels
except KeyError:
pass
# load array
data = np.genfromtxt(values, dtype=dtype, delimiter='\t')
return data, mdata
0
Example 49
def main():
from xy_model_system import XYModlelSystem
fname = "gmin.coords"
fname = "goldstone_mode.coords"
fname = "vortex2.coords"
reduced_coords = np.genfromtxt(fname)
print (len(reduced_coords))
print np.sqrt(len(reduced_coords))
dim = [24, 24]
print "dim", dim
system = XYModlelSystem(dim, phi_disorder=0)
coords = system.pot.coords_converter.get_full_coords(reduced_coords)
x = np.zeros(len(coords))
y = x.copy()
dx = x.copy()
dy = x.copy()
for node in system.pot.G.nodes():
xyz = system.node2xyz(node)
i = system.pot.indices[node]
x[i] = xyz[0]
y[i] = xyz[1]
theta = coords[i]
dx[i] = np.cos(theta)
dy[i] = np.sin(theta)
if True:
pot = system.get_potential()
energies = pot.get_spin_energies(coords)
else:
energies = None
# plt.quiver(x, y, dx, dy, energies, pivot="middle", cmap="YlOrRd_r")
if energies is not None:
plt.quiver(x, y, dx, dy, energies, pivot="middle", cmap="cool_r")
else:
plt.quiver(x, y, dx, dy, pivot="middle")
d = 0.5
plt.xlim([-d, dim[0]-d])
plt.ylim([-d, dim[1]-d])
plt.tight_layout()
plt.xticks([])
plt.yticks([])
plt.gcf().set_facecolor('white')
plt.box(on=False)
plt.savefig(fname + ".eps", format="eps")
0
Example 50
Project: quality-assessment-protocol Source File: temporal_qc.py
def fd_jenkinson(in_file, rmax=80., out_file=None):
'''
@ Krsna
May 2013
compute
1) Jenkinson FD from 3dvolreg's *.affmat12.1D file from -1Dmatrix_save
option input: subject ID, rest_number, name of 6 parameter motion
correction file (an output of 3dvolreg) output: FD_J.1D file
Assumptions: 1) subject is available in BASE_DIR
2) 3dvolreg is already performed and the 1D motion parameter and 1D_matrix
file file is present in sub?/rest_? called as --->'lfo_mc_affmat.1D'
Method to calculate Framewise Displacement (FD) calculations
(Jenkinson et al., 2002)
Parameters; in_file : string
rmax : float
The default radius (as in FSL) of a sphere represents the brain
Returns; out_file : string
NOTE: infile should have one 3dvolreg affine matrix in one row -
NOT the motion parameters
'''
import numpy as np
import os
import os.path as op
from shutil import copyfile
import sys
import math
if out_file is None:
fname, ext = op.splitext(op.basename(in_file))
out_file = op.abspath('%s_fdfile%s' % (fname, ext))
# if in_file (coordinate_transformation) is actually the rel_mean output
# of the MCFLIRT command, forward that file
if 'rel.rms' in in_file:
copyfile(in_file, out_file)
return out_file
pm_ = np.genfromtxt(in_file)
original_shape = pm_.shape
pm = np.zeros((pm_.shape[0], pm_.shape[1] + 4))
pm[:, :original_shape[1]] = pm_
pm[:, original_shape[1]:] = [0.0, 0.0, 0.0, 1.0]
# rigid body transformation matrix
T_rb_prev = np.matrix(np.eye(4))
flag = 0
X = [0] # First timepoint
for i in range(0, pm.shape[0]):
# making use of the fact that the order of aff12 matrix is "row-by-row"
T_rb = np.matrix(pm[i].reshape(4, 4))
if flag == 0:
flag = 1
else:
M = np.dot(T_rb, T_rb_prev.I) - np.eye(4)
A = M[0:3, 0:3]
b = M[0:3, 3]
FD_J = math.sqrt(
(rmax * rmax / 5) * np.trace(np.dot(A.T, A)) + np.dot(b.T, b))
X.append(FD_J)
T_rb_prev = T_rb
np.savetxt(out_file, np.array(X))
return out_file