numpy.genfromtxt

Here are the examples of the python api numpy.genfromtxt taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

136 Examples 7

Example 1

Project: statsmodels Source File: test_phreg.py
Function: load_file
    def load_file(self, fname):
        cur_dir = os.path.dirname(os.path.abspath(__file__))
        data = np.genfromtxt(os.path.join(cur_dir, 'results', fname),
                             delimiter=" ")
        time = data[:,0]
        status = data[:,1]
        entry = data[:,2]
        exog = data[:,3:]

        return time, status, entry, exog

Example 2

Project: gala Source File: test_sgr.py
def test_against_David_Law():
    """ Test my code against an output file from using David Law's cpp code. Do:

            g++ SgrCoord.cpp; ./a.out

        to generate the data file, SgrCoord_data.

    """
    filename = get_pkg_data_filename('SgrCoord_data')
    law_data = np.genfromtxt(filename, names=True, delimiter=',')

    c = coord.Galactic(law_data["l"]*u.deg, law_data["b"]*u.deg)
    sgr_coords = c.transform_to(Sagittarius)

    law_sgr_coords = Sagittarius(Lambda=law_data["lambda"]*u.deg, Beta=law_data["beta"]*u.deg)

    sep = sgr_coords.separation(law_sgr_coords).arcsec*u.arcsec
    assert np.all(sep < 1.*u.arcsec)

Example 3

Project: talon Source File: classifier.py
Function: train
def train(classifier, train_data_filename, save_classifier_filename=None):
    """Trains and saves classifier so that it could be easily loaded later."""
    file_data = genfromtxt(train_data_filename, delimiter=",")
    train_data, labels = file_data[:, :-1], file_data[:, -1]
    classifier.fit(train_data, labels)

    if save_classifier_filename:
        joblib.dump(classifier, save_classifier_filename)
    return classifier

Example 4

Project: solpy Source File: epw_thermal.py
    def read_EPW(self):
        EPW_vars  = ('year','month','day','hour','minute','datasource','t_amb','t_dp',
            'rh_amb','press','g_o','g_on','ir_sky','g','g_n','g_d','ill','ill_n','ill_d',
            'zen_lum','winddir','windspeed','sky_t','sky_o','visibility','ceiling',
            'presweathobs','presweathcodes','precipwtr','aerosoloptdepth','snowdepth',
            'dayslastsnow','albedo','rain','rain_hr')
        out_vars = ['time','t_amb','t_dp','rh_amb','press',
            'g_o','g_on','ir_sky','g','g_n','g_d','winddir','windspeed']  # Add as needed
        out_types = tuple(['object'] + ['float']*12)    
        self.units  = ('-','C','C','%','Pa','Wh/m2','Wh/m2','Wh/m2','Wh/m2',
            'Wh/m2','Wh/m2','deg','m/s')
        
        raw = np.genfromtxt(self.filename,dtype=None,delimiter=',',skiprows=8)  
        raw.dtype.names = EPW_vars
        data = raw.astype(zip(out_vars,out_types))
        data['time'] = [datetime(y,m,d,h-1,30) for y,m,d,h in zip(raw['year'],raw['month'],raw['day'],raw['hour'])]
        print('\n  Read from file %s:  data for %s' % (self.filename, data['time'][0].strftime("%b %d, %Y (%A)") ))
        return data

Example 5

Project: vsm Source File: FileReadWrite.py
def file_to_mat(filename):
    """
    Data to an array. works for theta, phi.
    Removes automatically added 'missing values' at the end of the rows.
    """
    arr = np.genfromtxt(filename, delimiter=',')

    return arr[:,:-1]

Example 6

Project: siphon Source File: ncss.py
def parse_csv_dataset(data, handle_units):
    fobj = BytesIO(data)
    names, units = parse_csv_header(fobj.readline().decode('utf-8'))
    arrs = np.genfromtxt(fobj, dtype=None, names=names, delimiter=',', unpack=True,
                         converters={'date': lambda s: parse_iso_date(s.decode('utf-8'))})
    d = dict()
    for f in arrs.dtype.fields:
        dat = arrs[f]
        if dat.dtype == np.object:
            dat = dat.tolist()
        d[f] = handle_units(dat, units.get(f, None))
    return d

Example 7

Project: gala Source File: test_core.py
Function: set_up
    def setup(self):
        with tempfile.NamedTemporaryFile(mode='w+b') as temp:
            temp.write(_txt.encode('utf-8'))
            temp.flush()
            temp.seek(0)
            self.data = np.genfromtxt(temp, names=True, skip_header=1)

Example 8

Project: osprey Source File: dataset_loaders.py
Function: loader
    def loader(self, fn):
        return np.genfromtxt(fn, delimiter=self.delimiter,
                             skip_header=self.skip_header,
                             skip_footer=self.skip_footer,
                             filling_values=self.filling_values,
                             usecols=self.usecols)

Example 9

Project: statsmodels Source File: test_kde.py
Function: setup_class
    @classmethod
    def setupClass(cls):
        cls.decimal_density = 2 # low accuracy because binning is different
        res1 = KDE(Xi)
        res1.fit(kernel="gau", fft=True, bw="silverman")
        cls.res1 = res1
        rfname2 = os.path.join(curdir,'results','results_kde_fft.csv')
        cls.res_density = np.genfromtxt(open(rfname2, 'rb'))

Example 10

Project: SciDB-Py Source File: _py3k_compat.py
def genfromstr(s, **kwargs):
    """Utility routine to create an array from a string.

    This uses either StringIO or BytesIO, depending on whether we're on
    Python 2 or Python 3.
    """
    return np.genfromtxt(stringio(s), **kwargs)

Example 11

Project: pyksc Source File: pop_predict.py
Function: main
@plac.annotations(features_fpath=plac.Annotation('Partial Features', 
                                                         type=str),
                  tag_categ_fpath=plac.Annotation('Tags file', type=str),
                  tseries_fpath=plac.Annotation('Time series file', type=str),
                  num_days_to_use=plac.Annotation('Num Days Series', type=int),
                  assign_fpath=plac.Annotation('Series assignment file', 
                                               type=str),
                  out_foldpath=plac.Annotation('Output folder', type=str))
def main(features_fpath, tag_categ_fpath, tseries_fpath, num_days_to_use, 
         assign_fpath, out_foldpath):
    
    X, feature_ids, _ = \
            create_input_table(features_fpath, tseries_fpath, tag_categ_fpath,
                               num_days_to_use)
   
    X = scale(X)
    y_clf = np.genfromtxt(assign_fpath)
    y_regr = scale(np.genfromtxt(tseries_fpath)[:,1:].sum(axis=1))
    run_experiment(X, y_clf, y_regr, feature_ids, out_foldpath)

Example 12

Project: Deep-Spying Source File: Label.py
Function: init
    def __init__(self, path):
        file_path = "{}labels.csv".format(path)
        try:
            data = np.genfromtxt(file_path, delimiter=',', skip_header=1,
                                 names=['timestamp', 'label'], dtype=[("timestamp", long), ('label', int)])
            self.has_label = True
        except IOError:
            self.has_label = False
            return

        self.timestamp = data['timestamp']
        label = data['label']

        self.label = []

        for i in range(0, len(label)):
            self.label.append(chr(int(label[i])))

        self.diff = np.diff(self.timestamp)

Example 13

Project: Deep-Spying Source File: View.py
    def plot_sensor_data_from_file(self, file_path):
        if not self.to_save and not self.to_show:
            return

        data = np.genfromtxt(file_path, delimiter=',', skip_header=1, names=['timestamp', 'x', 'y', 'z'])

        file_name = basename(file_path)
        file_name = file_name[:file_name.find(".")]

        View.plot_sensor_data(basename(file_name), data['timestamp'], data['x'], data['y'], data['z'])

Example 14

Project: pyLAR Source File: test_ealm.py
Function: test_recover
    def test_recover(self):
        """Test recovery from outliers.
        """
        # run recovery
        lr, sp, _ = ealm.recover(self._data, None)
        # load baseline (no outliers)
        file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "im_baseline.dat")
        baseline = np.genfromtxt(file_path)
        # Frobenius norm between recovered mat. and baseline
        d = np.linalg.norm(np.round(lr)-baseline, ord='fro')
        self.assertTrue(np.allclose(d,0.0))

Example 15

Project: ReliefF Source File: tests.py
Function: get_testing_data
def get_testing_data(file='data/GAMETES-test.csv.gz', target=-1,
                     return_header=False, head=0):
    """Loads the desired data for testing ReliefF's functions"""
    data = np.genfromtxt(file, delimiter=',')
    header, data = data[head], data[head+1:]  # Separate header (first) row
    y = data[:, target]  # labels of the data
    X = np.delete(data, target, axis=1)  # Remove labels from data

    X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                        random_state=34895)

    # The user can decide if they want the headers returned as first value
    if return_header:
        return header, X_train, X_test, y_train, y_test

    return X_train, X_test, y_train, y_test

Example 16

Project: ViTables Source File: import_csv.py
Function: get_array
def getArray(buf):
    """Fill an intermediate ``numpy`` array with data read from the `CSV` file.

    The dtypes are determined by the contents of each column.
    Multidimensional columns will have string datatype.

    :Parameter buf: the data buffer
    """

    # The dtypes are determined by the contents of each column
    # Multidimensional columns will have string datatype
    temp_file = tempfile.TemporaryFile()
    temp_file.writelines(buf)
    temp_file.seek(0)
    data = numpy.genfromtxt(temp_file, delimiter=',', dtype=None)
    temp_file.close()
    return data

Example 17

Project: ArcREST Source File: _geoenrichment.py
Function: read_csv
    def _readcsv(self, path_to_csv):
        """reads a csv column"""
        return np.genfromtxt(path_to_csv,
                             dtype=None,
                             delimiter=',',
                             names=True)

Example 18

Project: msmbuilder-legacy Source File: gen_ref_dat.py
Function: get_data
def get_data( fn ):

    data_file = open( fn, 'r' )

    lines = [ l for l in data_file.readlines() if not l[0] in ['#','@'] ]
    dat = np.genfromtxt(lines)

    return dat[:,1]

Example 19

Project: flopy Source File: util_list.py
Function: from_file
    def __fromfile(self, f):
        # d = np.fromfile(f,dtype=self.dtype,count=count)
        try:
            d = np.genfromtxt(f, dtype=self.dtype)
        except Exception as e:
            raise Exception("MfList.__fromfile() error reading recarray " + \
                            "from file " + str(e))
        return d

Example 20

Project: OpenBCI_Python Source File: mne_openbci.py
    def _get_data_dims(self, input_fname):
        """Briefly scan the data file for info"""
        # raw data formatting is nsamps by nchans + counter
        data = np.genfromtxt(input_fname, delimiter=',', comments='%',
                             skip_footer=1)
        diff = np.abs(np.diff(data[:, 0]))
        diff = np.mod(diff, 254) - 1
        missing_idx = np.where(diff != 0)[0]
        missing_samps = diff[missing_idx].astype(int)
        nsamps, nchan = data.shape
        # add the missing samples
        nsamps += sum(missing_samps)
        # remove the tracker column
        nchan -= 1
        del data

        return nsamps, nchan

Example 21

Project: pyksc Source File: learn_base.py
Function: load_referrers
def load_referrers(referrers_fpath, X = None, column_ids=None):
    X_ref = np.genfromtxt(referrers_fpath)[:,1:].copy()

    new_col_ids = {}
    with open(referrers_fpath) as referrers_file:
        for line in referrers_file:
            if '#' in line:
                spl = line.split()[1:]
                new_col_ids = dict((k, v) for k, v in enumerate(spl))

                return hstack_if_possible(X, X_ref), \
                    update_col_ids(new_col_ids, column_ids)

Example 22

Project: pyksc Source File: learn_base.py
Function: load_time_series
def load_time_series(tseries_fpath, num_pts = -1, X = None, column_ids=None):
    X_series = np.genfromtxt(tseries_fpath)[:,1:][:,range(num_pts)]
    
    new_col_ids = dict((i, 'POINT_%d'%pnt) \
                       for i, pnt in enumerate(range(num_pts)))

    return hstack_if_possible(X, X_series), \
        update_col_ids(new_col_ids, column_ids)

Example 23

Project: pele Source File: _pointgrouporder.py
def testlj75(): # pragma: no cover
    import numpy as np
    coords = np.genfromtxt("tests/coords.lj75.gmin")
    from pele.mindist import ExactMatchAtomicCluster
    
    permlist = [range(75)]
    match = ExactMatchAtomicCluster(permlist=permlist, can_invert=True)
    calculator = PointGroupOrderCluster(match)
    pgorder = calculator(coords)
    print pgorder

Example 24

Project: hmf Source File: test_genmf.py
Function: check_col
    def check_col(self, pert, fit, redshift, col):
        """ Able to check all columns"""
        data = np.genfromtxt(LOCATION + "/tests/data/" + fit + '_' + str(int(redshift)))[::-1][400:1201]

        # We have to do funky stuff to the data if its been cut by genmf
        if col is "sigma":
            assert max_diff_rel(pert.sigma, data[:, 5], 0.004)
        elif col is "lnsigma":  # We just do diff on this one because it passes through 0
            assert max_diff(pert.lnsigma, data[:, 3], 0.001)
        elif col is "n_eff":
            assert max_diff_rel(pert.n_eff, data[:, 6], 0.001)
        elif col is "dndlog10m":
            assert rms_diff(pert.dndlog10m, 10 ** data[:, 1], 0.004)
        elif col is "fsigma":
            assert rms_diff(pert.fsigma, data[:, 4], 0.004)
        elif col is "ngtm":
            # # The reason this is only good to 5% is GENMF's problem -- it uses
            # # poor integration.
            assert rms_diff(pert.ngtm, 10 ** data[:, 2], 0.047)

Example 25

Project: pyksc Source File: classify_pts_test.py
Function: main
def main(tseries_fpath, centroids_fpath, test_fpath, assign_fpath, out_folder, 	
	 gamma_max):
    gamma_max = int(gamma_max)
    
    C = np.genfromtxt(centroids_fpath)
    Xtest = ioutil.load_series(tseries_fpath, test_fpath)
    y_train = np.arange(C.shape[0])

    max_pts = gamma_max
    for num_pts in range(1, max_pts + 1):
    #for num_pts in [1, 25, 50, 75]:
        probs = fit(C, y_train, Xtest, num_pts)

        probs_fpath = os.path.join(out_folder, 'probs-%d-pts.dat' % num_pts)
        np.savetxt(probs_fpath, probs)

Example 26

Project: Loan_Default_Prediction Source File: predict.py
def load_train_fs():
    # In the validation process, the training data was randomly shuffled firstly. 
    # For the prediction process, there is no need to shuffle the dataset. 
    # Owing to out of memory problem, Gaussian process only use part of training data, the prediction of gaussian process
    # may be a little different from the model,which the training data was shuffled.
    train_fs = np.genfromtxt(open(dir + '/train_v2.csv','rb'), delimiter=',', skip_header=1)
    col_mean = stats.nanmean(train_fs, axis=0)
    inds = np.where(np.isnan(train_fs))
    train_fs[inds] = np.take(col_mean, inds[1])
    train_fs[np.isinf(train_fs)] = 0
    return train_fs

Example 27

Project: hyperspy Source File: protochips.py
Function: read_data
    def _read_data(self, header_line_number):
        names = [name.replace(' ', '_') for name in self.column_name]
        data = np.genfromtxt(self.filename, delimiter=',', dtype=None,
                             names=names,
                             skip_header=header_line_number,
                             unpack=True)

        self._data_dictionary = dict()
        for i, name, name_dtype in zip(range(len(names)), self.column_name,
                                       names):
            if name == 'Notes':
                self.notes = data[name_dtype].astype(str)
            elif name == 'Time':
                self.time_axis = data[name_dtype]
            else:
                self._data_dictionary[name] = data[name_dtype]

Example 28

Project: mlxtend Source File: three_blobs.py
def three_blobs_data():
    """A random dataset of 3 2D blobs for clustering.

    Number of samples : 150
    Suggested labels : {0, 1, 2}, distribution: [50, 50, 50]

    Returns
    --------
    X, y : [n_samples, n_features], [n_cluster_labels]
        X is the feature matrix with 159 samples as rows
        and 2 feature columns.
        y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2

    """
    tmp = np.genfromtxt(fname=DATA_PATH, delimiter=',')
    X, y = tmp[:, :-1], tmp[:, -1]
    y = y.astype(int)
    return X, y

Example 29

Project: python-rl Source File: compareParameters.py
def loadParameterData(filename, param_index):
    data = numpy.genfromtxt(filename, delimiter=',')[:,(0,1,param_index)] # Grab the mean, std, and parameter
    data = data[numpy.lexsort((data[:,2],)),:]

    if data[:,2].std() <= 1.e-10:
        xs = numpy.linspace(0, 1.0)
        ys = xs.copy()
        ys.fill(data[:,0].mean())
        stdv = xs.copy()
        xs.fill(0.0)
        return numpy.array([ys, stdv, xs]).T
    else:
        return data

Example 30

Project: battery-status Source File: battery-status-graph.py
def parse_csv_np():
    logging.debug('loading CSV file %s with NumPy', args.logfile)
    data = np.genfromtxt(args.logfile,
                         delimiter=',', names=True,
                         filling_values = 0.0)
    # convert timestamp to datetime, but also select only relevant
    # fields to avoid having to specify all
    # XXX: it seems that datetime is not what plot expects, so stick
    # with float and we convert later    
    #return data.astype([('timestamp', 'datetime64[s]'),
    #                    ('energy_full', 'f'),
    #                    ('energy_full_design', 'f'),
    #                    ('energy_now', 'f')])
    return data

Example 31

Project: pyLAR Source File: op.py
Function: main
def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        prog=argv[0],
        description=__doc__
        )
    parser.add_argument("-i", "--dat", required=True, help="data file")
    parser.add_argument("-g", "--gam", required=True, help="gamma value", type=float)
    options = parser.parse_args(argv[1:])

    X = np.genfromtxt(options.dat)
    t0 = time.clock()
    low_rank, sparse, term, n_iter = opursuit(X, None, options.gam)
    t1 = time.clock()
    print ("%d iterations (%.2g [sec]) to convergence (val=%.10f) on X=(%d x %d) " %
           (n_iter, (t1-t0), term, X.shape[0], X.shape[1]))

Example 32

Project: automl_gpu Source File: libscores.py
Function: read_array
def read_array(filename):
    ''' Read array and convert to 2d np arrays '''
    array = np.genfromtxt(filename, dtype=float)
    if len(array.shape)==1:
        array = array.reshape( -1, 1 ) 
    return array

Example 33

Project: pyLAR Source File: test_ialm.py
Function: test_recover
    def test_recover(self):
        """Test recovery from outliers.
        """
        # run recovery
        res = ialm.recover(self._data, None)
        lr = res[0]
        sp = res[1]

        # load baseline (no outliers)
        file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "im_baseline.dat")
        baseline = np.genfromtxt(file_path)
        # Frobenius norm between recovered mat. and baseline
        d = np.linalg.norm(np.round(lr)-baseline, ord='fro')
        self.assertTrue(np.allclose(d,0.0))

Example 34

Project: gala Source File: test_propermotion.py
Function: set_up
    def setup(self):
        with tempfile.NamedTemporaryFile() as temp:
            temp.write(_txt.encode('utf-8'))
            temp.flush()
            temp.seek(0)
            self.data = np.genfromtxt(temp, names=True, skip_header=1)

Example 35

Project: talon Source File: dataset_test.py
def test_build_extraction_dataset():
    if os.path.exists(os.path.join(TMP_DIR, 'extraction.data')):
        os.remove(os.path.join(TMP_DIR, 'extraction.data'))
    d.build_extraction_dataset(os.path.join(EMAILS_DIR, 'P'),
                               os.path.join(TMP_DIR,
                                            'extraction.data'), 1)

    filename = os.path.join(TMP_DIR, 'extraction.data')
    file_data = genfromtxt(filename, delimiter=",")
    test_data = file_data[:, :-1]

    # the result is a loadable signature extraction dataset
    # 32 comes from 3 emails in emails/P folder, 11 lines checked to be
    # a signature, one email has only 10 lines
    eq_(test_data.shape[0], 32)
    eq_(len(features('')), test_data.shape[1])

Example 36

Project: solpy Source File: solar_fun.py
Function: init
    def __init__(self, placename):
        self.name   = placename
        self.fname  = '/Users/daniel/work/solar/weather_data/EPW_data/BGD_%s/BGD_%s.epw' % \
                      (self.name,self.name)
        self.d      = np.genfromtxt(self.fname,dtype=None,delimiter=',',skiprows=8, 
                      usecols=(1,2,3,4,6,7,8,9,10,11,12,13,14,15,20,21))
        self.d.dtype.names = ('month','day','hour','minute','t_amb','t_dp','rh_amb','press',
            'g_o','g_on','ir_sky','g','g_n','g_d','winddir','windspeed')
        self.units  = ('-','-','-','-','C','C','%','Pa','Wh/m2','Wh/m2','Wh/m2','Wh/m2',
            'Wh/m2','Wh/m2','deg','m/s')
        self.time   = datetime(2010,1,1,1)
        self.step   = timedelta(hours=1)
        self.curr   = 0

Example 37

Project: SHARPpy Source File: pecan_decoder.py
    def _parseSection(self, section):
        parts = section.split('\n')
        dt_obj = datetime.strptime(parts[1], 'TIME = %y%m%d/%H%M')
        member = parts[0].split('=')[-1].strip()
        location = parts[2].split('SLAT')[0].split('=')[-1].strip()
        data = '\n'.join(parts[5:])
        sound_data = StringIO( data )
        p, h, t, td, wdir, wspd, omeg = np.genfromtxt( sound_data, delimiter=',', unpack=True)

        prof = profile.create_profile(profile='raw', pres=p[1:], hght=h[1:], tmpc=t[1:], dwpc=td[1:], wspd=wspd[1:],\
                                      wdir=wdir[1:], omeg=omeg[1:], location=location, date=dt_obj, missing=-999.0)
        return prof, dt_obj, member

Example 38

Project: sncosmo Source File: builtins.py
def load_csp(**kwargs):

    # this file contains the csp zeropoints and standards
    fname = get_pkg_data_filename('data/bandpasses/csp/csp_filter_info.dat')
    data = np.genfromtxt(fname, names=True, dtype=None, skip_header=3)
    bands = data['name']
    refsystems = data['reference_sed']
    offsets = data['natural_mag']

    # In Python 3, convert to native strings (Unicode)
    if six.PY3:
        bands = np.char.decode(bands)
        refsystems = np.char.decode(refsystems)

    return CompositeMagSystem(bands, refsystems, offsets, name='csp')

Example 39

Project: Loan_Default_Prediction Source File: predict.py
def load_test_fs():
    test_fs = np.genfromtxt(open(dir + '/test_v2.csv','rb'), delimiter=',', skip_header = 1)
    col_mean = stats.nanmean(test_fs, axis=0)
    inds = np.where(np.isnan(test_fs))
    test_fs[inds] = np.take(col_mean, inds[1])
    test_fs[np.isinf(test_fs)] = 0
    return test_fs

Example 40

Project: mlxtend Source File: mnist.py
def mnist_data():
    """5000 samples from the MNIST handwritten digits dataset.

    Data Source : http://yann.lecun.com/exdb/mnist/

    Returns
    --------
    X, y : [n_samples, n_features], [n_class_labels]
        X is the feature matrix with 5000 image samples as rows,
        each row consists of 28x28 pixels that were unrolled into
        784 pixel feature vectors.
        y contains the 10 unique class labels 0-9.

    """
    tmp = np.genfromtxt(fname=DATA_PATH, delimiter=',')
    X, y = tmp[:, :-1], tmp[:, -1]
    y = y.astype(int)
    return X, y

Example 41

Project: spotpy Source File: analyser.py
def load_csv_results(filename, usecols=None):
    """
    Get an array of your results in the given file, without the first and the 
    last column. The first line may have a different objectivefunction and the last 
    line may be incomplete, which would result in an error.
    
    :filename: Expects an available filename, without the csv, in your working directory
    :type: str
    
    :return: Result array
    :rtype: array
    """
    if usecols == None:
        #return np.genfromtxt(filename+'.csv',delimiter=',',names=True,skip_footer=1,invalid_raise=False)[1:]   
        return np.genfromtxt(filename+'.csv',delimiter=',',names=True,invalid_raise=False)  
    else:
        return np.genfromtxt(filename+'.csv',delimiter=',',names=True,skip_footer=1,invalid_raise=False,usecols=usecols)[1:]   

Example 42

Project: oq-hazardlib Source File: rupture_test.py
    def test_get_dppvalue(self):
        rupture = self.make_rupture_fordpp(
            ParametricProbabilisticRupture, occurrence_rate=0.01,
            temporal_occurrence_model=PoissonTOM(50))
        # Load the testing site.
        data_path = os.path.dirname(__file__)
        filename = os.path.join(
            data_path, "./data/geo_cycs_ss3_testing_site.csv")
        data = numpy.genfromtxt(filename,
                                dtype=float, delimiter=',', names=True,
                                skip_header=6675, skip_footer=6673)

        for loc in range(len(data)):
            lon = data[loc][0]
            lat = data[loc][1]
            ref_dpp = data[loc][2]
            dpp = rupture.get_dppvalue(Point(lon, lat))

            self.assertAlmostEqual(dpp, ref_dpp, delta=0.1)

Example 43

Project: python-rl Source File: neurostim.py
Function: init
    def __init__(self, filename=os.path.join(os.path.dirname(__file__),'configs', 'neurostim', 'params.dat'),
                noise=0.00001, stim_penalty=-1.0, seizure_penalty=-40.0):

        self.noise = noise
        self.stim_penalty = stim_penalty
        self.seizure_penalty = seizure_penalty
        self.embed = -8

        # Usefull utility function
        def samedir(file):
            return os.path.join(os.path.split(filename)[0], file)

        # Load model parameters from input file
        with open(filename, "rb") as f:
            lines = [line.rstrip() for line in f]

        self.features_filename = samedir(lines[0])
        self.labels_filename = samedir(lines[1])
        self.stimulation_filename = samedir(lines[2])
        self.N, self.E, self.L, self.seiz_label = map(int, lines[3:7])
        self.stim_magnitude = float(lines[7])
        self.Ndt = int(lines[8])

        # Load Data from Files...
        # Model features
        self.features = numpy.genfromtxt(self.features_filename)
        # Model labels
        self.labels = numpy.genfromtxt(self.labels_filename)
        # Prototype stimulation
        self.dstim = numpy.genfromtxt(self.stimulation_filename)

        # Initialize model state to zero
        self.state = numpy.zeros((self.E,))

        # Construct the Noise Model
        self.noise_mags = numpy.abs(self.features[1:,:] - self.features[:-1,:]).mean(0)

        # Construct K-d Tree (for Nearest Neighbor computation)
        self.knn = NearestNeighbors(n_neighbors=1, algorithm='kd_tree')
        self.knn.fit(self.features)

        # Initialize state
        self.reset()

Example 44

Project: GParML Source File: gd-example.py
def main():
    global N, Y, X_mu, X_S, flat_global_statistics_bounds, fix_beta, global_statistics_names

    iterations = 100

    # Load data
    Y = numpy.concatenate((
    numpy.genfromtxt('./easydata/inputs/easy_1', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_2', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_3', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_4', delimiter=',')))
    N = Y.shape[0]

    # We have several differet possible initialisations for the embeddings
    #X_mu = numpy.load(X_file)
    #X_mu = PCA(Y_file, Q)
    #X_mu = scipy.randn(N, Q)
    X_S = numpy.clip(numpy.ones((N, Q)) * 0.5
                        + 0.01 * scipy.randn(N, Q),
                    0.001, 1)
    #X_S = numpy.zeros((N, Q))

    X_mu = numpy.concatenate((
        scipy.load('./easydata/embeddings/easy_1.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_2.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_3.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_4.embedding.npy')))
    '''
    X_S = numpy.concatenate((
        scipy.load('./easydata/embeddings/easy_1.variance.npy'),
        scipy.load('./easydata/embeddings/easy_2.variance.npy'),
        scipy.load('./easydata/embeddings/easy_3.variance.npy'),
        scipy.load('./easydata/embeddings/easy_4.variance.npy')))
    '''
    # Initialise the inducing points
    Z = X_mu[numpy.random.permutation(N)[:M],:]
    #Z = X_mu[:M,:]
    Z += scipy.randn(M, Q) * 0.1


    global_statistics_names = {
        'Z' : (M, Q), 'sf2' : (1,1), 'alpha' : (1, Q), 'beta' : (1,1), 'X_mu' : (N, Q), 'X_S' : (N, Q)
    }
    # Initialise the global statistics
    global_statistics = {
        'Z' : Z, # see GPy models/bayesian_gplvm.py
        'sf2' : numpy.array([[1.0]]), # see GPy kern/rbf.py
        'alpha' : scipy.ones((1, Q)), # see GPy kern/rbf.py
        'beta' : numpy.array([[1.0]]), # see GPy likelihood/gaussian.py
        'X_Zmu' : X_mu,
        'X_S' : X_S,
    }

    # Initialise bounds for optimisation
    global_statistics_bounds = {
        'Z' : [(None, None) for i in range(M * Q)],
        'sf2' : [(0, None)],
        'alpha' : [(0, None) for i in range(Q)],
        'beta' : [(0, None)],
        'X_mu' : [(None, None) for i in range(N * Q)],
        'X_S' : [(0, None) for i in range(N * Q)],
    }
    flat_global_statistics_bounds = []
    for key, statistic in global_statistics_bounds.items():
        flat_global_statistics_bounds = flat_global_statistics_bounds+statistic


    ''' 
    Run the optimiser 
    '''
    x0 = flatten_global_statistics(global_statistics)
    # Transform the positiv parameters to be in the range (-Inf, Inf)
    x0 = numpy.array([transform_back(b, x) for b, x in zip(flat_global_statistics_bounds, x0)])
    
    ''' 
    SCG optimisation (adapted from GPy implementation to reduce function calls)
    The number of iterations might be greater than max_f_eval
    '''
    #fix_beta = True
    # We set the flag fixed_embeddings to true because we just need the globals (which now include the embeddings) optimised
    x = GD(likelihood_and_gradient, x0, './easydata/tmp/', fixed_embeddings=True, display=True, maxiters=iterations)
    #fix_beta = False
    #x = SC(likelihood_and_gradient, x[0], display=True, maxiters=iterations)
    flat_array = x[0]
    
    # Transform the parameters that have to be positive to be positive
    flat_array_transformed = numpy.array([transform(b, x) for b, x in zip(flat_global_statistics_bounds, flat_array)])
    global_statistics = rebuild_global_statistics(global_statistics_names, flat_array_transformed)
    print 'Final global_statistics'
    print global_statistics

Example 45

Project: GParML Source File: scg-example.py
def main():
    global N, Y, X_mu, X_S, flat_global_statistics_bounds, fix_beta, global_statistics_names

    iterations = 20

    # Load data
    Y = numpy.concatenate((
    numpy.genfromtxt('./easydata/inputs/easy_1', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_2', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_3', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_4', delimiter=',')))
    N = Y.shape[0]

    # We have several differet possible initialisations for the embeddings
    #X_mu = numpy.load(X_file)
    #X_mu = PCA(Y_file, Q)
    #X_mu = scipy.randn(N, Q)
    X_S = numpy.clip(numpy.ones((N, Q)) * 0.5
                        + 0.01 * scipy.randn(N, Q),
                    0.001, 1)
    #X_S = numpy.zeros((N, Q))

    X_mu = numpy.concatenate((
        scipy.load('./easydata/embeddings/easy_1.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_2.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_3.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_4.embedding.npy')))
    '''
    X_S = numpy.concatenate((
        scipy.load('./easydata/embeddings/easy_1.variance.npy'),
        scipy.load('./easydata/embeddings/easy_2.variance.npy'),
        scipy.load('./easydata/embeddings/easy_3.variance.npy'),
        scipy.load('./easydata/embeddings/easy_4.variance.npy')))
    '''
    # Initialise the inducing points
    Z = X_mu[numpy.random.permutation(N)[:M],:]
    #Z = X_mu[:M,:]
    Z += scipy.randn(M, Q) * 0.1


    global_statistics_names = {
        'Z' : (M, Q), 'sf2' : (1,1), 'alpha' : (1, Q), 'beta' : (1,1), 'X_mu' : (N, Q), 'X_S' : (N, Q)
    }
    # Initialise the global statistics
    global_statistics = {
        'Z' : Z, # see GPy models/bayesian_gplvm.py
        'sf2' : numpy.array([[1.0]]), # see GPy kern/rbf.py
        'alpha' : scipy.ones((1, Q)), # see GPy kern/rbf.py
        'beta' : numpy.array([[1.0]]), # see GPy likelihood/gaussian.py
        'X_Zmu' : X_mu,
        'X_S' : X_S
    }

    # Initialise bounds for optimisation
    global_statistics_bounds = {
        'Z' : [(None, None) for i in range(M * Q)],
        'sf2' : [(0, None)],
        'alpha' : [(0, None) for i in range(Q)],
        'beta' : [(0, None)],
        'X_mu' : [(None, None) for i in range(N * Q)],
        'X_S' : [(0, None) for i in range(N * Q)]
    }
    flat_global_statistics_bounds = []
    for key, statistic in global_statistics_bounds.items():
        flat_global_statistics_bounds = flat_global_statistics_bounds+statistic


    ''' 
    Run the optimiser 
    '''
    x0 = flatten_global_statistics(global_statistics)
    # Transform the positiv parameters to be in the range (-Inf, Inf)
    x0 = numpy.array([transform_back(b, x) for b, x in zip(flat_global_statistics_bounds, x0)])
    
    ''' 
    SCG optimisation (adapted from GPy implementation to reduce function calls)
    The number of iterations might be greater than max_f_eval
    '''
    #fix_beta = True
    x = SCG(likelihood, gradient, x0, display=True, maxiters=iterations)
    #fix_beta = False
    #x = SC(likelihood_and_gradient, x[0], display=True, maxiters=iterations)
    flat_array = x[0]
    
    # Transform the parameters that have to be positive to be positive
    flat_array_transformed = numpy.array([transform(b, x) for b, x in zip(flat_global_statistics_bounds, flat_array)])
    global_statistics = rebuild_global_statistics(global_statistics_names, flat_array_transformed)
    print 'Final global_statistics'
    print global_statistics

Example 46

Project: GParML Source File: scg_adapted-example.py
def main():
    global N, Y, X_mu, X_S, flat_global_statistics_bounds, fix_beta, global_statistics_names

    iterations = 20

    # Load data
    Y = numpy.concatenate((
    numpy.genfromtxt('./easydata/inputs/easy_1', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_2', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_3', delimiter=','),
    numpy.genfromtxt('./easydata/inputs/easy_4', delimiter=',')))
    N = Y.shape[0]

    # We have several differet possible initialisations for the embeddings
    #X_mu = numpy.load(X_file)
    #X_mu = PCA(Y_file, Q)
    #X_mu = scipy.randn(N, Q)
    X_S = numpy.clip(numpy.ones((N, Q)) * 0.5
                        + 0.01 * scipy.randn(N, Q),
                    0.001, 1)
    #X_S = numpy.zeros((N, Q))

    X_mu = numpy.concatenate((
        scipy.load('./easydata/embeddings/easy_1.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_2.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_3.embedding.npy'),
        scipy.load('./easydata/embeddings/easy_4.embedding.npy')))
    '''
    X_S = numpy.concatenate((
        scipy.load('./easydata/embeddings/easy_1.variance.npy'),
        scipy.load('./easydata/embeddings/easy_2.variance.npy'),
        scipy.load('./easydata/embeddings/easy_3.variance.npy'),
        scipy.load('./easydata/embeddings/easy_4.variance.npy')))
    '''
    # Initialise the inducing points
    Z = X_mu[numpy.random.permutation(N)[:M],:]
    #Z = X_mu[:M,:]
    Z += scipy.randn(M, Q) * 0.1


    global_statistics_names = {
        'Z' : (M, Q), 'sf2' : (1,1), 'alpha' : (1, Q), 'beta' : (1,1), 'X_mu' : (N, Q), 'X_S' : (N, Q)
    }
    # Initialise the global statistics
    global_statistics = {
        'Z' : Z, # see GPy models/bayesian_gplvm.py
        'sf2' : numpy.array([[1.0]]), # see GPy kern/rbf.py
        'alpha' : scipy.ones((1, Q)), # see GPy kern/rbf.py
        'beta' : numpy.array([[1.0]]), # see GPy likelihood/gaussian.py
        'X_Zmu' : X_mu,
        'X_S' : X_S
    }

    # Initialise bounds for optimisation
    global_statistics_bounds = {
        'Z' : [(None, None) for i in range(M * Q)],
        'sf2' : [(0, None)],
        'alpha' : [(0, None) for i in range(Q)],
        'beta' : [(0, None)],
        'X_mu' : [(None, None) for i in range(N * Q)],
        'X_S' : [(0, None) for i in range(N * Q)]
    }
    flat_global_statistics_bounds = []
    for key, statistic in global_statistics_bounds.items():
        flat_global_statistics_bounds = flat_global_statistics_bounds+statistic


    ''' 
    Run the optimiser 
    '''
    x0 = flatten_global_statistics(global_statistics)
    # Transform the positiv parameters to be in the range (-Inf, Inf)
    x0 = numpy.array([transform_back(b, x) for b, x in zip(flat_global_statistics_bounds, x0)])
    
    ''' 
    SCG optimisation (adapted from GPy implementation to reduce function calls)
    The number of iterations might be greater than max_f_eval
    '''
    #fix_beta = True
    # We set the flag fixed_embeddings to true because we just need the globals (which now include the embeddings) optimised
    x = SCG_adapted(likelihood_and_gradient, x0, './easydata/tmp/', fixed_embeddings=True, display=True, maxiters=iterations)
    #fix_beta = False
    #x = SC(likelihood_and_gradient, x[0], display=True, maxiters=iterations)
    flat_array = x[0]
    
    # Transform the parameters that have to be positive to be positive
    flat_array_transformed = numpy.array([transform(b, x) for b, x in zip(flat_global_statistics_bounds, flat_array)])
    global_statistics = rebuild_global_statistics(global_statistics_names, flat_array_transformed)
    print 'Final global_statistics'
    print global_statistics

Example 47

Project: pele Source File: test_optim_compatibility.py
    def test1(self):
        current_dir = os.path.dirname(__file__)
        db = Database()
        converter = OptimDBConverter(db, 
                                     mindata=os.path.join(current_dir, "min.data"),
                                     tsdata=os.path.join(current_dir, "ts.data"),
                                     pointsmin=os.path.join(current_dir, "points.min"),
                                     pointsts=os.path.join(current_dir, "points.ts"),
                                     endianness="<")
        
        converter.convert()
        
        mdata = tempfile.NamedTemporaryFile(delete=True)
        tsdata = tempfile.NamedTemporaryFile(delete=True)
        pm = tempfile.NamedTemporaryFile(delete=True)
        pts = tempfile.NamedTemporaryFile(delete=True)
        print mdata.name, tsdata.name
        writer = WritePathsampleDB(db,
                                   mindata=mdata.name,
                                   tsdata=tsdata.name,
                                   pointsmin=pm.name,
                                   pointsts=pts.name,
                                   endianness="<")
        
        writer.write_db()
        
        d1 = np.genfromtxt(os.path.join(current_dir, "min.data"))[:,:3]
        d2 = np.genfromtxt(mdata.name)[:,:3]
        _base_test.assert_arrays_almost_equal(self, d1, d2)
        
        d1 = np.genfromtxt(os.path.join(current_dir, "ts.data")).reshape(-1,8)[:,:5]
        d2 = np.genfromtxt(tsdata.name).reshape(-1,8)[:,:5]
        print d1, d2
        _base_test.assert_arrays_almost_equal(self, d1, d2)
        
        self.assertEqual(sha1_of_file(os.path.join(current_dir, "points.min")),
                         sha1_of_file(pm.name))
        self.assertEqual(sha1_of_file(os.path.join(current_dir, "points.ts")),
                         sha1_of_file(pts.name))

Example 48

Project: BioSPPy Source File: storage.py
Function: load_txt
def load_txt(path):
    """Load data from a text file.

    Parameters
    ----------
    path : str
        Path to file.

    Returns
    -------
    data : array
        Loaded data.
    mdata : dict
        Metadata.

    """

    # normalize path
    path = utils.normpath(path)

    with open(path, 'r') as fid:
        lines = fid.readlines()

    # extract header
    mdata_tmp = {}
    fields = ['Sampling Rate', 'Resolution', 'Date', 'Data Type', 'Labels']
    values = []
    for item in lines:
        if '#' in item:
            # parse comment
            for f in fields:
                if f in item:
                    mdata_tmp[f] = item.split(':= ')[1].strip()
                    fields.remove(f)
                    break
        else:
            values.append(item)

    # convert mdata
    mdata = {}
    df = '%Y-%m-%dT%H:%M:%S.%f'
    try:
        mdata['sampling_rate'] = float(mdata_tmp['Sampling Rate'])
    except KeyError:
        pass
    try:
        mdata['resolution'] = int(mdata_tmp['Resolution'])
    except KeyError:
        pass
    try:
        dtype = mdata_tmp['Data Type']
    except KeyError:
        dtype = None
    try:
        d = datetime.datetime.strptime(mdata_tmp['Date'], df)
        mdata['date'] = d
    except (KeyError, ValueError):
        pass
    try:
        labels = mdata_tmp['Labels'].split('\t')
        mdata['labels'] = labels
    except KeyError:
        pass

    # load array
    data = np.genfromtxt(values, dtype=dtype, delimiter='\t')

    return data, mdata

Example 49

Project: pele Source File: plot_xy_model.py
Function: main
def main():
    from xy_model_system import XYModlelSystem
    fname = "gmin.coords"
    fname = "goldstone_mode.coords"
    fname = "vortex2.coords"
    reduced_coords = np.genfromtxt(fname)
    print (len(reduced_coords))
    print np.sqrt(len(reduced_coords))
    dim = [24, 24]
    print "dim", dim
    
    system = XYModlelSystem(dim, phi_disorder=0)
    coords = system.pot.coords_converter.get_full_coords(reduced_coords)
    
    x = np.zeros(len(coords))
    y = x.copy()
    dx = x.copy()
    dy = x.copy()
    for node in system.pot.G.nodes():
        xyz = system.node2xyz(node)
        i = system.pot.indices[node]
        x[i] = xyz[0]
        y[i] = xyz[1]
        
        theta = coords[i]
        dx[i] = np.cos(theta)
        dy[i] = np.sin(theta)
    
    if True:
        pot = system.get_potential()
        energies = pot.get_spin_energies(coords)
    else:
        energies = None
    
#    plt.quiver(x, y, dx, dy, energies, pivot="middle", cmap="YlOrRd_r")
    if energies is not None:
        plt.quiver(x, y, dx, dy, energies, pivot="middle", cmap="cool_r")
    else:
        plt.quiver(x, y, dx, dy, pivot="middle")
    d = 0.5
    plt.xlim([-d, dim[0]-d])
    plt.ylim([-d, dim[1]-d])
    plt.tight_layout()
    plt.xticks([])
    plt.yticks([])
    plt.gcf().set_facecolor('white')
    plt.box(on=False)

    plt.savefig(fname + ".eps", format="eps")

Example 50

Project: quality-assessment-protocol Source File: temporal_qc.py
def fd_jenkinson(in_file, rmax=80., out_file=None):
    '''
    @ Krsna
    May 2013
    compute
    1) Jenkinson FD from 3dvolreg's *.affmat12.1D file from -1Dmatrix_save
    option input: subject ID, rest_number, name of 6 parameter motion
    correction file (an output of 3dvolreg) output: FD_J.1D file
    Assumptions: 1) subject is available in BASE_DIR
    2) 3dvolreg is already performed and the 1D motion parameter and 1D_matrix
    file file is present in sub?/rest_? called as --->'lfo_mc_affmat.1D'

    Method to calculate Framewise Displacement (FD) calculations
    (Jenkinson et al., 2002)
    Parameters; in_file : string
                rmax : float
                The default radius (as in FSL) of a sphere represents the brain
    Returns; out_file : string
    NOTE: infile should have one 3dvolreg affine matrix in one row -
    NOT the motion parameters
    '''

    import numpy as np
    import os
    import os.path as op
    from shutil import copyfile
    import sys
    import math

    if out_file is None:
        fname, ext = op.splitext(op.basename(in_file))
        out_file = op.abspath('%s_fdfile%s' % (fname, ext))

    # if in_file (coordinate_transformation) is actually the rel_mean output
    # of the MCFLIRT command, forward that file
    if 'rel.rms' in in_file:
        copyfile(in_file, out_file)
        return out_file

    pm_ = np.genfromtxt(in_file)
    original_shape = pm_.shape
    pm = np.zeros((pm_.shape[0], pm_.shape[1] + 4))
    pm[:, :original_shape[1]] = pm_
    pm[:, original_shape[1]:] = [0.0, 0.0, 0.0, 1.0]

    # rigid body transformation matrix
    T_rb_prev = np.matrix(np.eye(4))

    flag = 0
    X = [0]  # First timepoint
    for i in range(0, pm.shape[0]):
        # making use of the fact that the order of aff12 matrix is "row-by-row"
        T_rb = np.matrix(pm[i].reshape(4, 4))

        if flag == 0:
            flag = 1
        else:
            M = np.dot(T_rb, T_rb_prev.I) - np.eye(4)
            A = M[0:3, 0:3]
            b = M[0:3, 3]

            FD_J = math.sqrt(
                (rmax * rmax / 5) * np.trace(np.dot(A.T, A)) + np.dot(b.T, b))
            X.append(FD_J)

        T_rb_prev = T_rb

    np.savetxt(out_file, np.array(X))
    return out_file
See More Examples - Go to Next Page
Page 1 Selected Page 2 Page 3