numpy.save

Here are the examples of the python api numpy.save taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

143 Examples 7

Example 51

Project: deepnet Source File: datahandler.py
Function: dump_buffer
  def DumpBuffer(self, i):
    """Write the contents of buffer i to disk."""
    buf_index = self.buffer_index[i]
    if buf_index == 0:
      return
    buf = self.buffers[i]
    output_prefix = os.path.join(self.output_dir, self.names[i])
    output_filename = '%s-%.5d-of-%.5d' % (
      output_prefix, (self.dump_count[i]+1), self.max_dumps[i])
    self.dump_count[i] += 1
    np.save(output_filename, buf[:buf_index])
    self.buffer_index[i] = 0
    self.data_written[i] += buf_index

Example 52

Project: ndstore Source File: benchmark.py
Function: postnpz
def postNPZ (p, post_data):
  """Post data using the npz interface"""
  
  # Build the url and then create a npz object
  url = 'http://{}/{}/{}/npz/{}/{},{}/{},{}/{},{}/'.format(SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)

  fileobj = cStringIO.StringIO ()
  np.save (fileobj, post_data)
  cdz = zlib.compress (fileobj.getvalue())
  return (url, cdz)

Example 53

Project: deepnet Source File: split_reps.py
def DumpLabelSplit(data, output_dir, name, dataset_pb):
  data_pb = dataset_pb.data.add()
  output_file_name = os.path.join(output_dir, name)
  np.save(output_file_name, data)
  data_pb.name = name
  data_pb.file_pattern = '%s.npy' % output_file_name
  data_pb.size = data.shape[0]
  data_pb.dimensions.append(data.shape[1])

Example 54

Project: modl Source File: hcp.py
def _single_mask(masker, img, dest_data_dir, data_dir):
    dest_file = img.replace('HCP', 'HCP_unmasked')
    dest_file = dest_file.replace('.nii.gz', '')
    dest_dir = os.path.abspath(os.path.join(dest_file, os.pardir))
    if not os.path.exists(dest_dir):
        os.makedirs(dest_dir)
    print('Unmasking %s' % img)
    data = masker.transform(img)
    np.save(dest_file, data)
    origin = dict(array=dest_file + '.npy', img=img)
    with open(join(dest_dir, 'origin.json'), 'w+') as f:
        json.dump(origin, f)

Example 55

Project: pylearn-parsimony Source File: test_svd.py
    def get_err_fast_sparse_svd(self, nrow, ncol, density):
        X = generate_sparse_matrix(shape=(nrow, ncol),
                                   density=density)
        # For debug
#        np.save("/tmp/X_%d_%d.npy" % (nrow, ncol), X)
        fd = None
        try:
            fd, tmpfilename = tempfile.mkstemp(suffix=".npy",
                                               prefix="X_%d_%d" % (nrow, ncol))
            np.save(tmpfilename, X)
        finally:
            if fd is not None:
                os.close(fd)

        # svd from parsimony
        fast_sparse_svd = RankOneSparseSVD(max_iter=1000)
        parsimony_v = fast_sparse_svd.run(X)
#        return self.get_err_by_np_linalg_svd(parsimony_v, X)
        return self.get_err_by_sp_sparse_linalg_svds(parsimony_v, X)

Example 56

Project: ndstore Source File: cube.py
Function: to_npz
  def toNPZ ( self ):
    """Pickle and zip the object"""
    try:
      # Create the compressed cube
      fileobj = cStringIO.StringIO ()
      np.save ( fileobj, self.data )
      return  zlib.compress (fileobj.getvalue())
    except:
      logger.error ("Failed to compress database cube.  Data integrity concern.")
      raise

Example 57

Project: NeuroVault Source File: tasks.py
@shared_task
def save_resampled_transformation_single(pk1, resample_dim=[4, 4, 4]):
    from neurovault.apps.statmaps.models import Image
    from six import BytesIO
    import numpy as np

    img = get_object_or_404(Image, pk=pk1)
    nii_obj = nib.load(img.file.path)   # standard_mask=True is default
    image_vector = make_resampled_transformation_vector(nii_obj,resample_dim)

    f = BytesIO()
    np.save(f, image_vector)
    f.seek(0)
    content_file = ContentFile(f.read())
    img.reduced_representation.save("transform_%smm_%s.npy" %(resample_dim[0],img.pk), content_file)

    return img

Example 58

Project: deepnet Source File: split_reps.py
def DumpDataSplit(data, output_dir, name, dataset_pb, stats_file):
  data_pb = dataset_pb.data.add()
  output_file_name = os.path.join(output_dir, name)
  np.save(output_file_name, data)
  data_pb.name = name
  data_pb.file_pattern = '%s.npy' % output_file_name
  data_pb.size = data.shape[0]
  if stats_file:
    data_pb.stats_file = stats_file
  data_pb.dimensions.append(data.shape[1])

Example 59

Project: bluesky Source File: examples.py
    def trigger(self):
        # save file stash file name
        self._result.clear()
        for idx, (name, reading) in enumerate(super().read().items()):
            # Save the actual reading['value'] to disk and create a record
            # in FileStore.
            np.save('{}_{}.npy'.format(self._path_stem, idx), reading['value'])
            datum_id = new_uid()
            self.fs.insert_datum(self._resource_id, datum_id,
                                 dict(index=idx))
            # And now change the reading in place, replacing the value with
            # a reference to FileStore.
            reading['value'] = datum_id
            self._result[name] = reading
        return NullStatus()

Example 60

Project: pyAudioAnalysis Source File: audioFeatureExtraction.py
def mtFeatureExtractionToFile(fileName, midTermSize, midTermStep, shortTermSize, shortTermStep, outPutFile,
                              storeStFeatures=False, storeToCSV=False, PLOT=False):
    """
    This function is used as a wrapper to:
    a) read the content of a WAV file
    b) perform mid-term feature extraction on that signal
    c) write the mid-term feature sequences to a numpy file
    """
    [Fs, x] = audioBasicIO.readAudioFile(fileName)            # read the wav file
    x = audioBasicIO.stereo2mono(x)                           # convert to MONO if required
    if storeStFeatures:
        [mtF, stF] = mtFeatureExtraction(x, Fs, round(Fs * midTermSize), round(Fs * midTermStep), round(Fs * shortTermSize), round(Fs * shortTermStep))
    else:
        [mtF, _] = mtFeatureExtraction(x, Fs, round(Fs*midTermSize), round(Fs * midTermStep), round(Fs * shortTermSize), round(Fs * shortTermStep))

    numpy.save(outPutFile, mtF)                              # save mt features to numpy file
    if PLOT:
        print "Mid-term numpy file: " + outPutFile + ".npy saved"
    if storeToCSV:
        numpy.savetxt(outPutFile+".csv", mtF.T, delimiter=",")
        if PLOT:
            print "Mid-term CSV file: " + outPutFile + ".csv saved"

    if storeStFeatures:
        numpy.save(outPutFile+"_st", stF)                    # save st features to numpy file
        if PLOT:
            print "Short-term numpy file: " + outPutFile + "_st.npy saved"
        if storeToCSV:
            numpy.savetxt(outPutFile+"_st.csv", stF.T, delimiter=",")    # store st features to CSV file
            if PLOT:
                print "Short-term CSV file: " + outPutFile + "_st.csv saved"

Example 61

Project: LatentStrainAnalysis Source File: eigenhashes.py
Function: save_clusters
	def save_clusters(self,Clusters,Nonzeros):
		for i in range(len(Clusters)):
			c = [Nonzeros[x] for x in Clusters[i]]
			np.save(self.output_path+str(i)+'.cluster.npy',c)
		np.save(self.output_path+'kmer_cluster_sizes.npy',[len(c) for c in Clusters])

Example 62

Project: datafeed Source File: server.py
    def get_5minute(self, symbol, date, format='npy'):
        """Get 5min historical quotes.

        Arguments:
          symbol: String of security.
          date: Which day data to get.
          format: npy or json
        """
        try:
            if isinstance(date, str):
                date = datetime.datetime.strptime(date, '%Y%m%d').date()

            y = self.dbm.fiveminstore.get(symbol, date)

            if format == 'npy':
                memfile = StringIO()
                np.save(memfile, y)
                data = memfile.getvalue()
                del(y)
            else:
                data = json_encode(y.tolist())
            self._write_response(data)
        except KeyError:
            self.request.write("-ERR No data.\r\n")

Example 63

Project: tree-hmm Source File: vb_gmtkExact_continuous.py
def run_gmtk_lineagehmm(args):
    try:
        os.mkdir('gmtk_images')
    except:
        pass

    args.iteration = 'initial'
    args.observe = 'all'
    args.run_name = 'gmtk'
    args.out_params = 'gmtk_images/gmtk_{param}_{observe}'
    args.out_dir = '.'
    args.free_energy = []
    plot_params(args)
    # convert .npy parameters and data into gmtk format
    args.observe_txt = []
    for f in args.observe_matrix:
        obs, obs_txt = npy_observations_to_txt(f)
        args.observe_txt.append(obs_txt)

    # prepare and triangulate the graphical model
    strfile = 'lineagehmm.str'
    gmtk_master = 'dummy.master'
    with open(gmtk_master, 'w') as outfile:
        outfile.write('''

MEAN_IN_FILE inline 2
0 gauss_emit_1_mean   1   1.0
1 gauss_emit_2_mean   1   50.0

COVAR_IN_FILE inline 2
0 gauss_emit_1_covar   1   1.0
1 gauss_emit_2_covar   1   50.0

MC_IN_FILE inline 2
0 1 0 gauss_emit_probs_1   gauss_emit_1_mean   gauss_emit_1_covar
1 1 0 gauss_emit_probs_2   gauss_emit_2_mean   gauss_emit_2_covar

DT_IN_FILE inline 1
0 parent_val 1
    -1 { p0 }

''')
    make_lineage_model(strfile, obs.shape[0], args.K, obs.shape[2], vert_parent=args.vert_parent, mark_avail=mark_avail, separate_theta=args.separate_theta)
    cmd = 'gmtkTriangulate -strFile %s -inputMasterFile %s -rePart T -findBest T  -triangulation completed ' % (strfile, gmtk_master)
    #cmd = 'gmtkTriangulate -strFile %s' % strfile
    subprocess.check_call(cmd, shell=True)

    # populate theta parts if they don't exist
    if args.separate_theta and any(not hasattr(args, 'theta_%s' % i) for i in range(2,args.I+1)):
        for i in range(2,args.I+1):
            if len(args.theta.shape) == 3:
                setattr(args, 'theta_%s' % i, args.theta)
            else:
                setattr(args, 'theta_%s' % i, args.theta[i-2,:,:,:])
            args.params_to_save.append('theta_%s' % i)
        #del args.theta
        #args.params_to_save.remove('theta')


    # for each iteration...
    for args.iteration in range(1, args.max_iterations+1):
        # save args to disk
        w = npy_params_to_workspace(args)

        em_args = []
        for index in range(len(args.observe_txt)):
            gmtk_master = '%s.master' % args.observe_txt[index]
            gmtk_obs = '%s.observations' % args.observe_txt[index]
            write_workspace_simple_master(w, gmtk_master)
            em_args.append((index, args.iteration, args.observe_txt[index], args.I, args.L, gmtk_master, gmtk_obs))

        # run a gmtk em iteration on each file input, accuemulating results
        if args.run_local:
            try:
                pool = multiprocessing.Pool(maxtasksperchild=1)
                pool.map_async(do_em_from_args, em_args).get(99999999)
                #map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
            except KeyboardInterrupt:
                print "Caught KeyboardInterrupt, terminating workers"
                pool.terminate()
                pool.join()
            else:
                pool.close()
                pool.join()
        else:
            pool = sge.SGEPool()
            #jobs_handle = pool.map_async(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))], chunksize=10)
            #jobs_handle = pool.map_async(do_em_from_args, [(i, i, i) for i in range(len(args.observe_txt))], chunksize=10)
            #jobs_handle = pool.imap_unordered(do_em_from_args, em_args, chunksize=1)
            jobs_handle = pool.map_async(do_em_from_args, em_args, chunksize=1)
            # wait for all jobs to finish
            for j in jobs_handle:
                #pass
                j.wait()

        # run one final accuemulator to get params
        accuemulate_em_runs(args, gmtk_obs, gmtk_master)

        plot_params(args)
        plot_energy(args)

        #check convergence
        f = args.free_energy[-1]
        try:
            print 'free energy is', f, 'percent change ll:', abs(args.last_free_energy - f) / args.last_free_energy
        except AttributeError:
            print 'first iteration. free energy is', f
        else:
            if abs(abs(args.last_free_energy - f) / args.last_free_energy) < args.epsilon_e:
                print 'converged! free energy is', f
                break
        finally:
            args.last_free_energy = f

        for p in args.params_to_save:
            numpy.save(os.path.join(args.out_dir, args.out_params.format(param=p, **args.__dict__)),
                    args.__dict__[p])

    if args.run_local:
        try:
            pool = multiprocessing.Pool()
            pool.map_async(do_viterbi_from_args, em_args).get(99999999)
            #map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
        except KeyboardInterrupt:
            print "Caught KeyboardInterrupt, terminating workers"
            pool.terminate()
            pool.join()
        else:
            pool.close()
            pool.join()
    else:
        pool = sge.SGEPool()
        jobs_handle = pool.map_async(do_viterbi_from_args, em_args, chunksize=1)
        # wait for all jobs to finish
        for j in jobs_handle:
            j.wait()
    for a in em_args:
        numpy.save('viterbi_Q_' + os.path.split(a[2])[1], parse_viterbi_states_to_Q(args, a[-1]))

Example 64

Project: kaggle_diabetic Source File: config.py
    def save_features(self, X, n_iter, skip=0, test=False):
        np.save(open(self.get_features_fname(n_iter, skip=skip, 
                                              test=test), 'wb'), X)

Example 65

Project: windml Source File: aemo.py
    def convert(self):
        def time_to_unix(datestr):
            t = datetime.datetime.strptime(datestr, "%Y-%m-%d %H:%M:%S")
            return time.mktime(t.timetuple())

        if not os.path.exists(self.data_home_npy):
            os.makedirs(self.data_home_npy)

        # convert meta csv to meta numpy array
        csvf = open(self.data_home_raw + "meta.csv", "r")

        buf = csvf.readlines()
        reader = csv.reader(buf, delimiter=',')
        reader.next()

        data = []
        for row in reader:
            point = []
            point.append(self.park_id[row[0]])
            point.append(row[3])
            point.append(row[4])
            point.append(row[5])
            data.append(point)

        data_arr = array([(a,b,c,d) for (a,b,c,d) in data], dtype = self.AEMO_META_DTYPE)
        save(self.data_home_npy + "meta.npy", data_arr)

        # convert data to windml format
        turbine_arrays, turbine_npy_arrays = {}, {}
        for k in self.park_id.keys():
            turbine_arrays[k] = []

        print "The following procedures are only necessary for the first time."
        print "Converting AEMO data to lists and filtering missing data."

        for year in self.years:
            for month in self.months_in_year[year]:
                location = self.data_home_raw + self.filename(year, month)
                current = open(location, "r")
                buf = current.readlines()
                reader = csv.reader(buf, delimiter=',')
                keys = reader.next()

                for row in reader:
                    for i in range(1, len(row)):

                        # filter corrupt data
                        if(row[0] == ""):
                            break
                        if(row[i] == ""):
                            continue

                        timestamp = time_to_unix(row[0])
                        power = row[i]
                        turbine_arrays[keys[i]].append([timestamp, power])

                current.close()

        print "Converting to numpy arrays"
        for k in turbine_arrays.keys():
            data = turbine_arrays[k]
            a = array([(a,b,nan) for (a,b) in data], dtype = self.AEMO_DATA_DTYPE)
            turbine_npy_arrays[k] = a
            save(self.data_home_npy + "%i.npy" % self.park_id[k], a)

Example 66

Project: theano_alexnet Source File: layers.py
Function: save_weight
    def save_weight(self, dir, name):
        print 'weight saved: ' + name
        np.save(dir + name + '.npy', self.val.get_value())

Example 67

Project: pydem Source File: processing_manager.py
Function: save_data
    def save_data(self, data, name):
        fn = self.get_fn(name)
        np.save(fn, data)

Example 68

Project: rf_helicopter Source File: pytests.py
def test_saving_tracks():
    routes = Obstacle_Tracks(MAX_OBS_HEIGHT=MAX_OBS_HEIGHT,
                             MAX_OBS_WIDTH=MAX_OBS_WIDTH,
                             WINDOW_HEIGHT=WINDOW_HEIGHT,
                             WINDOW_WIDTH=WINDOW_WIDTH,
                             N_OBSTABLE_GEN=N_OBSTABLE_GEN,
                             MIN_GAP=MIN_GAP,
                             N_TRACKS_GEN=N_TRACKS_GEN)
    # Generate Obstacles
    output5 = routes.generate_tracks
    # Get the first obstacle
    saved = output5[0]
    # Save the obstacle
    np.save(os.path.join(os.getcwd(),
                         'Tests',
                         'Test_Track'), saved)
    # Load obstacle
    loaded = np.load(os.path.join(os.getcwd(),
                                  'Tests',
                                  'Test_Track.npy'))
    assert saved.shape == loaded.shape, 'Dimensions Incorrect'

Example 69

Project: kaggle_diabetic Source File: config.py
    def save_std(self, X, n_iter, skip=0, test=False):
        np.save(open(self.get_std_fname(n_iter, skip=skip,
                                        test=test), 'wb'), X)

Example 70

Project: cat-fancier Source File: train_model.py
def report(clf, testdata, testlabel, traindata_all, trainlabel_all, labels,
           reportdir, modeltype, istrain=False):
    print('# ----- Classification report -----')
    if hasattr(clf, 'best_estimator_'):
        print('## --- best estimator:')
        print(clf.best_estimator_)
    else:
        print('## --- estimator:')
        print(clf)

    print('## test data shape: %s' % (testdata.shape,))
    predlabel = clf.predict(testdata)
    
    print('## accuracy: %s' % (accuracy_score(testlabel, predlabel),))  ## == clf.score
    cm = confusion_matrix(testlabel, predlabel)
    print('## confusion matrix')
    print(cm)
    cr = classification_report(testlabel, predlabel, target_names=labels)
    print(cr)

    if istrain:
        print('save classification report files')
        np.save(reportdir + 'cm_' + modeltype, cm)
        np.save(reportdir + 'cr_' + modeltype, cr)

Example 71

Project: westpa Source File: gen_bstate.py
def run(platform_name, deviceid, two):
    not_obs = [True, True]

    system, coordinates = wcadimer.WCADimer()
    print "Time step: ", (wcadimer.stable_timestep * 2.0).in_units_of(units.femtoseconds)

    # Minimization
    platform = openmm.Platform.getPlatformByName('Reference')

    print 'Minimizing energy...'
    coordinates = minimize(platform, system, coordinates)
    print 'Separation distance: {}'.format(norm(coordinates[1,:] - coordinates[0,:]) / units.angstroms)

    print 'Equilibrating...'
    platform = openmm.Platform.getPlatformByName(platform_name)

    if platform_name == 'CUDA':
        platform.setPropertyDefaultValue('CudaDeviceIndex', '%d' % deviceid)
        platform.setPropertyDefaultValue('CudaPrecision', 'mixed')
    elif platform_name == 'OpenCL':
        platform.setPropertyDefaultValue('OpenCLDeviceIndex', '%d' % deviceid)
        platform.setPropertyDefaultValue('OpenCLPrecision', 'mixed')

    integrator = openmm.LangevinIntegrator(wcadimer.temperature, 
            wcadimer.collision_rate, 
            2.0 * wcadimer.stable_timestep)

    context = openmm.Context(system, integrator, platform)
    context.setPositions(coordinates)
    context.setVelocitiesToTemperature(wcadimer.temperature)

    if two:
        while not_obs[0] or not_obs[1]:
            integrator.step(5000)

            state = context.getState(getPositions=True)
            coordinates = state.getPositions(asNumpy=True)
            sep_dist = norm(coordinates[1,:] - coordinates[0,:]) / units.angstroms
            print 'Separation distance: {}'.format(sep_dist)
            if sep_dist < 5.7:
                not_obs[0] = False
                tag = '_a'
                sep_dist_a = sep_dist
            else:
                not_obs[1] = False
                tag = '_b'
                sep_dist_b = sep_dist

            if not os.path.isdir('bstates'):
                os.makedirs('bstates')

            np.save('bstates/init_coords{}.npy'.format(tag), coordinates / units.nanometers)

        print sep_dist_a, sep_dist_b

    else:
        integrator.step(5000)

        state = context.getState(getPositions=True)
        coordinates = state.getPositions(asNumpy=True)
        print 'Separation distance: {}'.format(norm(coordinates[1,:] - coordinates[0,:]) / units.angstroms)

        if not os.path.isdir('bstates'):
            os.makedirs('bstates')
            np.save('bstates/init_coords.npy', coordinates / units.nanometers)

Example 72

Project: cortex Source File: load_mri.py
Function: load_niftis
def load_niftis(source_dir, out_dir, name='mri', patterns=None):
    '''Loads niftis from a directory.

    Saves the data, paths, mask, and `sites`.

    Args:
        source_dir (str): Directory of nifti files.
        out_dir (str): Output directory for saving arrays, etc.
        name (str): Name of dataset.
        patterns (Optional[list]): list of glob for filtering files.

    '''

    if patterns is not None:
        file_lists = []
        for i, pattern in enumerate(patterns):
            file_list = glob(path.join(source_dir, pattern))
            file_lists.append(file_list)
    else:
        file_lists = [find_niftis(source_dir)]

    base_file = file_lists[0][0]
    paths_file = path.join(out_dir, name + '_file_paths.npy')
    sites_file = path.join(out_dir, name + '_sites.npy')
    mask_file = path.join(out_dir, name + '_mask.npy')
    yaml_file = path.join(out_dir, name + '.yaml')
    tmp_dir = path.join(out_dir, name + '_tmp')
    if not path.isdir(tmp_dir):
        os.mkdir(tmp_dir)

    readline.set_completer_delims(' \t\n;')
    readline.parse_and_bind('tab: complete')
    readline.set_completer(complete_path)
    print ('The MRI dataset requires an anatomical nifti file to visualize'
           ' properly. Enter the path for the anatomical file or leave blank'
           ' if you plan not to use visualization or will enter into the yaml'
           ' file later.')

    anat_file = raw_input('Anat file: ')
    if anat_file == '': yaml_file = None

    datas = []
    new_file_lists = []
    data_paths = []
    for i, file_list in enumerate(file_lists):
        data, new_file_list = read_niftis(file_list)
        new_file_lists.append(new_file_list)
        datas.append(data)
        data_path = path.join(out_dir, name + '_%d.npy' % i)
        data_paths.append(data_path)
        np.save(data_path, data)

    sites = [[0 if 'st' in f else 1 for f in fl] for fl in file_lists]
    sites = sites[0] + sites[1]

    save_mask(np.concatenate(datas, axis=0), mask_file)
    np.save(paths_file, new_file_lists)
    np.save(sites_file, sites)
    with open(yaml_file, 'w') as yf:
        yf.write(
            yaml.dump(
                dict(name=name,
                     data=data_paths,
                     mask=mask_file,
                     nifti=base_file,
                     sites=sites_file,
                     tmp_path=tmp_dir,
                     anat_file=anat_file
                     )
                )
            )

Example 73

Project: out-for-justice Source File: make_risks.py
Function: main
def main(input_file):
    with open(input_file) as f:
        graph = pickle.load(f)

    node_map = {int(node_id): i for i, node_id in enumerate(graph.nodes())}

    outcomes = []
    for fn, name in [
        ('data/sfnodesdtINTOXICATIONCRIME.csv', 'intoxication'),
        ('data/sfnodesdtPROPERTYCRIME.csv', 'property'),
        ('data/sfnodesdtVIOLENTCRIME.csv', 'violent'),
    ]:
        df = pd.read_csv(fn)
        df['crime_type'] = name
        outcomes.append(df)

    df = pd.concat(outcomes)
    df['id'] = df['id'].apply(node_map.get)

    df = df[df['id'].notnull()]

    for (tod, dow), time_df in df.groupby(['daytime', 'superday']):
        mat = time_df.set_index(['id', 'crime_type'])['preds'].unstack()

        outfile = 'data/sf_crime_risks_{}_{}.npy'.format(
            tod.lower().replace('-','_'),
            dow.lower()
        )

        np.save(outfile, mat.values)

Example 74

Project: models Source File: train_student.py
def prepare_student_data(dataset, nb_teachers, save=False):
  """
  Takes a dataset name and the size of the teacher ensemble and prepares
  training data for the student model, according to parameters indicated
  in flags above.
  :param dataset: string corresponding to mnist, cifar10, or svhn
  :param nb_teachers: number of teachers (in the ensemble) to learn from
  :param save: if set to True, will dump student training labels predicted by
               the ensemble of teachers (with Laplacian noise) as npy files.
               It also dumps the clean votes for each class (without noise) and
               the labels assigned by teachers
  :return: pairs of (data, labels) to be used for student training and testing
  """
  assert input.create_dir_if_needed(FLAGS.train_dir)

  # Load the dataset
  if dataset == 'svhn':
    test_data, test_labels = input.ld_svhn(test_only=True)
  elif dataset == 'cifar10':
    test_data, test_labels = input.ld_cifar10(test_only=True)
  elif dataset == 'mnist':
    test_data, test_labels = input.ld_mnist(test_only=True)
  else:
    print("Check value of dataset flag")
    return False

  # Make sure there is data leftover to be used as a test set
  assert FLAGS.stdnt_share < len(test_data)

  # Prepare [unlabeled] student training data (subset of test set)
  stdnt_data = test_data[:FLAGS.stdnt_share]

  # Compute teacher predictions for student training data
  teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)

  # Aggregate teacher predictions to get student training labels
  if not save:
    stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
  else:
    # Request clean votes and clean labels as well
    stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line)

    # Prepare filepath for numpy dump of clean votes
    filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_clean_votes_lap_' + str(FLAGS.lap_scale) + '.npy'  # NOLINT(long-line)

    # Prepare filepath for numpy dump of clean labels
    filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_teachers_labels_lap_' + str(FLAGS.lap_scale) + '.npy'  # NOLINT(long-line)

    # Dump clean_votes array
    with tf.gfile.Open(filepath, mode='w') as file_obj:
      np.save(file_obj, clean_votes)

    # Dump labels_for_dump array
    with tf.gfile.Open(filepath_labels, mode='w') as file_obj:
      np.save(file_obj, labels_for_dump)

  # Print accuracy of aggregated labels
  ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share])
  print("Accuracy of the aggregated labels: " + str(ac_ag_labels))

  # Store unused part of test set for use as a test set after student training
  stdnt_test_data = test_data[FLAGS.stdnt_share:]
  stdnt_test_labels = test_labels[FLAGS.stdnt_share:]

  if save:
    # Prepare filepath for numpy dump of labels produced by noisy aggregation
    filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_labels_lap_' + str(FLAGS.lap_scale) + '.npy' #NOLINT(long-line)

    # Dump student noisy labels array
    with tf.gfile.Open(filepath, mode='w') as file_obj:
      np.save(file_obj, stdnt_labels)

  return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels

Example 75

Project: histwords Source File: pmi2svd.py
def main():
    args = docopt("""
    Usage:
        pmi2svd.py [options] <pmi_path> <output_path>
    
    Options:
        --dim NUM    Dimensionality of eigenvectors [default: 500]
        --neg NUM    Number of negative samples; subtracts its log from PMI [default: 1]
    """)
    
    pmi_path = args['<pmi_path>']
    output_path = args['<output_path>']
    dim = int(args['--dim'])
    neg = int(args['--neg'])
    
    explicit = PositiveExplicit(pmi_path, normalize=False, neg=neg)

    ut, s, vt = sparsesvd(explicit.m.tocsc(), dim)

    np.save(output_path + '.ut.npy', ut)
    np.save(output_path + '.s.npy', s)
    np.save(output_path + '.vt.npy', vt)
    save_vocabulary(output_path + '.words.vocab', explicit.iw)
    save_vocabulary(output_path + '.contexts.vocab', explicit.ic)

Example 76

Project: histwords Source File: makelowdim.py
Function: worker
def worker(proc_num, queue, out_dir, in_dir, count_dir, words, dim, num_words, min_count=100):
    while True:
        if queue.empty():
            break
        year = queue.get()
        print "Loading embeddings for year", year
        time.sleep(random.random() * 120)
        valid_words = set(words_above_count(count_dir, year, min_count))
        print len(valid_words)
        words = list(valid_words.intersection(words[year][:num_words]))
        print len(words)
        base_embed = Explicit.load((in_dir + INPUT_FORMAT).format(year=year), normalize=False)
        base_embed = base_embed.get_subembed(words, restrict_context=True)
        print "SVD for year", year
        u, s, v = randomized_svd(base_embed.m, n_components=dim, n_iter=5)
        print "Saving year", year
        np.save((out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-u.npy", u)
        np.save((out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-v.npy", v)
        np.save((out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-s.npy", s)
        write_pickle(base_embed.iw, (out_dir + OUT_FORMAT).format(year=year, dim=dim) + "-vocab.pkl")

Example 77

Project: datafeed Source File: server.py
    def get_1minute(self, symbol, date, format='npy'):
        """Get 5min historical quotes.

        Arguments:
          symbol: String of security.
          date: Which day data to get.
          format: npy or json
        """
        try:
            if isinstance(date, str):
                date = datetime.datetime.strptime(date, '%Y%m%d').date()

            y = self.dbm.oneminstore.get(symbol, date)

            if format == 'npy':
                memfile = StringIO()
                np.save(memfile, y)
                data = memfile.getvalue()
                del(y)
            else:
                data = json_encode(y.tolist())
            self._write_response(data)
        except KeyError:
            self.request.write("-ERR No data.\r\n")

Example 78

Project: tree-hmm Source File: vb_gmtkExact.py
def run_gmtk_lineagehmm(args):
    try:
        os.mkdir('gmtk_images')
    except:
        pass

    args.iteration = 'initial'
    args.observe = 'all'
    args.run_name = 'gmtk'
    args.out_params = 'gmtk_images/gmtk_{param}_{observe}'
    args.out_dir = '.'
    args.free_energy = []
    plot_params(args)
    # convert .npy parameters and data into gmtk format
    args.observe_txt = []
    for f in args.observe_matrix:
        obs, obs_txt = npy_observations_to_txt(f)
        args.observe_txt.append(obs_txt)

    # prepare and triangulate the graphical model
    strfile = 'lineagehmm.str'
    make_lineage_model(strfile, obs.shape[0], args.K, obs.shape[2], vert_parent=args.vert_parent, mark_avail=mark_avail, separate_theta=args.separate_theta)
    cmd = 'gmtkTriangulate -strFile %s -rePart T -findBest T  -triangulation completed ' % strfile
    #cmd = 'gmtkTriangulate -strFile %s' % strfile
    subprocess.check_call(cmd, shell=True)

    # populate theta parts if they don't exist
    if args.separate_theta and any(not hasattr(args, 'theta_%s' % i) for i in range(2,args.I+1)):
        for i in range(2,args.I+1):
            if len(args.theta.shape) == 3:
                setattr(args, 'theta_%s' % i, args.theta)
            else:
                setattr(args, 'theta_%s' % i, args.theta[i-2,:,:,:])
            args.params_to_save.append('theta_%s' % i)
        #del args.theta
        #args.params_to_save.remove('theta')


    # for each iteration...
    for args.iteration in range(1, args.max_iterations+1):
        # save args to disk
        w = npy_params_to_workspace(args)

        em_args = []
        for index in range(len(args.observe_txt)):
            gmtk_master = '%s.master' % args.observe_txt[index]
            gmtk_obs = '%s.observations' % args.observe_txt[index]
            write_workspace_simple_master(w, gmtk_master)
            em_args.append((index, args.iteration, args.observe_txt[index], args.I, args.L, gmtk_master, gmtk_obs))

        # run a gmtk em iteration on each file input, accuemulating results
        if args.run_local:
            try:
                pool = multiprocessing.Pool(maxtasksperchild=1)
                pool.map_async(do_em_from_args, em_args).get(99999999)
                #map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
            except KeyboardInterrupt:
                print "Caught KeyboardInterrupt, terminating workers"
                pool.terminate()
                pool.join()
            else:
                pool.close()
                pool.join()
        else:
            pool = sge.SGEPool()
            #jobs_handle = pool.map_async(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))], chunksize=10)
            #jobs_handle = pool.map_async(do_em_from_args, [(i, i, i) for i in range(len(args.observe_txt))], chunksize=10)
            #jobs_handle = pool.imap_unordered(do_em_from_args, em_args, chunksize=1)
            jobs_handle = pool.map_async(do_em_from_args, em_args, chunksize=1)
            # wait for all jobs to finish
            for j in jobs_handle:
                #pass
                j.wait()

        # run one final accuemulator to get params
        accuemulate_em_runs(args, gmtk_obs, gmtk_master)

        plot_params(args)
        plot_energy(args)

        #check convergence
        f = args.free_energy[-1]
        try:
            print 'free energy is', f, 'percent change ll:', abs(args.last_free_energy - f) / args.last_free_energy
        except AttributeError:
            print 'first iteration. free energy is', f
        else:
            if abs(abs(args.last_free_energy - f) / args.last_free_energy) < args.epsilon_e:
                print 'converged! free energy is', f
                break
        finally:
            args.last_free_energy = f

        for p in args.params_to_save:
            numpy.save(os.path.join(args.out_dir, args.out_params.format(param=p, **args.__dict__)),
                    args.__dict__[p])

    if args.run_local:
        try:
            pool = multiprocessing.Pool()
            pool.map_async(do_viterbi_from_args, em_args).get(99999999)
            #map(do_em_from_args, [(i, w, args) for i in range(len(args.observe_txt))])
        except KeyboardInterrupt:
            print "Caught KeyboardInterrupt, terminating workers"
            pool.terminate()
            pool.join()
        else:
            pool.close()
            pool.join()
    else:
        pool = sge.SGEPool()
        jobs_handle = pool.map_async(do_viterbi_from_args, em_args, chunksize=1)
        # wait for all jobs to finish
        for j in jobs_handle:
            j.wait()
    for a in em_args:
        numpy.save('viterbi_Q_' + os.path.split(a[2])[1], parse_viterbi_states_to_Q(args, a[-1]))

Example 79

Project: datafeed Source File: server.py
Function: get_day
    def get_day(self, symbol, length_or_date, format='npy'):
        """Get OHLCs quotes.

        Return chronicle ordered quotes.
        """
        try:
            if len(length_or_date) == 8: # eg: 20101209
                date = datetime.datetime.strptime(length_or_date, '%Y%m%d').date()
                y = self.dbm.daystore.get_by_date(symbol, date)
            else:
                length = length_or_date
                y = self.dbm.daystore.get(symbol, int(length))
                if length == 1:
                    y = y[0]

            if format == 'npy':
                memfile = StringIO()
                np.save(memfile, y)
                data = memfile.getvalue()
                del(y)
            else:
                data = json_encode(y.tolist())
            self._write_response(data)
        except KeyError:
            self.request.write("-ERR Symbol %s not exists.\r\n" % symbol)

Example 80

Project: deep_nets_iclr04 Source File: layer_blocks.py
Function: save_weight
    def save_weight(self, dir, epoch = 0):
        print '- Saved weight: ' + self.name + '_ep'+str(epoch)
        numpy.save(dir + self.name + '_ep'+str(epoch), self.val.get_value())

Example 81

Project: variational-text-tensorflow Source File: utils.py
Function: save_npy
def save_npy(path, obj):
  np.save(path, obj)
  print(" [*] save %s" % path)

Example 82

Project: Theano-MPI Source File: googlenet.py
Function: save_weight
    def save_weight(self, dir, name):
        #print 'weight saved: ' + name
        np.save(dir + name + '.npy', self.val.get_value())

Example 83

Project: ptsa Source File: meld.py
Function: memmap_array
def _memmap_array(x, memmap_dir=None, use_h5py=False, unique_id=''):
    if memmap_dir is None:
        memmap_dir = tempfile.gettempdir()
    # generate the base filename
    filename = os.path.join(memmap_dir,
                            unique_id + '_' + str(id(x)))
    if use_h5py:
        import h5py
        filename += '.h5'
        h = h5py.File(filename)
        mmap_dat = h.create_dataset('mdat', data=x,
                                    compression='gzip')
        h.flush()
    else:
        # use normal memmap
        # filename = os.path.join(memmap_dir, str(id(x)) + '.npy')
        filename += '.npy'
        np.save(filename, x)
        mmap_dat = np.load(filename, 'r+')
    return mmap_dat

Example 84

Project: Emotion-Recognition-RNN Source File: model.py
Function: save_npy
    def save_npy(self, filename):
        np.save(filename, self.get_params())

Example 85

Project: rf_helicopter Source File: pytests.py
def test_saving_obstacles():
    routes = Obstacle_Tracks(MAX_OBS_HEIGHT=MAX_OBS_HEIGHT,
                             MAX_OBS_WIDTH=MAX_OBS_WIDTH,
                             WINDOW_HEIGHT=WINDOW_HEIGHT,
                             WINDOW_WIDTH=WINDOW_WIDTH,
                             N_OBSTABLE_GEN=N_OBSTABLE_GEN,
                             MIN_GAP=MIN_GAP,
                             N_TRACKS_GEN=N_TRACKS_GEN)
    # Generate Obstacles
    output4 = routes.generate_obstacles
    # Get the first obstacle
    saved = output4[0]
    # Save the obstacle
    np.save(os.path.join(os.getcwd(),
                         'Tests',
                         'Test_Obstacle'), saved)
    # Load obstacle
    loaded = np.load(os.path.join(os.getcwd(),
                                  'Tests',
                                  'Test_Obstacle.npy'))
    assert saved.shape == loaded.shape, 'Dimensions Incorrect'

Example 86

Project: theano_alexnet Source File: tools.py
Function: save_momentums
def save_momentums(vels, weights_dir, epoch):
    for ind in range(len(vels)):
        np.save(os.path.join(weights_dir, 'mom_' + str(ind) + '_' + str(epoch)),
                vels[ind].get_value())

Example 87

Project: models Source File: train_student.py
def prepare_student_data(dataset, nb_teachers, save=False):
  """
  Takes a dataset name and the size of the teacher ensemble and prepares
  training data for the student model, according to parameters indicated
  in flags above.
  :param dataset: string corresponding to mnist, cifar10, or svhn
  :param nb_teachers: number of teachers (in the ensemble) to learn from
  :param save: if set to True, will dump student training labels predicted by
               the ensemble of teachers (with Laplacian noise) as npy files.
               It also dumps the clean votes for each class (without noise) and
               the labels assigned by teachers
  :return: pairs of (data, labels) to be used for student training and testing
  """
  assert input.create_dir_if_needed(FLAGS.train_dir)

  # Load the dataset
  if dataset == 'svhn':
    test_data, test_labels = input.ld_svhn(test_only=True)
  elif dataset == 'cifar10':
    test_data, test_labels = input.ld_cifar10(test_only=True)
  elif dataset == 'mnist':
    test_data, test_labels = input.ld_mnist(test_only=True)
  else:
    print("Check value of dataset flag")
    return False

  # Make sure there is data leftover to be used as a test set
  assert FLAGS.stdnt_share < len(test_data)

  # Prepare [unlabeled] student training data (subset of test set)
  stdnt_data = test_data[:FLAGS.stdnt_share]

  # Compute teacher predictions for student training data
  teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)

  # Aggregate teacher predictions to get student training labels
  if not save:
    stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
  else:
    # Request clean votes and clean labels as well
    stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line)

    # Prepare filepath for numpy dump of clean votes
    filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_clean_votes_lap_' + str(FLAGS.lap_scale) + '.npy'  # NOLINT(long-line)

    # Prepare filepath for numpy dump of clean labels
    filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_teachers_labels_lap_' + str(FLAGS.lap_scale) + '.npy'  # NOLINT(long-line)

    # Dump clean_votes array
    with gfile.Open(filepath, mode='w') as file_obj:
      np.save(file_obj, clean_votes)

    # Dump labels_for_dump array
    with gfile.Open(filepath_labels, mode='w') as file_obj:
      np.save(file_obj, labels_for_dump)

  # Print accuracy of aggregated labels
  ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share])
  print("Accuracy of the aggregated labels: " + str(ac_ag_labels))

  # Store unused part of test set for use as a test set after student training
  stdnt_test_data = test_data[FLAGS.stdnt_share:]
  stdnt_test_labels = test_labels[FLAGS.stdnt_share:]

  if save:
    # Prepare filepath for numpy dump of labels produced by noisy aggregation
    filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_labels_lap_' + str(FLAGS.lap_scale) + '.npy' #NOLINT(long-line)

    # Dump student noisy labels array
    with gfile.Open(filepath, mode='w') as file_obj:
      np.save(file_obj, stdnt_labels)

  return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels

Example 88

Project: beacon-ml Source File: beacon-ml.py
Function: main
def main():
  fname = os.path.join(DATA_DIR, CSV_FNAME)
  if not os.path.exists(RESULTS_DIR):
    os.makedirs(RESULTS_DIR)

  # run this at first to make sure everything looks good
  if RUN_CHECKS:
    print 'Running checks'
    checks()
    sys.stdout.flush()

  # then run to to generate the vectorized data from the raw dump (already done)
  if REGEN_DATA:
    print 'Re-generating data'
    sys.stdout.flush()
    data, labels, ranges = subsample_and_vectorize_data(fname, LABEL, PRETTY_PRINT_LABEL)
    with open(VECTOR_DATA_PATH, 'wb') as file:
      np.save(file, data)
    data = None
    with open(VECTOR_LABELS_PATH, 'wb') as file:
      np.save(file, labels)
    labels = None
    with open(VALUE_RANGES_PATH, 'wb') as file:
      json.dump(ranges, file, indent=4)
    ranges = None

  # finally train the model
  if TRAIN == 'deep_model':
    print 'Training deep model'
    sys.stdout.flush()
    train_deep_model()
    test_deep_model()

  # you can also train a logreg
  if TRAIN == 'logistic_regression':
    print 'Training logistic regression'
    sys.stdout.flush()
    train_logistic_regression()

  # or train a random forest
  if TRAIN == 'random_forest':
    print 'Training random forest'
    sys.stdout.flush()
    top_features = train_random_forest()
    with open(IMPORTANCES_CSV, 'wb') as f:
      f.write('feature,importance\n')
      for feature, importance in top_features:
        f.write('%s,%.4f\n' % (feature, importance))

Example 89

Project: beacon-ml Source File: deep-learning.py
Function: main
def main():
  if not os.path.exists(RESULTS_DIR):
    os.makedirs(RESULTS_DIR)

  # then run to to generate the vectorized data from the raw dump (already done)
  if REGEN_DATA or \
      not os.path.exists(VECTOR_DATA_PATH) or \
      not os.path.exists(VECTOR_LABELS_PATH) or \
      not os.path.exists(VALUE_RANGES_PATH):
    print 'Re-generating data'
    csv_fname = os.path.join(DATA_DIR, CSV_FNAME)
    sys.stdout.flush()
    data, labels, ranges = subsample_and_vectorize_data(csv_fname, LABEL, PRETTY_PRINT_LABEL)
    with open(VECTOR_DATA_PATH, 'wb') as file:
      np.save(file, data)
    data = None
    with open(VECTOR_LABELS_PATH, 'wb') as file:
      np.save(file, labels)
    labels = None
    with open(VALUE_RANGES_PATH, 'wb') as file:
      json.dump(ranges, file, indent=4)
    ranges = None

  features = load_feature_names()
  (x_train_full, y_train), (x_val_full, y_val) = prepare_data()
  train_rows, train_cols = x_train_full.shape
  val_rows, val_cols = x_val_full.shape

  # Figure out how many columns we need for the known starting features
  fname = os.path.join(RESULTS_DIR, PRETTY_PRINT_LABEL + '_accuracy_test')
  base_columns = 0;
  for name in starting_features:
    if name in features:
      fname += "." + name
      base_columns += features[name]['end'] - features[name]['start'] + 1
  fname += ".csv"

  # Try training each feature against the data set individually
  feature_count = len(features)
  feature_num = 0
  with open(fname, 'wb', 1) as out:
    for name, feature in features.iteritems():
      if not len(test_features) or name in test_features:
        feature_num += 1

        #Build an input data set with just the columns we care about
        count = feature['end'] - feature['start'] + 1
        x_train = np.zeros((train_rows, base_columns + count))
        x_val = np.zeros((val_rows, base_columns + count))
        col = 0
        # Populate the starting features
        for n in starting_features:
          if n == name:
            continue
          if n in features:
            for column in xrange(features[n]['start'], features[n]['end'] + 1):
              x_train[:, col] = x_train_full[:, column]
              x_val[:, col] = x_val_full[:, column]
              col += 1
        # Populate the features we are testing
        for column in xrange(feature['start'], feature['end'] + 1):
          x_train[:,col] = x_train_full[:,column]
          x_val[:, col] = x_val_full[:, column]
          col += 1

        # normalize the data
        scaler = StandardScaler()
        x_train = scaler.fit_transform(x_train)
        x_val = scaler.transform(x_val)

        # Run the actual training
        print '[{0:d}/{1:d}] Training deep model on {2} ({3:d} columns)'.format(feature_num, feature_count, name, col)
        sys.stdout.flush()
        acc, model = train_deep_model(x_train, y_train, x_val, y_val)
        print '{0} Accuracy: {1:0.4f}'.format(name, acc)
        sys.stdout.flush()
        out.write('{0},{1:0.4f}\n'.format(name,acc))

        # Test the varous values for the feature
        if len(test_features):
          max_val = 100000
          min_val = 100
          step_size = 100
          count = (max_val - min_val) / step_size
          original_values = np.array([[0.0]] * count)
          row = 0
          for value in xrange(100, 100000, 100):
            original_values[row] = value
            row += 1
          data = scaler.transform(original_values)
          prob = model.predict_proba(data, verbose=0)
          with open(os.path.join(RESULTS_DIR, PRETTY_PRINT_LABEL + '_values_' + name), 'wb') as v:
            for row in xrange(0, count):
              value = original_values[row][0]
              probability = prob[row][0]
              v.write('{0:d},{1:f}\n'.format(int(value), probability))

Example 90

Project: Dracula Source File: train.py
Function: save
    def save(self, folder):
        for param, name in zip(self.params, self.names):
            np.save(os.path.join(folder, name + '.npy'), param.get_value())

Example 91

Project: maxfield Source File: timetest.py
Function: main
def main(min_portals=15,max_portals=60,d_portals=5,
         min_agents=1,max_agents=5,d_agents=1,
         trials=10,
         outfile='runtimes.csv',
         timeout=600):
    """
    Run a series of maxfields to estimate the runtime for a
    given number of portals and a given number of agents
    Run each test trials times
    Saves runtimes to outfile
    """
    args = []
    # Set up array of jobs
    # loop over portal number
    for nportals in range(min_portals,max_portals+1,d_portals):
        # loop over number of agents
        for nagents in range(min_agents,max_agents+1,d_agents):
            # Loop over each trial
            for trial in range(trials):
                # Once with and once without skipping plots
                for skipplot in [True,False]:
                    args.append((nportals,nagents,trial,skipplot))
    with process.Pool(4) as p:
        jobs = [p.schedule(worker,args=arg,timeout=timeout) for arg in args]
    runtimes = []
    for arg,job in zip(args,jobs):
        try:
            runtimes.append(job.get())
        except TimeoutError:
            runtimes.append((arg[0],arg[1],arg[2],arg[3],'nan'))
            cleanup(arg[0],arg[1],arg[2],arg[3])
    np.save('runtimes.npy',runtimes)
    with open(outfile,'w') as f:
        f.write('nportals, nagents, trial, skipplot, runtime\n')
        for foo in runtimes:
            nportals, nagents, trial, skipplot, runtime = foo
            f.write('{0}, {1}, {2}, {3}, {4}\n'.format(nportals,nagents,trial,skipplot,runtime))

Example 92

Project: datafeed Source File: client.py
    def put_minute(self, symbol, rawdata):
        memfile = StringIO()
        np.save(memfile, rawdata)
        return self.execute_command('PUT_MINUTE', symbol, memfile.getvalue(), 'npy')

Example 93

Project: datafeed Source File: client.py
    def put_1minute(self, symbol, rawdata):
        memfile = StringIO()
        np.save(memfile, rawdata)
        return self.execute_command('PUT_1MINUTE', symbol, memfile.getvalue(), 'npy')

Example 94

Project: datafeed Source File: client.py
    def put_5minute(self, symbol, rawdata):
        memfile = StringIO()
        np.save(memfile, rawdata)
        return self.execute_command('PUT_5MINUTE', symbol, memfile.getvalue(), 'npy')

Example 95

Project: datafeed Source File: client.py
    def put_day(self, symbol, rawdata):
        memfile = StringIO()
        np.save(memfile, rawdata)
        return self.execute_command('PUT_DAY', symbol, memfile.getvalue(), 'npy')

Example 96

Project: seisflows Source File: array.py
Function: save_npy
def savenpy(filename, v):
    """Saves numpy binary file."""
    np.save(filename, v)
    os.rename(filename + '.npy', filename)

Example 97

Project: cortex Source File: read_fmri.py
Function: load_niftis
def load_niftis(source_dir, out_dir, name='fmri', patterns=None):
    '''Loads niftis from a directory.

    Saves the data, paths, mask, and `sites`.

    Args:
        source_dir (str): Directory of nifti files.
        out_dir (str): Output directory for saving arrays, etc.
        name (str): Name of dataset.
        patterns (Optional[list]): list of glob for filtering files.

    '''

    if patterns is not None:
        file_lists = []
        for i, pattern in enumerate(patterns):
            file_list = glob(path.join(source_dir, pattern))
            file_lists.append(file_list)
    else:
        file_lists = [find_niftis(source_dir)]

    base_file = file_lists[0][0]
    paths_file = path.join(out_dir, name + '_file_paths.npy')
    sites_file = path.join(out_dir, name + '_sites.npy')
    mask_file = path.join(out_dir, name + '_mask.npy')
    yaml_file = path.join(out_dir, name + '.yaml')
    tmp_dir = path.join(out_dir, name + '_tmp')
    if not path.isdir(tmp_dir):
        os.mkdir(tmp_dir)

    readline.set_completer_delims(' \t\n;')
    readline.parse_and_bind('tab: complete')
    readline.set_completer(complete_path)
    print ('The fMRI dataset requires an anatomical nifti file to visualize'
           ' properly. Enter the path for the anatomical file or leave blank'
           ' if you plan not to use visualization or will enter into the yaml'
           ' file later.')

    anat_file = raw_input('Anat file: ')
    if anat_file == '': yaml_file = None

    datas = []
    new_file_lists = []
    data_paths = []
    for i, file_list in enumerate(file_lists):
        data, new_file_list = read_niftis(file_list)
        new_file_lists.append(new_file_list)
        datas.append(data)
        data_path = path.join(out_dir, name + '_%d.npy' % i)
        data_paths.append(data_path)
        np.save(data_path, data)

    sites = [[0 if 'st' in f else 1 for f in fl] for fl in file_lists]
    sites = sites[0] + sites[1]

    save_mask(np.concatenate(datas, axis=0), mask_file)
    np.save(paths_file, new_file_lists)
    np.save(sites_file, sites)
    with open(yaml_file, 'w') as yf:
        yf.write(
            yaml.dump(
                dict(name=name,
                     data=data_paths,
                     mask=mask_file,
                     nifti=base_file,
                     sites=sites_file,
                     tmp_path=tmp_dir,
                     anat_file=anat_file
                     )
                )
            )

Example 98

Project: simpeg Source File: Directives.py
Function: enditer
    def endIter(self):
        np.save('{0:03d}-{1!s}'.format(self.opt.iter, self.fileName), self.opt.xc)

Example 99

Project: D-GEX Source File: GTEx.py
Function: main
def main():
    GTEx_gctobj = gct.GCT(GTEx_GCTX)
    GTEx_gctobj.read()
    GTEx_genes = map(lambda x:x.split('.')[0], GTEx_gctobj.get_rids())

    lm_id = []
    infile = open(BGEDV2_LM_ID)
    for line in infile:
        ID = line.strip('\n').split('\t')[0]
        lm_id.append(ID)
    
    infile.close()
    lm_idx = map(GTEx_genes.index, lm_id)
    
    tg_id = []
    infile = open(BGEDV2_TG_ID)
    for line in infile:
        ID = line.strip('\n').split('\t')[0]
        tg_id.append(ID)
    
    infile.close()
    tg_idx = map(GTEx_genes.index, tg_id)
    
    genes_idx = lm_idx + tg_idx
    
    data = GTEx_gctobj.matrix[genes_idx, :].astype('float64')
    
    np.save('GTEx_float64.npy', data)

Example 100

Project: datafeed Source File: server.py
Function: get_minute
    def get_minute(self, symbol, timestamp, format='npy'):
        """Get daily minutes history.

        Arguments:
        symbol: String of security.
        timestamp: Which day data to get.
        format: npy or json
        """
        try:
            ts = int(timestamp)
            if ts > 0:
                store = self.dbm.get_minutestore_at(ts)
            else:
                store = self.dbm.minutestore
                
            y = store.get(symbol)

            if format == 'npy':
                memfile = StringIO()
                np.save(memfile, y)
                data = memfile.getvalue()
                del(y)
            else:
                data = json_encode(y.tolist())
            self._write_response(data)
        except KeyError:
            self.request.write("-ERR Symbol %s not exists.\r\n" % symbol)
See More Examples - Go to Next Page
Page 1 Page 2 Selected Page 3