Here are the examples of the python api numpy.asarray taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
160 Examples
0
Example 101
Project: neurokernel Source File: timing_connectome_demo_gpu_slow.py
def gen_sels(conn_mat, scaling=1):
"""
Generate port selectors for LPUs in benchmark test.
Parameters
----------
conn_mat : numpy.ndarray
Square array containing numbers of directed spiking port connections
between LPUs (which correspond to the row and column indices).
scaling : int
Scaling factor; multiply all connection numbers by this value.
Returns
-------
mod_sels : dict of tuples
Ports in module interfaces; the keys are the module IDs and the values are tuples
containing the respective selectors for all ports, all input ports, all
output ports, all graded potential, and all spiking ports.
pat_sels : dict of tuples
Ports in pattern interfaces; the keys are tuples containing the two
module IDs connected by the pattern and the values are pairs of tuples
containing the respective selectors for all source ports, all
destination ports, all input ports connected to the first module,
all output ports connected to the first module, all graded potential ports
connected to the first module, all spiking ports connected to the first
module, all input ports connected to the second module,
all output ports connected to the second module, all graded potential ports
connected to the second module, and all spiking ports connected to the second
module.
"""
conn_mat = np.asarray(conn_mat)
r, c = conn_mat.shape
assert r == c
n_lpu = r
assert scaling > 0 and isinstance(scaling, numbers.Integral)
conn_mat *= scaling
# Construct selectors describing the ports exposed by each module:
mod_sels = {}
for i in xrange(n_lpu):
lpu_id = 'lpu%s' % i
# Structure ports as
# /lpu_id/in_or_out/spike_or_gpot/other_lpu_id/[0:n_spike]
# where in_or_out is relative to module i:
sel_in_gpot = Selector('')
sel_out_gpot = Selector('')
sel_in_spike = \
Selector(','.join(['/lpu%i/in/spike/lpu%i/[0:%i]' % (i, j, n) for j, n in \
enumerate(conn_mat[:, i]) if (j != i and n != 0)]))
sel_out_spike = \
Selector(','.join(['/lpu%i/out/spike/lpu%i/[0:%i]' % (i, j, n) for j, n in \
enumerate(conn_mat[i, :]) if (j != i and n != 0)]))
mod_sels[lpu_id] = (Selector.union(sel_in_gpot, sel_in_spike,
sel_out_gpot, sel_out_spike),
Selector.union(sel_in_gpot, sel_in_spike),
Selector.union(sel_out_gpot, sel_out_spike),
Selector.union(sel_in_gpot, sel_out_gpot),
Selector.union(sel_in_spike, sel_out_spike))
# Construct selectors describing the ports connected by each pattern:
pat_sels = {}
for i, j in itertools.combinations(xrange(n_lpu), 2):
lpu_i = 'lpu%s' % i
lpu_j = 'lpu%s' % j
# The pattern's input ports are labeled "../out.." because that selector
# describes the output ports of the connected module's interface:
sel_in_gpot_i = Selector('')
sel_out_gpot_i = Selector('')
sel_in_gpot_j = Selector('')
sel_out_gpot_j = Selector('')
sel_in_spike_i = Selector('/%s/out/spike/%s[0:%i]' % (lpu_i, lpu_j,
conn_mat[i, j]))
sel_out_spike_i = Selector('/%s/in/spike/%s[0:%i]' % (lpu_i, lpu_j,
conn_mat[j, i]))
sel_in_spike_j = Selector('/%s/out/spike/%s[0:%i]' % (lpu_j, lpu_i,
conn_mat[j, i]))
sel_out_spike_j = Selector('/%s/in/spike/%s[0:%i]' % (lpu_j, lpu_i,
conn_mat[i, j]))
# The order of these two selectors is important; the individual 'from'
# and 'to' ports must line up properly for Pattern.from_concat to
# produce the right pattern:
sel_from = Selector.add(sel_in_gpot_i, sel_in_spike_i,
sel_in_gpot_j, sel_in_spike_j)
sel_to = Selector.add(sel_out_gpot_j, sel_out_spike_j,
sel_out_gpot_i, sel_out_spike_i)
# Exclude scenarios where the "from" or "to" selector is empty (and
# therefore cannot be used to construct a pattern):
if len(sel_from) and len(sel_to):
pat_sels[(lpu_i, lpu_j)] = \
(sel_from, sel_to,
Selector.union(sel_in_gpot_i, sel_in_spike_i),
Selector.union(sel_out_gpot_i, sel_out_spike_i),
Selector.union(sel_in_gpot_i, sel_out_gpot_i),
Selector.union(sel_in_spike_i, sel_out_spike_i),
Selector.union(sel_in_gpot_j, sel_in_spike_j),
Selector.union(sel_out_gpot_j, sel_out_spike_j),
Selector.union(sel_in_gpot_j, sel_out_gpot_j),
Selector.union(sel_in_spike_j, sel_out_spike_j))
return mod_sels, pat_sels
0
Example 102
def pca(data, axis=0, mask=None, ncomp=None, standardize=True,
design_keep=None, design_resid='mean', tol_ratio=0.01):
"""Compute the SVD PCA of an array-like thing over `axis`.
Parameters
----------
data : ndarray-like (np.float)
The array on which to perform PCA over axis `axis` (below)
axis : int, optional
The axis over which to perform PCA (axis identifying
observations). Default is 0 (first)
mask : ndarray-like (np.bool), optional
An optional mask, should have shape given by data axes, with
`axis` removed, i.e.: ``s = data.shape; s.pop(axis);
msk_shape=s``
ncomp : {None, int}, optional
How many component basis projections to return. If ncomp is None
(the default) then the number of components is given by the
calculated rank of the data, after applying `design_keep`,
`design_resid` and `tol_ratio` below. We always return all the
basis vectors and percent variance for each component; `ncomp`
refers only to the number of basis_projections returned.
standardize : bool, optional
If True, standardize so each time series (after application of
`design_keep` and `design_resid`) has the same standard
deviation, as calculated by the ``np.std`` function.
design_keep : None or ndarray, optional
Data is projected onto the column span of design_keep.
None (default) equivalent to ``np.identity(data.shape[axis])``
design_resid : str or None or ndarray, optional
After projecting onto the column span of design_keep, data is
projected perpendicular to the column span of this matrix. If
None, we do no such second projection. If a string 'mean', then
the mean of the data is removed, equivalent to passing a column
vector matrix of 1s.
tol_ratio : float, optional
If ``XZ`` is the vector of singular values of the projection
matrix from `design_keep` and `design_resid`, and S are the
singular values of ``XZ``, then `tol_ratio` is the value used to
calculate the effective rank of the projection of the design, as
in ``rank = ((S / S.max) > tol_ratio).sum()``
Returns
-------
results : dict
$G$ is the number of non-trivial components found after applying
`tol_ratio` to the projections of `design_keep` and
`design_resid`.
`results` has keys:
* ``basis_vectors``: series over `axis`, shape (data.shape[axis], G) -
the eigenvectors of the PCA
* ``pcnt_var``: percent variance explained by component, shape
(G,)
* ``basis_projections``: PCA components, with components varying
over axis `axis`; thus shape given by: ``s = list(data.shape);
s[axis] = ncomp``
* ``axis``: axis over which PCA has been performed.
Notes
-----
See ``pca_image.m`` from ``fmristat`` for Keith Worsley's code on
which some of this is based.
See: http://en.wikipedia.org/wiki/Principal_component_analysis for
some inspiration for naming - particularly 'basis_vectors' and
'basis_projections'
Examples
--------
>>> arr = np.random.normal(size=(17, 10, 12, 14))
>>> msk = np.all(arr > -2, axis=0)
>>> res = pca(arr, mask=msk, ncomp=9)
Basis vectors are columns. There is one column for each component. The
number of components is the calculated rank of the data matrix after
applying the various projections listed in the parameters. In this case we
are only removing the mean, so the number of components is one less than the
axis over which we do the PCA (here axis=0 by default).
>>> res['basis_vectors'].shape
(17, 16)
Basis projections are arrays with components in the dimension over which we
have done the PCA (axis=0 by default). Because we set `ncomp` above, we
only retain `ncomp` components.
>>> res['basis_projections'].shape
(9, 10, 12, 14)
"""
data = np.asarray(data)
# We roll the PCA axis to be first, for convenience
if axis is None:
raise ValueError('axis cannot be None')
data = np.rollaxis(data, axis)
if mask is not None:
mask = np.asarray(mask)
if not data.shape[1:] == mask.shape:
raise ValueError('Mask should match dimensions of data other than '
'the axis over which to do the PCA')
if design_resid == 'mean':
# equivalent to: design_resid = np.ones((data.shape[0], 1))
def project_resid(Y):
return Y - Y.mean(0)[None,...]
elif design_resid is None:
def project_resid(Y): return Y
else: # matrix passed, we hope
projector = np.dot(design_resid, npl.pinv(design_resid))
def project_resid(Y):
return Y - np.dot(projector, Y)
if standardize:
def rmse_scales_func(std_source):
# modifies array in place
resid = project_resid(std_source)
# root mean square of the residual
rmse = np.sqrt(np.square(resid).sum(axis=0) / resid.shape[0])
# positive 1/rmse
return np.where(rmse<=0, 0, 1. / rmse)
else:
rmse_scales_func = None
"""
Perform the computations needed for the PCA. This stores the
covariance/correlation matrix of the data in the attribute 'C'. The
components are stored as the attributes 'components', for an fMRI
image these are the time series explaining the most variance.
Now, we compute projection matrices. First, data is projected onto
the columnspace of design_keep, then it is projected perpendicular
to column space of design_resid.
"""
if design_keep is None:
X = np.eye(data.shape[0])
else:
X = np.dot(design_keep, npl.pinv(design_keep))
XZ = project_resid(X)
UX, SX, VX = npl.svd(XZ, full_matrices=0)
# The matrix UX has orthonormal columns and represents the
# final "column space" that the data will be projected onto.
rank = (SX/SX.max() > tol_ratio).sum()
UX = UX[:,:rank].T
# calculate covariance matrix in full-rank column space. The returned
# array is roughly: YX = dot(UX, data); C = dot(YX, YX.T), perhaps where the
# data has been standarized, perhaps summed over slices
C_full_rank = _get_covariance(data, UX, rmse_scales_func, mask)
# find the eigenvalues D and eigenvectors Vs of the covariance
# matrix
D, Vs = npl.eigh(C_full_rank)
# Compute basis vectors in original column space
basis_vectors = np.dot(UX.T, Vs).T
# sort both in descending order of eigenvalues
order = np.argsort(-D)
D = D[order]
basis_vectors = basis_vectors[order]
pcntvar = D * 100 / D.sum()
"""
Output the component basis_projections
"""
if ncomp is None:
ncomp = rank
subVX = basis_vectors[:ncomp]
out = _get_basis_projections(data, subVX, rmse_scales_func)
# Roll PCA image axis back to original position in data array
if axis < 0:
axis += data.ndim
out = np.rollaxis(out, 0, axis+1)
return {'basis_vectors': basis_vectors.T,
'pcnt_var': pcntvar,
'basis_projections': out,
'axis': axis}
0
Example 103
Project: mne-python Source File: montage.py
def read_montage(kind, ch_names=None, path=None, unit='m', transform=False):
"""Read a generic (built-in) montage from a file.
This function can be used to read electrode positions from a user specified
file using the `kind` and `path` parameters. Alternatively, use only the
`kind` parameter to load one of the built-in montages:
=================== =====================================================
Kind description
=================== =====================================================
standard_1005 Electrodes are named and positioned according to the
international 10-05 system.
standard_1020 Electrodes are named and positioned according to the
international 10-20 system.
standard_alphabetic Electrodes are named with LETTER-NUMBER combinations
(A1, B2, F4, etc.)
standard_postfixed Electrodes are named according to the international
10-20 system using postfixes for intermediate
positions.
standard_prefixed Electrodes are named according to the international
10-20 system using prefixes for intermediate
positions.
standard_primed Electrodes are named according to the international
10-20 system using prime marks (' and '') for
intermediate positions.
biosemi16 BioSemi cap with 16 electrodes
biosemi32 BioSemi cap with 32 electrodes
biosemi64 BioSemi cap with 64 electrodes
biosemi128 BioSemi cap with 128 electrodes
biosemi160 BioSemi cap with 160 electrodes
biosemi256 BioSemi cap with 256 electrodes
easycap-M10 Brainproducts EasyCap with electrodes named
according to the 10-05 system
easycap-M1 Brainproduct EasyCap with numbered electrodes
EGI_256 Geodesic Sensor Net with 256 channels
GSN-HydroCel-32 HydroCel Geodesic Sensor Net with 32 electrodes
GSN-HydroCel-64_1.0 HydroCel Geodesic Sensor Net with 64 electrodes
GSN-HydroCel-65_1.0 HydroCel Geodesic Sensor Net with 64 electrodes + Cz
GSN-HydroCel-128 HydroCel Geodesic Sensor Net with 128 electrodes
GSN-HydroCel-129 HydroCel Geodesic Sensor Net with 128 electrodes + Cz
GSN-HydroCel-256 HydroCel Geodesic Sensor Net with 256 electrodes
GSN-HydroCel-257 HydroCel Geodesic Sensor Net with 256 electrodes + Cz
=================== =====================================================
Parameters
----------
kind : str
The name of the montage file without the file extension (e.g.
kind='easycap-M10' for 'easycap-M10.txt'). Files with extensions
'.elc', '.txt', '.csd', '.elp', '.hpts', '.sfp' or '.loc' ('.locs' and
'.eloc') are supported.
ch_names : list of str | None
If not all electrodes defined in the montage are present in the EEG
data, use this parameter to select subset of electrode positions to
load. If None (default), all defined electrode positions are returned.
.. note:: ``ch_names`` are compared to channel names in the montage
file after converting them both to upper case. If a match is
found, the letter case in the original ``ch_names`` is used
in the returned montage.
path : str | None
The path of the folder containing the montage file. Defaults to the
mne/channels/data/montages folder in your mne-python installation.
unit : 'm' | 'cm' | 'mm'
Unit of the input file. If not 'm' (default), coordinates will be
rescaled to 'm'.
transform : bool
If True, points will be transformed to Neuromag space.
The fidicuals, 'nasion', 'lpa', 'rpa' must be specified in
the montage file. Useful for points captured using Polhemus FastSCAN.
Default is False.
Returns
-------
montage : instance of Montage
The montage.
See Also
--------
read_dig_montage : To read subject-specific digitization information.
Notes
-----
Built-in montages are not scaled or transformed by default.
Montages can contain fiducial points in addition to electrode
locations, e.g. ``biosemi-64`` contains 67 total channels.
.. versionadded:: 0.9.0
"""
if path is None:
path = op.join(op.dirname(__file__), 'data', 'montages')
if not op.isabs(kind):
supported = ('.elc', '.txt', '.csd', '.sfp', '.elp', '.hpts', '.loc',
'.locs', '.eloc')
montages = [op.splitext(f) for f in os.listdir(path)]
montages = [m for m in montages if m[1] in supported and kind == m[0]]
if len(montages) != 1:
raise ValueError('Could not find the montage. Please provide the '
'full path.')
kind, ext = montages[0]
else:
kind, ext = op.splitext(kind)
fname = op.join(path, kind + ext)
if ext == '.sfp':
# EGI geodesic
with open(fname, 'r') as f:
lines = f.read().replace('\t', ' ').splitlines()
ch_names_, pos = [], []
for ii, line in enumerate(lines):
line = line.strip().split()
if len(line) > 0: # skip empty lines
if len(line) != 4: # name, x, y, z
raise ValueError("Malformed .sfp file in line " + str(ii))
this_name, x, y, z = line
ch_names_.append(this_name)
pos.append([float(cord) for cord in (x, y, z)])
pos = np.asarray(pos)
elif ext == '.elc':
# 10-5 system
ch_names_ = []
pos = []
with open(fname) as fid:
# Default units are meters
for line in fid:
if 'UnitPosition' in line:
units = line.split()[1]
scale_factor = dict(m=1., mm=1e-3)[units]
break
else:
raise RuntimeError('Could not detect units in file %s' % fname)
for line in fid:
if 'Positions\n' in line:
break
pos = []
for line in fid:
if 'Labels\n' in line:
break
pos.append(list(map(float, line.split())))
for line in fid:
if not line or not set(line) - set([' ']):
break
ch_names_.append(line.strip(' ').strip('\n'))
pos = np.array(pos) * scale_factor
elif ext == '.txt':
# easycap
try: # newer version
data = np.genfromtxt(fname, dtype='str', skip_header=1)
except TypeError:
data = np.genfromtxt(fname, dtype='str', skiprows=1)
ch_names_ = list(data[:, 0])
az = np.deg2rad(data[:, 2].astype(float))
pol = np.deg2rad(data[:, 1].astype(float))
pos = _sph_to_cart(np.array([np.ones(len(az)) * 85., az, pol]).T)
elif ext == '.csd':
# CSD toolbox
dtype = [('label', 'S4'), ('theta', 'f8'), ('phi', 'f8'),
('radius', 'f8'), ('x', 'f8'), ('y', 'f8'), ('z', 'f8'),
('off_sph', 'f8')]
try: # newer version
table = np.loadtxt(fname, skip_header=2, dtype=dtype)
except TypeError:
table = np.loadtxt(fname, skiprows=2, dtype=dtype)
ch_names_ = table['label']
az = np.deg2rad(table['theta'])
pol = np.deg2rad(90. - table['phi'])
pos = _sph_to_cart(np.array([np.ones(len(az)), az, pol]).T)
elif ext == '.elp':
# standard BESA spherical
dtype = np.dtype('S8, S8, f8, f8, f8')
try:
data = np.loadtxt(fname, dtype=dtype, skip_header=1)
except TypeError:
data = np.loadtxt(fname, dtype=dtype, skiprows=1)
ch_names_ = data['f1'].astype(np.str)
az = data['f2']
horiz = data['f3']
radius = np.abs(az / 180.)
az = np.deg2rad(np.array([90. - h if a >= 0. else -90. - h
for h, a in zip(horiz, az)]))
pol = radius * np.pi
pos = _sph_to_cart(np.array([np.ones(len(az)) * 85., az, pol]).T)
elif ext == '.hpts':
# MNE-C specified format for generic digitizer data
dtype = [('type', 'S8'), ('name', 'S8'),
('x', 'f8'), ('y', 'f8'), ('z', 'f8')]
data = np.loadtxt(fname, dtype=dtype)
ch_names_ = data['name'].astype(np.str)
pos = np.vstack((data['x'], data['y'], data['z'])).T
elif ext in ('.loc', '.locs', '.eloc'):
ch_names_ = np.loadtxt(fname, dtype='S4',
usecols=[3]).astype(np.str).tolist()
dtype = {'names': ('angle', 'radius'), 'formats': ('f4', 'f4')}
topo = np.loadtxt(fname, dtype=float, usecols=[1, 2])
sph = _topo_to_sph(topo)
pos = _sph_to_cart(sph)
pos[:, [0, 1]] = pos[:, [1, 0]] * [-1, 1]
else:
raise ValueError('Currently the "%s" template is not supported.' %
kind)
selection = np.arange(len(pos))
if unit == 'mm':
pos /= 1e3
elif unit == 'cm':
pos /= 1e2
elif unit != 'm':
raise ValueError("'unit' should be either 'm', 'cm', or 'mm'.")
if transform:
names_lower = [name.lower() for name in list(ch_names_)]
if ext == '.hpts':
fids = ('2', '1', '3') # Alternate cardinal point names
else:
fids = ('nz', 'lpa', 'rpa')
missing = [name for name in fids
if name not in names_lower]
if missing:
raise ValueError("The points %s are missing, but are needed "
"to transform the points to the MNE coordinate "
"system. Either add the points, or read the "
"montage with transform=False. " % missing)
nasion = pos[names_lower.index(fids[0])]
lpa = pos[names_lower.index(fids[1])]
rpa = pos[names_lower.index(fids[2])]
neuromag_trans = get_ras_to_neuromag_trans(nasion, lpa, rpa)
pos = apply_trans(neuromag_trans, pos)
if ch_names is not None:
# Ensure channels with differing case are found.
upper_names = [ch_name.upper() for ch_name in ch_names]
sel, ch_names_ = zip(*[(i, ch_names[upper_names.index(e)]) for i, e in
enumerate([n.upper() for n in ch_names_])
if e in upper_names])
sel = list(sel)
pos = pos[sel]
selection = selection[sel]
else:
ch_names_ = list(ch_names_)
kind = op.split(kind)[-1]
return Montage(pos=pos, ch_names=ch_names_, kind=kind, selection=selection)
0
Example 104
Project: networkx Source File: nx_pylab.py
def draw_networkx_edges(G, pos,
edgelist=None,
width=1.0,
edge_color='k',
style='solid',
alpha=1.0,
edge_cmap=None,
edge_vmin=None,
edge_vmax=None,
ax=None,
arrows=True,
label=None,
**kwds):
"""Draw the edges of the graph G.
This draws only the edges of the graph G.
Parameters
----------
G : graph
A networkx graph
pos : dictionary
A dictionary with nodes as keys and positions as values.
Positions should be sequences of length 2.
edgelist : collection of edge tuples
Draw only specified edges(default=G.edges())
width : float, or array of floats
Line width of edges (default=1.0)
edge_color : color string, or array of floats
Edge color. Can be a single color format string (default='r'),
or a sequence of colors with the same length as edgelist.
If numeric values are specified they will be mapped to
colors using the edge_cmap and edge_vmin,edge_vmax parameters.
style : string
Edge line style (default='solid') (solid|dashed|dotted,dashdot)
alpha : float
The edge transparency (default=1.0)
edge_ cmap : Matplotlib colormap
Colormap for mapping intensities of edges (default=None)
edge_vmin,edge_vmax : floats
Minimum and maximum for edge colormap scaling (default=None)
ax : Matplotlib Axes object, optional
Draw the graph in the specified Matplotlib axes.
arrows : bool, optional (default=True)
For directed graphs, if True draw arrowheads.
label : [None| string]
Label for legend
Returns
-------
matplotlib.collection.LineCollection
`LineCollection` of the edges
Notes
-----
For directed graphs, "arrows" (actually just thicker stubs) are drawn
at the head end. Arrows can be turned off with keyword arrows=False.
Yes, it is ugly but drawing proper arrows with Matplotlib this
way is tricky.
Examples
--------
>>> G=nx.dodecahedral_graph()
>>> edges=nx.draw_networkx_edges(G,pos=nx.spring_layout(G))
Also see the NetworkX drawing examples at
http://networkx.github.io/docuementation/latest/gallery.html
See Also
--------
draw()
draw_networkx()
draw_networkx_nodes()
draw_networkx_labels()
draw_networkx_edge_labels()
"""
try:
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.cbook as cb
from matplotlib.colors import colorConverter, Colormap
from matplotlib.collections import LineCollection
import numpy
except ImportError:
raise ImportError("Matplotlib required for draw()")
except RuntimeError:
print("Matplotlib unable to open display")
raise
if ax is None:
ax = plt.gca()
if edgelist is None:
edgelist = list(G.edges())
if not edgelist or len(edgelist) == 0: # no edges!
return None
# set edge positions
edge_pos = numpy.asarray([(pos[e[0]], pos[e[1]]) for e in edgelist])
if not cb.iterable(width):
lw = (width,)
else:
lw = width
if not cb.is_string_like(edge_color) \
and cb.iterable(edge_color) \
and len(edge_color) == len(edge_pos):
if numpy.alltrue([cb.is_string_like(c)
for c in edge_color]):
# (should check ALL elements)
# list of color letters such as ['k','r','k',...]
edge_colors = tuple([colorConverter.to_rgba(c, alpha)
for c in edge_color])
elif numpy.alltrue([not cb.is_string_like(c)
for c in edge_color]):
# If color specs are given as (rgb) or (rgba) tuples, we're OK
if numpy.alltrue([cb.iterable(c) and len(c) in (3, 4)
for c in edge_color]):
edge_colors = tuple(edge_color)
else:
# numbers (which are going to be mapped with a colormap)
edge_colors = None
else:
raise ValueError('edge_color must consist of either color names or numbers')
else:
if cb.is_string_like(edge_color) or len(edge_color) == 1:
edge_colors = (colorConverter.to_rgba(edge_color, alpha), )
else:
raise ValueError('edge_color must be a single color or list of exactly m colors where m is the number or edges')
edge_collection = LineCollection(edge_pos,
colors=edge_colors,
linewidths=lw,
antialiaseds=(1,),
linestyle=style,
transOffset = ax.transData,
)
edge_collection.set_zorder(1) # edges go behind nodes
edge_collection.set_label(label)
ax.add_collection(edge_collection)
# Note: there was a bug in mpl regarding the handling of alpha values for
# each line in a LineCollection. It was fixed in matplotlib in r7184 and
# r7189 (June 6 2009). We should then not set the alpha value globally,
# since the user can instead provide per-edge alphas now. Only set it
# globally if provided as a scalar.
if cb.is_numlike(alpha):
edge_collection.set_alpha(alpha)
if edge_colors is None:
if edge_cmap is not None:
assert(isinstance(edge_cmap, Colormap))
edge_collection.set_array(numpy.asarray(edge_color))
edge_collection.set_cmap(edge_cmap)
if edge_vmin is not None or edge_vmax is not None:
edge_collection.set_clim(edge_vmin, edge_vmax)
else:
edge_collection.autoscale()
arrow_collection = None
if G.is_directed() and arrows:
# a directed graph hack
# draw thick line segments at head end of edge
# waiting for someone else to implement arrows that will work
arrow_colors = edge_colors
a_pos = []
p = 1.0-0.25 # make head segment 25 percent of edge length
for src, dst in edge_pos:
x1, y1 = src
x2, y2 = dst
dx = x2-x1 # x offset
dy = y2-y1 # y offset
d = numpy.sqrt(float(dx**2 + dy**2)) # length of edge
if d == 0: # source and target at same position
continue
if dx == 0: # vertical edge
xa = x2
ya = dy*p+y1
if dy == 0: # horizontal edge
ya = y2
xa = dx*p+x1
else:
theta = numpy.arctan2(dy, dx)
xa = p*d*numpy.cos(theta)+x1
ya = p*d*numpy.sin(theta)+y1
a_pos.append(((xa, ya), (x2, y2)))
arrow_collection = LineCollection(a_pos,
colors=arrow_colors,
linewidths=[4*ww for ww in lw],
antialiaseds=(1,),
transOffset = ax.transData,
)
arrow_collection.set_zorder(1) # edges go behind nodes
arrow_collection.set_label(label)
ax.add_collection(arrow_collection)
# update view
minx = numpy.amin(numpy.ravel(edge_pos[:, :, 0]))
maxx = numpy.amax(numpy.ravel(edge_pos[:, :, 0]))
miny = numpy.amin(numpy.ravel(edge_pos[:, :, 1]))
maxy = numpy.amax(numpy.ravel(edge_pos[:, :, 1]))
w = maxx-minx
h = maxy-miny
padx, pady = 0.05*w, 0.05*h
corners = (minx-padx, miny-pady), (maxx+padx, maxy+pady)
ax.update_datalim(corners)
ax.autoscale_view()
# if arrow_collection:
return edge_collection
0
Example 105
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: _minimize.py
def minimize(fun, x0, args=(), method=None, jac=None, hess=None,
hessp=None, bounds=None, constraints=(), tol=None,
callback=None, options=None):
"""Minimization of scalar function of one or more variables.
In general, the optimization problems are of the form::
minimize f(x) subject to
g_i(x) >= 0, i = 1,...,m
h_j(x) = 0, j = 1,...,p
where x is a vector of one or more variables.
``g_i(x)`` are the inequality constraints.
``h_j(x)`` are the equality constrains.
Optionally, the lower and upper bounds for each element in x can also be specified
using the `bounds` argument.
Parameters
----------
fun : callable
Objective function.
x0 : ndarray
Initial guess.
args : tuple, optional
Extra arguments passed to the objective function and its
derivatives (Jacobian, Hessian).
method : str or callable, optional
Type of solver. Should be one of
- 'Nelder-Mead' :ref:`(see here) <optimize.minimize-neldermead>`
- 'Powell' :ref:`(see here) <optimize.minimize-powell>`
- 'CG' :ref:`(see here) <optimize.minimize-cg>`
- 'BFGS' :ref:`(see here) <optimize.minimize-bfgs>`
- 'Newton-CG' :ref:`(see here) <optimize.minimize-newtoncg>`
- 'L-BFGS-B' :ref:`(see here) <optimize.minimize-lbfgsb>`
- 'TNC' :ref:`(see here) <optimize.minimize-tnc>`
- 'COBYLA' :ref:`(see here) <optimize.minimize-cobyla>`
- 'SLSQP' :ref:`(see here) <optimize.minimize-slsqp>`
- 'dogleg' :ref:`(see here) <optimize.minimize-dogleg>`
- 'trust-ncg' :ref:`(see here) <optimize.minimize-trustncg>`
- custom - a callable object (added in version 0.14.0),
see below for description.
If not given, chosen to be one of ``BFGS``, ``L-BFGS-B``, ``SLSQP``,
depending if the problem has constraints or bounds.
jac : bool or callable, optional
Jacobian (gradient) of objective function. Only for CG, BFGS,
Newton-CG, L-BFGS-B, TNC, SLSQP, dogleg, trust-ncg.
If `jac` is a Boolean and is True, `fun` is assumed to return the
gradient along with the objective function. If False, the
gradient will be estimated numerically.
`jac` can also be a callable returning the gradient of the
objective. In this case, it must accept the same arguments as `fun`.
hess, hessp : callable, optional
Hessian (matrix of second-order derivatives) of objective function or
Hessian of objective function times an arbitrary vector p. Only for
Newton-CG, dogleg, trust-ncg.
Only one of `hessp` or `hess` needs to be given. If `hess` is
provided, then `hessp` will be ignored. If neither `hess` nor
`hessp` is provided, then the Hessian product will be approximated
using finite differences on `jac`. `hessp` must compute the Hessian
times an arbitrary vector.
bounds : sequence, optional
Bounds for variables (only for L-BFGS-B, TNC and SLSQP).
``(min, max)`` pairs for each element in ``x``, defining
the bounds on that parameter. Use None for one of ``min`` or
``max`` when there is no bound in that direction.
constraints : dict or sequence of dict, optional
Constraints definition (only for COBYLA and SLSQP).
Each constraint is defined in a dictionary with fields:
type : str
Constraint type: 'eq' for equality, 'ineq' for inequality.
fun : callable
The function defining the constraint.
jac : callable, optional
The Jacobian of `fun` (only for SLSQP).
args : sequence, optional
Extra arguments to be passed to the function and Jacobian.
Equality constraint means that the constraint function result is to
be zero whereas inequality means that it is to be non-negative.
Note that COBYLA only supports inequality constraints.
tol : float, optional
Tolerance for termination. For detailed control, use solver-specific
options.
options : dict, optional
A dictionary of solver options. All methods accept the following
generic options:
maxiter : int
Maximum number of iterations to perform.
disp : bool
Set to True to print convergence messages.
For method-specific options, see :func:`show_options()`.
callback : callable, optional
Called after each iteration, as ``callback(xk)``, where ``xk`` is the
current parameter vector.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes.
See also
--------
minimize_scalar : Interface to minimization algorithms for scalar
univariate functions
show_options : Additional options accepted by the solvers
Notes
-----
This section describes the available solvers that can be selected by the
'method' parameter. The default method is *BFGS*.
**Unconstrained minimization**
Method :ref:`Nelder-Mead <optimize.minimize-neldermead>` uses the
Simplex algorithm [1]_, [2]_. This algorithm is robust in many
applications. However, if numerical computation of derivative can be
trusted, other algorithms using the first and/or second derivatives
information might be preferred for their better performance in
general.
Method :ref:`Powell <optimize.minimize-powell>` is a modification
of Powell's method [3]_, [4]_ which is a conjugate direction
method. It performs sequential one-dimensional minimizations along
each vector of the directions set (`direc` field in `options` and
`info`), which is updated at each iteration of the main
minimization loop. The function need not be differentiable, and no
derivatives are taken.
Method :ref:`CG <optimize.minimize-cg>` uses a nonlinear conjugate
gradient algorithm by Polak and Ribiere, a variant of the
Fletcher-Reeves method described in [5]_ pp. 120-122. Only the
first derivatives are used.
Method :ref:`BFGS <optimize.minimize-bfgs>` uses the quasi-Newton
method of Broyden, Fletcher, Goldfarb, and Shanno (BFGS) [5]_
pp. 136. It uses the first derivatives only. BFGS has proven good
performance even for non-smooth optimizations. This method also
returns an approximation of the Hessian inverse, stored as
`hess_inv` in the OptimizeResult object.
Method :ref:`Newton-CG <optimize.minimize-newtoncg>` uses a
Newton-CG algorithm [5]_ pp. 168 (also known as the truncated
Newton method). It uses a CG method to the compute the search
direction. See also *TNC* method for a box-constrained
minimization with a similar algorithm.
Method :ref:`dogleg <optimize.minimize-dogleg>` uses the dog-leg
trust-region algorithm [5]_ for unconstrained minimization. This
algorithm requires the gradient and Hessian; furthermore the
Hessian is required to be positive definite.
Method :ref:`trust-ncg <optimize.minimize-trustncg>` uses the
Newton conjugate gradient trust-region algorithm [5]_ for
unconstrained minimization. This algorithm requires the gradient
and either the Hessian or a function that computes the product of
the Hessian with a given vector.
**Constrained minimization**
Method :ref:`L-BFGS-B <optimize.minimize-lbfgsb>` uses the L-BFGS-B
algorithm [6]_, [7]_ for bound constrained minimization.
Method :ref:`TNC <optimize.minimize-tnc>` uses a truncated Newton
algorithm [5]_, [8]_ to minimize a function with variables subject
to bounds. This algorithm uses gradient information; it is also
called Newton Conjugate-Gradient. It differs from the *Newton-CG*
method described above as it wraps a C implementation and allows
each variable to be given upper and lower bounds.
Method :ref:`COBYLA <optimize.minimize-cobyla>` uses the
Constrained Optimization BY Linear Approximation (COBYLA) method
[9]_, [10]_, [11]_. The algorithm is based on linear
approximations to the objective function and each constraint. The
method wraps a FORTRAN implementation of the algorithm. The
constraints functions 'fun' may return either a single number
or an array or list of numbers.
Method :ref:`SLSQP <optimize.minimize-slsqp>` uses Sequential
Least SQuares Programming to minimize a function of several
variables with any combination of bounds, equality and inequality
constraints. The method wraps the SLSQP Optimization subroutine
originally implemented by Dieter Kraft [12]_. Note that the
wrapper handles infinite values in bounds by converting them into
large floating values.
**Custom minimizers**
It may be useful to pass a custom minimization method, for example
when using a frontend to this method such as `scipy.optimize.basinhopping`
or a different library. You can simply pass a callable as the ``method``
parameter.
The callable is called as ``method(fun, x0, args, **kwargs, **options)``
where ``kwargs`` corresponds to any other parameters passed to `minimize`
(such as `callback`, `hess`, etc.), except the `options` dict, which has
its contents also passed as `method` parameters pair by pair. Also, if
`jac` has been passed as a bool type, `jac` and `fun` are mangled so that
`fun` returns just the function values and `jac` is converted to a function
returning the Jacobian. The method shall return an ``OptimizeResult``
object.
The provided `method` callable must be able to accept (and possibly ignore)
arbitrary parameters; the set of parameters accepted by `minimize` may
expand in future versions and then these parameters will be passed to
the method. You can find an example in the scipy.optimize tutorial.
.. versionadded:: 0.11.0
References
----------
.. [1] Nelder, J A, and R Mead. 1965. A Simplex Method for Function
Minimization. The Computer Journal 7: 308-13.
.. [2] Wright M H. 1996. Direct search methods: Once scorned, now
respectable, in Numerical Analysis 1995: Proceedings of the 1995
Dundee Biennial Conference in Numerical Analysis (Eds. D F
Griffiths and G A Watson). Addison Wesley Longman, Harlow, UK.
191-208.
.. [3] Powell, M J D. 1964. An efficient method for finding the minimum of
a function of several variables without calculating derivatives. The
Computer Journal 7: 155-162.
.. [4] Press W, S A Teukolsky, W T Vetterling and B P Flannery.
Numerical Recipes (any edition), Cambridge University Press.
.. [5] Nocedal, J, and S J Wright. 2006. Numerical Optimization.
Springer New York.
.. [6] Byrd, R H and P Lu and J. Nocedal. 1995. A Limited Memory
Algorithm for Bound Constrained Optimization. SIAM Journal on
Scientific and Statistical Computing 16 (5): 1190-1208.
.. [7] Zhu, C and R H Byrd and J Nocedal. 1997. L-BFGS-B: Algorithm
778: L-BFGS-B, FORTRAN routines for large scale bound constrained
optimization. ACM Transactions on Mathematical Software 23 (4):
550-560.
.. [8] Nash, S G. Newton-Type Minimization Via the Lanczos Method.
1984. SIAM Journal of Numerical Analysis 21: 770-778.
.. [9] Powell, M J D. A direct search optimization method that models
the objective and constraint functions by linear interpolation.
1994. Advances in Optimization and Numerical Analysis, eds. S. Gomez
and J-P Hennart, Kluwer Academic (Dordrecht), 51-67.
.. [10] Powell M J D. Direct search algorithms for optimization
calculations. 1998. Acta Numerica 7: 287-336.
.. [11] Powell M J D. A view of algorithms for optimization without
derivatives. 2007.Cambridge University Technical Report DAMTP
2007/NA03
.. [12] Kraft, D. A software package for sequential quadratic
programming. 1988. Tech. Rep. DFVLR-FB 88-28, DLR German Aerospace
Center -- Institute for Flight Mechanics, Koln, Germany.
Examples
--------
Let us consider the problem of minimizing the Rosenbrock function. This
function (and its respective derivatives) is implemented in `rosen`
(resp. `rosen_der`, `rosen_hess`) in the `scipy.optimize`.
>>> from scipy.optimize import minimize, rosen, rosen_der
A simple application of the *Nelder-Mead* method is:
>>> x0 = [1.3, 0.7, 0.8, 1.9, 1.2]
>>> res = minimize(rosen, x0, method='Nelder-Mead', tol=1e-6)
>>> res.x
array([ 1., 1., 1., 1., 1.])
Now using the *BFGS* algorithm, using the first derivative and a few
options:
>>> res = minimize(rosen, x0, method='BFGS', jac=rosen_der,
... options={'gtol': 1e-6, 'disp': True})
Optimization terminated successfully.
Current function value: 0.000000
Iterations: 52
Function evaluations: 64
Gradient evaluations: 64
>>> res.x
array([ 1., 1., 1., 1., 1.])
>>> print(res.message)
Optimization terminated successfully.
>>> res.hess_inv
array([[ 0.00749589, 0.01255155, 0.02396251, 0.04750988, 0.09495377], # may vary
[ 0.01255155, 0.02510441, 0.04794055, 0.09502834, 0.18996269],
[ 0.02396251, 0.04794055, 0.09631614, 0.19092151, 0.38165151],
[ 0.04750988, 0.09502834, 0.19092151, 0.38341252, 0.7664427 ],
[ 0.09495377, 0.18996269, 0.38165151, 0.7664427, 1.53713523]])
Next, consider a minimization problem with several constraints (namely
Example 16.4 from [5]_). The objective function is:
>>> fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2
There are three constraints defined as:
>>> cons = ({'type': 'ineq', 'fun': lambda x: x[0] - 2 * x[1] + 2},
... {'type': 'ineq', 'fun': lambda x: -x[0] - 2 * x[1] + 6},
... {'type': 'ineq', 'fun': lambda x: -x[0] + 2 * x[1] + 2})
And variables must be positive, hence the following bounds:
>>> bnds = ((0, None), (0, None))
The optimization problem is solved using the SLSQP method as:
>>> res = minimize(fun, (2, 0), method='SLSQP', bounds=bnds,
... constraints=cons)
It should converge to the theoretical solution (1.4 ,1.7).
"""
x0 = np.asarray(x0)
if x0.dtype.kind in np.typecodes["AllInteger"]:
x0 = np.asarray(x0, dtype=float)
if not isinstance(args, tuple):
args = (args,)
if method is None:
# Select automatically
if constraints:
method = 'SLSQP'
elif bounds is not None:
method = 'L-BFGS-B'
else:
method = 'BFGS'
if callable(method):
meth = "_custom"
else:
meth = method.lower()
if options is None:
options = {}
# check if optional parameters are supported by the selected method
# - jac
if meth in ['nelder-mead', 'powell', 'cobyla'] and bool(jac):
warn('Method %s does not use gradient information (jac).' % method,
RuntimeWarning)
# - hess
if meth not in ('newton-cg', 'dogleg', 'trust-ncg', '_custom') and hess is not None:
warn('Method %s does not use Hessian information (hess).' % method,
RuntimeWarning)
# - hessp
if meth not in ('newton-cg', 'dogleg', 'trust-ncg', '_custom') and hessp is not None:
warn('Method %s does not use Hessian-vector product '
'information (hessp).' % method, RuntimeWarning)
# - constraints or bounds
if (meth in ['nelder-mead', 'powell', 'cg', 'bfgs', 'newton-cg', 'dogleg',
'trust-ncg'] and (bounds is not None or np.any(constraints))):
warn('Method %s cannot handle constraints nor bounds.' % method,
RuntimeWarning)
if meth in ['l-bfgs-b', 'tnc'] and np.any(constraints):
warn('Method %s cannot handle constraints.' % method,
RuntimeWarning)
if meth == 'cobyla' and bounds is not None:
warn('Method %s cannot handle bounds.' % method,
RuntimeWarning)
# - callback
if (meth in ['cobyla'] and callback is not None):
warn('Method %s does not support callback.' % method, RuntimeWarning)
# - return_all
if (meth in ['l-bfgs-b', 'tnc', 'cobyla', 'slsqp'] and
options.get('return_all', False)):
warn('Method %s does not support the return_all option.' % method,
RuntimeWarning)
# fun also returns the jacobian
if not callable(jac):
if bool(jac):
fun = MemoizeJac(fun)
jac = fun.derivative
else:
jac = None
# set default tolerances
if tol is not None:
options = dict(options)
if meth in ['nelder-mead', 'newton-cg', 'powell', 'tnc']:
options.setdefault('xtol', tol)
if meth in ['nelder-mead', 'powell', 'l-bfgs-b', 'tnc', 'slsqp']:
options.setdefault('ftol', tol)
if meth in ['bfgs', 'cg', 'l-bfgs-b', 'tnc', 'dogleg', 'trust-ncg']:
options.setdefault('gtol', tol)
if meth in ['cobyla', '_custom']:
options.setdefault('tol', tol)
if meth == '_custom':
return method(fun, x0, args=args, jac=jac, hess=hess, hessp=hessp,
bounds=bounds, constraints=constraints,
callback=callback, **options)
elif meth == 'nelder-mead':
return _minimize_neldermead(fun, x0, args, callback, **options)
elif meth == 'powell':
return _minimize_powell(fun, x0, args, callback, **options)
elif meth == 'cg':
return _minimize_cg(fun, x0, args, jac, callback, **options)
elif meth == 'bfgs':
return _minimize_bfgs(fun, x0, args, jac, callback, **options)
elif meth == 'newton-cg':
return _minimize_newtoncg(fun, x0, args, jac, hess, hessp, callback,
**options)
elif meth == 'l-bfgs-b':
return _minimize_lbfgsb(fun, x0, args, jac, bounds,
callback=callback, **options)
elif meth == 'tnc':
return _minimize_tnc(fun, x0, args, jac, bounds, callback=callback,
**options)
elif meth == 'cobyla':
return _minimize_cobyla(fun, x0, args, constraints, **options)
elif meth == 'slsqp':
return _minimize_slsqp(fun, x0, args, jac, bounds,
constraints, callback=callback, **options)
elif meth == 'dogleg':
return _minimize_dogleg(fun, x0, args, jac, hess,
callback=callback, **options)
elif meth == 'trust-ncg':
return _minimize_trust_ncg(fun, x0, args, jac, hess, hessp,
callback=callback, **options)
else:
raise ValueError('Unknown solver %s' % method)
0
Example 106
def test_mlp(
initial_learning_rate,
learning_rate_decay,
squared_filter_length_limit,
n_epochs,
batch_size,
mom_params,
activations,
dropout,
dropout_rates,
results_file_name,
layer_sizes,
dataset,
use_bias,
random_seed=1234):
"""
The dataset is the one from the mlp demo on deeplearning.net. This training
function is lifted from there almost exactly.
:type dataset: string
:param dataset: the path of the MNIST dataset file from
http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
"""
assert len(layer_sizes) - 1 == len(dropout_rates)
# extract the params for momentum
mom_start = mom_params["start"]
mom_end = mom_params["end"]
mom_epoch_interval = mom_params["interval"]
datasets = load_mnist(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
epoch = T.scalar()
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
learning_rate = theano.shared(np.asarray(initial_learning_rate,
dtype=theano.config.floatX))
rng = np.random.RandomState(random_seed)
# construct the MLP class
classifier = MLP(rng=rng, input=x,
layer_sizes=layer_sizes,
dropout_rates=dropout_rates,
activations=activations,
use_bias=use_bias)
# Build the expresson for the cost function.
cost = classifier.negative_log_likelihood(y)
dropout_cost = classifier.dropout_negative_log_likelihood(y)
# Compile theano function for testing.
test_model = theano.function(inputs=[index],
outputs=classifier.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]})
#theano.printing.pydotprint(test_model, outfile="test_file.png",
# var_with_name_simple=True)
# Compile theano function for validation.
validate_model = theano.function(inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]})
#theano.printing.pydotprint(validate_model, outfile="validate_file.png",
# var_with_name_simple=True)
# Compute gradients of the model wrt parameters
gparams = []
for param in classifier.params:
# Use the right cost function here to train with or without dropout.
gparam = T.grad(dropout_cost if dropout else cost, param)
gparams.append(gparam)
# ... and allocate mmeory for momentum'd versions of the gradient
gparams_mom = []
for param in classifier.params:
gparam_mom = theano.shared(np.zeros(param.get_value(borrow=True).shape,
dtype=theano.config.floatX))
gparams_mom.append(gparam_mom)
# Compute momentum for the current epoch
mom = ifelse(epoch < mom_epoch_interval,
mom_start*(1.0 - epoch/mom_epoch_interval) + mom_end*(epoch/mom_epoch_interval),
mom_end)
# Update the step direction using momentum
updates = OrderedDict()
for gparam_mom, gparam in zip(gparams_mom, gparams):
# Misha Denil's original version
#updates[gparam_mom] = mom * gparam_mom + (1. - mom) * gparam
# change the update rule to match Hinton's dropout paper
updates[gparam_mom] = mom * gparam_mom - (1. - mom) * learning_rate * gparam
# ... and take a step along that direction
for param, gparam_mom in zip(classifier.params, gparams_mom):
# Misha Denil's original version
#stepped_param = param - learning_rate * updates[gparam_mom]
# since we have included learning_rate in gparam_mom, we don't need it
# here
stepped_param = param + updates[gparam_mom]
# This is a silly hack to constrain the norms of the rows of the weight
# matrices. This just checks if there are two dimensions to the
# parameter and constrains it if so... maybe this is a bit silly but it
# should work for now.
if param.get_value(borrow=True).ndim == 2:
#squared_norms = T.sum(stepped_param**2, axis=1).reshape((stepped_param.shape[0],1))
#scale = T.clip(T.sqrt(squared_filter_length_limit / squared_norms), 0., 1.)
#updates[param] = stepped_param * scale
# constrain the norms of the COLUMNs of the weight, according to
# https://github.com/BVLC/caffe/issues/109
col_norms = T.sqrt(T.sum(T.sqr(stepped_param), axis=0))
desired_norms = T.clip(col_norms, 0, T.sqrt(squared_filter_length_limit))
scale = desired_norms / (1e-7 + col_norms)
updates[param] = stepped_param * scale
else:
updates[param] = stepped_param
# Compile theano function for training. This returns the training cost and
# updates the model parameters.
output = dropout_cost if dropout else cost
train_model = theano.function(inputs=[epoch, index], outputs=output,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]})
#theano.printing.pydotprint(train_model, outfile="train_file.png",
# var_with_name_simple=True)
# Theano function to decay the learning rate, this is separate from the
# training function because we only want to do this once each epoch instead
# of after each minibatch.
decay_learning_rate = theano.function(inputs=[], outputs=learning_rate,
updates={learning_rate: learning_rate * learning_rate_decay})
###############
# TRAIN MODEL #
###############
print '... training'
best_params = None
best_validation_errors = np.inf
best_iter = 0
test_score = 0.
epoch_counter = 0
start_time = time.clock()
results_file = open(results_file_name, 'wb')
while epoch_counter < n_epochs:
# Train this epoch
epoch_counter = epoch_counter + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_model(epoch_counter, minibatch_index)
# Compute loss on validation set
validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
this_validation_errors = np.sum(validation_losses)
# Report and save progress.
print "epoch {}, test error {}, learning_rate={}{}".format(
epoch_counter, this_validation_errors,
learning_rate.get_value(borrow=True),
" **" if this_validation_errors < best_validation_errors else "")
best_validation_errors = min(best_validation_errors,
this_validation_errors)
results_file.write("{0}\n".format(this_validation_errors))
results_file.flush()
new_learning_rate = decay_learning_rate()
end_time = time.clock()
print(('Optimization complete. Best validation score of %f %% '
'obtained at iteration %i, with test performance %f %%') %
(best_validation_errors * 100., best_iter, test_score * 100.))
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
0
Example 107
def draw_networkx_edges(G, pos,
edgelist=None,
width=1.0,
edge_color='k',
style='solid',
alpha=None,
edge_cmap=None,
edge_vmin=None,
edge_vmax=None,
ax=None,
arrows=True,
**kwds):
"""Draw the edges of the graph G
This draws only the edges of the graph G.
pos is a dictionary keyed by vertex with a two-tuple
of x-y positions as the value.
See networkx.layout for functions that compute node positions.
edgelist is an optional list of the edges in G to be drawn.
If provided, only the edges in edgelist will be drawn.
edgecolor can be a list of matplotlib color letters such as 'k' or
'b' that lists the color of each edge; the list must be ordered in
the same way as the edge list. Alternatively, this list can contain
numbers and those number are mapped to a color scale using the color
map edge_cmap. Finally, it can also be a list of (r,g,b) or (r,g,b,a)
tuples, in which case these will be used directly to color the edges. If
the latter mode is used, you should not provide a value for alpha, as it
would be applied globally to all lines.
For directed graphs, "arrows" (actually just thicker stubs) are drawn
at the head end. Arrows can be turned off with keyword arrows=False.
See draw_networkx for the list of other optional parameters.
"""
try:
import matplotlib
import matplotlib.pylab as pylab
import numpy as np
from matplotlib.colors import colorConverter,Colormap
from matplotlib.collections import LineCollection
except ImportError:
raise ImportError("Matplotlib required for draw()")
except RuntimeError:
pass # unable to open display
if ax is None:
ax=pylab.gca()
if edgelist is None:
edgelist=G.edges()
if not edgelist or len(edgelist)==0: # no edges!
return None
# set edge positions
edge_pos=np.asarray([(pos[e[0]],pos[e[1]]) for e in edgelist])
if not cb.iterable(width):
lw = (width,)
else:
lw = width
if not cb.is_string_like(edge_color) \
and cb.iterable(edge_color) \
and len(edge_color)==len(edge_pos):
if np.alltrue([cb.is_string_like(c)
for c in edge_color]):
# (should check ALL elements)
# list of color letters such as ['k','r','k',...]
edge_colors = tuple([colorConverter.to_rgba(c,alpha)
for c in edge_color])
elif np.alltrue([not cb.is_string_like(c)
for c in edge_color]):
# If color specs are given as (rgb) or (rgba) tuples, we're OK
if np.alltrue([cb.iterable(c) and len(c) in (3,4)
for c in edge_color]):
edge_colors = tuple(edge_color)
alpha=None
else:
# numbers (which are going to be mapped with a colormap)
edge_colors = None
else:
raise ValueError('edge_color must consist of either color names or numbers')
else:
if len(edge_color)==1:
edge_colors = ( colorConverter.to_rgba(edge_color, alpha), )
else:
raise ValueError('edge_color must be a single color or list of exactly m colors where m is the number or edges')
edge_collection = LineCollection(edge_pos,
colors = edge_colors,
linewidths = lw,
antialiaseds = (1,),
linestyle = style,
transOffset = ax.transData,
)
# Note: there was a bug in mpl regarding the handling of alpha values for
# each line in a LineCollection. It was fixed in matplotlib in r7184 and
# r7189 (June 6 2009). We should then not set the alpha value globally,
# since the user can instead provide per-edge alphas now. Only set it
# globally if provided as a scalar.
if cb.is_numlike(alpha):
edge_collection.set_alpha(alpha)
# need 0.87.7 or greater for edge colormaps. No checks done, this will
# just not work with an older mpl
if edge_colors is None:
if edge_cmap is not None: assert(isinstance(edge_cmap, Colormap))
edge_collection.set_array(np.asarray(edge_color))
edge_collection.set_cmap(edge_cmap)
if edge_vmin is not None or edge_vmax is not None:
edge_collection.set_clim(edge_vmin, edge_vmax)
else:
edge_collection.autoscale()
pylab.sci(edge_collection)
arrow_collection=None
if G.is_directed() and arrows:
# a directed graph hack
# draw thick line segments at head end of edge
# waiting for someone else to implement arrows that will work
arrow_colors = ( colorConverter.to_rgba('k', alpha), )
a_pos=[]
p=1.0-0.25 # make head segment 25 percent of edge length
for src,dst in edge_pos:
x1,y1=src
x2,y2=dst
dx=x2-x1 # x offset
dy=y2-y1 # y offset
d=np.sqrt(float(dx**2+dy**2)) # length of edge
if d==0: # source and target at same position
continue
if dx==0: # vertical edge
xa=x2
ya=dy*p+y1
if dy==0: # horizontal edge
ya=y2
xa=dx*p+x1
else:
theta=np.arctan2(dy,dx)
xa=p*d*np.cos(theta)+x1
ya=p*d*np.sin(theta)+y1
a_pos.append(((xa,ya),(x2,y2)))
arrow_collection = LineCollection(a_pos,
colors = arrow_colors,
linewidths = [4*ww for ww in lw],
antialiaseds = (1,),
transOffset = ax.transData,
)
# update view
minx = np.amin(np.ravel(edge_pos[:,:,0]))
maxx = np.amax(np.ravel(edge_pos[:,:,0]))
miny = np.amin(np.ravel(edge_pos[:,:,1]))
maxy = np.amax(np.ravel(edge_pos[:,:,1]))
w = maxx-minx
h = maxy-miny
padx, pady = 0.05*w, 0.05*h
corners = (minx-padx, miny-pady), (maxx+padx, maxy+pady)
ax.update_datalim( corners)
ax.autoscale_view()
edge_collection.set_zorder(1) # edges go behind nodes
ax.add_collection(edge_collection)
if arrow_collection:
arrow_collection.set_zorder(1) # edges go behind nodes
ax.add_collection(arrow_collection)
return edge_collection
0
Example 108
Project: volumina Source File: volumeEditorWidget.py
def _initViewMenu(self):
self._viewMenu = QMenu("View", parent=self)
self._viewMenu.setObjectName( "view_menu" )
self._debugActions = []
ActionInfo = ShortcutManager.ActionInfo
# This action is saved as a member so it can be triggered from tests
self._viewMenu.actionFitToScreen = self._viewMenu.addAction( "&Zoom to &fit" )
self._viewMenu.actionFitToScreen.triggered.connect(self._fitToScreen)
def toggleHud():
hide = not self.editor.imageViews[0]._hud.isVisible()
for v in self.editor.imageViews:
v.setHudVisible(hide)
# This action is saved as a member so it can be triggered from tests
self._viewMenu.actionToggleAllHuds = self._viewMenu.addAction( "Toggle huds" )
self._viewMenu.actionToggleAllHuds.triggered.connect(toggleHud)
def resetAllAxes():
for s in self.editor.imageScenes:
s.resetAxes()
self._viewMenu.addAction( "Reset all axes" ).triggered.connect(resetAllAxes)
def centerAllImages():
for v in self.editor.imageViews:
v.centerImage()
self._viewMenu.addAction( "Center images" ).triggered.connect(centerAllImages)
def toggleDebugPatches(show):
self.editor.showDebugPatches = show
actionShowTiling = self._viewMenu.addAction( "Show Tiling" )
actionShowTiling.setCheckable(True)
actionShowTiling.toggled.connect(toggleDebugPatches)
ShortcutManager().register( "Ctrl+D", ActionInfo(
"Navigation",
"Show tiling",
"Show tiling",
actionShowTiling.toggle,
self,
None ) )
self._debugActions.append( actionShowTiling )
def setCacheSize( cache_size ):
dlg = QDialog(self)
layout = QHBoxLayout()
layout.addWidget( QLabel("Cached Slices Per View:") )
spinBox = QSpinBox( parent=dlg )
spinBox.setRange( 0, 1000 )
spinBox.setValue( self.editor.cacheSize )
layout.addWidget( spinBox )
okButton = QPushButton( "OK", parent=dlg )
okButton.clicked.connect( dlg.accept )
layout.addWidget( okButton )
dlg.setLayout( layout )
dlg.setModal(True)
if dlg.exec_() == QDialog.Accepted:
self.editor.cacheSize = spinBox.value()
self._viewMenu.addAction( "Set layer cache size" ).triggered.connect(setCacheSize)
def enablePrefetching( enable ):
# Enable for Z view only
self.editor.imageScenes[2].setPrefetchingEnabled( enable )
# for scene in self.editor.imageScenes:
# scene.setPrefetchingEnabled( enable )
actionUsePrefetching = self._viewMenu.addAction( "Use prefetching" )
actionUsePrefetching.setCheckable(True)
actionUsePrefetching.toggled.connect(enablePrefetching)
def blockGuiForRendering():
for v in self.editor.imageViews:
v.scene().joinRenderingAllTiles()
v.repaint()
QApplication.processEvents()
actionBlockGui = self._viewMenu.addAction( "Block for rendering" )
actionBlockGui.triggered.connect(blockGuiForRendering)
ShortcutManager().register("Ctrl+B", ActionInfo(
"Navigation",
"Block gui for rendering",
"Block gui for rendering",
actionBlockGui.trigger,
self,
None ) )
self._debugActions.append( actionBlockGui )
# ------ Separator ------
self._viewMenu.addAction("").setSeparator(True)
# Text only
actionOnlyForSelectedView = self._viewMenu.addAction( "Only for selected view" )
actionOnlyForSelectedView.setIconVisibleInMenu(True)
font = actionOnlyForSelectedView.font()
font.setItalic(True)
font.setBold(True)
actionOnlyForSelectedView.setFont(font)
def setCurrentAxisIcon():
"""Update the icon that shows the currently selected axis."""
actionOnlyForSelectedView.setIcon(QIcon(self.editor.imageViews[self.editor._lastImageViewFocus]._hud.axisLabel.pixmap()))
self.editor.newImageView2DFocus.connect(setCurrentAxisIcon)
setCurrentAxisIcon()
actionFitImage = self._viewMenu.addAction( "Fit image" )
actionFitImage.triggered.connect(self._fitImage)
ShortcutManager().register( "K", ActionInfo( "Navigation",
"Fit image on screen",
"Fit image on screen",
actionFitImage.trigger,
self,
None ) )
def toggleSelectedHud():
self.editor.imageViews[self.editor._lastImageViewFocus].toggleHud()
actionToggleSelectedHud = self._viewMenu.addAction( "Toggle hud" )
actionToggleSelectedHud.triggered.connect(toggleSelectedHud)
def resetAxes():
self.editor.imageScenes[self.editor._lastImageViewFocus].resetAxes()
self._viewMenu.addAction( "Reset axes" ).triggered.connect(resetAxes)
def centerImage():
self.editor.imageViews[self.editor._lastImageViewFocus].centerImage()
actionCenterImage = self._viewMenu.addAction( "Center image" )
actionCenterImage.triggered.connect(centerImage)
ShortcutManager().register( "C", ActionInfo( "Navigation",
"Center image",
"Center image",
actionCenterImage.trigger,
self,
None ) )
def restoreImageToOriginalSize():
self.editor.imageViews[self.editor._lastImageViewFocus].doScaleTo()
actionResetZoom = self._viewMenu.addAction( "Reset zoom" )
actionResetZoom.triggered.connect(restoreImageToOriginalSize)
ShortcutManager().register( "W", ActionInfo( "Navigation",
"Reset zoom",
"Reset zoom",
actionResetZoom.trigger,
self,
None ) )
def updateHudActions():
dataShape = self.editor.dataShape
# if the image is 2D, do not show the HUD action (issue #190)
is2D = numpy.sum(numpy.asarray(dataShape[1:4]) == 1) == 1
actionToggleSelectedHud.setVisible(not is2D)
self._viewMenu.actionToggleAllHuds.setVisible(not is2D)
self.editor.shapeChanged.connect( updateHudActions )
0
Example 109
Project: colorspacious Source File: ciecam02.py
def CIECAM02_to_XYZ100(self, J=None, C=None, h=None,
Q=None, M=None, s=None, H=None):
"""Return the unique tristimulus values that have the given CIECAM02
appearance correlates under these viewing conditions.
You must specify 3 arguments:
* Exactly one of ``J`` and ``Q``
* Exactly one of ``C``, ``M``, and ``s``
* Exactly one of ``h`` and ``H``.
Arguments can be vectors, in which case they will be broadcast against
each other.
Returned tristimulus values will be on the 0-100 scale, not the 0-1
scale.
"""
#### Argument checking
require_exactly_one(J=J, Q=Q)
require_exactly_one(C=C, M=M, s=s)
require_exactly_one(h=h, H=H)
if J is not None:
J = np.asarray(J, dtype=float)
if C is not None:
C = np.asarray(C, dtype=float)
if h is not None:
h = np.asarray(h, dtype=float)
if Q is not None:
Q = np.asarray(Q, dtype=float)
if M is not None:
M = np.asarray(M, dtype=float)
if s is not None:
s = np.asarray(s, dtype=float)
if H is not None:
H = np.asarray(H, dtype=float)
#### Step 1: conversions to get JCh
if J is None:
J = 6.25 * ((self.c * Q) / ((self.A_w + 4) * self.F_L**0.25)) ** 2
if C is None:
if M is not None:
C = M / self.F_L**0.25
else:
assert s is not None
# when starting from s, we need Q
if Q is None:
Q = self._J_to_Q(J)
C = (s / 100) ** 2 * (Q / self.F_L**0.25)
if h is None:
i = np.searchsorted(H_i, H, side="right") - 1
# BROKEN:
num1 = (H - H_i[i]) * (e_i[i + 1] * h_i[i] - e_i[i] * h_i[i + 1])
num2 = -100 * h_i[i] * e_i[i + 1]
denom1 = (H - H_i[i]) * (e_i[i + 1] - e_i[i])
denom2 = -100 * e_i[i + 1]
hprime = (num1 + num2) / (denom1 + denom2)
h = np.select([hprime > 360, True], [hprime - 360, hprime])
J, C, h = np.broadcast_arrays(J, C, h)
target_shape = J.shape
# 0d arrays break indexing stuff
if J.ndim == 0:
J = np.atleast_1d(J)
C = np.atleast_1d(C)
h = np.atleast_1d(h)
#### Step 2
t = (C
/ (np.sqrt(J / 100) * (1.64 - 0.29**self.n) ** 0.73)
) ** (1 / 0.9)
e_t = 0.25 * (np.cos(np.deg2rad(h) + 2) + 3.8)
A = self.A_w * (J / 100) ** (1 / (self.c * self.z))
# an awkward way of calculating 1/t such that 1/0 -> inf
with np.errstate(divide="ignore"):
one_over_t = 1 / t
one_over_t = np.select([np.isnan(one_over_t), True],
[np.inf, one_over_t])
p_1 = (50000. / 13) * self.N_c * self.N_cb * e_t * one_over_t
p_2 = A / self.N_bb + 0.305
p_3 = 21. / 20
#### Step 3
sin_h = np.sin(np.deg2rad(h))
cos_h = np.cos(np.deg2rad(h))
# to avoid divide-by-zero (or divide-by-eps) issues, we use different
# computations when |sin_h| > |cos_h| and vice-versa
num = p_2 * (2 + p_3) * (460. / 1403)
denom_part2 = (2 + p_3) * (220. / 1403)
denom_part3 = (-27. / 1403) + p_3 * (6300. / 1403)
a = np.empty_like(h)
b = np.empty_like(h)
small_cos = (np.abs(sin_h) >= np.abs(cos_h))
# NB denom_part2 and denom_part3 are scalars
b[small_cos] = (num[small_cos]
/ (p_1[small_cos] / sin_h[small_cos]
+ (denom_part2
* cos_h[small_cos] / sin_h[small_cos])
+ denom_part3))
a[small_cos] = b[small_cos] * cos_h[small_cos] / sin_h[small_cos]
a[~small_cos] = (num[~small_cos]
/ (p_1[~small_cos] / cos_h[~small_cos]
+ denom_part2
+ (denom_part3
* sin_h[~small_cos] / cos_h[~small_cos])))
b[~small_cos] = a[~small_cos] * sin_h[~small_cos] / cos_h[~small_cos]
#### Step 4
p2ab = np.concatenate((p_2[..., np.newaxis],
a[..., np.newaxis],
b[..., np.newaxis]),
axis=-1)
RGBprime_a_matrix = (1. / 1403
* np.asarray([[ 460, 451, 288],
[ 460, -891, -261],
[ 460, -220, -6300]], dtype=float))
RGBprime_a = broadcasting_matvec(RGBprime_a_matrix, p2ab)
#### Step 5
RGBprime = (np.sign(RGBprime_a - 0.1)
* (100 / self.F_L)
* ((27.13 * np.abs(RGBprime_a - 0.1))
/ (400 - np.abs(RGBprime_a - 0.1))) ** (1 / 0.42))
#### Step 6
RGB_C = broadcasting_matvec(M_CAT02_M_HPE_inv, RGBprime)
#### Step 7
RGB = RGB_C / self.D_RGB
#### Step 8
XYZ100 = broadcasting_matvec(M_CAT02_inv, RGB)
XYZ100 = XYZ100.reshape(target_shape + (3,))
return XYZ100
0
Example 110
Project: machine-learning Source File: sv.py
def sv_model(model, kernel_type, session_id, feature_request, list_error):
'''@sv_model
This method generates an sv (i.e. svm, or svr) model using feature data,
retrieved from the database. The generated model, is then stored within the
NoSQL datastore.
@grouped_features, a matrix of observations, where each nested vector,
or python list, is a collection of features within the containing
observation.
@encoded_labels, observation labels (dependent variable labels),
encoded into a unique integer representation.
'''
# local variables
dataset = feature_request.get_dataset(session_id, model)
get_feature_count = feature_request.get_count(session_id)
label_encoder = preprocessing.LabelEncoder()
logger = Logger(__name__, 'error', 'error')
list_model_type = current_app.config.get('MODEL_TYPE')
# get dataset
if dataset['error']:
logger.log(dataset['error'])
list_error.append(dataset['error'])
dataset = None
else:
dataset = numpy.asarray(dataset['result'])
# get feature count
if get_feature_count['error']:
logger.log(get_feature_count['error'])
list_error.append(get_feature_count['error'])
feature_count = None
else:
feature_count = get_feature_count['result'][0][0]
# check dataset integrity, build model
if len(dataset) % feature_count == 0:
features_list = dataset[:, [[0], [2], [1]]]
current_features = []
grouped_features = []
observation_labels = []
feature_labels = []
# group features into observation instances, record labels
for index, feature in enumerate(features_list):
# svm: observation labels
if model == list_model_type[0]:
current_features.append(feature[1][0])
if (index+1) % feature_count == 0:
grouped_features.append(current_features)
observation_labels.append(feature[0][0])
current_features = []
# svr: observation labels
elif model == list_model_type[1]:
current_features.append(float(feature[1][0]))
if (index+1) % feature_count == 0:
grouped_features.append(current_features)
observation_labels.append(float(feature[0][0]))
current_features = []
# general feature labels in every observation
if not len(feature_labels) == feature_count:
feature_labels.append(feature[2][0])
# case 1: svm model
if model == list_model_type[0]:
# convert observation labels to a unique integer representation
label_encoder = preprocessing.LabelEncoder()
label_encoder.fit(dataset[:, 0])
encoded_labels = label_encoder.transform(observation_labels)
# create model
clf = svm.SVC(kernel=kernel_type, probability=True)
# cache encoded labels
Cache_Model(label_encoder).cache(model + '_labels', session_id)
# fit model
clf.fit(grouped_features, encoded_labels)
# case 2: svr model
elif model == list_model_type[1]:
# create model
clf = svm.SVR(kernel=kernel_type)
# fit model
clf.fit(grouped_features, observation_labels)
# compute, and cache coefficient of determination
r2 = clf.score(grouped_features, observation_labels)
Cache_Hset().cache(
model + '_r2',
session_id,
r2
)
# get title
entity = Retrieve_Entity()
title = entity.get_title(session_id)['result'][0][0]
# cache model, title
Cache_Model(clf).cache(
model + '_model',
str(session_id) + '_' + title
)
Cache_Hset().cache(model + '_title', session_id, title)
# cache feature labels, with respect to given session id
Cache_Hset().cache(
model + '_feature_labels',
str(session_id),
json.dumps(feature_labels)
)
# return error(s) if exists
return {'error': list_error}
0
Example 111
Project: seaborn Source File: timeseries.py
def tsplot(data, time=None, unit=None, condition=None, value=None,
err_style="ci_band", ci=68, interpolate=True, color=None,
estimator=np.mean, n_boot=5000, err_palette=None, err_kws=None,
legend=True, ax=None, **kwargs):
"""Plot one or more timeseries with flexible representation of uncertainty.
This function is intended to be used with data where observations are
nested within sampling units that were measured at multiple timepoints.
It can take data specified either as a long-form (tidy) DataFrame or as an
ndarray with dimensions (unit, time) The interpretation of some of the
other parameters changes depending on the type of object passed as data.
Parameters
----------
data : DataFrame or ndarray
Data for the plot. Should either be a "long form" dataframe or an
array with dimensions (unit, time, condition). In both cases, the
condition field/dimension is optional. The type of this argument
determines the interpretation of the next few parameters. When
using a DataFrame, the index has to be sequential.
time : string or series-like
Either the name of the field corresponding to time in the data
DataFrame or x values for a plot when data is an array. If a Series,
the name will be used to label the x axis.
unit : string
Field in the data DataFrame identifying the sampling unit (e.g.
subject, neuron, etc.). The error representation will collapse over
units at each time/condition observation. This has no role when data
is an array.
value : string
Either the name of the field corresponding to the data values in
the data DataFrame (i.e. the y coordinate) or a string that forms
the y axis label when data is an array.
condition : string or Series-like
Either the name of the field identifying the condition an observation
falls under in the data DataFrame, or a sequence of names with a length
equal to the size of the third dimension of data. There will be a
separate trace plotted for each condition. If condition is a Series
with a name attribute, the name will form the title for the plot
legend (unless legend is set to False).
err_style : string or list of strings or None
Names of ways to plot uncertainty across units from set of
{ci_band, ci_bars, boot_traces, boot_kde, unit_traces, unit_points}.
Can use one or more than one method.
ci : float or list of floats in [0, 100]
Confidence interval size(s). If a list, it will stack the error
plots for each confidence interval. Only relevant for error styles
with "ci" in the name.
interpolate : boolean
Whether to do a linear interpolation between each timepoint when
plotting. The value of this parameter also determines the marker
used for the main plot traces, unless marker is specified as a keyword
argument.
color : seaborn palette or matplotlib color name or dictionary
Palette or color for the main plots and error representation (unless
plotting by unit, which can be separately controlled with err_palette).
If a dictionary, should map condition name to color spec.
estimator : callable
Function to determine central tendency and to pass to bootstrap
must take an ``axis`` argument.
n_boot : int
Number of bootstrap iterations.
err_palette : seaborn palette
Palette name or list of colors used when plotting data for each unit.
err_kws : dict, optional
Keyword argument dictionary passed through to matplotlib function
generating the error plot,
legend : bool, optional
If ``True`` and there is a ``condition`` variable, add a legend to
the plot.
ax : axis object, optional
Plot in given axis; if None creates a new figure
kwargs :
Other keyword arguments are passed to main plot() call
Returns
-------
ax : matplotlib axis
axis with plot data
Examples
--------
Plot a trace with translucent confidence bands:
.. plot::
:context: close-figs
>>> import numpy as np; np.random.seed(22)
>>> import seaborn as sns; sns.set(color_codes=True)
>>> x = np.linspace(0, 15, 31)
>>> data = np.sin(x) + np.random.rand(10, 31) + np.random.randn(10, 1)
>>> ax = sns.tsplot(data=data)
Plot a long-form dataframe with several conditions:
.. plot::
:context: close-figs
>>> gammas = sns.load_dataset("gammas")
>>> ax = sns.tsplot(time="timepoint", value="BOLD signal",
... unit="subject", condition="ROI",
... data=gammas)
Use error bars at the positions of the observations:
.. plot::
:context: close-figs
>>> ax = sns.tsplot(data=data, err_style="ci_bars", color="g")
Don't interpolate between the observations:
.. plot::
:context: close-figs
>>> import matplotlib.pyplot as plt
>>> ax = sns.tsplot(data=data, err_style="ci_bars", interpolate=False)
Show multiple confidence bands:
.. plot::
:context: close-figs
>>> ax = sns.tsplot(data=data, ci=[68, 95], color="m")
Use a different estimator:
.. plot::
:context: close-figs
>>> ax = sns.tsplot(data=data, estimator=np.median)
Show each bootstrap resample:
.. plot::
:context: close-figs
>>> ax = sns.tsplot(data=data, err_style="boot_traces", n_boot=500)
Show the trace from each sampling unit:
.. plot::
:context: close-figs
>>> ax = sns.tsplot(data=data, err_style="unit_traces")
"""
# Sort out default values for the parameters
if ax is None:
ax = plt.gca()
if err_kws is None:
err_kws = {}
# Handle different types of input data
if isinstance(data, pd.DataFrame):
xlabel = time
ylabel = value
# Condition is optional
if condition is None:
condition = pd.Series(1, index=data.index)
legend = False
legend_name = None
n_cond = 1
else:
legend = True and legend
legend_name = condition
n_cond = len(data[condition].unique())
else:
data = np.asarray(data)
# Data can be a timecourse from a single unit or
# several observations in one condition
if data.ndim == 1:
data = data[np.newaxis, :, np.newaxis]
elif data.ndim == 2:
data = data[:, :, np.newaxis]
n_unit, n_time, n_cond = data.shape
# Units are experimental observations. Maybe subjects, or neurons
if unit is None:
units = np.arange(n_unit)
unit = "unit"
units = np.repeat(units, n_time * n_cond)
ylabel = None
# Time forms the xaxis of the plot
if time is None:
times = np.arange(n_time)
else:
times = np.asarray(time)
xlabel = None
if hasattr(time, "name"):
xlabel = time.name
time = "time"
times = np.tile(np.repeat(times, n_cond), n_unit)
# Conditions split the timeseries plots
if condition is None:
conds = range(n_cond)
legend = False
if isinstance(color, dict):
err = "Must have condition names if using color dict."
raise ValueError(err)
else:
conds = np.asarray(condition)
legend = True and legend
if hasattr(condition, "name"):
legend_name = condition.name
else:
legend_name = None
condition = "cond"
conds = np.tile(conds, n_unit * n_time)
# Value forms the y value in the plot
if value is None:
ylabel = None
else:
ylabel = value
value = "value"
# Convert to long-form DataFrame
data = pd.DataFrame(dict(value=data.ravel(),
time=times,
unit=units,
cond=conds))
# Set up the err_style and ci arguments for the loop below
if isinstance(err_style, string_types):
err_style = [err_style]
elif err_style is None:
err_style = []
if not hasattr(ci, "__iter__"):
ci = [ci]
# Set up the color palette
if color is None:
current_palette = utils.get_color_cycle()
if len(current_palette) < n_cond:
colors = color_palette("husl", n_cond)
else:
colors = color_palette(n_colors=n_cond)
elif isinstance(color, dict):
colors = [color[c] for c in data[condition].unique()]
else:
try:
colors = color_palette(color, n_cond)
except ValueError:
color = mpl.colors.colorConverter.to_rgb(color)
colors = [color] * n_cond
# Do a groupby with condition and plot each trace
for c, (cond, df_c) in enumerate(data.groupby(condition, sort=False)):
df_c = df_c.pivot(unit, time, value)
x = df_c.columns.values.astype(np.float)
# Bootstrap the data for confidence intervals
boot_data = algo.bootstrap(df_c.values, n_boot=n_boot,
axis=0, func=estimator)
cis = [utils.ci(boot_data, v, axis=0) for v in ci]
central_data = estimator(df_c.values, axis=0)
# Get the color for this condition
color = colors[c]
# Use subroutines to plot the uncertainty
for style in err_style:
# Allow for null style (only plot central tendency)
if style is None:
continue
# Grab the function from the global environment
try:
plot_func = globals()["_plot_%s" % style]
except KeyError:
raise ValueError("%s is not a valid err_style" % style)
# Possibly set up to plot each observation in a different color
if err_palette is not None and "unit" in style:
orig_color = color
color = color_palette(err_palette, len(df_c.values))
# Pass all parameters to the error plotter as keyword args
plot_kwargs = dict(ax=ax, x=x, data=df_c.values,
boot_data=boot_data,
central_data=central_data,
color=color, err_kws=err_kws)
# Plot the error representation, possibly for multiple cis
for ci_i in cis:
plot_kwargs["ci"] = ci_i
plot_func(**plot_kwargs)
if err_palette is not None and "unit" in style:
color = orig_color
# Plot the central trace
kwargs.setdefault("marker", "" if interpolate else "o")
ls = kwargs.pop("ls", "-" if interpolate else "")
kwargs.setdefault("linestyle", ls)
label = cond if legend else "_nolegend_"
ax.plot(x, central_data, color=color, label=label, **kwargs)
# Pad the sides of the plot only when not interpolating
ax.set_xlim(x.min(), x.max())
x_diff = x[1] - x[0]
if not interpolate:
ax.set_xlim(x.min() - x_diff, x.max() + x_diff)
# Add the plot labels
if xlabel is not None:
ax.set_xlabel(xlabel)
if ylabel is not None:
ax.set_ylabel(ylabel)
if legend:
ax.legend(loc=0, title=legend_name)
return ax
0
Example 112
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: hierarchical.py
def ward_tree(X, connectivity=None, n_components=None, n_clusters=None,
return_distance=False):
"""Ward clustering based on a Feature matrix.
Recursively merges the pair of clusters that minimally increases
within-cluster variance.
The inertia matrix uses a Heapq-based representation.
This is the structured version, that takes into account some topological
structure between samples.
Read more in the :ref:`User Guide <hierarchical_clustering>`.
Parameters
----------
X : array, shape (n_samples, n_features)
feature matrix representing n_samples samples to be clustered
connectivity : sparse matrix (optional).
connectivity matrix. Defines for each sample the neighboring samples
following a given structure of the data. The matrix is assumed to
be symmetric and only the upper triangular half is used.
Default is None, i.e, the Ward algorithm is unstructured.
n_components : int (optional)
Number of connected components. If None the number of connected
components is estimated from the connectivity matrix.
NOTE: This parameter is now directly determined directly
from the connectivity matrix and will be removed in 0.18
n_clusters : int (optional)
Stop early the construction of the tree at n_clusters. This is
useful to decrease computation time if the number of clusters is
not small compared to the number of samples. In this case, the
complete tree is not computed, thus the 'children' output is of
limited use, and the 'parents' output should rather be used.
This option is valid only when specifying a connectivity matrix.
return_distance: bool (optional)
If True, return the distance between the clusters.
Returns
-------
children : 2D array, shape (n_nodes-1, 2)
The children of each non-leaf node. Values less than `n_samples`
correspond to leaves of the tree which are the original samples.
A node `i` greater than or equal to `n_samples` is a non-leaf
node and has children `children_[i - n_samples]`. Alternatively
at the i-th iteration, children[i][0] and children[i][1]
are merged to form node `n_samples + i`
n_components : int
The number of connected components in the graph.
n_leaves : int
The number of leaves in the tree
parents : 1D array, shape (n_nodes, ) or None
The parent of each node. Only returned when a connectivity matrix
is specified, elsewhere 'None' is returned.
distances : 1D array, shape (n_nodes-1, )
Only returned if return_distance is set to True (for compatibility).
The distances between the centers of the nodes. `distances[i]`
corresponds to a weighted euclidean distance between
the nodes `children[i, 1]` and `children[i, 2]`. If the nodes refer to
leaves of the tree, then `distances[i]` is their unweighted euclidean
distance. Distances are updated in the following way
(from scipy.hierarchy.linkage):
The new entry :math:`d(u,v)` is computed as follows,
.. math::
d(u,v) = \\sqrt{\\frac{|v|+|s|}
{T}d(v,s)^2
+ \\frac{|v|+|t|}
{T}d(v,t)^2
- \\frac{|v|}
{T}d(s,t)^2}
where :math:`u` is the newly joined cluster consisting of
clusters :math:`s` and :math:`t`, :math:`v` is an unused
cluster in the forest, :math:`T=|v|+|s|+|t|`, and
:math:`|*|` is the cardinality of its argument. This is also
known as the incremental algorithm.
"""
X = np.asarray(X)
if X.ndim == 1:
X = np.reshape(X, (-1, 1))
n_samples, n_features = X.shape
if connectivity is None:
from scipy.cluster import hierarchy # imports PIL
if n_clusters is not None:
warnings.warn('Partial build of the tree is implemented '
'only for structured clustering (i.e. with '
'explicit connectivity). The algorithm '
'will build the full tree and only '
'retain the lower branches required '
'for the specified number of clusters',
stacklevel=2)
out = hierarchy.ward(X)
children_ = out[:, :2].astype(np.intp)
if return_distance:
distances = out[:, 2]
return children_, 1, n_samples, None, distances
else:
return children_, 1, n_samples, None
if n_components is not None:
warnings.warn(
"n_components is now directly calculated from the connectivity "
"matrix and will be removed in 0.18",
DeprecationWarning)
connectivity, n_components = _fix_connectivity(X, connectivity)
if n_clusters is None:
n_nodes = 2 * n_samples - 1
else:
if n_clusters > n_samples:
raise ValueError('Cannot provide more clusters than samples. '
'%i n_clusters was asked, and there are %i samples.'
% (n_clusters, n_samples))
n_nodes = 2 * n_samples - n_clusters
# create inertia matrix
coord_row = []
coord_col = []
A = []
for ind, row in enumerate(connectivity.rows):
A.append(row)
# We keep only the upper triangular for the moments
# Generator expressions are faster than arrays on the following
row = [i for i in row if i < ind]
coord_row.extend(len(row) * [ind, ])
coord_col.extend(row)
coord_row = np.array(coord_row, dtype=np.intp, order='C')
coord_col = np.array(coord_col, dtype=np.intp, order='C')
# build moments as a list
moments_1 = np.zeros(n_nodes, order='C')
moments_1[:n_samples] = 1
moments_2 = np.zeros((n_nodes, n_features), order='C')
moments_2[:n_samples] = X
inertia = np.empty(len(coord_row), dtype=np.float, order='C')
_hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col,
inertia)
inertia = list(six.moves.zip(inertia, coord_row, coord_col))
heapify(inertia)
# prepare the main fields
parent = np.arange(n_nodes, dtype=np.intp)
used_node = np.ones(n_nodes, dtype=bool)
children = []
if return_distance:
distances = np.empty(n_nodes - n_samples)
not_visited = np.empty(n_nodes, dtype=np.int8, order='C')
# recursive merge loop
for k in range(n_samples, n_nodes):
# identify the merge
while True:
inert, i, j = heappop(inertia)
if used_node[i] and used_node[j]:
break
parent[i], parent[j] = k, k
children.append((i, j))
used_node[i] = used_node[j] = False
if return_distance: # store inertia value
distances[k - n_samples] = inert
# update the moments
moments_1[k] = moments_1[i] + moments_1[j]
moments_2[k] = moments_2[i] + moments_2[j]
# update the structure matrix A and the inertia matrix
coord_col = []
not_visited.fill(1)
not_visited[k] = 0
_hierarchical._get_parents(A[i], coord_col, parent, not_visited)
_hierarchical._get_parents(A[j], coord_col, parent, not_visited)
# List comprehension is faster than a for loop
[A[l].append(k) for l in coord_col]
A.append(coord_col)
coord_col = np.array(coord_col, dtype=np.intp, order='C')
coord_row = np.empty(coord_col.shape, dtype=np.intp, order='C')
coord_row.fill(k)
n_additions = len(coord_row)
ini = np.empty(n_additions, dtype=np.float, order='C')
_hierarchical.compute_ward_dist(moments_1, moments_2,
coord_row, coord_col, ini)
# List comprehension is faster than a for loop
[heappush(inertia, (ini[idx], k, coord_col[idx]))
for idx in range(n_additions)]
# Separate leaves in children (empty lists up to now)
n_leaves = n_samples
# sort children to get consistent output with unstructured version
children = [c[::-1] for c in children]
children = np.array(children) # return numpy array for efficient caching
if return_distance:
# 2 is scaling factor to compare w/ unstructured version
distances = np.sqrt(2. * distances)
return children, n_components, n_leaves, parent, distances
else:
return children, n_components, n_leaves, parent
0
Example 113
def build_model(self, train_set_unlabeled, train_set_labeled, test_set, validation_set=None):
"""
Build the auxiliary deep generative model from the initialized hyperparameters.
Define the lower bound term and compile it into a training function.
:param train_set_unlabeled: Unlabeled train set containing variables x, t.
:param train_set_labeled: Unlabeled train set containing variables x, t.
:param test_set: Test set containing variables x, t.
:param validation_set: Validation set containing variables x, t.
:return: train, test, validation function and dicts of arguments.
"""
super(SDGMSSL, self).build_model(train_set_unlabeled, test_set, validation_set)
sh_train_x_l = theano.shared(np.asarray(train_set_labeled[0], dtype=theano.config.floatX), borrow=True)
sh_train_t_l = theano.shared(np.asarray(train_set_labeled[1], dtype=theano.config.floatX), borrow=True)
n = self.sh_train_x.shape[0].astype(theano.config.floatX) # no. of data points
n_l = sh_train_x_l.shape[0].astype(theano.config.floatX) # no. of labeled data points
# Define the layers for the density estimation used in the lower bound.
l_log_qa = GaussianLogDensityLayer(self.l_qa, self.l_qa_mu, self.l_qa_logvar)
l_log_qz = GaussianLogDensityLayer(self.l_qz, self.l_qz_mu, self.l_qz_logvar)
l_log_qy = MultinomialLogDensityLayer(self.l_qy, self.l_y_in, eps=1e-8)
l_log_pz = StandardNormalLogDensityLayer(self.l_qz)
l_log_pa = GaussianLogDensityLayer(self.l_qa, self.l_pa_mu, self.l_pa_logvar)
if self.x_dist == 'bernoulli':
l_log_px = BernoulliLogDensityLayer(self.l_px, self.l_x_in)
elif self.x_dist == 'multinomial':
l_log_px = MultinomialLogDensityLayer(self.l_px, self.l_x_in)
elif self.x_dist == 'gaussian':
l_log_px = GaussianLogDensityLayer(self.l_x_in, self.l_px_mu, self.l_px_logvar)
def lower_bound(log_pa, log_qa, log_pz, log_qz, log_py, log_px):
lb = log_px + log_py + log_pz + log_pa - log_qa - log_qz
return lb
# Lower bound for labeled data
out_layers = [l_log_pa, l_log_pz, l_log_qa, l_log_qz, l_log_px, l_log_qy]
inputs = {self.l_x_in: self.sym_x_l, self.l_y_in: self.sym_t_l}
out = get_output(out_layers, inputs, batch_norm_update_averages=False, batch_norm_use_averages=False)
log_pa_l, log_pz_l, log_qa_x_l, log_qz_axy_l, log_px_zy_l, log_qy_ax_l = out
# Prior p(y) expecting that all classes are evenly distributed
py_l = softmax(T.zeros((self.sym_x_l.shape[0], self.n_y)))
log_py_l = -categorical_crossentropy(py_l, self.sym_t_l).reshape((-1, 1)).dimshuffle((0, 'x', 'x', 1))
lb_l = lower_bound(log_pa_l, log_qa_x_l, log_pz_l, log_qz_axy_l, log_py_l, log_px_zy_l)
lb_l = lb_l.mean(axis=(1, 2)) # Mean over the sampling dimensions
log_qy_ax_l *= (self.sym_beta * (n / n_l)) # Scale the supervised cross entropy with the alpha constant
lb_l -= log_qy_ax_l.mean(axis=(1, 2)) # Collect the lower bound term and mean over sampling dimensions
# Lower bound for unlabeled data
bs_u = self.sym_x_u.shape[0]
# For the integrating out approach, we repeat the input matrix x, and construct a target (bs * n_y) x n_y
# Example of input and target matrix for a 3 class problem and batch_size=2. 2D tensors of the form
# x_repeat t_repeat
# [[x[0,0], x[0,1], ..., x[0,n_x]] [[1, 0, 0]
# [x[1,0], x[1,1], ..., x[1,n_x]] [1, 0, 0]
# [x[0,0], x[0,1], ..., x[0,n_x]] [0, 1, 0]
# [x[1,0], x[1,1], ..., x[1,n_x]] [0, 1, 0]
# [x[0,0], x[0,1], ..., x[0,n_x]] [0, 0, 1]
# [x[1,0], x[1,1], ..., x[1,n_x]]] [0, 0, 1]]
t_eye = T.eye(self.n_y, k=0)
t_u = t_eye.reshape((self.n_y, 1, self.n_y)).repeat(bs_u, axis=1).reshape((-1, self.n_y))
x_u = self.sym_x_u.reshape((1, bs_u, self.n_x)).repeat(self.n_y, axis=0).reshape((-1, self.n_x))
# Since the expectation of var a is outside the integration we calculate E_q(a|x) first
a_x_u = get_output(self.l_qa, self.sym_x_u, batch_norm_update_averages=True, batch_norm_use_averages=False)
a_x_u_rep = a_x_u.reshape((1, bs_u * self.sym_samples, self.n_a)).repeat(self.n_y, axis=0).reshape(
(-1, self.n_a))
out_layers = [l_log_pa, l_log_pz, l_log_qa, l_log_qz, l_log_px]
inputs = {self.l_x_in: x_u, self.l_y_in: t_u, self.l_a_in: a_x_u_rep}
out = get_output(out_layers, inputs, batch_norm_update_averages=False, batch_norm_use_averages=False)
log_pa_u, log_pz_u, log_qa_x_u, log_qz_axy_u, log_px_zy_u = out
# Prior p(y) expecting that all classes are evenly distributed
py_u = softmax(T.zeros((bs_u * self.n_y, self.n_y)))
log_py_u = -categorical_crossentropy(py_u, t_u).reshape((-1, 1)).dimshuffle((0, 'x', 'x', 1))
lb_u = lower_bound(log_pa_u, log_qa_x_u, log_pz_u, log_qz_axy_u, log_py_u, log_px_zy_u)
lb_u = lb_u.reshape((self.n_y, 1, 1, bs_u)).transpose(3, 1, 2, 0).mean(axis=(1, 2))
inputs = {self.l_x_in: self.sym_x_u, self.l_a_in: a_x_u.reshape((-1, self.n_a))}
y_u = get_output(self.l_qy, inputs, batch_norm_update_averages=True, batch_norm_use_averages=False).mean(
axis=(1, 2))
y_u += 1e-8 # Ensure that we get no NANs when calculating the entropy
y_u /= T.sum(y_u, axis=1, keepdims=True)
lb_u = (y_u * (lb_u - T.log(y_u))).sum(axis=1)
if self.batchnorm:
# TODO: implement the BN layer correctly.
inputs = {self.l_x_in: self.sym_x_u, self.l_y_in: y_u, self.l_a_in: a_x_u}
get_output(out_layers, inputs, weighting=None, batch_norm_update_averages=True,
batch_norm_use_averages=False)
# Regularizing with weight priors p(theta|N(0,1)), collecting and clipping gradients
weight_priors = 0.0
for p in self.trainable_model_params:
if 'W' not in str(p):
continue
weight_priors += log_normal(p, 0, 1).sum()
# Collect the lower bound and scale it with the weight priors.
elbo = ((lb_l.mean() + lb_u.mean()) * n + weight_priors) / -n
lb_labeled = -lb_l.mean()
lb_unlabeled = -lb_u.mean()
grads_collect = T.grad(elbo, self.trainable_model_params)
params_collect = self.trainable_model_params
sym_beta1 = T.scalar('beta1')
sym_beta2 = T.scalar('beta2')
clip_grad, max_norm = 1, 5
mgrads = total_norm_constraint(grads_collect, max_norm=max_norm)
mgrads = [T.clip(g, -clip_grad, clip_grad) for g in mgrads]
updates = adam(mgrads, params_collect, self.sym_lr, sym_beta1, sym_beta2)
# Training function
indices = self._srng.choice(size=[self.sym_bs_l], a=sh_train_x_l.shape[0], replace=False)
x_batch_l = sh_train_x_l[indices]
t_batch_l = sh_train_t_l[indices]
x_batch_u = self.sh_train_x[self.batch_slice]
if self.x_dist == 'bernoulli': # Sample bernoulli input.
x_batch_u = self._srng.binomial(size=x_batch_u.shape, n=1, p=x_batch_u, dtype=theano.config.floatX)
x_batch_l = self._srng.binomial(size=x_batch_l.shape, n=1, p=x_batch_l, dtype=theano.config.floatX)
givens = {self.sym_x_l: x_batch_l,
self.sym_x_u: x_batch_u,
self.sym_t_l: t_batch_l}
inputs = [self.sym_index, self.sym_batchsize, self.sym_bs_l, self.sym_beta,
self.sym_lr, sym_beta1, sym_beta2, self.sym_samples]
outputs = [elbo, lb_labeled, lb_unlabeled]
f_train = theano.function(inputs=inputs, outputs=outputs, givens=givens, updates=updates)
# Default training args. Note that these can be changed during or prior to training.
self.train_args['inputs']['batchsize_unlabeled'] = 100
self.train_args['inputs']['batchsize_labeled'] = 100
self.train_args['inputs']['beta'] = 0.1
self.train_args['inputs']['learningrate'] = 3e-4
self.train_args['inputs']['beta1'] = 0.9
self.train_args['inputs']['beta2'] = 0.999
self.train_args['inputs']['samples'] = 1
self.train_args['outputs']['lb'] = '%0.4f'
self.train_args['outputs']['lb-labeled'] = '%0.4f'
self.train_args['outputs']['lb-unlabeled'] = '%0.4f'
# Validation and test function
y = get_output(self.l_qy, self.sym_x_l, deterministic=True).mean(axis=(1, 2))
class_err = (1. - categorical_accuracy(y, self.sym_t_l).mean()) * 100
givens = {self.sym_x_l: self.sh_test_x,
self.sym_t_l: self.sh_test_t}
f_test = theano.function(inputs=[self.sym_samples], outputs=[class_err], givens=givens)
# Test args. Note that these can be changed during or prior to training.
self.test_args['inputs']['samples'] = 1
self.test_args['outputs']['test'] = '%0.2f%%'
f_validate = None
if validation_set is not None:
givens = {self.sym_x_l: self.sh_valid_x,
self.sym_t_l: self.sh_valid_t}
f_validate = theano.function(inputs=[self.sym_samples], outputs=[class_err], givens=givens)
# Default validation args. Note that these can be changed during or prior to training.
self.validate_args['inputs']['samples'] = 1
self.validate_args['outputs']['validation'] = '%0.2f%%'
return f_train, f_test, f_validate, self.train_args, self.test_args, self.validate_args
0
Example 114
Project: elephant Source File: spectral.py
def _welch(x, y, fs=1.0, window='hanning', nperseg=256, noverlap=None,
nfft=None, detrend='constant', scaling='density', axis=-1):
"""
A helper function to estimate cross spectral density using Welch's method.
This function is a slightly modified version of `scipy.signal.welch()` with
modifications based on `matplotlib.mlab._spectral_helper()`.
Welch's method [1]_ computes an estimate of the cross spectral density
by dividing the data into overlapping segments, computing a modified
periodogram for each segment and averaging the cross-periodograms.
Parameters
----------
x, y : array_like
Time series of measurement values
fs : float, optional
Sampling frequency of the `x` and `y` time series in units of Hz.
Defaults to 1.0.
window : str or tuple or array_like, optional
Desired window to use. See `get_window` for a list of windows and
required parameters. If `window` is array_like it will be used
directly as the window and its length will be used for nperseg.
Defaults to 'hanning'.
nperseg : int, optional
Length of each segment. Defaults to 256.
noverlap: int, optional
Number of points to overlap between segments. If None,
``noverlap = nperseg / 2``. Defaults to None.
nfft : int, optional
Length of the FFT used, if a zero padded FFT is desired. If None,
the FFT length is `nperseg`. Defaults to None.
detrend : str or function, optional
Specifies how to detrend each segment. If `detrend` is a string,
it is passed as the ``type`` argument to `detrend`. If it is a
function, it takes a segment and returns a detrended segment.
Defaults to 'constant'.
scaling : { 'density', 'spectrum' }, optional
Selects between computing the power spectral density ('density')
where Pxx has units of V**2/Hz if x is measured in V and computing
the power spectrum ('spectrum') where Pxx has units of V**2 if x is
measured in V. Defaults to 'density'.
axis : int, optional
Axis along which the periodogram is computed; the default is over
the last axis (i.e. ``axis=-1``).
Returns
-------
f : ndarray
Array of sample frequencies.
Pxy : ndarray
Cross spectral density or cross spectrum of x and y.
Notes
-----
An appropriate amount of overlap will depend on the choice of window
and on your requirements. For the default 'hanning' window an
overlap of 50% is a reasonable trade off between accurately estimating
the signal power, while not over counting any of the data. Narrower
windows may require a larger overlap.
If `noverlap` is 0, this method is equivalent to Bartlett's method [2]_.
References
----------
.. [1] P. Welch, "The use of the fast Fourier transform for the
estimation of power spectra: A method based on time averaging
over short, modified periodograms", IEEE Trans. Audio
Electroacoust. vol. 15, pp. 70-73, 1967.
.. [2] M.S. Bartlett, "Periodogram Analysis and Continuous Spectra",
Biometrika, vol. 37, pp. 1-16, 1950.
"""
# TODO: This function should be replaced by `scipy.signal.csd()`, which
# will appear in SciPy 0.16.0.
# The checks for if y is x are so that we can use the same function to
# obtain both power spectrum and cross spectrum without doing extra
# calculations.
same_data = y is x
# Make sure we're dealing with a numpy array. If y and x were the same
# object to start with, keep them that way
x = np.asarray(x)
if same_data:
y = x
else:
if x.shape != y.shape:
raise ValueError("x and y must be of the same shape.")
y = np.asarray(y)
if x.size == 0:
return np.empty(x.shape), np.empty(x.shape)
if axis != -1:
x = np.rollaxis(x, axis, len(x.shape))
if not same_data:
y = np.rollaxis(y, axis, len(y.shape))
if x.shape[-1] < nperseg:
warnings.warn('nperseg = %d, is greater than x.shape[%d] = %d, using '
'nperseg = x.shape[%d]'
% (nperseg, axis, x.shape[axis], axis))
nperseg = x.shape[-1]
if isinstance(window, string_types) or type(window) is tuple:
win = get_window(window, nperseg)
else:
win = np.asarray(window)
if len(win.shape) != 1:
raise ValueError('window must be 1-D')
if win.shape[0] > x.shape[-1]:
raise ValueError('window is longer than x.')
nperseg = win.shape[0]
if scaling == 'density':
scale = 1.0 / (fs * (win * win).sum())
elif scaling == 'spectrum':
scale = 1.0 / win.sum()**2
else:
raise ValueError('Unknown scaling: %r' % scaling)
if noverlap is None:
noverlap = nperseg // 2
elif noverlap >= nperseg:
raise ValueError('noverlap must be less than nperseg.')
if nfft is None:
nfft = nperseg
elif nfft < nperseg:
raise ValueError('nfft must be greater than or equal to nperseg.')
if not hasattr(detrend, '__call__'):
detrend_func = lambda seg: signaltools.detrend(seg, type=detrend)
elif axis != -1:
# Wrap this function so that it receives a shape that it could
# reasonably expect to receive.
def detrend_func(seg):
seg = np.rollaxis(seg, -1, axis)
seg = detrend(seg)
return np.rollaxis(seg, axis, len(seg.shape))
else:
detrend_func = detrend
step = nperseg - noverlap
indices = np.arange(0, x.shape[-1] - nperseg + 1, step)
for k, ind in enumerate(indices):
x_dt = detrend_func(x[..., ind:ind + nperseg])
xft = fftpack.fft(x_dt * win, nfft)
if same_data:
yft = xft
else:
y_dt = detrend_func(y[..., ind:ind + nperseg])
yft = fftpack.fft(y_dt * win, nfft)
if k == 0:
Pxy = (xft * yft.conj())
else:
Pxy *= k / (k + 1.0)
Pxy += (xft * yft.conj()) / (k + 1.0)
Pxy *= scale
f = fftpack.fftfreq(nfft, 1.0 / fs)
if axis != -1:
Pxy = np.rollaxis(Pxy, -1, axis)
return f, Pxy
0
Example 115
def savetxt(fname, X, fmt='%.18e',delimiter=' '):
"""
Save an array to file.
Parameters
----------
fname : filename or a file handle
If the filename ends in .gz, the file is automatically saved in
compressed gzip format. The load() command understands gzipped
files transparently.
X : array_like
Data.
fmt : string or sequence of strings
A single format (%10.5f), a sequence of formats, or a
multi-format string, e.g. 'Iteration %d -- %10.5f', in which
case delimiter is ignored.
delimiter : str
Character separating columns.
Notes
-----
Further explanation of the `fmt` parameter
(``%[flag]width[.precision]specifier``):
flags:
``-`` : left justify
``+`` : Forces to preceed result with + or -.
``0`` : Left pad the number with zeros instead of space (see width).
width:
Minimum number of characters to be printed. The value is not truncated
if it has more characters.
precision:
- For integer specifiers (eg. ``d,i,o,x``), the minimum number of
digits.
- For ``e, E`` and ``f`` specifiers, the number of digits to print
after the decimal point.
- For ``g`` and ``G``, the maximum number of significant digits.
- For ``s``, the maximum number of characters.
specifiers:
``c`` : character
``d`` or ``i`` : signed decimal integer
``e`` or ``E`` : scientific notation with ``e`` or ``E``.
``f`` : decimal floating point
``g,G`` : use the shorter of ``e,E`` or ``f``
``o`` : signed octal
``s`` : string of characters
``u`` : unsigned decimal integer
``x,X`` : unsigned hexadecimal integer
This is not an exhaustive specification.
Examples
--------
>>> savetxt('test.out', x, delimiter=',') # X is an array
>>> savetxt('test.out', (x,y,z)) # x,y,z equal sized 1D arrays
>>> savetxt('test.out', x, fmt='%1.4e') # use exponential notation
"""
if _string_like(fname):
if fname.endswith('.gz'):
import gzip
fh = gzip.open(fname,'wb')
else:
fh = file(fname,'w')
elif hasattr(fname, 'seek'):
fh = fname
else:
raise ValueError('fname must be a string or file handle')
X = np.asarray(X)
# Handle 1-dimensional arrays
if X.ndim == 1:
# Common case -- 1d array of numbers
if X.dtype.names is None:
X = np.atleast_2d(X).T
ncol = 1
# Complex dtype -- each field indicates a separate column
else:
ncol = len(X.dtype.descr)
else:
ncol = X.shape[1]
# `fmt` can be a string with multiple insertion points or a list of formats.
# E.g. '%10.5f\t%10d' or ('%10.5f', '$10d')
if type(fmt) in (list, tuple):
if len(fmt) != ncol:
raise AttributeError('fmt has wrong shape. %s' % str(fmt))
format = delimiter.join(fmt)
elif type(fmt) is str:
if fmt.count('%') == 1:
fmt = [fmt,]*ncol
format = delimiter.join(fmt)
elif fmt.count('%') != ncol:
raise AttributeError('fmt has wrong number of %% formats. %s'
% fmt)
else:
format = fmt
for row in X:
fh.write(format % tuple(row) + '\n')
0
Example 116
Project: spinalcordtoolbox Source File: msct_register.py
def register2d(fname_src, fname_dest, fname_mask='', fname_warp='warp_forward.nii.gz', fname_warp_inv='warp_inverse.nii.gz', paramreg=Paramreg(step='0', type='im', algo='Translation', metric='MI', iter='5', shrink='1', smooth='0', gradStep='0.5'),
ants_registration_params={'rigid': '', 'affine': '', 'compositeaffine': '', 'similarity': '', 'translation': '','bspline': ',10', 'gaussiandisplacementfield': ',3,0',
'bsplinedisplacementfield': ',5,10', 'syn': ',3,0', 'bsplinesyn': ',1,3'}, verbose=0):
"""Slice-by-slice registration of two images.
We first split the 3D images into 2D images (and the mask if inputted). Then we register slices of the two images
that physically correspond to one another looking at the physical origin of each image. The images can be of
different sizes but the destination image must be smaller thant the input image. We do that using antsRegistration
in 2D. Once this has been done for each slices, we gather the results and return them.
Algorithms implemented: translation, rigid, affine, syn and BsplineSyn.
N.B.: If the mask is inputted, it must also be 3D and it must be in the same space as the destination image.
input:
fname_source: name of moving image (type: string)
fname_dest: name of fixed image (type: string)
mask[optional]: name of mask file (type: string) (parameter -x of antsRegistration)
fname_warp: name of output 3d forward warping field
fname_warp_inv: name of output 3d inverse warping field
paramreg[optional]: parameters of antsRegistration (type: Paramreg class from sct_register_multimodal)
ants_registration_params[optional]: specific algorithm's parameters for antsRegistration (type: dictionary)
output:
if algo==translation:
x_displacement: list of translation along x axis for each slice (type: list)
y_displacement: list of translation along y axis for each slice (type: list)
if algo==rigid:
x_displacement: list of translation along x axis for each slice (type: list)
y_displacement: list of translation along y axis for each slice (type: list)
theta_rotation: list of rotation angle in radian (and in ITK's coordinate system) for each slice (type: list)
if algo==affine or algo==syn or algo==bsplinesyn:
creation of two 3D warping fields (forward and inverse) that are the concatenations of the slice-by-slice
warps.
"""
# set metricSize
if paramreg.metric == 'MI':
metricSize = '32' # corresponds to number of bins
else:
metricSize = '4' # corresponds to radius (for CC, MeanSquares...)
# Get image dimensions and retrieve nz
sct.printv('\nGet image dimensions of destination image...', verbose)
nx, ny, nz, nt, px, py, pz, pt = Image(fname_dest).dim
sct.printv('.. matrix size: '+str(nx)+' x '+str(ny)+' x '+str(nz), verbose)
sct.printv('.. voxel size: '+str(px)+'mm x '+str(py)+'mm x '+str(pz)+'mm', verbose)
# Split input volume along z
sct.printv('\nSplit input volume...', verbose)
from sct_image import split_data
im_src = Image('src.nii')
split_source_list = split_data(im_src, 2)
for im in split_source_list:
im.save()
# Split destination volume along z
sct.printv('\nSplit destination volume...', verbose)
im_dest = Image('dest.nii')
split_dest_list = split_data(im_dest, 2)
for im in split_dest_list:
im.save()
# Split mask volume along z
if fname_mask != '':
sct.printv('\nSplit mask volume...', verbose)
im_mask = Image('mask.nii.gz')
split_mask_list = split_data(im_mask, 2)
for im in split_mask_list:
im.save()
# coord_origin_dest = im_dest.transfo_pix2phys([[0,0,0]])
# coord_origin_input = im_src.transfo_pix2phys([[0,0,0]])
# coord_diff_origin = (np.asarray(coord_origin_dest[0]) - np.asarray(coord_origin_input[0])).tolist()
# [x_o, y_o, z_o] = [coord_diff_origin[0] * 1.0/px, coord_diff_origin[1] * 1.0/py, coord_diff_origin[2] * 1.0/pz]
# initialization
if paramreg.algo in ['Translation']:
x_displacement = [0 for i in range(nz)]
y_displacement = [0 for i in range(nz)]
theta_rotation = [0 for i in range(nz)]
if paramreg.algo in ['Rigid', 'Affine', 'BSplineSyN', 'SyN']:
list_warp = []
list_warp_inv = []
# loop across slices
for i in range(nz):
# set masking
sct.printv('Registering slice '+str(i)+'/'+str(nz-1)+'...', verbose)
num = numerotation(i)
prefix_warp2d = 'warp2d_'+num
# if mask is used, prepare command for ANTs
if fname_mask != '':
masking = '-x mask_Z' +num+ '.nii.gz'
else:
masking = ''
# main command for registration
cmd = ('isct_antsRegistration '
'--dimensionality 2 '
'--transform '+paramreg.algo+'['+str(paramreg.gradStep) +
ants_registration_params[paramreg.algo.lower()]+'] '
'--metric '+paramreg.metric+'[dest_Z' + num + '.nii' + ',src_Z' + num + '.nii' +',1,'+metricSize+'] ' #[fixedImage,movingImage,metricWeight +nb_of_bins (MI) or radius (other)
'--convergence '+str(paramreg.iter)+' '
'--shrink-factors '+str(paramreg.shrink)+' '
'--smoothing-sigmas '+str(paramreg.smooth)+'mm '
'--output ['+prefix_warp2d+',src_Z'+ num +'_reg.nii] ' #--> file.mat (contains Tx,Ty, theta)
'--interpolation BSpline[3] '
+ masking)
# add init translation
if not paramreg.init == '':
init_dict = {'geometric': '0', 'centermass': '1', 'origin': '2'}
cmd += ' -r [dest_Z'+num+'.nii'+',src_Z'+num+'.nii,'+init_dict[paramreg.init]+']'
try:
# run registration
sct.run(cmd)
if paramreg.algo in ['Translation']:
file_mat = prefix_warp2d+'0GenericAffine.mat'
matfile = loadmat(file_mat, struct_as_record=True)
array_transfo = matfile['AffineTransform_double_2_2']
x_displacement[i] = array_transfo[4][0] # Tx in ITK'S coordinate system
y_displacement[i] = array_transfo[5][0] # Ty in ITK'S and fslview's coordinate systems
theta_rotation[i] = asin(array_transfo[2]) # angle of rotation theta in ITK'S coordinate system (minus theta for fslview)
if paramreg.algo in ['Rigid', 'Affine', 'BSplineSyN', 'SyN']:
# List names of 2d warping fields for subsequent merge along Z
file_warp2d = prefix_warp2d+'0Warp.nii.gz'
file_warp2d_inv = prefix_warp2d+'0InverseWarp.nii.gz'
list_warp.append(file_warp2d)
list_warp_inv.append(file_warp2d_inv)
if paramreg.algo in ['Rigid', 'Affine']:
# Generating null 2d warping field (for subsequent concatenation with affine transformation)
sct.run('isct_antsRegistration -d 2 -t SyN[1, 1, 1] -c 0 -m MI[dest_Z'+num+'.nii, src_Z'+num+'.nii, 1, 32] -o warp2d_null -f 1 -s 0')
# --> outputs: warp2d_null0Warp.nii.gz, warp2d_null0InverseWarp.nii.gz
file_mat = prefix_warp2d + '0GenericAffine.mat'
# Concatenating mat transfo and null 2d warping field to obtain 2d warping field of affine transformation
sct.run('isct_ComposeMultiTransform 2 ' + file_warp2d + ' -R dest_Z'+num+'.nii warp2d_null0Warp.nii.gz ' + file_mat)
sct.run('isct_ComposeMultiTransform 2 ' + file_warp2d_inv + ' -R src_Z'+num+'.nii warp2d_null0InverseWarp.nii.gz -i ' + file_mat)
# if an exception occurs with ants, take the last value for the transformation
# TODO: DO WE NEED TO DO THAT??? (julien 2016-03-01)
except Exception, e:
sct.printv('ERROR: Exception occurred.\n'+str(e), 1, 'error')
# Merge warping field along z
sct.printv('\nMerge warping fields along z...', verbose)
if paramreg.algo in ['Translation']:
# convert to array
x_disp_a = np.asarray(x_displacement)
y_disp_a = np.asarray(y_displacement)
theta_rot_a = np.asarray(theta_rotation)
# Generate warping field
generate_warping_field('dest.nii', x_disp_a, y_disp_a, theta_rot_a, fname=fname_warp) #name_warp= 'step'+str(paramreg.step)
# Inverse warping field
generate_warping_field('src.nii', -x_disp_a, -y_disp_a, theta_rot_a, fname=fname_warp_inv)
if paramreg.algo in ['Rigid', 'Affine', 'BSplineSyN', 'SyN']:
from sct_image import concat_warp2d
# concatenate 2d warping fields along z
concat_warp2d(list_warp, fname_warp, 'dest.nii')
concat_warp2d(list_warp_inv, fname_warp_inv, 'src.nii')
0
Example 117
Project: faststats Source File: fastkde.py
def fastkde1D(xin, gridsize=200, extents=None, weights=None, adjust=1.):
"""
A fft-based Gaussian kernel density estimate (KDE)
for computing the KDE on a regular grid
Note that this is a different use case than scipy's original
scipy.stats.kde.gaussian_kde
IMPLEMENTATION
--------------
Performs a gaussian kernel density estimate over a regular grid using a
convolution of the gaussian kernel with a 2D histogram of the data.
It computes the sparse bi-dimensional histogram of two data samples where
*x*, and *y* are 1-D sequences of the same length. If *weights* is None
(default), this is a histogram of the number of occurences of the
observations at (x[i], y[i]).
histogram of the data is a faster implementation than numpy.histogram as it
avoids intermediate copies and excessive memory usage!
This function is typically *several orders of magnitude faster* than
scipy.stats.kde.gaussian_kde. For large (>1e7) numbers of points, it
produces an essentially identical result.
**Example usage and timing**
from scipy.stats import gaussian_kde
def npkde(x, xe):
kde = gaussian_kde(x)
r = kde.evaluate(xe)
return r
x = np.random.normal(0, 1, 1e6)
%timeit fastkde1D(x)
10 loops, best of 3: 31.9 ms per loop
%timeit npkde(x, xe)
1 loops, best of 3: 11.8 s per loop
~ 1e4 speed up!!! However gaussian_kde is not optimized for this application
Boundary conditions on the data is corrected by using a symmetric /
reflection condition. Hence the limits of the dataset does not affect the
pdf estimate.
INPUTS
------
xin: ndarray[ndim=1]
The x-coords, y-coords of the input data points respectively
gridsize: int
A nx integer of the size of the output grid (default: 200x200)
extents: (xmin, xmax) tuple
tuple of the extents of output grid (default: extent of input data)
weights: ndarray[ndim=1]
An array of the same shape as x that weights each sample x_i
by w_i. Defaults to an array of ones the same size as x (default: None)
adjust : float
An adjustment factor for the bw. Bandwidth becomes bw * adjust.
OUTPUTS
-------
g: ndarray[ndim=2]
A gridded 2D kernel density estimate of the input points.
e: (xmin, xmax, ymin, ymax) tuple
Extents of g
"""
# Variable check
x = np.squeeze(np.asarray(xin))
# Default extents are the extent of the data
if extents is None:
xmin, xmax = x.min(), x.max()
else:
xmin, xmax = map(float, extents)
x = x[ (x <= xmax) & (x >= xmin) ]
n = x.size
if weights is None:
# Default: Weight all points equally
weights = np.ones(n)
else:
weights = np.squeeze(np.asarray(weights))
if weights.size != x.size:
raise ValueError('Input weights must be an array of the same size as input x & y arrays!')
# Optimize gridsize ------------------------------------------------------
#Make grid and discretize the data and round it to the next power of 2
# to optimize with the fft usage
if gridsize is None:
gridsize = np.max((len(x), 512.))
gridsize = 2 ** np.ceil(np.log2(gridsize)) # round to next power of 2
nx = gridsize
# Make the sparse 2d-histogram -------------------------------------------
dx = (xmax - xmin) / (nx - 1)
# Basically, this is just doing what np.digitize does with one less copy
# xyi contains the bins of each point as a 2d array [(xi,yi)]
xyi = x - xmin
xyi /= dx
xyi = np.floor(xyi, xyi)
xyi = np.vstack((xyi, np.zeros(n, dtype=int)))
# Next, make a 2D histogram of x & y.
# Exploit a sparse coo_matrix avoiding np.histogram2d due to excessive
# memory usage with many points
grid = coo_matrix((weights, xyi), shape=(nx, 1)).toarray()
# Kernel Preliminary Calculations ---------------------------------------
std_x = np.std(xyi[0])
# Scaling factor for bandwidth
scotts_factor = n ** (-1. / 5.) * adjust # For n ** (-1. / (d + 4))
#Silvermann n * (d + 2) / 4.)**(-1. / (d + 4)).
# Make the gaussian kernel ---------------------------------------------
# First, determine the bandwidth using Scott's rule
# (note that Silvermann's rule gives the # same value for 2d datasets)
kern_nx = np.round(scotts_factor * 2 * np.pi * std_x)
# Then evaluate the gaussian function on the kernel grid
kernel = np.reshape(gaussian(kern_nx, scotts_factor * std_x), (kern_nx, 1))
#---- Produce the kernel density estimate --------------------------------
# Convolve the histogram with the gaussian kernel
# use symmetric padding to correct for data boundaries in the kde
npad = np.min((nx, 2 * kern_nx))
grid = np.vstack( [grid[npad: 0: -1], grid, grid[nx: nx - npad: -1]] )
grid = convolve(grid, kernel, mode='same')[npad: npad + nx]
# Normalization factor to divide result by so that units are in the same
# units as scipy.stats.kde.gaussian_kde's output.
norm_factor = 2 * np.pi * std_x * std_x * scotts_factor ** 2
norm_factor = n * dx * np.sqrt(norm_factor)
# Normalize the result
grid /= norm_factor
return np.squeeze(grid), (xmin, xmax)
0
Example 118
Project: dipy Source File: test_sumsqdiff.py
def test_compute_residual_displacement_field_ssd_2d():
# Select arbitrary images' shape (same shape for both images)
sh = (20, 10)
# Select arbitrary centers
c_f = np.asarray(sh) / 2
c_g = c_f + 0.5
# Compute the identity vector field I(x) = x in R^2
x_0 = np.asarray(range(sh[0]))
x_1 = np.asarray(range(sh[1]))
X = np.ndarray(sh + (2,), dtype=np.float64)
O = np.ones(sh)
X[..., 0] = x_0[:, None] * O
X[..., 1] = x_1[None, :] * O
# Compute the gradient fields of F and G
np.random.seed(5512751)
grad_F = X - c_f
grad_G = X - c_g
Fnoise = np.random.ranf(
np.size(grad_F)).reshape(
grad_F.shape) * grad_F.max() * 0.1
Fnoise = Fnoise.astype(floating)
grad_F += Fnoise
Gnoise = np.random.ranf(
np.size(grad_G)).reshape(
grad_G.shape) * grad_G.max() * 0.1
Gnoise = Gnoise.astype(floating)
grad_G += Gnoise
# The squared norm of grad_G
sq_norm_grad_G = np.sum(grad_G**2, -1)
# Compute F and G
F = 0.5 * np.sum(grad_F**2, -1)
G = 0.5 * sq_norm_grad_G
Fnoise = np.random.ranf(np.size(F)).reshape(F.shape) * F.max() * 0.1
Fnoise = Fnoise.astype(floating)
F += Fnoise
Gnoise = np.random.ranf(np.size(G)).reshape(G.shape) * G.max() * 0.1
Gnoise = Gnoise.astype(floating)
G += Gnoise
delta_field = np.array(F - G, dtype=floating)
sigma_field = np.random.randn(delta_field.size).reshape(delta_field.shape)
sigma_field = sigma_field.astype(floating)
# Select some pixels to force sigma_field = infinite
inf_sigma = np.random.randint(0, 2, sh[0] * sh[1])
inf_sigma = inf_sigma.reshape(sh)
sigma_field[inf_sigma == 1] = np.inf
# Select an initial displacement field
d = np.random.randn(grad_G.size).reshape(grad_G.shape).astype(floating)
# d = np.zeros_like(grad_G, dtype=floating)
lambda_param = 1.5
# Implementation under test
iut = ssd.compute_residual_displacement_field_ssd_2d
# In the first iteration we test the case target=None
# In the second iteration, target is not None
target = None
rtol = 1e-9
atol = 1e-4
for it in range(2):
# Sum of differences with the neighbors
s = np.zeros_like(d, dtype=np.float64)
s[:, :-1] += d[:, :-1] - d[:, 1:] # right
s[:, 1:] += d[:, 1:] - d[:, :-1] # left
s[:-1, :] += d[:-1, :] - d[1:, :] # down
s[1:, :] += d[1:, :] - d[:-1, :] # up
s *= lambda_param
# Dot product of displacement and gradient
dp = d[..., 0] * grad_G[..., 0] + \
d[..., 1] * grad_G[..., 1]
dp = dp.astype(np.float64)
# Compute expected residual
expected = None
if target is None:
expected = np.zeros_like(grad_G)
expected[..., 0] = delta_field * grad_G[..., 0]
expected[..., 1] = delta_field * grad_G[..., 1]
else:
expected = target.copy().astype(np.float64)
# Expected residuals when sigma != infinte
expected[inf_sigma == 0, 0] -= grad_G[inf_sigma == 0, 0] * \
dp[inf_sigma == 0] + sigma_field[inf_sigma == 0] * s[inf_sigma == 0, 0]
expected[inf_sigma == 0, 1] -= grad_G[inf_sigma == 0, 1] * \
dp[inf_sigma == 0] + sigma_field[inf_sigma == 0] * s[inf_sigma == 0, 1]
# Expected residuals when sigma == infinte
expected[inf_sigma == 1] = -1.0 * s[inf_sigma == 1]
# Test residual field computation starting with residual = None
actual = iut(delta_field, sigma_field, grad_G.astype(floating),
target, lambda_param, d, None)
assert_allclose(actual, expected, rtol=rtol, atol=atol)
# destroy previous result
actual = np.ndarray(actual.shape, dtype=floating)
# Test residual field computation starting with residual is not None
iut(delta_field, sigma_field, grad_G.astype(floating),
target, lambda_param, d, actual)
assert_allclose(actual, expected, rtol=rtol, atol=atol)
# Set target for next iteration
target = actual
# Test Gauss-Seidel step with residual=None and residual=target
for residual in [None, target]:
expected = d.copy()
iterate_residual_field_ssd_2d(
delta_field,
sigma_field,
grad_G.astype(floating),
residual,
lambda_param,
expected)
actual = d.copy()
ssd.iterate_residual_displacement_field_ssd_2d(
delta_field,
sigma_field,
grad_G.astype(floating),
residual,
lambda_param,
actual)
assert_allclose(actual, expected, rtol=rtol, atol=atol)
0
Example 119
def main(model, raw_fpath):
store = pd.HDFStore(model)
from_ = store['from_'][0][0]
to = store['to'][0][0]
assert from_ == 0
trace_fpath = store['trace_fpath'][0][0]
Theta_zh = store['Theta_zh'].values
Psi_oz = store['Psi_sz'].values
count_z = store['count_z'].values[:, 0]
Psi_oz = Psi_oz / Psi_oz.sum(axis=0)
Psi_zo = (Psi_oz * count_z).T
Psi_zo = Psi_zo / Psi_zo.sum(axis=0)
obj2id = dict(store['source2id'].values)
hyper2id = dict(store['hyper2id'].values)
id2obj = dict((v, k) for k, v in obj2id.items())
for name in ['Miles Davis', 'Eric Dolphy', 'Ron Carter', 'Dave Holland', 'John Coltrane']:
try:
f = open(raw_fpath)
series_ts = []
for l in f:
ts, c, s, d = l.strip().split('\t')
ts = float(ts)
if d == name:
series_ts.append([ts, c, s, d])
PZs = []
for ts, c, s, d in series_ts:
PZs.append([])
for z in xrange(Psi_oz.shape[1]):
pzu = Theta_zh[z, hyper2id[c]]
psz = Psi_oz[obj2id[s], z]
pdz = Psi_oz[obj2id[d], z]
pz = (pzu * psz * pdz) / (1 - psz)
PZs[-1].append(pz)
data = []
index = []
for i in xrange(len(series_ts)):
p = PZs[i]
p = np.asarray(p)
p = p / p.sum()
data.append(p)
index.append(series_ts[i][0])
heights = []
x = set()
for z in xrange(Psi_oz.shape[1]):
series = pd.Series(data=np.array(data)[:,z], index=pd.DatetimeIndex(np.array(index) * 1e9))
series = series.resample('%dD'% (2 * 365), how='mean', fill_method='pad')
heights.append(series.values)
x.update(series.index)
x = np.array([arrow.get(d).timestamp for d in sorted(x)])
heights = np.column_stack(heights)
canvas = toyplot.Canvas(width=500, height=300)
axes = canvas.axes(label=name, ylabel='Likelihood (Avg over 2 years)', xlabel='Year')
color = np.arange(heights.shape[1])
labels = ['Z_%d' % z for z in xrange(30)]
to_use = []
for i in range(heights.shape[1]):
to_use.append(heights[:, i].max())
to_use = np.array(to_use)
print(to_use.argsort()[-5:])
#heights = heights[:, to_use.argsort()[-5:]]
#m = axes.plot(x, heights)#, color=color)#, baseline="stacked", color=color)
m = axes.fill(x, heights, color=color, baseline="stacked")#, color=color)
label_style = {"text-anchor":"start", "-toyplot-anchor-shift":"0px"}
#m(0)
#for i in range(heights.shape[1]):
# p = heights[:, i].argmax()
# axes.text(x[p], heights[:, i].max(), "Z_%s" % to_use.argsort()[-5:][i], style=label_style, color='black')
except:
pass
#print(m)
#seen = set()
#for i, h in enumerate(heights):
# top = h.argsort()[::-1]
# for t in largest_area:
# if t not in seen and len(seen) < 8:
# seen.add(t)
# axes.text(x[i], h[t], 'Z_%d' % t, \
# style=label_style,
# color='Orange')
# break
#for h in heights:
# print(h.argsort()[::-1][:4])
axes.x.ticks.show = True
axes.x.ticks.locator = toyplot.locator.Timestamp()
toyplot.pdf.render(canvas, '%s-stack-2y.pdf' % name)
toyplot.html.render(canvas, '%s-stack-2y.html' % name)
f.close()
#plt.savefig('series.pdf')
#ZtZ = Psi_zo.dot(Psi_oz)
#ZtZ = ZtZ / ZtZ.sum(axis=0)
#L = ZtZ
#colormap = toyplot.color.brewer.map("Purples", domain_min=L.min(), domain_max=L.max(), reverse=True)
#canvas = toyplot.matrix((L, colormap), label="P[Z = d | Z = s]", \
# colorshow=True)[0]
#toyplot.pdf.render(canvas, 'test.pdf')
#AtA = Psi_oz.dot(Psi_zo)
#np.fill_diagonal(AtA, 0)
#AtA = AtA / AtA.sum(axis=0)
store.close()
0
Example 120
def draw_networkx_edges(G, pos,
edgelist=None,
width=1.0,
edge_color='k',
style='solid',
alpha=None,
edge_cmap=None,
edge_vmin=None,
edge_vmax=None,
ax=None,
arrows=True,
**kwds):
"""Draw the edges of the graph G
This draws only the edges of the graph G.
pos is a dictionary keyed by vertex with a two-tuple
of x-y positions as the value.
See networkx.layout for functions that compute node positions.
edgelist is an optional list of the edges in G to be drawn.
If provided, only the edges in edgelist will be drawn.
edgecolor can be a list of matplotlib color letters such as 'k' or
'b' that lists the color of each edge; the list must be ordered in
the same way as the edge list. Alternatively, this list can contain
numbers and those number are mapped to a color scale using the color
map edge_cmap. Finally, it can also be a list of (r,g,b) or (r,g,b,a)
tuples, in which case these will be used directly to color the edges. If
the latter mode is used, you should not provide a value for alpha, as it
would be applied globally to all lines.
For directed graphs, 'arrows' (actually just thicker stubs) are drawn
at the head end. Arrows can be turned off with keyword arrows=False.
See draw_networkx for the list of other optional parameters.
"""
try:
import matplotlib.pylab as pylab
import matplotlib.cbook as cb
from matplotlib.colors import colorConverter, Colormap
from matplotlib.collections import LineCollection
except ImportError:
raise ImportError("Matplotlib required for draw()")
except RuntimeError:
pass # unable to open display
if ax is None:
ax = pylab.gca()
if edgelist is None:
edgelist = G.edges()
if not edgelist or len(edgelist) == 0: # no edges!
return None
# set edge positions
edge_pos = np.asarray([(pos[e[0]], pos[e[1]]) for e in edgelist])
if not cb.iterable(width):
lw = (width,)
else:
lw = width
if not cb.is_string_like(edge_color) \
and cb.iterable(edge_color) \
and len(edge_color) == len(edge_pos):
if np.alltrue([cb.is_string_like(c)
for c in edge_color]):
# (should check ALL elements)
# list of color letters such as ['k','r','k',...]
edge_colors = tuple([colorConverter.to_rgba(c, alpha)
for c in edge_color])
elif np.alltrue([not cb.is_string_like(c)
for c in edge_color]):
# If color specs are given as (rgb) or (rgba) tuples, we're OK
if np.alltrue([cb.iterable(c) and len(c) in (3, 4)
for c in edge_color]):
edge_colors = tuple(edge_color)
alpha = None
else:
# numbers (which are going to be mapped with a colormap)
edge_colors = None
else:
e_s = 'edge_color must consist of either color names or numbers'
raise ValueError(e_s)
else:
if len(edge_color) == 1:
edge_colors = (colorConverter.to_rgba(edge_color, alpha),)
else:
e_s = 'edge_color must be a single color or list of exactly'
e_s += 'm colors where m is the number or edges'
raise ValueError(e_s)
edge_collection = LineCollection(edge_pos,
colors=edge_colors,
linewidths=lw,
antialiaseds=(1,),
linestyle=style,
transOffset=ax.transData,
)
# Note: there was a bug in mpl regarding the handling of alpha values for
# each line in a LineCollection. It was fixed in matplotlib in r7184 and
# r7189 (June 6 2009). We should then not set the alpha value globally,
# since the user can instead provide per-edge alphas now. Only set it
# globally if provided as a scalar.
if cb.is_numlike(alpha):
edge_collection.set_alpha(alpha)
# need 0.87.7 or greater for edge colormaps
if edge_colors is None:
if edge_cmap is not None:
assert(isinstance(edge_cmap, Colormap))
edge_collection.set_array(np.asarray(edge_color))
edge_collection.set_cmap(edge_cmap)
if edge_vmin is not None or edge_vmax is not None:
edge_collection.set_clim(edge_vmin, edge_vmax)
else:
edge_collection.autoscale()
pylab.sci(edge_collection)
# else:
# sys.stderr.write(\
# """matplotlib version >= 0.87.7 required for colormapped edges.
# (version %s detected)."""%matplotlib.__version__)
# raise UserWarning(\
# """matplotlib version >= 0.87.7 required for colormapped edges.
# (version %s detected)."""%matplotlib.__version__)
arrow_collection = None
if G.is_directed() and arrows:
# a directed graph hack
# draw thick line segments at head end of edge
# waiting for someone else to implement arrows that will work
arrow_colors = (colorConverter.to_rgba('k', alpha),)
a_pos = []
p = 1.0 - 0.25 # make head segment 25 percent of edge length
for src, dst in edge_pos:
x1, y1 = src
x2, y2 = dst
dx = x2 - x1 # x offset
dy = y2 - y1 # y offset
d = np.sqrt(float(dx ** 2 + dy ** 2)) # length of edge
if d == 0: # source and target at same position
continue
if dx == 0: # vertical edge
xa = x2
ya = dy * p + y1
if dy == 0: # horizontal edge
ya = y2
xa = dx * p + x1
else:
theta = np.arctan2(dy, dx)
xa = p * d * np.cos(theta) + x1
ya = p * d * np.sin(theta) + y1
a_pos.append(((xa, ya), (x2, y2)))
arrow_collection = LineCollection(a_pos,
colors=arrow_colors,
linewidths=[4 * ww for ww in lw],
antialiaseds=(1,),
transOffset=ax.transData,
)
# update view
minx = np.amin(np.ravel(edge_pos[:, :, 0]))
maxx = np.amax(np.ravel(edge_pos[:, :, 0]))
miny = np.amin(np.ravel(edge_pos[:, :, 1]))
maxy = np.amax(np.ravel(edge_pos[:, :, 1]))
w = maxx - minx
h = maxy - miny
padx, pady = 0.05 * w, 0.05 * h
corners = (minx - padx, miny - pady), (maxx + padx, maxy + pady)
ax.update_datalim(corners)
ax.autoscale_view()
edge_collection.set_zorder(1) # edges go behind nodes
ax.add_collection(edge_collection)
if arrow_collection:
arrow_collection.set_zorder(1) # edges go behind nodes
ax.add_collection(arrow_collection)
return ax
0
Example 121
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: minpack.py
def curve_fit(f, xdata, ydata, p0=None, sigma=None, absolute_sigma=False,
check_finite=True, bounds=(-np.inf, np.inf), method=None,
**kwargs):
"""
Use non-linear least squares to fit a function, f, to data.
Assumes ``ydata = f(xdata, *params) + eps``
Parameters
----------
f : callable
The model function, f(x, ...). It must take the independent
variable as the first argument and the parameters to fit as
separate remaining arguments.
xdata : An M-length sequence or an (k,M)-shaped array
for functions with k predictors.
The independent variable where the data is measured.
ydata : M-length sequence
The dependent data --- nominally f(xdata, ...)
p0 : None, scalar, or N-length sequence, optional
Initial guess for the parameters. If None, then the initial
values will all be 1 (if the number of parameters for the function
can be determined using introspection, otherwise a ValueError
is raised).
sigma : None or M-length sequence, optional
If not None, the uncertainties in the ydata array. These are used as
weights in the least-squares problem
i.e. minimising ``np.sum( ((f(xdata, *popt) - ydata) / sigma)**2 )``
If None, the uncertainties are assumed to be 1.
absolute_sigma : bool, optional
If False, `sigma` denotes relative weights of the data points.
The returned covariance matrix `pcov` is based on *estimated*
errors in the data, and is not affected by the overall
magnitude of the values in `sigma`. Only the relative
magnitudes of the `sigma` values matter.
If True, `sigma` describes one standard deviation errors of
the input data points. The estimated covariance in `pcov` is
based on these values.
check_finite : bool, optional
If True, check that the input arrays do not contain nans of infs,
and raise a ValueError if they do. Setting this parameter to
False may silently produce nonsensical results if the input arrays
do contain nans. Default is True.
bounds : 2-tuple of array_like, optional
Lower and upper bounds on independent variables. Defaults to no bounds.
Each element of the tuple must be either an array with the length equal
to the number of parameters, or a scalar (in which case the bound is
taken to be the same for all parameters.) Use ``np.inf`` with an
appropriate sign to disable bounds on all or some parameters.
.. versionadded:: 0.17
method : {'lm', 'trf', 'dogbox'}, optional
Method to use for optimization. See `least_squares` for more details.
Default is 'lm' for unconstrained problems and 'trf' if `bounds` are
provided. The method 'lm' won't work when the number of observations
is less than the number of variables, use 'trf' or 'dogbox' in this
case.
.. versionadded:: 0.17
kwargs
Keyword arguments passed to `leastsq` for ``method='lm'`` or
`least_squares` otherwise.
Returns
-------
popt : array
Optimal values for the parameters so that the sum of the squared error
of ``f(xdata, *popt) - ydata`` is minimized
pcov : 2d array
The estimated covariance of popt. The diagonals provide the variance
of the parameter estimate. To compute one standard deviation errors
on the parameters use ``perr = np.sqrt(np.diag(pcov))``.
How the `sigma` parameter affects the estimated covariance
depends on `absolute_sigma` argument, as described above.
If the Jacobian matrix at the solution doesn't have a full rank, then
'lm' method returns a matrix filled with ``np.inf``, on the other hand
'trf' and 'dogbox' methods use Moore-Penrose pseudoinverse to compute
the covariance matrix.
Raises
------
OptimizeWarning
if covariance of the parameters can not be estimated.
ValueError
if either `ydata` or `xdata` contain NaNs.
See Also
--------
least_squares : Minimize the sum of squares of nonlinear functions.
stats.linregress : Calculate a linear least squares regression for two sets
of measurements.
Notes
-----
With ``method='lm'``, the algorithm uses the Levenberg-Marquardt algorithm
through `leastsq`. Note that this algorithm can only deal with
unconstrained problems.
Box constraints can be handled by methods 'trf' and 'dogbox'. Refer to
the docstring of `least_squares` for more information.
Examples
--------
>>> import numpy as np
>>> from scipy.optimize import curve_fit
>>> def func(x, a, b, c):
... return a * np.exp(-b * x) + c
>>> xdata = np.linspace(0, 4, 50)
>>> y = func(xdata, 2.5, 1.3, 0.5)
>>> ydata = y + 0.2 * np.random.normal(size=len(xdata))
>>> popt, pcov = curve_fit(func, xdata, ydata)
Constrain the optimization to the region of ``0 < a < 3``, ``0 < b < 2``
and ``0 < c < 1``:
>>> popt, pcov = curve_fit(func, xdata, ydata, bounds=(0, [3., 2., 1.]))
"""
if p0 is None:
# determine number of parameters by inspecting the function
from scipy._lib._util import getargspec_no_self as _getargspec
args, varargs, varkw, defaults = _getargspec(f)
if len(args) < 2:
raise ValueError("Unable to determine number of fit parameters.")
n = len(args) - 1
else:
p0 = np.atleast_1d(p0)
n = p0.size
lb, ub = prepare_bounds(bounds, n)
if p0 is None:
p0 = _initialize_feasible(lb, ub)
bounded_problem = np.any((lb > -np.inf) | (ub < np.inf))
if method is None:
if bounded_problem:
method = 'trf'
else:
method = 'lm'
if method == 'lm' and bounded_problem:
raise ValueError("Method 'lm' only works for unconstrained problems. "
"Use 'trf' or 'dogbox' instead.")
# NaNs can not be handled
if check_finite:
ydata = np.asarray_chkfinite(ydata)
else:
ydata = np.asarray(ydata)
if isinstance(xdata, (list, tuple, np.ndarray)):
# `xdata` is passed straight to the user-defined `f`, so allow
# non-array_like `xdata`.
if check_finite:
xdata = np.asarray_chkfinite(xdata)
else:
xdata = np.asarray(xdata)
args = (xdata, ydata, f)
if sigma is None:
func = _general_function
else:
func = _weighted_general_function
args += (1.0 / asarray(sigma),)
if method == 'lm':
# Remove full_output from kwargs, otherwise we're passing it in twice.
return_full = kwargs.pop('full_output', False)
res = leastsq(func, p0, args=args, full_output=1, **kwargs)
popt, pcov, infodict, errmsg, ier = res
cost = np.sum(infodict['fvec'] ** 2)
if ier not in [1, 2, 3, 4]:
raise RuntimeError("Optimal parameters not found: " + errmsg)
else:
res = least_squares(func, p0, args=args, bounds=bounds, method=method,
**kwargs)
if not res.success:
raise RuntimeError("Optimal parameters not found: " + res.message)
cost = 2 * res.cost # res.cost is half sum of squares!
popt = res.x
# Do Moore-Penrose inverse discarding zero singular values.
_, s, VT = svd(res.jac, full_matrices=False)
threshold = np.finfo(float).eps * max(res.jac.shape) * s[0]
s = s[s > threshold]
VT = VT[:s.size]
pcov = np.dot(VT.T / s**2, VT)
return_full = False
warn_cov = False
if pcov is None:
# indeterminate covariance
pcov = zeros((len(popt), len(popt)), dtype=float)
pcov.fill(inf)
warn_cov = True
elif not absolute_sigma:
if ydata.size > p0.size:
s_sq = cost / (ydata.size - p0.size)
pcov = pcov * s_sq
else:
pcov.fill(inf)
warn_cov = True
if warn_cov:
warnings.warn('Covariance of the parameters could not be estimated',
category=OptimizeWarning)
if return_full:
return popt, pcov, infodict, errmsg, ier
else:
return popt, pcov
0
Example 122
def _linprog_simplex(c, A_ub=None, b_ub=None, A_eq=None, b_eq=None,
bounds=None, maxiter=1000, disp=False, callback=None,
tol=1.0E-12, bland=False, **unknown_options):
"""
Solve the following linear programming problem via a two-phase
simplex algorithm.
maximize: c^T * x
subject to: A_ub * x <= b_ub
A_eq * x == b_eq
Parameters
----------
c : array_like
Coefficients of the linear objective function to be maximized.
A_ub : array_like
2-D array which, when matrix-multiplied by x, gives the values of the
upper-bound inequality constraints at x.
b_ub : array_like
1-D array of values representing the upper-bound of each inequality
constraint (row) in A_ub.
A_eq : array_like
2-D array which, when matrix-multiplied by x, gives the values of the
equality constraints at x.
b_eq : array_like
1-D array of values representing the RHS of each equality constraint
(row) in A_eq.
bounds : array_like
The bounds for each independent variable in the solution, which can take
one of three forms::
None : The default bounds, all variables are non-negative.
(lb, ub) : If a 2-element sequence is provided, the same
lower bound (lb) and upper bound (ub) will be applied
to all variables.
[(lb_0, ub_0), (lb_1, ub_1), ...] : If an n x 2 sequence is provided,
each variable x_i will be bounded by lb[i] and ub[i].
Infinite bounds are specified using -np.inf (negative)
or np.inf (positive).
callback : callable
If a callback function is provide, it will be called within each
iteration of the simplex algorithm. The callback must have the
signature `callback(xk, **kwargs)` where xk is the current solution
vector and kwargs is a dictionary containing the following::
"tableau" : The current Simplex algorithm tableau
"nit" : The current iteration.
"pivot" : The pivot (row, column) used for the next iteration.
"phase" : Whether the algorithm is in Phase 1 or Phase 2.
"bv" : A structured array containing a string representation of each
basic variable and its current value.
Options
-------
maxiter : int
The maximum number of iterations to perform.
disp : bool
If True, print exit status message to sys.stdout
tol : float
The tolerance which determines when a solution is "close enough" to zero
in Phase 1 to be considered a basic feasible solution or close enough
to positive to to serve as an optimal solution.
bland : bool
If True, use Bland's anti-cycling rule [3] to choose pivots to
prevent cycling. If False, choose pivots which should lead to a
converged solution more quickly. The latter method is subject to
cycling (non-convergence) in rare instances.
Returns
-------
A scipy.optimize.OptimizeResult consisting of the following fields::
x : ndarray
The independent variable vector which optimizes the linear
programming problem.
slack : ndarray
The values of the slack variables. Each slack variable corresponds
to an inequality constraint. If the slack is zero, then the
corresponding constraint is active.
success : bool
Returns True if the algorithm succeeded in finding an optimal
solution.
status : int
An integer representing the exit status of the optimization::
0 : Optimization terminated successfully
1 : Iteration limit reached
2 : Problem appears to be infeasible
3 : Problem appears to be unbounded
nit : int
The number of iterations performed.
message : str
A string descriptor of the exit status of the optimization.
Examples
--------
Consider the following problem:
Minimize: f = -1*x[0] + 4*x[1]
Subject to: -3*x[0] + 1*x[1] <= 6
1*x[0] + 2*x[1] <= 4
x[1] >= -3
where: -inf <= x[0] <= inf
This problem deviates from the standard linear programming problem. In
standard form, linear programming problems assume the variables x are
non-negative. Since the variables don't have standard bounds where
0 <= x <= inf, the bounds of the variables must be explicitly set.
There are two upper-bound constraints, which can be expressed as
dot(A_ub, x) <= b_ub
The input for this problem is as follows:
>>> from scipy.optimize import linprog
>>> c = [-1, 4]
>>> A = [[-3, 1], [1, 2]]
>>> b = [6, 4]
>>> x0_bnds = (None, None)
>>> x1_bnds = (-3, None)
>>> res = linprog(c, A, b, bounds=(x0_bnds, x1_bnds))
>>> print(res)
fun: -22.0
message: 'Optimization terminated successfully.'
nit: 1
slack: array([ 39., 0.])
status: 0
success: True
x: array([ 10., -3.])
References
----------
.. [1] Dantzig, George B., Linear programming and extensions. Rand
Corporation Research Study Princeton Univ. Press, Princeton, NJ, 1963
.. [2] Hillier, S.H. and Lieberman, G.J. (1995), "Introduction to
Mathematical Programming", McGraw-Hill, Chapter 4.
.. [3] Bland, Robert G. New finite pivoting rules for the simplex method.
Mathematics of Operations Research (2), 1977: pp. 103-107.
"""
_check_unknown_options(unknown_options)
status = 0
messages = {0: "Optimization terminated successfully.",
1: "Iteration limit reached.",
2: "Optimization failed. Unable to find a feasible"
" starting point.",
3: "Optimization failed. The problem appears to be unbounded.",
4: "Optimization failed. Singular matrix encountered."}
have_floor_variable = False
cc = np.asarray(c)
# The initial value of the objective function element in the tableau
f0 = 0
# The number of variables as given by c
n = len(c)
# Convert the input arguments to arrays (sized to zero if not provided)
Aeq = np.asarray(A_eq) if A_eq is not None else np.empty([0, len(cc)])
Aub = np.asarray(A_ub) if A_ub is not None else np.empty([0, len(cc)])
beq = np.ravel(np.asarray(b_eq)) if b_eq is not None else np.empty([0])
bub = np.ravel(np.asarray(b_ub)) if b_ub is not None else np.empty([0])
# Analyze the bounds and determine what modifications to be made to
# the constraints in order to accommodate them.
L = np.zeros(n, dtype=np.float64)
U = np.ones(n, dtype=np.float64)*np.inf
if bounds is None or len(bounds) == 0:
pass
elif len(bounds) == 2 and not hasattr(bounds[0], '__len__'):
# All bounds are the same
a = bounds[0] if bounds[0] is not None else -np.inf
b = bounds[1] if bounds[1] is not None else np.inf
L = np.asarray(n*[a], dtype=np.float64)
U = np.asarray(n*[b], dtype=np.float64)
else:
if len(bounds) != n:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Length of bounds is inconsistent with the length of c")
else:
try:
for i in range(n):
if len(bounds[i]) != 2:
raise IndexError()
L[i] = bounds[i][0] if bounds[i][0] is not None else -np.inf
U[i] = bounds[i][1] if bounds[i][1] is not None else np.inf
except IndexError:
status = -1
message = ("Invalid input for linprog with "
"method = 'simplex'. bounds must be a n x 2 "
"sequence/array where n = len(c).")
if np.any(L == -np.inf):
# If any lower-bound constraint is a free variable
# add the first column variable as the "floor" variable which
# accommodates the most negative variable in the problem.
n = n + 1
L = np.concatenate([np.array([0]), L])
U = np.concatenate([np.array([np.inf]), U])
cc = np.concatenate([np.array([0]), cc])
Aeq = np.hstack([np.zeros([Aeq.shape[0], 1]), Aeq])
Aub = np.hstack([np.zeros([Aub.shape[0], 1]), Aub])
have_floor_variable = True
# Now before we deal with any variables with lower bounds < 0,
# deal with finite bounds which can be simply added as new constraints.
# Also validate bounds inputs here.
for i in range(n):
if(L[i] > U[i]):
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Lower bound %d is greater than upper bound %d" % (i, i))
if np.isinf(L[i]) and L[i] > 0:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Lower bound may not be +infinity")
if np.isinf(U[i]) and U[i] < 0:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Upper bound may not be -infinity")
if np.isfinite(L[i]) and L[i] > 0:
# Add a new lower-bound (negative upper-bound) constraint
Aub = np.vstack([Aub, np.zeros(n)])
Aub[-1, i] = -1
bub = np.concatenate([bub, np.array([-L[i]])])
L[i] = 0
if np.isfinite(U[i]):
# Add a new upper-bound constraint
Aub = np.vstack([Aub, np.zeros(n)])
Aub[-1, i] = 1
bub = np.concatenate([bub, np.array([U[i]])])
U[i] = np.inf
# Now find negative lower bounds (finite or infinite) which require a
# change of variables or free variables and handle them appropriately
for i in range(0, n):
if L[i] < 0:
if np.isfinite(L[i]) and L[i] < 0:
# Add a change of variables for x[i]
# For each row in the constraint matrices, we take the
# coefficient from column i in A,
# and subtract the product of that and L[i] to the RHS b
beq = beq - Aeq[:, i] * L[i]
bub = bub - Aub[:, i] * L[i]
# We now have a nonzero initial value for the objective
# function as well.
f0 = f0 - cc[i] * L[i]
else:
# This is an unrestricted variable, let x[i] = u[i] - v[0]
# where v is the first column in all matrices.
Aeq[:, 0] = Aeq[:, 0] - Aeq[:, i]
Aub[:, 0] = Aub[:, 0] - Aub[:, i]
cc[0] = cc[0] - cc[i]
if np.isinf(U[i]):
if U[i] < 0:
status = -1
message = ("Invalid input for linprog with "
"method = 'simplex'. Upper bound may not be -inf.")
# The number of upper bound constraints (rows in A_ub and elements in b_ub)
mub = len(bub)
# The number of equality constraints (rows in A_eq and elements in b_eq)
meq = len(beq)
# The total number of constraints
m = mub+meq
# The number of slack variables (one for each of the upper-bound constraints)
n_slack = mub
# The number of artificial variables (one for each lower-bound and equality
# constraint)
n_artificial = meq + np.count_nonzero(bub < 0)
try:
Aub_rows, Aub_cols = Aub.shape
except ValueError:
raise ValueError("Invalid input. A_ub must be two-dimensional")
try:
Aeq_rows, Aeq_cols = Aeq.shape
except ValueError:
raise ValueError("Invalid input. A_eq must be two-dimensional")
if Aeq_rows != meq:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"The number of rows in A_eq must be equal "
"to the number of values in b_eq")
if Aub_rows != mub:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"The number of rows in A_ub must be equal "
"to the number of values in b_ub")
if Aeq_cols > 0 and Aeq_cols != n:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Number of columns in A_eq must be equal "
"to the size of c")
if Aub_cols > 0 and Aub_cols != n:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Number of columns in A_ub must be equal to the size of c")
if status != 0:
# Invalid inputs provided
raise ValueError(message)
# Create the tableau
T = np.zeros([m+2, n+n_slack+n_artificial+1])
# Insert objective into tableau
T[-2, :n] = cc
T[-2, -1] = f0
b = T[:-2, -1]
if meq > 0:
# Add Aeq to the tableau
T[:meq, :n] = Aeq
# Add beq to the tableau
b[:meq] = beq
if mub > 0:
# Add Aub to the tableau
T[meq:meq+mub, :n] = Aub
# At bub to the tableau
b[meq:meq+mub] = bub
# Add the slack variables to the tableau
np.fill_diagonal(T[meq:m, n:n+n_slack], 1)
# Further set up the tableau.
# If a row corresponds to an equality constraint or a negative b (a lower
# bound constraint), then an artificial variable is added for that row.
# Also, if b is negative, first flip the signs in that constraint.
slcount = 0
avcount = 0
basis = np.zeros(m, dtype=int)
r_artificial = np.zeros(n_artificial, dtype=int)
for i in range(m):
if i < meq or b[i] < 0:
# basic variable i is in column n+n_slack+avcount
basis[i] = n+n_slack+avcount
r_artificial[avcount] = i
avcount += 1
if b[i] < 0:
b[i] *= -1
T[i, :-1] *= -1
T[i, basis[i]] = 1
T[-1, basis[i]] = 1
else:
# basic variable i is in column n+slcount
basis[i] = n+slcount
slcount += 1
# Make the artificial variables basic feasible variables by subtracting
# each row with an artificial variable from the Phase 1 objective
for r in r_artificial:
T[-1, :] = T[-1, :] - T[r, :]
nit1, status = _solve_simplex(T, n, basis, phase=1, callback=callback,
maxiter=maxiter, tol=tol, bland=bland)
# if pseudo objective is zero, remove the last row from the tableau and
# proceed to phase 2
if abs(T[-1, -1]) < tol:
# Remove the pseudo-objective row from the tableau
T = T[:-1, :]
# Remove the artificial variable columns from the tableau
T = np.delete(T, np.s_[n+n_slack:n+n_slack+n_artificial], 1)
else:
# Failure to find a feasible starting point
status = 2
if status != 0:
message = messages[status]
if disp:
print(message)
return OptimizeResult(x=np.nan, fun=-T[-1, -1], nit=nit1, status=status,
message=message, success=False)
# Phase 2
nit2, status = _solve_simplex(T, n, basis, maxiter=maxiter-nit1, phase=2,
callback=callback, tol=tol, nit0=nit1,
bland=bland)
solution = np.zeros(n+n_slack+n_artificial)
solution[basis[:m]] = T[:m, -1]
x = solution[:n]
slack = solution[n:n+n_slack]
# For those variables with finite negative lower bounds,
# reverse the change of variables
masked_L = np.ma.array(L, mask=np.isinf(L), fill_value=0.0).filled()
x = x + masked_L
# For those variables with infinite negative lower bounds,
# take x[i] as the difference between x[i] and the floor variable.
if have_floor_variable:
for i in range(1, n):
if np.isinf(L[i]):
x[i] -= x[0]
x = x[1:]
# Optimization complete at this point
obj = -T[-1, -1]
if status in (0, 1):
if disp:
print(messages[status])
print(" Current function value: {0: <12.6f}".format(obj))
print(" Iterations: {0:d}".format(nit2))
else:
if disp:
print(messages[status])
print(" Iterations: {0:d}".format(nit2))
return OptimizeResult(x=x, fun=obj, nit=int(nit2), status=status, slack=slack,
message=messages[status], success=(status == 0))
0
Example 123
Project: tribeflow Source File: dataio.py
def initialize_trace(trace_fpath, num_topics, num_iter, \
from_=0, to=np.inf, initial_assign=None):
'''
Given a trace (user trajectories) to learn from, this method will
initialize the necessary matrices and dicts to learn tribeflow.
Using from_, to_ the trace can be sliced. initial_assign is useful to
pickup learning from a previous model.
Parameters
----------
trace_fpath : string
The location of the trace
num_topics : int
The number of latent spaces
from_ : int
Where to begin reading the trace from. 0 is the first line.
to : int
We will stop reading the file here
initial_assign : array-like
Initial topic assignments.
Returns
-------
Count matrices and dicts used to learn tribeflow.
'''
count_zh_dict = defaultdict(int)
count_oz_dict = defaultdict(int)
count_z_dict = defaultdict(int)
count_h_dict = defaultdict(int)
hyper2id = OrderedDict()
obj2id = OrderedDict()
if initial_assign:
initial_assign = np.asarray(initial_assign, dtype='i')
assert initial_assign.min() >= 0
assert initial_assign.max() < num_topics
Dts = []
Trace = []
with open(trace_fpath, 'r') as trace_file:
for i, line in enumerate(trace_file):
if i < from_:
continue
if i >= to:
break
spl = line.strip().split('\t')
assert len(spl) >= 4
assert (len(spl) - 2) % 2 == 0
mem_size = (len(spl) - 2) // 2
line_dts = []
for j in xrange(mem_size):
line_dts.append(float(spl[j]))
Dts.append(line_dts)
hyper_str = spl[mem_size]
if hyper_str not in hyper2id:
hyper2id[hyper_str] = len(hyper2id)
if not initial_assign:
z = np.random.randint(num_topics)
else:
z = initial_assign[i]
h = hyper2id[hyper_str]
count_zh_dict[z, h] += 1
count_h_dict[h] += 1
line_int = [h]
for j in xrange(mem_size + 1, len(spl)):
obj_str = spl[j]
if obj_str not in obj2id:
obj2id[obj_str] = len(obj2id)
o = obj2id[obj_str]
line_int.append(o)
count_oz_dict[o, z] += 1
count_z_dict[z] += 1
line_int.append(z)
Trace.append(line_int)
#Sort by the last residency time.
Dts = np.asarray(Dts)
argsort = Dts[:, -1].argsort()
assert Dts.shape[1] == mem_size
#Create contiguous arrays, not needed but adds a small speedup
Dts = np.asanyarray(Dts[argsort], order='C')
Trace = np.asarray(Trace)
Trace = np.asanyarray(Trace[argsort], dtype='i4', order='C')
nh = len(hyper2id)
no = len(obj2id)
nz = num_topics
previous_stamps = StampLists(num_topics)
for z in xrange(nz):
idx = Trace[:, -1] == z
topic_stamps = Dts[:, -1][idx]
previous_stamps._extend(z, topic_stamps)
Count_zh = np.zeros(shape=(nz, nh), dtype='i4')
Count_oz = np.zeros(shape=(no, nz), dtype='i4')
count_h = np.zeros(shape=(nh,), dtype='i4')
count_z = np.zeros(shape=(nz,), dtype='i4')
for z in xrange(Count_zh.shape[0]):
count_z[z] = count_z_dict[z]
for h in xrange(Count_zh.shape[1]):
count_h[h] = count_h_dict[h]
Count_zh[z, h] = count_zh_dict[z, h]
for o in xrange(Count_oz.shape[0]):
Count_oz[o, z] = count_oz_dict[o, z]
assert (Count_oz.sum(axis=0) == count_z).all()
prob_topics_aux = np.zeros(nz, dtype='f8')
Theta_zh = np.zeros(shape=(nz, nh), dtype='f8')
Psi_oz = np.zeros(shape=(no, nz), dtype='f8')
return Dts, Trace, previous_stamps, Count_zh, Count_oz, \
count_h, count_z, prob_topics_aux, Theta_zh, Psi_oz, \
hyper2id, obj2id
0
Example 124
def _spectral_helper(x, y, fs=1.0, window='hanning', nperseg=256,
noverlap=None, nfft=None, detrend='constant',
return_onesided=True, scaling='spectrum', axis=-1,
mode='psd'):
"""
Calculate various forms of windowed FFTs for PSD, CSD, etc.
This is a helper function that implements the commonality between the
psd, csd, and spectrogram functions. It is not designed to be called
externally. The windows are not averaged over; the result from each window
is returned.
Parameters
---------
x : array_like
Array or sequence containing the data to be analyzed.
y : array_like
Array or sequence containing the data to be analyzed. If this is
the same object in memoery as x (i.e. _spectral_helper(x, x, ...)),
the extra computations are spared.
fs : float, optional
Sampling frequency of the time series. Defaults to 1.0.
window : str or tuple or array_like, optional
Desired window to use. See `get_window` for a list of windows and
required parameters. If `window` is array_like it will be used
directly as the window and its length will be used for nperseg.
Defaults to 'hanning'.
nperseg : int, optional
Length of each segment. Defaults to 256.
noverlap : int, optional
Number of points to overlap between segments. If None,
``noverlap = nperseg // 2``. Defaults to None.
nfft : int, optional
Length of the FFT used, if a zero padded FFT is desired. If None,
the FFT length is `nperseg`. Defaults to None.
detrend : str or function or False, optional
Specifies how to detrend each segment. If `detrend` is a string,
it is passed as the ``type`` argument to `detrend`. If it is a
function, it takes a segment and returns a detrended segment.
If `detrend` is False, no detrending is done. Defaults to 'constant'.
return_onesided : bool, optional
If True, return a one-sided spectrum for real data. If False return
a two-sided spectrum. Note that for complex data, a two-sided
spectrum is always returned.
scaling : { 'density', 'spectrum' }, optional
Selects between computing the cross spectral density ('density')
where `Pxy` has units of V**2/Hz and computing the cross spectrum
('spectrum') where `Pxy` has units of V**2, if `x` and `y` are
measured in V and fs is measured in Hz. Defaults to 'density'
axis : int, optional
Axis along which the periodogram is computed; the default is over
the last axis (i.e. ``axis=-1``).
mode : str, optional
Defines what kind of return values are expected. Options are ['psd',
'complex', 'magnitude', 'angle', 'phase'].
Returns
-------
freqs : ndarray
Array of sample frequencies.
result : ndarray
Array of output data, contents dependant on *mode* kwarg.
t : ndarray
Array of times corresponding to each data segment
References
----------
.. [1] Stack Overflow, "Rolling window for 1D arrays in Numpy?",
http://stackoverflow.com/a/6811241
.. [2] Stack Overflow, "Using strides for an efficient moving average
filter", http://stackoverflow.com/a/4947453
Notes
-----
Adapted from matplotlib.mlab
.. versionadded:: 0.16.0
"""
if mode not in ['psd', 'complex', 'magnitude', 'angle', 'phase']:
raise ValueError("Unknown value for mode %s, must be one of: "
"'default', 'psd', 'complex', "
"'magnitude', 'angle', 'phase'" % mode)
# If x and y are the same object we can save ourselves some computation.
same_data = y is x
if not same_data and mode != 'psd':
raise ValueError("x and y must be equal if mode is not 'psd'")
axis = int(axis)
# Ensure we have np.arrays, get outdtype
x = np.asarray(x)
if not same_data:
y = np.asarray(y)
outdtype = np.result_type(x,y,np.complex64)
else:
outdtype = np.result_type(x,np.complex64)
if not same_data:
# Check if we can broadcast the outer axes together
xouter = list(x.shape)
youter = list(y.shape)
xouter.pop(axis)
youter.pop(axis)
try:
outershape = np.broadcast(np.empty(xouter), np.empty(youter)).shape
except ValueError:
raise ValueError('x and y cannot be broadcast together.')
if same_data:
if x.size == 0:
return np.empty(x.shape), np.empty(x.shape), np.empty(x.shape)
else:
if x.size == 0 or y.size == 0:
outshape = outershape + (min([x.shape[axis], y.shape[axis]]),)
emptyout = np.rollaxis(np.empty(outshape), -1, axis)
return emptyout, emptyout, emptyout
if x.ndim > 1:
if axis != -1:
x = np.rollaxis(x, axis, len(x.shape))
if not same_data and y.ndim > 1:
y = np.rollaxis(y, axis, len(y.shape))
# Check if x and y are the same length, zero-pad if neccesary
if not same_data:
if x.shape[-1] != y.shape[-1]:
if x.shape[-1] < y.shape[-1]:
pad_shape = list(x.shape)
pad_shape[-1] = y.shape[-1] - x.shape[-1]
x = np.concatenate((x, np.zeros(pad_shape)), -1)
else:
pad_shape = list(y.shape)
pad_shape[-1] = x.shape[-1] - y.shape[-1]
y = np.concatenate((y, np.zeros(pad_shape)), -1)
# X and Y are same length now, can test nperseg with either
if x.shape[-1] < nperseg:
warnings.warn('nperseg = {0:d}, is greater than input length = {1:d}, '
'using nperseg = {1:d}'.format(nperseg, x.shape[-1]))
nperseg = x.shape[-1]
nperseg = int(nperseg)
if nperseg < 1:
raise ValueError('nperseg must be a positive integer')
if nfft is None:
nfft = nperseg
elif nfft < nperseg:
raise ValueError('nfft must be greater than or equal to nperseg.')
else:
nfft = int(nfft)
if noverlap is None:
noverlap = nperseg//2
elif noverlap >= nperseg:
raise ValueError('noverlap must be less than nperseg.')
else:
noverlap = int(noverlap)
# Handle detrending and window functions
if not detrend:
def detrend_func(d):
return d
elif not hasattr(detrend, '__call__'):
def detrend_func(d):
return signaltools.detrend(d, type=detrend, axis=-1)
elif axis != -1:
# Wrap this function so that it receives a shape that it could
# reasonably expect to receive.
def detrend_func(d):
d = np.rollaxis(d, -1, axis)
d = detrend(d)
return np.rollaxis(d, axis, len(d.shape))
else:
detrend_func = detrend
if isinstance(window, string_types) or type(window) is tuple:
win = get_window(window, nperseg)
else:
win = np.asarray(window)
if len(win.shape) != 1:
raise ValueError('window must be 1-D')
if win.shape[0] != nperseg:
raise ValueError('window must have length of nperseg')
if np.result_type(win,np.complex64) != outdtype:
win = win.astype(outdtype)
if mode == 'psd':
if scaling == 'density':
scale = 1.0 / (fs * (win*win).sum())
elif scaling == 'spectrum':
scale = 1.0 / win.sum()**2
else:
raise ValueError('Unknown scaling: %r' % scaling)
else:
scale = 1
if return_onesided is True:
if np.iscomplexobj(x):
sides = 'twosided'
else:
sides = 'onesided'
if not same_data:
if np.iscomplexobj(y):
sides = 'twosided'
else:
sides = 'twosided'
if sides == 'twosided':
num_freqs = nfft
elif sides == 'onesided':
if nfft % 2:
num_freqs = (nfft + 1)//2
else:
num_freqs = nfft//2 + 1
# Perform the windowed FFTs
result = _fft_helper(x, win, detrend_func, nperseg, noverlap, nfft)
result = result[..., :num_freqs]
freqs = fftpack.fftfreq(nfft, 1/fs)[:num_freqs]
if not same_data:
# All the same operations on the y data
result_y = _fft_helper(y, win, detrend_func, nperseg, noverlap, nfft)
result_y = result_y[..., :num_freqs]
result = np.conjugate(result) * result_y
elif mode == 'psd':
result = np.conjugate(result) * result
elif mode == 'magnitude':
result = np.absolute(result)
elif mode == 'angle' or mode == 'phase':
result = np.angle(result)
elif mode == 'complex':
pass
result *= scale
if sides == 'onesided':
if nfft % 2:
result[...,1:] *= 2
else:
# Last point is unpaired Nyquist freq point, don't double
result[...,1:-1] *= 2
t = np.arange(nperseg/2, x.shape[-1] - nperseg/2 + 1, nperseg - noverlap)/float(fs)
if sides != 'twosided' and not nfft % 2:
# get the last value correctly, it is negative otherwise
freqs[-1] *= -1
# we unwrap the phase here to handle the onesided vs. twosided case
if mode == 'phase':
result = np.unwrap(result, axis=-1)
result = result.astype(outdtype)
# All imaginary parts are zero anyways
if same_data and mode != 'complex':
result = result.real
# Output is going to have new last axis for window index
if axis != -1:
# Specify as positive axis index
if axis < 0:
axis = len(result.shape)-1-axis
# Roll frequency axis back to axis where the data came from
result = np.rollaxis(result, -1, axis)
else:
# Make sure window/time index is last axis
result = np.rollaxis(result, -1, -2)
return freqs, t, result
0
Example 125
def minimize(func, x0, args=(),
bounds=None, scaling=None,
redfactor=2.0, deltainit=1.0, deltatol=1e-3, feps=1e-15,
errorcontrol=True, funcNinit=30, funcmultfactor=2.0,
paired=True, alpha=0.05, disp=False, callback=None, **kwargs):
"""
Minimization of an objective function by a pattern search.
The search algorithm is a simple compass search along coordinate directions.
If the function evaluation contains a stochastic element, then the function
is called repeatedly to average over the stochasticity in the function
evaluation. The number of evaluations is adapted dynamically to ensure
convergence.
Parameters
----------
func: callable
objective function to be minimized
x0: array-like
starting point
args: tuple
extra arguments to be supplied to func
bounds: array-like
bounds on the variables
scaling: array-like
scaling by which to multiply step size and tolerances along different dimensions
redfactor: float
reduction factor by which to reduce delta if no reduction direction found
deltainit: float
inital pattern size
deltatol: float
smallest pattern size
feps: float
smallest difference in function value to resolve
errorcontrol: boolean
whether to control error of simulation by repeated sampling
funcNinit: int, only for errorcontrol=True
initial number of iterations to use for the function, do not set much lower
than 30 as otherwise there is no sufficient statistics for function comparisons
funcmultfactor: float, only for errorcontrol=True
multiplication factor by which to increase number of iterations of function
paired: boolean, only for errorcontrol=True
compare for same random seeds
alpha: float, only for errorcontrol=True
significance level of tests, the higher this value the more statistics
is acquired, which decreases the risk of taking a step in a non-descent
direction at the expense of higher computational cost per iteration
disp: boolean
whether to output status updates during the optimization
callback: callable
called after each iteration, as callback(xk), where xk is the current parameter vector.
Returns
-------
scipy.optimize.OptimizeResult object
special entry: free
Boolean array indicating whether the variable is free (within feps) at the optimum
"""
if disp:
print('minimization starting')
print('args', args)
print('errorcontrol', errorcontrol)
print('paired', paired)
# absolute tolerance for float comparisons
floatcompatol = 1e-14
x0 = np.asarray(x0)
if scaling is None:
scaling = np.ones(x0.shape)
else:
scaling = np.asarray(scaling)
# ensure initial point lies within bounds
if bounds is not None:
bounds = np.asarray(bounds)
np.clip(x0, bounds[:, 0], bounds[:, 1], out=x0)
def clip(x, d):
"""clip x+d to respect bounds
returns clipped result and effective distance"""
xnew = x + d
if bounds is not None:
# if test point depasses set to boundary instead
xclipped = np.clip(xnew, bounds[:, 0], bounds[:, 1])
deltaeff = np.abs(x - xclipped).sum()
return xclipped, deltaeff
else:
return xnew, delta
# generate set of search directions (+- s_i e_i | i = 1, ..., N)
def unit(i, N):
"return ith unit vector in R^N"
arr = np.zeros(N)
arr[i] = 1.0
return arr
N = len(x0)
generatingset = [unit(i, N)*direction*scaling[i] for i in np.arange(N) for direction in [+1, -1]]
# memoize function
if errorcontrol:
funcm = AveragedFunction(
func, fargs=args, paired=paired, N=funcNinit)
# apply Bonferroni correction to confidence level
# (need more statistics in higher dimensions)
alpha = alpha/len(generatingset)
else:
# freeze function arguments
def funcf(x, **kwargs):
return func(x, *args, **kwargs)
funcm = memoized(funcf)
x = x0
delta = deltainit
# number of iterations
nit = 0
# continue as long as delta is larger than tolerance
# or if there was an update during the last iteration
found = False
while delta >= deltatol-floatcompatol or found:
nit += 1
# if delta gets close to deltatol, do iteration with step size deltatol instead
if delta/redfactor < deltatol:
delta = deltatol
if disp:
print('nit %i, Delta %g' % (nit, delta))
found = False
np.random.shuffle(generatingset)
for d in generatingset:
xtest, deltaeff = clip(x, delta*d)
if deltaeff < floatcompatol:
continue
# Does xtest improve upon previous function value?
if ((not errorcontrol and (funcm(xtest) < funcm(x)-feps))
or (errorcontrol
and funcm.test(xtest, x, type_='smaller', alpha=alpha))):
x = xtest
found = True
if disp:
print(x)
# Is non-improvement due to too large step size or missing statistics?
elif ((deltaeff >= deltatol*np.sum(np.abs(d))) # no refinement for boundary steps smaller than tolerance
and ((not errorcontrol and (funcm(xtest) < funcm(x)+feps))
or (errorcontrol
and funcm.test(xtest, x, type_='equality', alpha=alpha)
and (funcm.diffse(xtest, x) > feps)))):
# If there is no significant difference the step size might
# correspond to taking a step to the other side of the minimum.
# Therefore test if middle point is better
xmid = 0.5*(x+xtest)
if ((not errorcontrol and funcm(xmid) < funcm(x)-feps)
or (errorcontrol
and funcm.test(xmid, x, type_='smaller', alpha=alpha))):
x = xmid
delta /= redfactor
found = True
if disp:
print('mid', x)
# otherwise increase accuracy of simulation to try to get to significance
elif errorcontrol:
funcm.N *= funcmultfactor
if disp:
print('new N %i' % funcm.N)
found = True
if callback is not None:
callback(x)
if not found:
delta /= redfactor
message = 'convergence within deltatol'
# check if any of the directions are free at the optimum
delta = deltatol
free = np.zeros(x.shape, dtype=bool)
for d in generatingset:
dim = np.argmax(np.abs(d))
xtest, deltaeff = clip(x, delta*d)
if deltaeff < deltatol*np.sum(np.abs(d))-floatcompatol: # do not consider as free for boundary steps
continue
if not free[dim] and (((not errorcontrol and funcm(xtest) - feps < funcm(x)) or
(errorcontrol and funcm.test(xtest, x, type_='equality', alpha=alpha)
and (funcm.diffse(xtest, x) < feps)))):
free[dim] = True
message += '. dim %i is free at optimum' % dim
reskwargs = dict(x=x, nit=nit, nfev=funcm.nev, message=message, free=free,
success=True)
if errorcontrol:
f, funse = funcm(x)
res = OptimizeResult(fun=f, funse=funse, **reskwargs)
else:
f = funcm(x)
res = OptimizeResult(fun=f, **reskwargs)
if disp:
print(res)
return res
0
Example 126
Project: msmbuilder Source File: discrete_approx.py
def discrete_approx_mvn(X, means, covars, match_variances=True):
"""Find a discrete approximation to a multivariate normal distribution.
The method employs find the discrete distribution with support only at the
supplied points X with minimal K-L divergence to a target multivariate
normal distribution under the constraints that the mean and variance
of the discrete distribution match the normal distribution exactly.
Parameters
----------
X : np.ndarray, shape=(n_points, n_features)
The allowable points
means : np.ndarray, shape=(n_features)
The mean vector of the MVN
covars : np.ndarray, shape=(n_features, n_features) or shape=(n_features,)
If covars is 2D, it's interpreted as the covariance matrix for
the model. If 1D, we assume a diagonal covariance matrix with the
specified diagonal entries.
match_variances : bool, optimal
When True, both the means and the variances of the discrete distribution
are constrained. Under some circuemstances, this is not satisfiable (e.g.
if there aren't enough samples
Returns
-------
weights : np.ndarray, shape=(n_samples,)
The weight for each of the points in X in the resulting
discrete probability distribution
Notes
-----
The discrete distribution is one that has mass only at the specified
points. It can therefore be parameterized by a set of weights on each
point. If :math:`\{X_i\}` is the set of allowable points, and
:math:`\{w_i\}` are the weights, then our discrete distribution has
the form
.. math::
p(y; w) = w_i \sum \delta(y - X_i).
We chose the :math:`w_i` by minimizing the K-L divergence from the our
discrete distribution to the desired multivariate normal subject to a
constraint that the first moments of the discrete distribution match
the mean of the multivariate normal exactly, and that the variances
also match. Let :math:`q(x)` be the target distribution. The optimal
weights are then
.. math::
min_{\{w_i\}} \sum_i p(X_i; w) \log \frac{p(X_i; w)}{q(X_i)}
subject to
.. math::
\sum_i (X_i) p(X_i; w) = \int_\Omega (x) q(x) = \mu,
\sum_i (X_i-mu)**2 p(X_i; w) = \int_\Omega (x-mu) q(x).
References
----------
.. [1] Tanaka, Ken'ichiro, and Alexis Akira Toda. "Discrete approximations
of continuous distributions by maximum entropy." Economics Letters 118.3
(2013): 445-450.
"""
X = ensure_type(np.asarray(X), dtype=np.float32, ndim=2, name='X', warn_on_cast=False)
means = ensure_type(np.asarray(means), np.float64, ndim=1, name='means', warn_on_cast=False)
covars = np.asarray(covars)
# Get the un-normalized probability of each point X_i in the MVN
# `prob` are the q(X_i) in the mathematics
# `moments` are the \bar{T} that we want to match.
if covars.ndim == 1:
# diagonal covariance case
if not len(covars) == len(means):
raise ValueError('Shape Error: covars and means musth have the same length')
prob = np.exp(-0.5 * np.sum(1. / np.sqrt(covars) * (X - means) ** 2, axis=1))
moments = np.concatenate((means, covars)) if match_variances else means
elif covars.ndim == 2:
if not (covars.shape[0] == len(means) and covars.shape[1] == len(means)):
raise ValueError('Shape Error: covars must be square, with size = len(means)')
# full 2d covariance matrix
cv_chol = scipy.linalg.cholesky(covars, lower=True)
cv_sol = scipy.linalg.solve_triangular(cv_chol, (X - means).T, lower=True).T
prob = np.exp(-0.5 * (np.sum(cv_sol ** 2, axis=1)))
moments = np.concatenate((means, np.diag(covars))) if match_variances else means
else:
raise ValueError('covars must be 1D or 2D')
# this is T(x_i) for each X_i
moment_contributions = np.hstack((X, (X - means) ** 2)) if match_variances else X
def objective_and_grad(l):
dot = np.dot(moment_contributions, l)
lse = scipy.misc.logsumexp(dot, b=prob)
# value of the objective function
obj_value = lse - np.dot(l, moments)
# gradient of objective function
dot_max = dot.max(axis=0)
exp_term = np.sum(moment_contributions * (prob * np.exp(dot - dot_max)).reshape(-1, 1), axis=0)
log_numerator = np.log(exp_term) + dot_max
grad_value = np.exp(log_numerator - lse) - moments
return obj_value, grad_value
result = scipy.optimize.minimize(
objective_and_grad, jac=True, x0=np.ones_like(moments), method='BFGS')
if not result['success']:
raise NotSatisfiableError()
dot = np.dot(moment_contributions, result['x'])
log_denominator = scipy.misc.logsumexp(dot, b=prob)
weights = prob * np.exp(dot - log_denominator)
if not np.all(np.isfinite(weights)):
raise NotSatisfiableError()
weights = weights / np.sum(weights)
return weights
0
Example 127
Project: AWS-Lambda-ML-Microservice-Skeleton Source File: hierarchical.py
def linkage_tree(X, connectivity=None, n_components=None,
n_clusters=None, linkage='complete', affinity="euclidean",
return_distance=False):
"""Linkage agglomerative clustering based on a Feature matrix.
The inertia matrix uses a Heapq-based representation.
This is the structured version, that takes into account some topological
structure between samples.
Read more in the :ref:`User Guide <hierarchical_clustering>`.
Parameters
----------
X : array, shape (n_samples, n_features)
feature matrix representing n_samples samples to be clustered
connectivity : sparse matrix (optional).
connectivity matrix. Defines for each sample the neighboring samples
following a given structure of the data. The matrix is assumed to
be symmetric and only the upper triangular half is used.
Default is None, i.e, the Ward algorithm is unstructured.
n_components : int (optional)
Number of connected components. If None the number of connected
components is estimated from the connectivity matrix.
NOTE: This parameter is now directly determined directly
from the connectivity matrix and will be removed in 0.18
n_clusters : int (optional)
Stop early the construction of the tree at n_clusters. This is
useful to decrease computation time if the number of clusters is
not small compared to the number of samples. In this case, the
complete tree is not computed, thus the 'children' output is of
limited use, and the 'parents' output should rather be used.
This option is valid only when specifying a connectivity matrix.
linkage : {"average", "complete"}, optional, default: "complete"
Which linkage critera to use. The linkage criterion determines which
distance to use between sets of observation.
- average uses the average of the distances of each observation of
the two sets
- complete or maximum linkage uses the maximum distances between
all observations of the two sets.
affinity : string or callable, optional, default: "euclidean".
which metric to use. Can be "euclidean", "manhattan", or any
distance know to paired distance (see metric.pairwise)
return_distance : bool, default False
whether or not to return the distances between the clusters.
Returns
-------
children : 2D array, shape (n_nodes-1, 2)
The children of each non-leaf node. Values less than `n_samples`
correspond to leaves of the tree which are the original samples.
A node `i` greater than or equal to `n_samples` is a non-leaf
node and has children `children_[i - n_samples]`. Alternatively
at the i-th iteration, children[i][0] and children[i][1]
are merged to form node `n_samples + i`
n_components : int
The number of connected components in the graph.
n_leaves : int
The number of leaves in the tree.
parents : 1D array, shape (n_nodes, ) or None
The parent of each node. Only returned when a connectivity matrix
is specified, elsewhere 'None' is returned.
distances : ndarray, shape (n_nodes-1,)
Returned when return_distance is set to True.
distances[i] refers to the distance between children[i][0] and
children[i][1] when they are merged.
See also
--------
ward_tree : hierarchical clustering with ward linkage
"""
X = np.asarray(X)
if X.ndim == 1:
X = np.reshape(X, (-1, 1))
n_samples, n_features = X.shape
linkage_choices = {'complete': _hierarchical.max_merge,
'average': _hierarchical.average_merge}
try:
join_func = linkage_choices[linkage]
except KeyError:
raise ValueError(
'Unknown linkage option, linkage should be one '
'of %s, but %s was given' % (linkage_choices.keys(), linkage))
if connectivity is None:
from scipy.cluster import hierarchy # imports PIL
if n_clusters is not None:
warnings.warn('Partial build of the tree is implemented '
'only for structured clustering (i.e. with '
'explicit connectivity). The algorithm '
'will build the full tree and only '
'retain the lower branches required '
'for the specified number of clusters',
stacklevel=2)
if affinity == 'precomputed':
# for the linkage function of hierarchy to work on precomputed
# data, provide as first argument an ndarray of the shape returned
# by pdist: it is a flat array containing the upper triangular of
# the distance matrix.
i, j = np.triu_indices(X.shape[0], k=1)
X = X[i, j]
elif affinity == 'l2':
# Translate to something understood by scipy
affinity = 'euclidean'
elif affinity in ('l1', 'manhattan'):
affinity = 'cityblock'
elif callable(affinity):
X = affinity(X)
i, j = np.triu_indices(X.shape[0], k=1)
X = X[i, j]
out = hierarchy.linkage(X, method=linkage, metric=affinity)
children_ = out[:, :2].astype(np.int)
if return_distance:
distances = out[:, 2]
return children_, 1, n_samples, None, distances
return children_, 1, n_samples, None
if n_components is not None:
warnings.warn(
"n_components is now directly calculated from the connectivity "
"matrix and will be removed in 0.18",
DeprecationWarning)
connectivity, n_components = _fix_connectivity(X, connectivity)
connectivity = connectivity.tocoo()
# Put the diagonal to zero
diag_mask = (connectivity.row != connectivity.col)
connectivity.row = connectivity.row[diag_mask]
connectivity.col = connectivity.col[diag_mask]
connectivity.data = connectivity.data[diag_mask]
del diag_mask
if affinity == 'precomputed':
distances = X[connectivity.row, connectivity.col]
else:
# FIXME We compute all the distances, while we could have only computed
# the "interesting" distances
distances = paired_distances(X[connectivity.row],
X[connectivity.col],
metric=affinity)
connectivity.data = distances
if n_clusters is None:
n_nodes = 2 * n_samples - 1
else:
assert n_clusters <= n_samples
n_nodes = 2 * n_samples - n_clusters
if return_distance:
distances = np.empty(n_nodes - n_samples)
# create inertia heap and connection matrix
A = np.empty(n_nodes, dtype=object)
inertia = list()
# LIL seems to the best format to access the rows quickly,
# without the numpy overhead of slicing CSR indices and data.
connectivity = connectivity.tolil()
# We are storing the graph in a list of IntFloatDict
for ind, (data, row) in enumerate(zip(connectivity.data,
connectivity.rows)):
A[ind] = IntFloatDict(np.asarray(row, dtype=np.intp),
np.asarray(data, dtype=np.float64))
# We keep only the upper triangular for the heap
# Generator expressions are faster than arrays on the following
inertia.extend(_hierarchical.WeightedEdge(d, ind, r)
for r, d in zip(row, data) if r < ind)
del connectivity
heapify(inertia)
# prepare the main fields
parent = np.arange(n_nodes, dtype=np.intp)
used_node = np.ones(n_nodes, dtype=np.intp)
children = []
# recursive merge loop
for k in xrange(n_samples, n_nodes):
# identify the merge
while True:
edge = heappop(inertia)
if used_node[edge.a] and used_node[edge.b]:
break
i = edge.a
j = edge.b
if return_distance:
# store distances
distances[k - n_samples] = edge.weight
parent[i] = parent[j] = k
children.append((i, j))
# Keep track of the number of elements per cluster
n_i = used_node[i]
n_j = used_node[j]
used_node[k] = n_i + n_j
used_node[i] = used_node[j] = False
# update the structure matrix A and the inertia matrix
# a clever 'min', or 'max' operation between A[i] and A[j]
coord_col = join_func(A[i], A[j], used_node, n_i, n_j)
for l, d in coord_col:
A[l].append(k, d)
# Here we use the information from coord_col (containing the
# distances) to update the heap
heappush(inertia, _hierarchical.WeightedEdge(d, k, l))
A[k] = coord_col
# Clear A[i] and A[j] to save memory
A[i] = A[j] = 0
# Separate leaves in children (empty lists up to now)
n_leaves = n_samples
# # return numpy array for efficient caching
children = np.array(children)[:, ::-1]
if return_distance:
return children, n_components, n_leaves, parent, distances
return children, n_components, n_leaves, parent
0
Example 128
def create_layout(self, G, pos=None, fixed=None, scale=1.0,
min_distance=None):
"""Position nodes using Fruchterman-Reingold force-directed algorithm.
Parameters
----------
G : NetworkX graph
dim : int
Dimension of layout
k : float (default=None)
Optimal distance between nodes. If None the distance is set to
1/sqrt(n) where n is the number of nodes. Increase this value
to move nodes farther apart.
pos : dict or None optional (default=None)
Initial positions for nodes as a dictionary with node as keys
and values as a list or tuple. If None, then nuse random initial
positions.
fixed : list or None optional (default=None)
Nodes to keep fixed at initial position.
iterations : int optional (default=50)
Number of iterations of spring-force relaxation
weight : string or None optional (default='weight')
The edge attribute that holds the numerical value used for
the edge weight. If None, then all edge weights are 1.
scale : float (default=1.0)
Scale factor for positions. The nodes are positioned
in a box of size [0,scale] x [0,scale].
min_distance : float optional (default=None)
Minimum distance to enforce between nodes. If passed with scale,
this may cause the returned positions to go outside the scale.
Returns
-------
dict :
A dictionary of positions keyed by node
Examples
--------
>>> G=nx.path_graph(4)
>>> pos=nx.spring_layout(G)
# The same using longer function name
>>> pos=nx.fruchterman_reingold_layout(G)
"""
# This is a modification of the networkx.layout library's
# fruchterman_reingold_layout to work well with fixed positions
# and large inital positions (not near 1.0). This involved
# modification to what the optimal "k" is and the removal of
# the resize when fixed is passed
dim = 2
try:
import numpy as np
except ImportError:
raise ImportError("fruchterman_reingold_layout() requires numpy: http://scipy.org/ ")
if fixed is not None:
nfixed=dict(zip(G,range(len(G))))
fixed=np.asarray([nfixed[v] for v in fixed])
if pos is not None:
# Determine size of exisiting domain
dom_size = max(flatten(pos.values()))
pos_arr=np.asarray(np.random.random((len(G),dim)))*dom_size
for i,n in enumerate(G):
if n in pos:
pos_arr[i]=np.asarray(pos[n])
else:
pos_arr=None
dom_size = 1.0
if len(G)==0:
return {}
if len(G)==1:
return {G.nodes()[0]:(1,)*dim}
A=nx.to_numpy_matrix(G)
nnodes,_ = A.shape
# I've found you want to occupy about a two-thirds of the window size
if fixed is not None:
k=(min(self.winfo_width(), self.winfo_height())*.66)/np.sqrt(nnodes)
else:
k = None
# Alternate k, for when vieweing the whole graph, not a subset
#k=dom_size/np.sqrt(nnodes)
pos=self._fruchterman_reingold(A,dim,k,pos_arr,fixed)
if fixed is None:
# Only rescale non fixed layouts
pos= nx.layout._rescale_layout(pos,scale=scale)
if min_distance and fixed is None:
# Find min distance between any two nodes and scale such that
# this distance = min_distance
# matrix of difference between points
delta = np.zeros((pos.shape[0],pos.shape[0],pos.shape[1]),
dtype=A.dtype)
for i in range(pos.shape[1]):
delta[:,:,i]= pos[:,i,None]-pos[:,i]
# distance between points
distance=np.sqrt((delta**2).sum(axis=-1))
cur_min_dist = np.where(distance==0, np.inf, distance).min()
if cur_min_dist < min_distance:
# calculate scaling factor and rescale
rescale = (min_distance / cur_min_dist) * pos.max()
pos = nx.layout._rescale_layout(pos, scale=rescale)
return dict(zip(G,pos))
0
Example 129
def algo_action(self):
close = np.asarray(self.close_buffer)
if len(close) < self.life_timeperiod:
self.logger.info('data not enough! len = {0}'.format(len(close)))
return
sma = SMA({'close':close}, timeperiod=self.short_timeperiod)
lma = SMA({'close':close}, timeperiod=self.long_timeperiod) ## make sure last lma is a number
life = SMA({'close':close}, timeperiod=self.life_timeperiod)
s_l_ma_delta = sma[-1] - lma[-1] ## 最新的短长MA差值
last_ma = sma[-1] ## 最新的短MA
#self.logger.info('sma length = {0}'.format(len(sma)))
#if len(sma) < 3:
# return
#momentum = self.momentum = sma[-1] - sma[-2] ## MA冲量
momentum = self.momentum = self.last_price - sma[-1] ## 当前价格相对MA冲量
# #self.logger.info( 'close: ', close)
## 短MA变动趋势,true表示进入趋势,false表示震荡
moving_long = self.moving_long = moving_short = self.moving_short = False
sma_diff = sma[-1] - sma[-2]
sma_diff_1 = sma[-2] - sma[-3]
moving = self.moving = sma_diff * sma_diff_1 > 0
if moving and (sma_diff * momentum > 0 or momentum >= self.momentum_factor* s_l_ma_delta):
moving_long = self.moving_long = True
elif moving and (sma_diff * momentum > 0 or momentum <= self.momentum_factor* s_l_ma_delta):
moving_short = self.moving_short = True
## 均线是多头还是空头排列
long_trends = self.long_trends = (sma[-1] > lma[-1] > life[-1])
short_trends = self.short_trends = (sma[-1] < lma[-1] < life[-1])
self.logger.info('short ma: {0}, long ma: {1}, life line: {2}'.format(round(sma[-1],4), round(lma[-1],4), round(life[-1],4)))
self.logger.info('short long ma delta: {0}, last_ma: {1}, momentum: {2}'.format(round(s_l_ma_delta,4), round(last_ma,4), round(momentum,4)))
self.logger.info('## short ma moving long: {0}, moving short: {1}; Trends is long: {2}, is short: {3}.'.format(moving_long, moving_short, long_trends, short_trends))
if self.analyse_only:
return
## 下单基准价,为了能够成交,所有开平仓都会根据配置的跳数追价,这里先计算基准价格
price_base = max(self.last_price, close[-1]) if long_trends else (min(self.last_price, close[-1]) if short_trends else self.last_price)
self.logger.info("current trade count = {0}, price base = {1} ".format(self.trade_count, round(price_base, 1)))
self.care_positions()
a_p = self.a_p
b_p = self.b_p
# cancel unfinished orders
#self.cancel_unfinished_orders()
threshold = self.threshold
significant_diff = self.significant_diff
stop_lose_threshold = self.stop_lose_threshold
stop_profit_threshold = self.stop_profit_threshold
## 多信号
if long_trends: ## short ma cross up long ma
## 没有多仓
if (a_p is None or a_p.volume < eps):
signal_filter = (moving_long or momentum >= significant_diff) and s_l_ma_delta > threshold and self.trade_count < self.trade_limit
vol = self.trade_unit[self.trade_count] ## get order volume from configured list
if signal_filter and vol > eps:
self.trade_count += 1
ord_price = price_base + self.hops*self.tick_size
self.logger.info("open long ... count {0}, {1} @ {2}".format(self.trade_count, vol, ord_price))
self.open_long(self.exchange, self.sec_id, ord_price, vol)
## 有多仓,且有止赢要求
elif b_p and b_p.volume > eps:
checking_limit = self.trade_count == self.trade_limit ## 最大交易次数限制止赢
b_pnl = self.last_price - b_p.vwap ## 多仓浮赢
if self.positive_stop and (checking_limit or b_pnl >= stop_profit_threshold):
ord_price = price_base - self.hops*self.tick_size
self.logger.info("trade count {0} to limit time[{1}], stop profit, close long @ {2}".format(self.trade_count, self.trade_limit, ord_price))
self.close_long_positions(b_p, ord_price)
self.trade_count = 0
elif moving_short: ## 短线趋势反转,检查是否要riskoff
self.try_stop_action()
## 有空仓
elif a_p and a_p.volume > eps:
self.logger.info("trend changed, stop lose, close short")
self.close_short_positions(a_p)
self.trade_count = 0
else:
pass
## 空信号
elif short_trends : ## short ma cross down long ma
## 没有空仓
if (b_p is None or b_p.volume < eps):
signal_filter = (moving_short or momentum <= -significant_diff) and s_l_ma_delta < -threshold and self.trade_count < self.trade_limit
vol = self.trade_unit[self.trade_count] ## get order volume from configured list
if signal_filter and vol > eps:
ord_price = price_base - self.hops*self.tick_size
self.logger.info("open short ... count {0}, {1} @ {2}".format(self.trade_count, vol, ord_price))
self.trade_count += 1 ## 开仓计数器
self.open_short(self.exchange, self.sec_id, ord_price, vol)
## 有空仓,且有止赢要求
elif a_p and a_p.volume > eps:
checking_limit = self.trade_count == self.trade_limit ## 最大交易次数限制止赢
a_pnl = -1.0 * (self.last_price - a_p.vwap) ## 空仓浮赢
if self.positive_stop and (checking_limit or a_pnl >= stop_profit_threshold):
ord_price = price_base + self.hops*self.tick_size
self.logger.info("trade count {0} to limit time [{1}], stop profit, close short @ {2}".format(self.trade_count, self.trade_limit, ord_price))
self.close_short_positions(a_p, ord_price)
self.trade_count = 0
elif moving_long: ## 短线趋势反转,检查是否要riskoff
self.try_stop_action()
## 有多仓
elif b_p and b_p.volume > eps:
self.logger.info("trend changed, stop lose, close long")
self.close_long_positions(b_p)
self.trade_count = 0
else:
pass
else: ## check if need to stop trading, close all positions
self.logger.info("no trends, here check if need to stop ...")
self.try_stop_action()
0
Example 130
def fit(self, X):
X = self.clamp(X)
n_rows, n_cols = X.shape
dtype = X.dtype
# To avoid inefficient memory access we keep around two copies
# of the array, one contiguous in the rows and the other
# contiguous in the columns
X_row_major = np.asarray(X, order="C")
X_column_major = np.asarray(X, order="F")
observed_row_major = ~np.isnan(X_row_major)
n_observed_per_row = observed_row_major.sum(axis=1)
n_empty_rows = (n_observed_per_row == 0).sum()
if n_empty_rows > 0:
raise ValueError("%d rows have no observed values" % n_empty_rows)
observed_column_major = np.asarray(observed_row_major, order="F")
n_observed_per_column = observed_column_major.sum(axis=0)
n_empty_columns = (n_observed_per_column == 0).sum()
if n_empty_columns > 0:
raise ValueError("%d columns have no observed values" % (
n_empty_columns,))
# initialize by assuming that rows are zero-mean/unit variance and
# with a direct estimate of mean and standard deviation
# of each column
row_means = np.zeros(n_rows, dtype=dtype)
row_scales = np.ones(n_rows, dtype=dtype)
if self.center_columns:
column_means = np.nanmean(X, axis=0)
else:
column_means = np.zeros(n_cols, dtype=dtype)
if self.scale_columns:
column_scales = np.nanstd(X, axis=0)
column_scales[column_scales == 0] = 1.0
else:
column_scales = np.ones(n_cols, dtype=dtype)
last_residual = self.residual(X)
if self.verbose:
print("[BiScaler] Initial log residual value = %f" % (
np.log(last_residual),))
for i in range(self.max_iters):
if last_residual == 0:
# already have a perfect fit, so let's get out of here
print("[BiScaler] No room for improvement")
break
assert len(column_means) == n_cols, \
"Wrong number of column means, expected %d but got %d" % (
n_cols,
len(column_means))
assert len(column_scales) == n_cols, \
"Wrong number of column scales, expected %d but got %d" % (
n_cols,
len(column_scales))
assert len(row_means) == n_rows, \
"Wrong number of row means, expected %d but got %d" % (
n_rows,
len(row_means))
assert len(row_scales) == n_rows, \
"Wrong number of row scales, expected %d but got %d" % (
n_rows,
len(row_scales))
if self.center_rows:
row_means = self.estimate_row_means(
X=X_row_major,
observed=observed_row_major,
column_means=column_means,
column_scales=column_scales)
if self.center_columns:
column_means = self.estimate_column_means(
X=X_column_major,
observed=observed_column_major,
row_means=row_means,
row_scales=row_scales)
X_centered = self.center(
X,
row_means,
column_means)
if self.scale_rows:
row_scales = self.estimate_row_scales(
X_centered=X_centered,
column_scales=column_scales)
if self.scale_columns:
column_scales = self.estimate_column_scales(
X_centered=X_centered,
row_scales=row_scales)
X_normalized = self.rescale(X_centered, row_scales, column_scales)
residual = self.residual(X_normalized)
change_in_residual = last_residual - residual
if self.verbose:
print("[BiScaler] Iter %d: log residual = %f, log improvement ratio=%f" % (
i + 1,
np.log(residual),
np.log(last_residual / residual)))
if change_in_residual / last_residual < self.tolerance:
break
last_residual = residual
self.row_means = row_means
self.row_scales = row_scales
self.column_means = column_means
self.column_scales = column_scales
0
Example 131
Project: networkx Source File: nx_pylab.py
def draw_networkx_nodes(G, pos,
nodelist=None,
node_size=300,
node_color='r',
node_shape='o',
alpha=1.0,
cmap=None,
vmin=None,
vmax=None,
ax=None,
linewidths=None,
label=None,
**kwds):
"""Draw the nodes of the graph G.
This draws only the nodes of the graph G.
Parameters
----------
G : graph
A networkx graph
pos : dictionary
A dictionary with nodes as keys and positions as values.
Positions should be sequences of length 2.
ax : Matplotlib Axes object, optional
Draw the graph in the specified Matplotlib axes.
nodelist : list, optional
Draw only specified nodes (default G.nodes())
node_size : scalar or array
Size of nodes (default=300). If an array is specified it must be the
same length as nodelist.
node_color : color string, or array of floats
Node color. Can be a single color format string (default='r'),
or a sequence of colors with the same length as nodelist.
If numeric values are specified they will be mapped to
colors using the cmap and vmin,vmax parameters. See
matplotlib.scatter for more details.
node_shape : string
The shape of the node. Specification is as matplotlib.scatter
marker, one of 'so^>v<dph8' (default='o').
alpha : float
The node transparency (default=1.0)
cmap : Matplotlib colormap
Colormap for mapping intensities of nodes (default=None)
vmin,vmax : floats
Minimum and maximum for node colormap scaling (default=None)
linewidths : [None | scalar | sequence]
Line width of symbol border (default =1.0)
label : [None| string]
Label for legend
Returns
-------
matplotlib.collections.PathCollection
`PathCollection` of the nodes.
Examples
--------
>>> G=nx.dodecahedral_graph()
>>> nodes=nx.draw_networkx_nodes(G,pos=nx.spring_layout(G))
Also see the NetworkX drawing examples at
http://networkx.github.io/docuementation/latest/gallery.html
See Also
--------
draw()
draw_networkx()
draw_networkx_edges()
draw_networkx_labels()
draw_networkx_edge_labels()
"""
import collections
try:
import matplotlib.pyplot as plt
import numpy
except ImportError:
raise ImportError("Matplotlib required for draw()")
except RuntimeError:
print("Matplotlib unable to open display")
raise
if ax is None:
ax = plt.gca()
if nodelist is None:
nodelist = list(G)
if not nodelist or len(nodelist) == 0: # empty nodelist, no drawing
return None
try:
xy = numpy.asarray([pos[v] for v in nodelist])
except KeyError as e:
raise nx.NetworkXError('Node %s has no position.'%e)
except ValueError:
raise nx.NetworkXError('Bad value in node positions.')
if isinstance(alpha, collections.Iterable):
node_color = apply_alpha(node_color, alpha, nodelist, cmap, vmin, vmax)
alpha = None
node_collection = ax.scatter(xy[:, 0], xy[:, 1],
s=node_size,
c=node_color,
marker=node_shape,
cmap=cmap,
vmin=vmin,
vmax=vmax,
alpha=alpha,
linewidths=linewidths,
label=label)
node_collection.set_zorder(2)
return node_collection
0
Example 132
def main():
"""
.. todo::
WRITEME
"""
parser = argparse.ArgumentParser()
parser.add_argument("--out")
parser.add_argument("model_paths", nargs='+')
parser.add_argument("--yrange", help='The y-range to be used for plotting, e.g. 0:1')
options = parser.parse_args()
model_paths = options.model_paths
if options.out is not None:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
print('generating names...')
model_names = [model_path.replace('.pkl', '!') for model_path in
model_paths]
model_names = unique_substrings(model_names, min_size=10)
model_names = [model_name.replace('!','') for model_name in
model_names]
print('...done')
for i, arg in enumerate(model_paths):
try:
model = serial.load(arg)
except Exception:
if arg.endswith('.yaml'):
print(sys.stderr, arg + " is a yaml config file," +
"you need to load a trained model.", file=sys.stderr)
quit(-1)
raise
this_model_channels = model.monitor.channels
if len(sys.argv) > 2:
postfix = ":" + model_names[i]
else:
postfix = ""
for channel in this_model_channels:
channels[channel+postfix] = this_model_channels[channel]
del model
gc.collect()
while True:
# Make a list of short codes for each channel so user can specify them
# easily
tag_generator = _TagGenerator()
codebook = {}
sorted_codes = []
for channel_name in sorted(channels,
key = number_aware_alphabetical_key):
code = tag_generator.get_tag()
codebook[code] = channel_name
codebook['<'+channel_name+'>'] = channel_name
sorted_codes.append(code)
x_axis = 'example'
print('set x_axis to example')
if len(channels.values()) == 0:
print("there are no channels to plot")
break
# If there is more than one channel in the monitor ask which ones to
# plot
prompt = len(channels.values()) > 1
if prompt:
# Display the codebook
for code in sorted_codes:
print(code + '. ' + codebook[code])
print()
print("Put e, b, s or h in the list somewhere to plot " +
"epochs, batches, seconds, or hours, respectively.")
response = input('Enter a list of channels to plot ' + \
'(example: A, C,F-G, h, <test_err>) or q to quit' + \
' or o for options: ')
if response == 'o':
print('1: smooth all channels')
print('any other response: do nothing, go back to plotting')
response = input('Enter your choice: ')
if response == '1':
for channel in channels.values():
k = 5
new_val_record = []
for i in xrange(len(channel.val_record)):
new_val = 0.
count = 0.
for j in xrange(max(0, i-k), i+1):
new_val += channel.val_record[j]
count += 1.
new_val_record.append(new_val / count)
channel.val_record = new_val_record
continue
if response == 'q':
break
#Remove spaces
response = response.replace(' ','')
#Split into list
codes = response.split(',')
final_codes = set([])
for code in codes:
if code == 'e':
x_axis = 'epoch'
continue
elif code == 'b':
x_axis = 'batche'
elif code == 's':
x_axis = 'second'
elif code == 'h':
x_axis = 'hour'
elif code.startswith('<'):
assert code.endswith('>')
final_codes.add(code)
elif code.find('-') != -1:
#The current list element is a range of codes
rng = code.split('-')
if len(rng) != 2:
print("Input not understood: "+code)
quit(-1)
found = False
for i in xrange(len(sorted_codes)):
if sorted_codes[i] == rng[0]:
found = True
break
if not found:
print("Invalid code: "+rng[0])
quit(-1)
found = False
for j in xrange(i,len(sorted_codes)):
if sorted_codes[j] == rng[1]:
found = True
break
if not found:
print("Invalid code: "+rng[1])
quit(-1)
final_codes = final_codes.union(set(sorted_codes[i:j+1]))
else:
#The current list element is just a single code
final_codes = final_codes.union(set([code]))
# end for code in codes
else:
final_codes ,= set(codebook.keys())
colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
styles = list(colors)
styles += [color+'--' for color in colors]
styles += [color+':' for color in colors]
fig = plt.figure()
ax = plt.subplot(1,1,1)
# plot the requested channels
for idx, code in enumerate(sorted(final_codes)):
channel_name= codebook[code]
channel = channels[channel_name]
y = np.asarray(channel.val_record)
if contains_nan(y):
print(channel_name + ' contains NaNs')
if contains_inf(y):
print(channel_name + 'contains infinite values')
if x_axis == 'example':
x = np.asarray(channel.example_record)
elif x_axis == 'batche':
x = np.asarray(channel.batch_record)
elif x_axis == 'epoch':
try:
x = np.asarray(channel.epoch_record)
except AttributeError:
# older saved monitors won't have epoch_record
x = np.arange(len(channel.batch_record))
elif x_axis == 'second':
x = np.asarray(channel.time_record)
elif x_axis == 'hour':
x = np.asarray(channel.time_record) / 3600.
else:
assert False
ax.plot( x,
y,
styles[idx % len(styles)],
marker = '.', # add point margers to lines
label = channel_name)
plt.xlabel('# '+x_axis+'s')
ax.ticklabel_format( scilimits = (-3,3), axis = 'both')
handles, labels = ax.get_legend_handles_labels()
lgd = ax.legend(handles, labels, loc = 'upper left',
bbox_to_anchor = (1.05, 1.02))
# Get the axis positions and the height and width of the legend
plt.draw()
ax_pos = ax.get_position()
pad_width = ax_pos.x0 * fig.get_size_inches()[0]
pad_height = ax_pos.y0 * fig.get_size_inches()[1]
dpi = fig.get_dpi()
lgd_width = ax.get_legend().get_frame().get_width() / dpi
lgd_height = ax.get_legend().get_frame().get_height() / dpi
# Adjust the bounding box to encompass both legend and axis. Axis should be 3x3 inches.
# I had trouble getting everything to align vertically.
ax_width = 3
ax_height = 3
total_width = 2*pad_width + ax_width + lgd_width
total_height = 2*pad_height + np.maximum(ax_height, lgd_height)
fig.set_size_inches(total_width, total_height)
ax.set_position([pad_width/total_width, 1-6*pad_height/total_height, ax_width/total_width, ax_height/total_height])
if(options.yrange is not None):
ymin, ymax = map(float, options.yrange.split(':'))
plt.ylim(ymin, ymax)
if options.out is None:
plt.show()
else:
plt.savefig(options.out)
if not prompt:
break
0
Example 133
Project: elephant Source File: spectral.py
def welch_cohere(x, y, num_seg=8, len_seg=None, freq_res=None, overlap=0.5,
fs=1.0, window='hanning', nfft=None, detrend='constant',
scaling='density', axis=-1):
"""
Estimates coherence between a given pair of analog signals. The estimation
is performed with Welch's method: the given pair of data are cut into short
segments, cross-spectra are calculated for each pair of segments, and the
cross-spectra are averaged and normalized by respective auto_spectra. By
default the data are cut into 8 segments with 50% overlap between
neighboring segments. These numbers can be changed through respective
parameters.
Parameters
----------
x, y: Neo AnalogSignalArray or Quantity array or Numpy ndarray
A pair of time series data, between which coherence is computed. The
shapes and the sampling frequencies of `x` and `y` must be identical.
When `x` and `y` are not of AnalogSignalArray, sampling frequency
should be specified through the keyword argument `fs`, otherwise the
default value (`fs=1.0`) is used.
num_seg: int, optional
Number of segments. The length of segments is adjusted so that
overlapping segments cover the entire stretch of the given data. This
parameter is ignored if *len_seg* or *freq_res* is given. Default is 8.
len_seg: int, optional
Length of segments. This parameter is ignored if *freq_res* is given.
Default is None (determined from other parameters).
freq_res: Quantity or float, optional
Desired frequency resolution of the obtained coherence estimate in
terms of the interval between adjacent frequency bins. When given as a
float, it is taken as frequency in Hz. Default is None (determined from
other parameters).
overlap: float, optional
Overlap between segments represented as a float number between 0 (no
overlap) and 1 (complete overlap). Default is 0.5 (half-overlapped).
fs: Quantity array or float, optional
Specifies the sampling frequency of the input time series. When the
input time series are given as AnalogSignalArray, the sampling
frequency is taken from their attribute and this parameter is ignored.
Default is 1.0.
window, nfft, detrend, scaling, axis: optional
These arguments are directly passed on to a helper function
`elephant.spectral._welch()`. See the respective descriptions in the
docstring of `elephant.spectral._welch()` for usage.
Returns
-------
freqs: Quantity array or Numpy ndarray
Frequencies associated with the estimates of coherency and phase lag.
`freqs` is always a 1-dimensional array irrespective of the shape of
the input data. Quantity array is returned if `x` and `y` are of
AnalogSignalArray or Quantity array. Otherwise Numpy ndarray containing
frequency in Hz is returned.
coherency: Numpy ndarray
Estimate of coherency between the input time series. For each frequency
coherency takes a value between 0 and 1, with 0 or 1 representing no or
perfect coherence, respectively. When the input arrays `x` and `y` are
multi-dimensional, `coherency` is of the same shape as the inputs and
frequency is indexed along either the first or the last axis depending
on the type of the input: when the input is AnalogSignalArray, the
first axis indexes frequency, otherwise the last axis does.
phase_lag: Quantity array or Numpy ndarray
Estimate of phase lag in radian between the input time series. For each
frequency phase lag takes a value between -PI and PI, positive values
meaning phase precession of `x` ahead of `y` and vice versa. Quantity
array is returned if `x` and `y` are of AnalogSignalArray or Quantity
array. Otherwise Numpy ndarray containing phase lag in radian is
returned. The axis for frequency index is determined in the same way as
for `coherency`.
"""
# initialize a parameter dict (to be given to _welch()) with
# the parameters directly passed on to _welch()
params = {'window': window, 'nfft': nfft,
'detrend': detrend, 'scaling': scaling, 'axis': axis}
# When the input is AnalogSignalArray, the axis for time index is rolled to
# the last
xdata = np.asarray(x)
ydata = np.asarray(y)
if isinstance(x, neo.AnalogSignalArray):
xdata = np.rollaxis(xdata, 0, len(xdata.shape))
ydata = np.rollaxis(ydata, 0, len(ydata.shape))
# if the data is given as AnalogSignalArray, use its attribute to specify
# the sampling frequency
if hasattr(x, 'sampling_rate'):
params['fs'] = x.sampling_rate.rescale('Hz').magnitude
else:
params['fs'] = fs
if overlap < 0:
raise ValueError("overlap must be greater than or equal to 0")
elif 1 <= overlap:
raise ValueError("overlap must be less then 1")
# determine the length of segments (i.e. *nperseg*) according to given
# parameters
if freq_res is not None:
if freq_res <= 0:
raise ValueError("freq_res must be positive")
dF = freq_res.rescale('Hz').magnitude \
if isinstance(freq_res, pq.quantity.Quantity) else freq_res
nperseg = int(params['fs'] / dF)
if nperseg > xdata.shape[axis]:
raise ValueError("freq_res is too high for the given data size")
elif len_seg is not None:
if len_seg <= 0:
raise ValueError("len_seg must be a positive number")
elif xdata.shape[axis] < len_seg:
raise ValueError("len_seg must be shorter than the data length")
nperseg = len_seg
else:
if num_seg <= 0:
raise ValueError("num_seg must be a positive number")
elif xdata.shape[axis] < num_seg:
raise ValueError("num_seg must be smaller than the data length")
# when only *num_seg* is given, *nperseg* is determined by solving the
# following equation:
# num_seg * nperseg - (num_seg-1) * overlap * nperseg = data.shape[-1]
# ----------------- =============================== ^^^^^^^^^^^
# summed segment lengths total overlap data length
nperseg = int(xdata.shape[axis] / (num_seg - overlap * (num_seg - 1)))
params['nperseg'] = nperseg
params['noverlap'] = int(nperseg * overlap)
freqs, Pxy = _welch(xdata, ydata, **params)
freqs, Pxx = _welch(xdata, xdata, **params)
freqs, Pyy = _welch(ydata, ydata, **params)
coherency = np.abs(Pxy)**2 / (np.abs(Pxx) * np.abs(Pyy))
phase_lag = np.angle(Pxy)
# attach proper units to return values
if isinstance(x, pq.quantity.Quantity):
freqs = freqs * pq.Hz
phase_lag = phase_lag * pq.rad
# When the input is AnalogSignalArray, the axis for frequency index is
# rolled to the first to comply with the Neo convention about time axis
if isinstance(x, neo.AnalogSignalArray):
coherency = np.rollaxis(coherency, -1)
phase_lag = np.rollaxis(phase_lag, -1)
return freqs, coherency, phase_lag
0
Example 134
def draw_networkx_edges(G, pos,
edgelist=None,
width=1.0,
edge_color='k',
style='solid',
alpha=None,
edge_cmap=None,
edge_vmin=None,
edge_vmax=None,
ax=None,
arrows=True,
label=None,
**kwds):
"""Draw the edges of the graph G.
This draws only the edges of the graph G.
Parameters
----------
G : graph
A networkx graph
pos : dictionary
A dictionary with nodes as keys and positions as values.
If not specified a spring layout positioning will be computed.
See networkx.layout for functions that compute node positions.
edgelist : collection of edge tuples
Draw only specified edges(default=G.edges())
width : float
Line width of edges (default =1.0)
edge_color : color string, or array of floats
Edge color. Can be a single color format string (default='r'),
or a sequence of colors with the same length as edgelist.
If numeric values are specified they will be mapped to
colors using the edge_cmap and edge_vmin,edge_vmax parameters.
style : string
Edge line style (default='solid') (solid|dashed|dotted,dashdot)
alpha : float
The edge transparency (default=1.0)
edge_ cmap : Matplotlib colormap
Colormap for mapping intensities of edges (default=None)
edge_vmin,edge_vmax : floats
Minimum and maximum for edge colormap scaling (default=None)
ax : Matplotlib Axes object, optional
Draw the graph in the specified Matplotlib axes.
arrows : bool, optional (default=True)
For directed graphs, if True draw arrowheads.
label : [None| string]
Label for legend
Notes
-----
For directed graphs, "arrows" (actually just thicker stubs) are drawn
at the head end. Arrows can be turned off with keyword arrows=False.
Yes, it is ugly but drawing proper arrows with Matplotlib this
way is tricky.
Examples
--------
>>> G=nx.dodecahedral_graph()
>>> edges=nx.draw_networkx_edges(G,pos=nx.spring_layout(G))
Also see the NetworkX drawing examples at
http://networkx.lanl.gov/gallery.html
See Also
--------
draw()
draw_networkx()
draw_networkx_nodes()
draw_networkx_labels()
draw_networkx_edge_labels()
"""
try:
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.cbook as cb
from matplotlib.colors import colorConverter,Colormap
from matplotlib.collections import LineCollection
import numpy
except ImportError:
raise ImportError("Matplotlib required for draw()")
except RuntimeError:
print("Matplotlib unable to open display")
raise
if ax is None:
ax=plt.gca()
if edgelist is None:
edgelist=G.edges()
if not edgelist or len(edgelist)==0: # no edges!
return None
# set edge positions
edge_pos=numpy.asarray([(pos[e[0]],pos[e[1]]) for e in edgelist])
if not cb.iterable(width):
lw = (width,)
else:
lw = width
if not cb.is_string_like(edge_color) \
and cb.iterable(edge_color) \
and len(edge_color)==len(edge_pos):
if numpy.alltrue([cb.is_string_like(c)
for c in edge_color]):
# (should check ALL elements)
# list of color letters such as ['k','r','k',...]
edge_colors = tuple([colorConverter.to_rgba(c,alpha)
for c in edge_color])
elif numpy.alltrue([not cb.is_string_like(c)
for c in edge_color]):
# If color specs are given as (rgb) or (rgba) tuples, we're OK
if numpy.alltrue([cb.iterable(c) and len(c) in (3,4)
for c in edge_color]):
edge_colors = tuple(edge_color)
else:
# numbers (which are going to be mapped with a colormap)
edge_colors = None
else:
raise ValueError('edge_color must consist of either color names or numbers')
else:
if cb.is_string_like(edge_color) or len(edge_color)==1:
edge_colors = ( colorConverter.to_rgba(edge_color, alpha), )
else:
raise ValueError('edge_color must be a single color or list of exactly m colors where m is the number or edges')
edge_collection = LineCollection(edge_pos,
colors = edge_colors,
linewidths = lw,
antialiaseds = (1,),
linestyle = style,
transOffset = ax.transData,
)
edge_collection.set_zorder(1) # edges go behind nodes
edge_collection.set_label(label)
ax.add_collection(edge_collection)
# Note: there was a bug in mpl regarding the handling of alpha values for
# each line in a LineCollection. It was fixed in matplotlib in r7184 and
# r7189 (June 6 2009). We should then not set the alpha value globally,
# since the user can instead provide per-edge alphas now. Only set it
# globally if provided as a scalar.
if cb.is_numlike(alpha):
edge_collection.set_alpha(alpha)
if edge_colors is None:
if edge_cmap is not None:
assert(isinstance(edge_cmap, Colormap))
edge_collection.set_array(numpy.asarray(edge_color))
edge_collection.set_cmap(edge_cmap)
if edge_vmin is not None or edge_vmax is not None:
edge_collection.set_clim(edge_vmin, edge_vmax)
else:
edge_collection.autoscale()
arrow_collection=None
if G.is_directed() and arrows:
# a directed graph hack
# draw thick line segments at head end of edge
# waiting for someone else to implement arrows that will work
arrow_colors = edge_colors
a_pos=[]
p=1.0-0.25 # make head segment 25 percent of edge length
for src,dst in edge_pos:
x1,y1=src
x2,y2=dst
dx=x2-x1 # x offset
dy=y2-y1 # y offset
d=numpy.sqrt(float(dx**2+dy**2)) # length of edge
if d==0: # source and target at same position
continue
if dx==0: # vertical edge
xa=x2
ya=dy*p+y1
if dy==0: # horizontal edge
ya=y2
xa=dx*p+x1
else:
theta=numpy.arctan2(dy,dx)
xa=p*d*numpy.cos(theta)+x1
ya=p*d*numpy.sin(theta)+y1
a_pos.append(((xa,ya),(x2,y2)))
arrow_collection = LineCollection(a_pos,
colors = arrow_colors,
linewidths = [4*ww for ww in lw],
antialiaseds = (1,),
transOffset = ax.transData,
)
arrow_collection.set_zorder(1) # edges go behind nodes
arrow_collection.set_label(label)
ax.add_collection(arrow_collection)
# update view
minx = numpy.amin(numpy.ravel(edge_pos[:,:,0]))
maxx = numpy.amax(numpy.ravel(edge_pos[:,:,0]))
miny = numpy.amin(numpy.ravel(edge_pos[:,:,1]))
maxy = numpy.amax(numpy.ravel(edge_pos[:,:,1]))
w = maxx-minx
h = maxy-miny
padx, pady = 0.05*w, 0.05*h
corners = (minx-padx, miny-pady), (maxx+padx, maxy+pady)
ax.update_datalim( corners)
ax.autoscale_view()
# if arrow_collection:
return edge_collection
0
Example 135
def register_images(fname_source, fname_dest, mask='', paramreg=Paramreg(step='0', type='im', algo='Translation', metric='MI', iter='5', shrink='1', smooth='0', gradStep='0.5'),
ants_registration_params={'rigid': '', 'affine': '', 'compositeaffine': '', 'similarity': '', 'translation': '','bspline': ',10', 'gaussiandisplacementfield': ',3,0',
'bsplinedisplacementfield': ',5,10', 'syn': ',3,0', 'bsplinesyn': ',1,3'}, remove_tmp_folder = 1):
"""Slice-by-slice registration of two images.
We first split the 3D images into 2D images (and the mask if inputted). Then we register slices of the two images
that physically correspond to one another looking at the physical origin of each image. The images can be of
different sizes but the destination image must be smaller thant the input image. We do that using antsRegistration
in 2D. Once this has been done for each slices, we gather the results and return them.
Algorithms implemented: translation, rigid, affine, syn and BsplineSyn.
N.B.: If the mask is inputted, it must also be 3D and it must be in the same space as the destination image.
input:
fname_source: name of moving image (type: string)
fname_dest: name of fixed image (type: string)
mask[optional]: name of mask file (type: string) (parameter -x of antsRegistration)
paramreg[optional]: parameters of antsRegistration (type: Paramreg class from sct_register_multimodal)
ants_registration_params[optional]: specific algorithm's parameters for antsRegistration (type: dictionary)
output:
if algo==translation:
x_displacement: list of translation along x axis for each slice (type: list)
y_displacement: list of translation along y axis for each slice (type: list)
if algo==rigid:
x_displacement: list of translation along x axis for each slice (type: list)
y_displacement: list of translation along y axis for each slice (type: list)
theta_rotation: list of rotation angle in radian (and in ITK's coordinate system) for each slice (type: list)
if algo==affine or algo==syn or algo==bsplinesyn:
creation of two 3D warping fields (forward and inverse) that are the concatenations of the slice-by-slice
warps.
"""
# Extracting names
path_i, root_i, ext_i = sct.extract_fname(fname_source)
path_d, root_d, ext_d = sct.extract_fname(fname_dest)
# set metricSize
if paramreg.metric == 'MI':
metricSize = '32' # corresponds to number of bins
else:
metricSize = '4' # corresponds to radius (for CC, MeanSquares...)
# Get image dimensions and retrieve nz
print '\nGet image dimensions of destination image...'
nx, ny, nz, nt, px, py, pz, pt = Image(fname_dest).dim
print '.. matrix size: '+str(nx)+' x '+str(ny)+' x '+str(nz)
print '.. voxel size: '+str(px)+'mm x '+str(py)+'mm x '+str(pz)+'mm'
# Define x and y displacement as list
x_displacement = [0 for i in range(nz)]
y_displacement = [0 for i in range(nz)]
theta_rotation = [0 for i in range(nz)]
# create temporary folder
print('\nCreate temporary folder...')
path_tmp = 'tmp.'+time.strftime("%y%m%d%H%M%S")
sct.create_folder(path_tmp)
print '\nCopy input data...'
sct.run('cp '+fname_source+ ' ' + path_tmp +'/'+ root_i+ext_i)
sct.run('cp '+fname_dest+ ' ' + path_tmp +'/'+ root_d+ext_d)
if mask:
sct.run('cp '+mask+ ' '+path_tmp +'/mask.nii.gz')
# go to temporary folder
os.chdir(path_tmp)
# Split input volume along z
print '\nSplit input volume...'
from sct_split_data import split_data
split_data(fname_source, 2, '_z')
# Split destination volume along z
print '\nSplit destination volume...'
split_data(fname_dest, 2, '_z')
# Split mask volume along z
if mask:
print '\nSplit mask volume...'
split_data('mask.nii.gz', 2, '_z')
im_dest_img = Image(fname_dest)
im_input_img = Image(fname_source)
coord_origin_dest = im_dest_img.transfo_pix2phys([[0,0,0]])
coord_origin_input = im_input_img.transfo_pix2phys([[0,0,0]])
coord_diff_origin = (asarray(coord_origin_dest[0]) - asarray(coord_origin_input[0])).tolist()
[x_o, y_o, z_o] = [coord_diff_origin[0] * 1.0/px, coord_diff_origin[1] * 1.0/py, coord_diff_origin[2] * 1.0/pz]
if paramreg.algo == 'BSplineSyN' or paramreg.algo == 'SyN' or paramreg.algo == 'Affine':
list_warp_x = []
list_warp_x_inv = []
list_warp_y = []
list_warp_y_inv = []
name_warp_final = 'Warp_total_step_'+str(paramreg.step) #if modified, name should also be modified in msct_register (algo slicereg2d_bsplinesyn and slicereg2d_syn)
# loop across slices
for i in range(nz):
# set masking
num = numerotation(i)
num_2 = numerotation(int(num) + int(z_o))
if mask:
masking = '-x mask_z' +num+ '.nii'
else:
masking = ''
cmd = ('isct_antsRegistration '
'--dimensionality 2 '
'--transform '+paramreg.algo+'['+str(paramreg.gradStep) +
ants_registration_params[paramreg.algo.lower()]+'] '
'--metric '+paramreg.metric+'['+root_d+'_z'+ num +'.nii' +','+root_i+'_z'+ num_2 +'.nii' +',1,'+metricSize+'] ' #[fixedImage,movingImage,metricWeight +nb_of_bins (MI) or radius (other)
'--convergence '+str(paramreg.iter)+' '
'--shrink-factors '+str(paramreg.shrink)+' '
'--smoothing-sigmas '+str(paramreg.smooth)+'mm '
#'--restrict-deformation 1x1x0 ' # how to restrict? should not restrict here, if transform is precised...?
'--output [transform_' + num + ','+root_i+'_z'+ num_2 +'reg.nii] ' #--> file.mat (contains Tx,Ty, theta)
'--interpolation BSpline[3] '
+masking)
try:
sct.run(cmd)
if paramreg.algo == 'Rigid' or paramreg.algo == 'Translation':
f = 'transform_' +num+ '0GenericAffine.mat'
matfile = loadmat(f, struct_as_record=True)
array_transfo = matfile['AffineTransform_double_2_2']
x_displacement[i] = array_transfo[4][0] # Tx in ITK'S coordinate system
y_displacement[i] = array_transfo[5][0] # Ty in ITK'S and fslview's coordinate systems
theta_rotation[i] = asin(array_transfo[2]) # angle of rotation theta in ITK'S coordinate system (minus theta for fslview)
if paramreg.algo == 'Affine':
# New process added for generating total nifti warping field from mat warp
name_dest = root_d+'_z'+ num +'.nii'
name_reg = root_i+'_z'+ num +'reg.nii'
name_output_warp = 'warp_from_mat_' + num_2 + '.nii.gz'
name_output_warp_inverse = 'warp_from_mat_' + num + '_inverse.nii.gz'
name_warp_null = 'warp_null_' + num + '.nii.gz'
name_warp_null_dest = 'warp_null_dest' + num + '.nii.gz'
name_warp_mat = 'transform_' + num + '0GenericAffine.mat'
# Generating null nifti warping fields
nx, ny, nz, nt, px, py, pz, pt = Image(name_reg).dim
nx_d, ny_d, nz_d, nt_d, px_d, py_d, pz_d, pt_d = Image(name_dest).dim
x_trans = [0 for i in range(nz)]
x_trans_d = [0 for i in range(nz_d)]
y_trans= [0 for i in range(nz)]
y_trans_d = [0 for i in range(nz_d)]
generate_warping_field(name_reg, x_trans=x_trans, y_trans=y_trans, fname=name_warp_null, verbose=0)
generate_warping_field(name_dest, x_trans=x_trans_d, y_trans=y_trans_d, fname=name_warp_null_dest, verbose=0)
# Concatenating mat wrp and null nifti warp to obtain equivalent nifti warp to mat warp
sct.run('isct_ComposeMultiTransform 2 ' + name_output_warp + ' -R ' + name_reg + ' ' + name_warp_null + ' ' + name_warp_mat)
sct.run('isct_ComposeMultiTransform 2 ' + name_output_warp_inverse + ' -R ' + name_dest + ' ' + name_warp_null_dest + ' -i ' + name_warp_mat)
# Split the warping fields into two for displacement along x and y before merge
# sct.run('isct_c3d -mcs ' + name_output_warp + ' -oo transform_'+num+'0Warp_x.nii.gz transform_'+num+'0Warp_y.nii.gz')
# sct.run('isct_c3d -mcs ' + name_output_warp_inverse + ' -oo transform_'+num+'0InverseWarp_x.nii.gz transform_'+num+'0InverseWarp_y.nii.gz')
sct.run('sct_maths -i ' + name_output_warp + ' -w -mcs -o transform_'+num+'0Warp_x.nii.gz,transform_'+num+'0Warp_y.nii.gz')
sct.run('sct_maths -i ' + name_output_warp_inverse + ' -w -mcs -o transform_'+num+'0InverseWarp_x.nii.gz,transform_'+num+'0InverseWarp_y.nii.gz')
# List names of warping fields for futur merge
list_warp_x.append('transform_'+num+'0Warp_x.nii.gz')
list_warp_x_inv.append('transform_'+num+'0InverseWarp_x.nii.gz')
list_warp_y.append('transform_'+num+'0Warp_y.nii.gz')
list_warp_y_inv.append('transform_'+num+'0InverseWarp_y.nii.gz')
if paramreg.algo == 'BSplineSyN' or paramreg.algo == 'SyN':
# Split the warping fields into two for displacement along x and y before merge
# Need to separate the merge for x and y displacement as merge of 3d warping fields does not work properly
# sct.run('isct_c3d -mcs transform_'+num+'0Warp.nii.gz -oo transform_'+num+'0Warp_x.nii.gz transform_'+num+'0Warp_y.nii.gz')
# sct.run('isct_c3d -mcs transform_'+num+'0InverseWarp.nii.gz -oo transform_'+num+'0InverseWarp_x.nii.gz transform_'+num+'0InverseWarp_y.nii.gz')
sct.run('sct_maths -i transform_'+num+'0Warp.nii.gz -w -mcs -o transform_'+num+'0Warp_x.nii.gz,transform_'+num+'0Warp_y.nii.gz')
sct.run('sct_maths -i transform_'+num+'0InverseWarp.nii.gz -w -mcs -o transform_'+num+'0InverseWarp_x.nii.gz,transform_'+num+'0InverseWarp_y.nii.gz')
# List names of warping fields for futur merge
list_warp_x.append('transform_'+num+'0Warp_x.nii.gz')
list_warp_x_inv.append('transform_'+num+'0InverseWarp_x.nii.gz')
list_warp_y.append('transform_'+num+'0Warp_y.nii.gz')
list_warp_y_inv.append('transform_'+num+'0InverseWarp_y.nii.gz')
# if an exception occurs with ants, take the last value for the transformation
except:
if paramreg.algo == 'Rigid' or paramreg.algo == 'Translation':
x_displacement[i] = x_displacement[i-1]
y_displacement[i] = y_displacement[i-1]
theta_rotation[i] = theta_rotation[i-1]
if paramreg.algo == 'BSplineSyN' or paramreg.algo == 'SyN' or paramreg.algo == 'Affine':
print'Problem with ants for slice '+str(i)+'. Copy of the last warping field.'
sct.run('cp transform_' + numerotation(i-1) + '0Warp.nii.gz transform_' + num + '0Warp.nii.gz')
sct.run('cp transform_' + numerotation(i-1) + '0InverseWarp.nii.gz transform_' + num + '0InverseWarp.nii.gz')
# Split the warping fields into two for displacement along x and y before merge
# sct.run('isct_c3d -mcs transform_'+num+'0Warp.nii.gz -oo transform_'+num+'0Warp_x.nii.gz transform_'+num+'0Warp_y.nii.gz')
# sct.run('isct_c3d -mcs transform_'+num+'0InverseWarp.nii.gz -oo transform_'+num+'0InverseWarp_x.nii.gz transform_'+num+'0InverseWarp_y.nii.gz')
sct.run('sct_maths -i transform_'+num+'0Warp.nii.gz -w -mcs -o transform_'+num+'0Warp_x.nii.gz,transform_'+num+'0Warp_y.nii.gz')
sct.run('sct_maths -i transform_'+num+'0InverseWarp.nii.gz -w -mcs -o transform_'+num+'0InverseWarp_x.nii.gz,transform_'+num+'0InverseWarp_y.nii.gz')
# List names of warping fields for futur merge
list_warp_x.append('transform_'+num+'0Warp_x.nii.gz')
list_warp_x_inv.append('transform_'+num+'0InverseWarp_x.nii.gz')
list_warp_y.append('transform_'+num+'0Warp_y.nii.gz')
list_warp_y_inv.append('transform_'+num+'0InverseWarp_y.nii.gz')
if paramreg.algo == 'BSplineSyN' or paramreg.algo == 'SyN' or paramreg.algo == 'Affine':
print'\nMerge along z of the warping fields...'
# from sct_concat_data import concat_data
sct.run('sct_concat_data -i '+','.join(list_warp_x)+' -o '+name_warp_final+'_x.nii.gz -dim z')
sct.run('sct_concat_data -i '+','.join(list_warp_x_inv)+' -o '+name_warp_final+'_x_inverse.nii.gz -dim z')
sct.run('sct_concat_data -i '+','.join(list_warp_y)+' -o '+name_warp_final+'_y.nii.gz -dim z')
sct.run('sct_concat_data -i '+','.join(list_warp_y_inv)+' -o '+name_warp_final+'_y_inverse.nii.gz -dim z')
# concat_data(','.join(list_warp_x), name_warp_final+'_x.nii.gz', 2)
# concat_data(','.join(list_warp_x_inv), name_warp_final+'_x_inverse.nii.gz', 2)
# concat_data(','.join(list_warp_y), name_warp_final+'_y.nii.gz', 2)
# concat_data(','.join(list_warp_y_inv), name_warp_final+'_y_inverse.nii.gz', 2)
# sct.run('fslmerge -z ' + name_warp_final + '_x ' + " ".join(list_warp_x))
# sct.run('fslmerge -z ' + name_warp_final + '_x_inverse ' + " ".join(list_warp_x_inv))
# sct.run('fslmerge -z ' + name_warp_final + '_y ' + " ".join(list_warp_y))
# sct.run('fslmerge -z ' + name_warp_final + '_y_inverse ' + " ".join(list_warp_y_inv))
print'\nChange resolution of warping fields to match the resolution of the destination image...'
from sct_copy_header import copy_header
copy_header(fname_dest, name_warp_final + '_x.nii.gz')
copy_header(fname_source, name_warp_final + '_x_inverse.nii.gz')
copy_header(fname_dest, name_warp_final + '_y.nii.gz')
copy_header(fname_source, name_warp_final + '_y_inverse.nii.gz')
# nx_d, ny_d, nz_d, nt, px, py, pz, pt = Image(fname_dest).dim
# nx_s, ny_s, nz_s, nt, px, py, pz, pt = Image(fname_source).dim
# sct.run('sct_resample -i '+name_warp_final + '_x.nii.gz -x spline -vox '+str(nx_d)+'x'+str(ny_d)+'x'+str(nz_d)+' -o '+name_warp_final + '_x.nii.gz')
# sct.run('sct_resample -i '+name_warp_final + '_x_inverse.nii.gz -x spline -vox '+str(nx_s)+'x'+str(ny_s)+'x'+str(nz_s)+' -o '+name_warp_final + '_x_inverse.nii.gz')
# sct.run('sct_resample -i '+name_warp_final + '_y.nii.gz -x spline -vox '+str(nx_d)+'x'+str(ny_d)+'x'+str(nz_d)+' -o '+name_warp_final + '_y.nii.gz')
# sct.run('sct_resample -i '+name_warp_final + '_y_inverse.nii.gz -x spline -vox '+str(nx_s)+'x'+str(ny_s)+'x'+str(nz_s)+' -o '+name_warp_final + '_y_inverse.nii.gz')
print'\nMerge translation fields along x and y into one global warping field '
print 'cuem*****************************************************************************************************************\n'
print 'name_warp _x and _y', name_warp_final
print '*********************************************************************************************************************\n'
nx, ny, nz, nt, px, py, pz, pt = Image(name_warp_final + '_x.nii.gz').dim
Image(param=zeros((nx, ny, nz)), absolutepath='warp_null_z.nii.gz').save()
print 'SCT_MATHS -OMC'
sct.run('sct_maths -i ' + name_warp_final + '_x.nii.gz,' + name_warp_final + '_y.nii.gz,warp_null_z.nii.gz -w -omc -o ' + name_warp_final + '.nii.gz')
sct.run('sct_maths -i ' + name_warp_final + '_x_inverse.nii.gz,' + name_warp_final + '_y_inverse.nii.gz,warp_null_z.nii.gz -w -omc -o ' + name_warp_final + '_inverse.nii.gz')
# print 'C3D'
# sct.run('isct_c3d ' + name_warp_final + '_x.nii.gz ' + name_warp_final + '_y.nii.gz -omc 2 ' + name_warp_final + '.nii.gz')
# sct.run('isct_c3d ' + name_warp_final + '_x_inverse.nii.gz ' + name_warp_final + '_y_inverse.nii.gz -omc 2 ' + name_warp_final + '_inverse.nii.gz')
print'\nCopy to parent folder...'
sct.run('cp ' + name_warp_final + '.nii.gz ../')
sct.run('cp ' + name_warp_final + '_inverse.nii.gz ../')
#Delete tmp folder
os.chdir('../')
if remove_tmp_folder:
print('\nRemove temporary files...')
sct.run('rm -rf '+path_tmp)
if paramreg.algo == 'Rigid':
return x_displacement, y_displacement, theta_rotation
if paramreg.algo == 'Translation':
return x_displacement, y_displacement
0
Example 136
Project: faststats Source File: fastkde.py
def fastkde(x, y, gridsize=(200, 200), extents=None, nocorrelation=False, weights=None, adjust=1.):
"""
A fft-based Gaussian kernel density estimate (KDE)
for computing the KDE on a regular grid
Note that this is a different use case than scipy's original
scipy.stats.kde.gaussian_kde
IMPLEMENTATION
--------------
Performs a gaussian kernel density estimate over a regular grid using a
convolution of the gaussian kernel with a 2D histogram of the data.
It computes the sparse bi-dimensional histogram of two data samples where
*x*, and *y* are 1-D sequences of the same length. If *weights* is None
(default), this is a histogram of the number of occurences of the
observations at (x[i], y[i]).
histogram of the data is a faster implementation than numpy.histogram as it
avoids intermediate copies and excessive memory usage!
This function is typically *several orders of magnitude faster* than
scipy.stats.kde.gaussian_kde. For large (>1e7) numbers of points, it
produces an essentially identical result.
Boundary conditions on the data is corrected by using a symmetric /
reflection condition. Hence the limits of the dataset does not affect the
pdf estimate.
INPUTS
------
x, y: ndarray[ndim=1]
The x-coords, y-coords of the input data points respectively
gridsize: tuple
A (nx,ny) tuple of the size of the output grid (default: 200x200)
extents: (xmin, xmax, ymin, ymax) tuple
tuple of the extents of output grid (default: extent of input data)
nocorrelation: bool
If True, the correlation between the x and y coords will be ignored
when preforming the KDE. (default: False)
weights: ndarray[ndim=1]
An array of the same shape as x & y that weights each sample (x_i,
y_i) by each value in weights (w_i). Defaults to an array of ones
the same size as x & y. (default: None)
adjust : float
An adjustment factor for the bw. Bandwidth becomes bw * adjust.
OUTPUTS
-------
g: ndarray[ndim=2]
A gridded 2D kernel density estimate of the input points.
e: (xmin, xmax, ymin, ymax) tuple
Extents of g
"""
# Variable check
x, y = np.asarray(x), np.asarray(y)
x, y = np.squeeze(x), np.squeeze(y)
if x.size != y.size:
raise ValueError('Input x & y arrays must be the same size!')
n = x.size
if weights is None:
# Default: Weight all points equally
weights = np.ones(n)
else:
weights = np.squeeze(np.asarray(weights))
if weights.size != x.size:
raise ValueError('Input weights must be an array of the same size as input x & y arrays!')
# Optimize gridsize ------------------------------------------------------
#Make grid and discretize the data and round it to the next power of 2
# to optimize with the fft usage
if gridsize is None:
gridsize = np.asarray([np.max((len(x), 512.)), np.max((len(y), 512.))])
gridsize = 2 ** np.ceil(np.log2(gridsize)) # round to next power of 2
nx, ny = gridsize
# Make the sparse 2d-histogram -------------------------------------------
# Default extents are the extent of the data
if extents is None:
xmin, xmax = x.min(), x.max()
ymin, ymax = y.min(), y.max()
else:
xmin, xmax, ymin, ymax = map(float, extents)
dx = (xmax - xmin) / (nx - 1)
dy = (ymax - ymin) / (ny - 1)
# Basically, this is just doing what np.digitize does with one less copy
# xyi contains the bins of each point as a 2d array [(xi,yi)]
xyi = np.vstack((x,y)).T
xyi -= [xmin, ymin]
xyi /= [dx, dy]
xyi = np.floor(xyi, xyi).T
# Next, make a 2D histogram of x & y.
# Exploit a sparse coo_matrix avoiding np.histogram2d due to excessive
# memory usage with many points
grid = coo_matrix((weights, xyi), shape=(nx, ny)).toarray()
# Kernel Preliminary Calculations ---------------------------------------
# Calculate the covariance matrix (in pixel coords)
cov = np.cov(xyi)
if nocorrelation:
cov[1,0] = 0
cov[0,1] = 0
# Scaling factor for bandwidth
scotts_factor = n ** (-1.0 / 6.) * adjust # For 2D
# Make the gaussian kernel ---------------------------------------------
# First, determine the bandwidth using Scott's rule
# (note that Silvermann's rule gives the # same value for 2d datasets)
std_devs = np.diag(np.sqrt(cov))
kern_nx, kern_ny = np.round(scotts_factor * 2 * np.pi * std_devs)
# Determine the bandwidth to use for the gaussian kernel
inv_cov = np.linalg.inv(cov * scotts_factor ** 2)
# x & y (pixel) coords of the kernel grid, with <x,y> = <0,0> in center
xx = np.arange(kern_nx, dtype=np.float) - kern_nx / 2.0
yy = np.arange(kern_ny, dtype=np.float) - kern_ny / 2.0
xx, yy = np.meshgrid(xx, yy)
# Then evaluate the gaussian function on the kernel grid
kernel = np.vstack((xx.flatten(), yy.flatten()))
kernel = np.dot(inv_cov, kernel) * kernel
kernel = np.sum(kernel, axis=0) / 2.0
kernel = np.exp(-kernel)
kernel = kernel.reshape((kern_ny, kern_nx))
#---- Produce the kernel density estimate --------------------------------
# Convolve the histogram with the gaussian kernel
# use boundary=symm to correct for data boundaries in the kde
grid = convolve2d(grid, kernel, mode='same', boundary='symm')
# Normalization factor to divide result by so that units are in the same
# units as scipy.stats.kde.gaussian_kde's output.
norm_factor = 2 * np.pi * cov * scotts_factor ** 2
norm_factor = np.linalg.det(norm_factor)
norm_factor = n * dx * dy * np.sqrt(norm_factor)
# Normalize the result
grid /= norm_factor
return grid, (xmin, xmax, ymin, ymax)
0
Example 137
def load():
X_agric = np.asarray([[86.3, 98.2, 3.52],
[92.9, 99.6, 3.27],
[74.0, 97.4, 2.46],
[58.7, 85.8, 4.15],
[93.8, 97.7, 3.04],
[83.7, 98.5, 2.31],
[49.7, 82.9, 2.10],
[93.8, 99.7, 2.67],
[84.9, 98.1, 2.57],
[88.1, 99.1, 1.86],
[79.2, 97.8, 4.00],
[45.8, 79.3, 1.50],
[79.5, 98.5, 3.08],
[86.4, 99.3, 2.75],
[74.0, 98.1, 2.53],
[82.8, 98.8, 2.78],
[59.9, 86.3, 1.22],
[58.3, 86.1, 3.30],
[86.0, 99.7, 2.89],
[74.7, 99.4, 2.93],
[75.7, 97.4, 2.87],
[52.2, 86.9, 3.99],
[88.1, 99.3, 4.33],
[59.8, 85.9, 1.25],
[80.3, 98.0, 3.21],
[47.0, 81.5, 1.36],
[70.0, 93.0, 2.25],
[63.8, 87.7, 2.99],
[60.5, 86.2, 3.99],
[77.3, 95.5, 3.15],
[75.7, 96.4, 2.39],
[66.9, 87.5, 2.14],
[73.7, 95.0, 2.59],
[87.5, 96.9, 2.61],
[56.4, 88.2, 3.65],
[45.0, 77.7, 0.00],
[67.1, 94.6, 3.04],
[78.0, 99.5, 3.80],
[57.7, 87.2, 2.99],
[49.8, 81.5, 2.99],
[65.2, 94.1, 3.71],
[71.0, 93.4, 3.82],
[70.5, 95.4, 3.06],
[81.7, 96.6, 3.58],
[90.9, 99.3, 3.07],
[67.4, 93.0, 1.90],
[43.7, 79.8, 0.00]])
X_ind = np.asarray([[5.92, 3.22],
[7.10, 2.64],
[6.28, 3.47],
[6.92, 2.30],
[4.19, 4.28],
[5.57, 4.11],
[7.42, 2.48],
[5.19, 3.40],
[5.80, 4.01],
[5.73, 4.01],
[5.89, 3.74],
[6.82, 3.14],
[5.32, 4.03],
[5.32, 3.97],
[4.89, 4.16],
[5.50, 4.14],
[6.85, 3.83],
[6.95, 3.26],
[5.19, 4.22],
[5.48, 3.87],
[4.92, 4.19],
[4.28, 4.26],
[5.27, 4.39],
[6.23, 3.69],
[6.09, 3.37],
[5.48, 3.69],
[4.50, 4.32],
[7.09, 3.14],
[6.56, 2.40],
[7.14, 2.77],
[5.54, 4.22],
[6.88, 3.26],
[5.86, 3.99],
[4.94, 4.09],
[5.30, 4.08],
[6.15, 4.04],
[4.89, 4.17],
[5.54, 3.91],
[7.06, 2.56],
[7.11, 2.30],
[4.88, 3.91],
[6.91, 1.61],
[7.76, 2.30],
[6.34, 3.61],
[6.64, 3.74],
[6.64, 2.64],
[5.69, 4.20]])
X_polit = np.asarray([[0, 0],
[1, 0],
[0, 0],
[1, 0],
[0, 1],
[0, 0],
[1, 0],
[0, 0],
[0, 0],
[0, 0],
[0, 1],
[1, 0],
[0, 1],
[0, 1],
[0, 1],
[0, 1],
[0, 0],
[0, 0],
[0, 1],
[0, 0],
[0, 1],
[1, 0],
[0, 1],
[1, 0],
[0, 0],
[0, 0],
[0, 1],
[1, 0],
[1, 0],
[1, 0],
[0, 1],
[1, 0],
[0, 1],
[0, 1],
[0, 1],
[0, 1],
[0, 1],
[0, 1],
[1, 0],
[1, 0],
[0, 1],
[1, 0],
[1, 0],
[1, 0],
[0, 1],
[0, 0],
[0, 1]])
return X_agric, X_ind, X_polit
0
Example 138
def bayesian_blocks(t, x=None, sigma=None,
fitness='events', **kwargs):
"""Bayesian Blocks Implementation
This is a flexible implementation of the Bayesian Blocks algorithm
described in Scargle 2012 [1]_
Parameters
----------
t : array_like
data times (one dimensional, length N)
x : array_like (optional)
data values
sigma : array_like or float (optional)
data errors
fitness : str or object
the fitness function to use.
If a string, the following options are supported:
- 'events' : binned or unbinned event data
extra arguments are `p0`, which gives the false alarm probability
to compute the prior, or `gamma` which gives the slope of the
prior on the number of bins.
- 'regular_events' : non-overlapping events measured at multiples
of a fundamental tick rate, `dt`, which must be specified as an
additional argument. The prior can be specified through `gamma`,
which gives the slope of the prior on the number of bins.
- 'measures' : fitness for a measured sequence with Gaussian errors
The prior can be specified using `gamma`, which gives the slope
of the prior on the number of bins. If `gamma` is not specified,
then a simulation-derived prior will be used.
Alternatively, the fitness can be a user-specified object of
type derived from the FitnessFunc class.
Returns
-------
edges : ndarray
array containing the (N+1) bin edges
Examples
--------
Event data:
>>> t = np.random.normal(size=100)
>>> bins = bayesian_blocks(t, fitness='events', p0=0.01)
Event data with repeats:
>>> t = np.random.normal(size=100)
>>> t[80:] = t[:20]
>>> bins = bayesian_blocks(t, fitness='events', p0=0.01)
Regular event data:
>>> dt = 0.01
>>> t = dt * np.arange(1000)
>>> x = np.zeros(len(t))
>>> x[np.random.randint(0, len(t), len(t) / 10)] = 1
>>> bins = bayesian_blocks(t, fitness='regular_events', dt=dt, gamma=0.9)
Measured point data with errors:
>>> t = 100 * np.random.random(100)
>>> x = np.exp(-0.5 * (t - 50) ** 2)
>>> sigma = 0.1
>>> x_obs = np.random.normal(x, sigma)
>>> bins = bayesian_blocks(t, fitness='measures')
References
----------
.. [1] Scargle, J `et al.` (2012)
http://adsabs.harvard.edu/abs/2012arXiv1207.5578S
See Also
--------
astroML.plotting.hist : histogram plotting function which can make use
of bayesian blocks.
"""
# validate array input
t = np.asarray(t, dtype=float)
if x is not None:
x = np.asarray(x)
if sigma is not None:
sigma = np.asarray(sigma)
# verify the fitness function
if fitness == 'events':
if x is not None and np.any(x % 1 > 0):
raise ValueError("x must be integer counts for fitness='events'")
fitfunc = Events(**kwargs)
elif fitness == 'regular_events':
if x is not None and (np.any(x % 1 > 0) or np.any(x > 1)):
raise ValueError("x must be 0 or 1 for fitness='regular_events'")
fitfunc = RegularEvents(**kwargs)
elif fitness == 'measures':
if x is None:
raise ValueError("x must be specified for fitness='measures'")
fitfunc = PointMeasures(**kwargs)
else:
if not (hasattr(fitness, 'args') and
hasattr(fitness, 'fitness') and
hasattr(fitness, 'prior')):
raise ValueError("fitness not understood")
fitfunc = fitness
# find unique values of t
t = np.array(t, dtype=float)
assert t.ndim == 1
unq_t, unq_ind, unq_inv = np.unique(t, return_index=True,
return_inverse=True)
# if x is not specified, x will be counts at each time
if x is None:
if sigma is not None:
raise ValueError("If sigma is specified, x must be specified")
if len(unq_t) == len(t):
x = np.ones_like(t)
else:
x = np.bincount(unq_inv)
t = unq_t
sigma = 1
# if x is specified, then we need to sort t and x together
else:
x = np.asarray(x)
if len(t) != len(x):
raise ValueError("Size of t and x does not match")
if len(unq_t) != len(t):
raise ValueError("Repeated values in t not supported when "
"x is specified")
t = unq_t
x = x[unq_ind]
# verify the given sigma value
N = t.size
if sigma is not None:
sigma = np.asarray(sigma)
if sigma.shape not in [(), (1,), (N,)]:
raise ValueError('sigma does not match the shape of x')
else:
sigma = 1
# validate the input
fitfunc.validate_input(t, x, sigma)
# compute values needed for computation, below
if 'a_k' in fitfunc.args:
ak_raw = np.ones_like(x) / (sigma * sigma)
if 'b_k' in fitfunc.args:
bk_raw = x / (sigma * sigma)
if 'c_k' in fitfunc.args:
ck_raw = x * x / (sigma * sigma)
# create length-(N + 1) array of cell edges
edges = np.concatenate([t[:1],
0.5 * (t[1:] + t[:-1]),
t[-1:]])
block_length = t[-1] - edges
# arrays to store the best configuration
best = np.zeros(N, dtype=float)
last = np.zeros(N, dtype=int)
#-----------------------------------------------------------------
# Start with first data cell; add one cell at each iteration
#-----------------------------------------------------------------
for R in range(N):
# Compute fit_vec : fitness of putative last block (end at R)
kwds = {}
# T_k: width/duration of each block
if 'T_k' in fitfunc.args:
kwds['T_k'] = block_length[:R + 1] - block_length[R + 1]
# N_k: number of elements in each block
if 'N_k' in fitfunc.args:
kwds['N_k'] = np.cuemsum(x[:R + 1][::-1])[::-1]
# a_k: eq. 31
if 'a_k' in fitfunc.args:
kwds['a_k'] = 0.5 * np.cuemsum(ak_raw[:R + 1][::-1])[::-1]
# b_k: eq. 32
if 'b_k' in fitfunc.args:
kwds['b_k'] = - np.cuemsum(bk_raw[:R + 1][::-1])[::-1]
# c_k: eq. 33
if 'c_k' in fitfunc.args:
kwds['c_k'] = 0.5 * np.cuemsum(ck_raw[:R + 1][::-1])[::-1]
# evaluate fitness function
fit_vec = fitfunc.fitness(**kwds)
A_R = fit_vec - fitfunc.prior(R + 1, N)
A_R[1:] += best[:R]
i_max = np.argmax(A_R)
last[R] = i_max
best[R] = A_R[i_max]
#-----------------------------------------------------------------
# Now find changepoints by iteratively peeling off the last block
#-----------------------------------------------------------------
change_points = np.zeros(N, dtype=int)
i_cp = N
ind = N
while True:
i_cp -= 1
change_points[i_cp] = ind
if ind == 0:
break
ind = last[ind - 1]
change_points = change_points[i_cp:]
return edges[change_points]
0
Example 139
Project: dipy Source File: test_expectmax.py
def test_compute_em_demons_step_3d():
r"""
Compares the output of the demons step in 3d against an analytical
step. The fixed image is given by $F(x) = \frac{1}{2}||x - c_f||^2$, the
moving image is given by $G(x) = \frac{1}{2}||x - c_g||^2$,
$x, c_f, c_g \in R^{3}$
References
----------
[Vercauteren09] Vercauteren, T., Pennec, X., Perchant, A., & Ayache, N.
(2009). Diffeomorphic demons: efficient non-parametric
image registration. NeuroImage, 45(1 Suppl), S61-72.
doi:10.1016/j.neuroimage.2008.10.040
"""
# Select arbitrary images' shape (same shape for both images)
sh = (20, 15, 10)
# Select arbitrary centers
c_f = np.asarray(sh) / 2
c_g = c_f + 0.5
# Compute the identity vector field I(x) = x in R^2
x_0 = np.asarray(range(sh[0]))
x_1 = np.asarray(range(sh[1]))
x_2 = np.asarray(range(sh[2]))
X = np.ndarray(sh + (3,), dtype=np.float64)
O = np.ones(sh)
X[..., 0] = x_0[:, None, None] * O
X[..., 1] = x_1[None, :, None] * O
X[..., 2] = x_2[None, None, :] * O
# Compute the gradient fields of F and G
grad_F = X - c_f
grad_G = X - c_g
# The squared norm of grad_G to be used later
sq_norm_grad_G = np.sum(grad_G**2, -1)
# Compute F and G
F = 0.5 * np.sum(grad_F**2, -1)
G = 0.5 * sq_norm_grad_G
delta_field = G - F
# Now select an arbitrary parameter for
# $\sigma_x$ (eq 4 in [Vercauteren09])
sigma_x_sq = 1.5
# Set arbitrary values for $\sigma_i$ (eq. 4 in [Vercauteren09])
# The original Demons algorithm used simply |F(x) - G(x)| as an
# estimator, so let's use it as well
sigma_i_sq = (F - G)**2
# Select some pixels to have special values
np.random.seed(1346491)
random_labels = np.random.randint(0, 5, sh[0] * sh[1] * sh[2])
random_labels = random_labels.reshape(sh)
# this label is used to set sigma_i_sq == 0 below
random_labels[sigma_i_sq == 0] = 2
# this label is used to set gradient == 0 below
random_labels[sq_norm_grad_G == 0] = 2
expected = np.zeros_like(grad_G)
# Pixels with sigma_i_sq = inf
sigma_i_sq[random_labels == 0] = np.inf
expected[random_labels == 0, ...] = 0
# Pixels with gradient!=0 and sigma_i_sq=0
sqnrm = sq_norm_grad_G[random_labels == 1]
sigma_i_sq[random_labels == 1] = 0
expected[random_labels == 1, 0] = (delta_field[random_labels == 1] *
grad_G[random_labels == 1, 0] / sqnrm)
expected[random_labels == 1, 1] = (delta_field[random_labels == 1] *
grad_G[random_labels == 1, 1] / sqnrm)
expected[random_labels == 1, 2] = (delta_field[random_labels == 1] *
grad_G[random_labels == 1, 2] / sqnrm)
# Pixels with gradient=0 and sigma_i_sq=0
sigma_i_sq[random_labels == 2] = 0
grad_G[random_labels == 2, ...] = 0
expected[random_labels == 2, ...] = 0
# Pixels with gradient=0 and sigma_i_sq!=0
grad_G[random_labels == 3, ...] = 0
# Directly compute the demons step according to eq. 4 in [Vercauteren09]
num = (sigma_x_sq * (F - G))[random_labels >= 3]
den = (sigma_x_sq * sq_norm_grad_G + sigma_i_sq)[random_labels >= 3]
# This is $J^{P}$ in eq. 4 [Vercauteren09]
expected[random_labels >= 3] = -1 * np.array(grad_G[random_labels >= 3])
expected[random_labels >= 3, ...] *= (num / den)[..., None]
# Now compute it using the implementation under test
actual = np.empty_like(expected, dtype=floating)
em.compute_em_demons_step_3d(np.array(delta_field, dtype=floating),
np.array(sigma_i_sq, dtype=floating),
np.array(grad_G, dtype=floating),
sigma_x_sq,
actual)
# Test sigma_i_sq == inf
try:
assert_array_almost_equal(actual[random_labels == 0],
expected[random_labels == 0])
except AssertionError:
raise AssertionError("Failed for sigma_i_sq == inf")
# Test sigma_i_sq == 0 and gradient != 0
try:
assert_array_almost_equal(actual[random_labels == 1],
expected[random_labels == 1])
except AssertionError:
raise AssertionError("Failed for sigma_i_sq == 0 and gradient != 0")
# Test sigma_i_sq == 0 and gradient == 0
try:
assert_array_almost_equal(actual[random_labels == 2],
expected[random_labels == 2])
except AssertionError:
raise AssertionError("Failed for sigma_i_sq == 0 and gradient == 0")
# Test sigma_i_sq != 0 and gradient == 0
try:
assert_array_almost_equal(actual[random_labels == 3],
expected[random_labels == 3])
except AssertionError:
raise AssertionError("Failed for sigma_i_sq != 0 and gradient == 0 ")
# Test sigma_i_sq != 0 and gradient != 0
try:
assert_array_almost_equal(actual[random_labels == 4],
expected[random_labels == 4])
except AssertionError:
raise AssertionError("Failed for sigma_i_sq != 0 and gradient != 0")
0
Example 140
Project: neural-network-animation Source File: tripcolor.py
def tripcolor(ax, *args, **kwargs):
"""
Create a pseudocolor plot of an unstructured triangular grid.
The triangulation can be specified in one of two ways; either::
tripcolor(triangulation, ...)
where triangulation is a :class:`matplotlib.tri.Triangulation`
object, or
::
tripcolor(x, y, ...)
tripcolor(x, y, triangles, ...)
tripcolor(x, y, triangles=triangles, ...)
tripcolor(x, y, mask=mask, ...)
tripcolor(x, y, triangles, mask=mask, ...)
in which case a Triangulation object will be created. See
:class:`~matplotlib.tri.Triangulation` for a explanation of these
possibilities.
The next argument must be *C*, the array of color values, either
one per point in the triangulation if color values are defined at
points, or one per triangle in the triangulation if color values
are defined at triangles. If there are the same number of points
and triangles in the triangulation it is assumed that color
values are defined at points; to force the use of color values at
triangles use the kwarg *facecolors*=C instead of just *C*.
*shading* may be 'flat' (the default) or 'gouraud'. If *shading*
is 'flat' and C values are defined at points, the color values
used for each triangle are from the mean C of the triangle's
three points. If *shading* is 'gouraud' then color values must be
defined at points. *shading* of 'faceted' is deprecated;
please use *edgecolors* instead.
The remaining kwargs are the same as for
:meth:`~matplotlib.axes.Axes.pcolor`.
**Example:**
.. plot:: mpl_examples/pylab_examples/tripcolor_demo.py
"""
if not ax._hold:
ax.cla()
alpha = kwargs.pop('alpha', 1.0)
norm = kwargs.pop('norm', None)
cmap = kwargs.pop('cmap', None)
vmin = kwargs.pop('vmin', None)
vmax = kwargs.pop('vmax', None)
shading = kwargs.pop('shading', 'flat')
facecolors = kwargs.pop('facecolors', None)
tri, args, kwargs = Triangulation.get_from_args_and_kwargs(*args, **kwargs)
# C is the colors array defined at either points or faces (i.e. triangles).
# If facecolors is None, C are defined at points.
# If facecolors is not None, C are defined at faces.
if facecolors is not None:
C = facecolors
else:
C = np.asarray(args[0])
# If there are a different number of points and triangles in the
# triangulation, can omit facecolors kwarg as it is obvious from
# length of C whether it refers to points or faces.
# Do not do this for gouraud shading.
if (facecolors is None and len(C) == len(tri.triangles) and
len(C) != len(tri.x) and shading != 'gouraud'):
facecolors = C
# Check length of C is OK.
if ((facecolors is None and len(C) != len(tri.x)) or
(facecolors is not None and len(C) != len(tri.triangles))):
raise ValueError('Length of color values array must be the same '
'as either the number of triangulation points '
'or triangles')
# Handling of linewidths, shading, edgecolors and antialiased as
# in Axes.pcolor
linewidths = (0.25,)
if 'linewidth' in kwargs:
kwargs['linewidths'] = kwargs.pop('linewidth')
kwargs.setdefault('linewidths', linewidths)
if shading == 'faceted': # Deprecated.
edgecolors = 'k'
else:
edgecolors = 'none'
if 'edgecolor' in kwargs:
kwargs['edgecolors'] = kwargs.pop('edgecolor')
ec = kwargs.setdefault('edgecolors', edgecolors)
if 'antialiased' in kwargs:
kwargs['antialiaseds'] = kwargs.pop('antialiased')
if 'antialiaseds' not in kwargs and ec.lower() == "none":
kwargs['antialiaseds'] = False
if shading == 'gouraud':
if facecolors is not None:
raise ValueError('Gouraud shading does not support the use '
'of facecolors kwarg')
if len(C) != len(tri.x):
raise ValueError('For gouraud shading, the length of color '
'values array must be the same as the '
'number of triangulation points')
collection = TriMesh(tri, **kwargs)
else:
# Vertices of triangles.
maskedTris = tri.get_masked_triangles()
verts = np.concatenate((tri.x[maskedTris][..., np.newaxis],
tri.y[maskedTris][..., np.newaxis]), axis=2)
# Color values.
if facecolors is None:
# One color per triangle, the mean of the 3 vertex color values.
C = C[maskedTris].mean(axis=1)
elif tri.mask is not None:
# Remove color values of masked triangles.
C = C.compress(1-tri.mask)
collection = PolyCollection(verts, **kwargs)
collection.set_alpha(alpha)
collection.set_array(C)
if norm is not None:
assert(isinstance(norm, Normalize))
collection.set_cmap(cmap)
collection.set_norm(norm)
if vmin is not None or vmax is not None:
collection.set_clim(vmin, vmax)
else:
collection.autoscale_None()
ax.grid(False)
minx = tri.x.min()
maxx = tri.x.max()
miny = tri.y.min()
maxy = tri.y.max()
corners = (minx, miny), (maxx, maxy)
ax.update_datalim(corners)
ax.autoscale_view()
ax.add_collection(collection)
return collection
0
Example 141
Project: dipy Source File: test_sumsqdiff.py
def test_compute_residual_displacement_field_ssd_3d():
# Select arbitrary images' shape (same shape for both images)
sh = (20, 15, 10)
# Select arbitrary centers
c_f = np.asarray(sh) / 2
c_g = c_f + 0.5
# Compute the identity vector field I(x) = x in R^2
x_0 = np.asarray(range(sh[0]))
x_1 = np.asarray(range(sh[1]))
x_2 = np.asarray(range(sh[2]))
X = np.ndarray(sh + (3,), dtype=np.float64)
O = np.ones(sh)
X[..., 0] = x_0[:, None, None] * O
X[..., 1] = x_1[None, :, None] * O
X[..., 2] = x_2[None, None, :] * O
# Compute the gradient fields of F and G
np.random.seed(9223102)
grad_F = X - c_f
grad_G = X - c_g
Fnoise = np.random.ranf(
np.size(grad_F)).reshape(
grad_F.shape) * grad_F.max() * 0.1
Fnoise = Fnoise.astype(floating)
grad_F += Fnoise
Gnoise = np.random.ranf(
np.size(grad_G)).reshape(
grad_G.shape) * grad_G.max() * 0.1
Gnoise = Gnoise.astype(floating)
grad_G += Gnoise
# The squared norm of grad_G
sq_norm_grad_G = np.sum(grad_G**2, -1)
# Compute F and G
F = 0.5 * np.sum(grad_F**2, -1)
G = 0.5 * sq_norm_grad_G
Fnoise = np.random.ranf(np.size(F)).reshape(F.shape) * F.max() * 0.1
Fnoise = Fnoise.astype(floating)
F += Fnoise
Gnoise = np.random.ranf(np.size(G)).reshape(G.shape) * G.max() * 0.1
Gnoise = Gnoise.astype(floating)
G += Gnoise
delta_field = np.array(F - G, dtype=floating)
sigma_field = np.random.randn(delta_field.size).reshape(delta_field.shape)
sigma_field = sigma_field.astype(floating)
# Select some pixels to force sigma_field = infinite
inf_sigma = np.random.randint(0, 2, sh[0] * sh[1] * sh[2])
inf_sigma = inf_sigma.reshape(sh)
sigma_field[inf_sigma == 1] = np.inf
# Select an initial displacement field
d = np.random.randn(grad_G.size).reshape(grad_G.shape).astype(floating)
# d = np.zeros_like(grad_G, dtype=floating)
lambda_param = 1.5
# Implementation under test
iut = ssd.compute_residual_displacement_field_ssd_3d
# In the first iteration we test the case target=None
# In the second iteration, target is not None
target = None
rtol = 1e-9
atol = 1e-4
for it in range(2):
# Sum of differences with the neighbors
s = np.zeros_like(d, dtype=np.float64)
s[:, :, :-1] += d[:, :, :-1] - d[:, :, 1:] # right
s[:, :, 1:] += d[:, :, 1:] - d[:, :, :-1] # left
s[:, :-1, :] += d[:, :-1, :] - d[:, 1:, :] # down
s[:, 1:, :] += d[:, 1:, :] - d[:, :-1, :] # up
s[:-1, :, :] += d[:-1, :, :] - d[1:, :, :] # below
s[1:, :, :] += d[1:, :, :] - d[:-1, :, :] # above
s *= lambda_param
# Dot product of displacement and gradient
dp = d[..., 0] * grad_G[..., 0] + \
d[..., 1] * grad_G[..., 1] + \
d[..., 2] * grad_G[..., 2]
# Compute expected residual
expected = None
if target is None:
expected = np.zeros_like(grad_G)
for i in range(3):
expected[..., i] = delta_field * grad_G[..., i]
else:
expected = target.copy().astype(np.float64)
# Expected residuals when sigma != infinte
for i in range(3):
expected[inf_sigma == 0, i] -= grad_G[inf_sigma == 0, i] * \
dp[inf_sigma == 0] + sigma_field[inf_sigma == 0] * s[inf_sigma == 0, i]
# Expected residuals when sigma == infinte
expected[inf_sigma == 1] = -1.0 * s[inf_sigma == 1]
# Test residual field computation starting with residual = None
actual = iut(delta_field, sigma_field, grad_G.astype(floating),
target, lambda_param, d, None)
assert_allclose(actual, expected, rtol=rtol, atol=atol)
# destroy previous result
actual = np.ndarray(actual.shape, dtype=floating)
# Test residual field computation starting with residual is not None
iut(delta_field, sigma_field, grad_G.astype(floating),
target, lambda_param, d, actual)
assert_allclose(actual, expected, rtol=rtol, atol=atol)
# Set target for next iteration
target = actual
# Test Gauss-Seidel step with residual=None and residual=target
for residual in [None, target]:
expected = d.copy()
iterate_residual_field_ssd_3d(
delta_field,
sigma_field,
grad_G.astype(floating),
residual,
lambda_param,
expected)
actual = d.copy()
ssd.iterate_residual_displacement_field_ssd_3d(
delta_field,
sigma_field,
grad_G.astype(floating),
residual,
lambda_param,
actual)
# the numpy linear solver may differ from our custom implementation
# we need to increase the tolerance a bit
assert_allclose(actual, expected, rtol=rtol, atol=atol * 5)
0
Example 142
Project: minirank Source File: logistic.py
def ordinal_logistic_fit(X, y, alpha=0, l1_ratio=0, n_class=None, max_iter=10000,
verbose=False, solver='TNC', w0=None):
"""
Ordinal logistic regression or proportional odds model.
Uses scipy's optimize.fmin_slsqp solver.
Parameters
----------
X : {array, sparse matrix}, shape (n_samples, n_feaures)
Input data
y : array-like
Target values
max_iter : int
Maximum number of iterations
verbose: bool
Print convergence information
Returns
-------
w : array, shape (n_features,)
coefficients of the linear model
theta : array, shape (k,), where k is the different values of y
vector of thresholds
"""
X = np.asarray(X)
y = np.asarray(y)
w0 = None
if not X.shape[0] == y.shape[0]:
raise ValueError('Wrong shape for X and y')
# .. order input ..
idx = np.argsort(y)
idx_inv = np.zeros_like(idx)
idx_inv[idx] = np.arange(idx.size)
X = X[idx]
y = y[idx].astype(np.int)
# make them continuous and start at zero
unique_y = np.unique(y)
for i, u in enumerate(unique_y):
y[y == u] = i
unique_y = np.unique(y)
# .. utility arrays used in f_grad ..
alpha = 0.
k1 = np.sum(y == unique_y[0])
E0 = (y[:, np.newaxis] == np.unique(y)).astype(np.int)
E1 = np.roll(E0, -1, axis=-1)
E1[:, -1] = 0.
E0, E1 = map(sparse.csr_matrix, (E0.T, E1.T))
def f_obj(x0, X, y):
"""
Objective function
"""
w, theta_0 = np.split(x0, [X.shape[1]])
theta_1 = np.roll(theta_0, 1)
t0 = theta_0[y]
z = np.diff(theta_0)
Xw = X.dot(w)
a = t0 - Xw
b = t0[k1:] - X[k1:].dot(w)
c = (theta_1 - theta_0)[y][k1:]
if np.any(c > 0):
return BIG
#loss = -(c[idx] + np.log(np.exp(-c[idx]) - 1)).sum()
loss = -np.log(1 - np.exp(c)).sum()
loss += b.sum() + log_logistic(b).sum() \
+ log_logistic(a).sum() \
+ .5 * alpha * w.dot(w) - np.log(z).sum() # penalty
if np.isnan(loss):
pass
#import ipdb; ipdb.set_trace()
return loss
def f_grad(x0, X, y):
"""
Gradient of the objective function
"""
w, theta_0 = np.split(x0, [X.shape[1]])
theta_1 = np.roll(theta_0, 1)
t0 = theta_0[y]
t1 = theta_1[y]
z = np.diff(theta_0)
Xw = X.dot(w)
a = t0 - Xw
b = t0[k1:] - X[k1:].dot(w)
c = (theta_1 - theta_0)[y][k1:]
# gradient for w
phi_a = phi(a)
phi_b = phi(b)
grad_w = -X[k1:].T.dot(phi_b) + X.T.dot(1 - phi_a) + alpha * w
# gradient for theta
idx = c > 0
tmp = np.empty_like(c)
tmp[idx] = 1. / (np.exp(-c[idx]) - 1)
tmp[~idx] = np.exp(c[~idx]) / (1 - np.exp(c[~idx])) # should not need
grad_theta = (E1 - E0)[:, k1:].dot(tmp) \
+ E0[:, k1:].dot(phi_b) - E0.dot(1 - phi_a)
grad_theta[:-1] += 1. / np.diff(theta_0)
grad_theta[1:] -= 1. / np.diff(theta_0)
out = np.concatenate((grad_w, grad_theta))
return out
def f_hess(x0, s, X, y):
x0 = np.asarray(x0)
w, theta_0 = np.split(x0, [X.shape[1]])
theta_1 = np.roll(theta_0, 1)
t0 = theta_0[y]
t1 = theta_1[y]
z = np.diff(theta_0)
Xw = X.dot(w)
a = t0 - Xw
b = t0[k1:] - X[k1:].dot(w)
c = (theta_1 - theta_0)[y][k1:]
D = np.diag(phi(a) * (1 - phi(a)))
D_= np.diag(phi(b) * (1 - phi(b)))
D1 = np.diag(np.exp(-c) / (np.exp(-c) - 1) ** 2)
Ex = (E1 - E0)[:, k1:].toarray()
Ex0 = E0.toarray()
H_A = X[k1:].T.dot(D_).dot(X[k1:]) + X.T.dot(D).dot(X)
H_C = - X[k1:].T.dot(D_).dot(E0[:, k1:].T.toarray()) \
- X.T.dot(D).dot(E0.T.toarray())
H_B = Ex.dot(D1).dot(Ex.T) + Ex0[:, k1:].dot(D_).dot(Ex0[:, k1:].T) \
- Ex0.dot(D).dot(Ex0.T)
p_w = H_A.shape[0]
tmp0 = H_A.dot(s[:p_w]) + H_C.dot(s[p_w:])
tmp1 = H_C.T.dot(s[:p_w]) + H_B.dot(s[p_w:])
return np.concatenate((tmp0, tmp1))
import ipdb; ipdb.set_trace()
import pylab as pl
pl.matshow(H_B)
pl.colorbar()
pl.title('True')
import numdifftools as nd
Hess = nd.Hessian(lambda x: f_obj(x, X, y))
H = Hess(x0)
pl.matshow(H[H_A.shape[0]:, H_A.shape[0]:])
#pl.matshow()
pl.title('estimated')
pl.colorbar()
pl.show()
def grad_hess(x0, X, y):
grad = f_grad(x0, X, y)
hess = lambda x: f_hess(x0, x, X, y)
return grad, hess
x0 = np.random.randn(X.shape[1] + unique_y.size) / X.shape[1]
if w0 is not None:
x0[:X.shape[1]] = w0
else:
x0[:X.shape[1]] = 0.
x0[X.shape[1]:] = np.sort(unique_y.size * np.random.rand(unique_y.size))
#print('Check grad: %s' % optimize.check_grad(f_obj, f_grad, x0, X, y))
#print(optimize.approx_fprime(x0, f_obj, 1e-6, X, y))
#print(f_grad(x0, X, y))
#print(optimize.approx_fprime(x0, f_obj, 1e-6, X, y) - f_grad(x0, X, y))
#import ipdb; ipdb.set_trace()
def callback(x0):
x0 = np.asarray(x0)
# print('Check grad: %s' % optimize.check_grad(f_obj, f_grad, x0, X, y))
if verbose:
# check that gradient is correctly computed
print('OBJ: %s' % f_obj(x0, X, y))
if solver == 'TRON':
import pytron
out = pytron.minimize(f_obj, grad_hess, x0, args=(X, y))
else:
options = {'maxiter' : max_iter, 'disp': 0, 'maxfun':10000}
out = optimize.minimize(f_obj, x0, args=(X, y), method=solver,
jac=f_grad, hessp=f_hess, options=options, callback=callback)
if not out.success:
warnings.warn(out.message)
w, theta = np.split(out.x, [X.shape[1]])
return w, theta
0
Example 143
def plot_map(map, affine, cut_coords=None, anat=None, anat_affine=None,
slicer='ortho',
figure=None, axes=None, title=None,
threshold=None, annotate=True, draw_cross=True,
do3d=False, threshold_3d=None,
view_3d=(38.5, 70.5, 300, (-2.7, -12, 9.1)),
black_bg=False, **imshow_kwargs):
""" Plot three cuts of a given activation map (Frontal, Axial, and Lateral)
Parameters
----------
map : 3D ndarray
The activation map, as a 3D image.
affine : 4x4 ndarray
The affine matrix going from image voxel space to MNI space.
cut_coords: None, int, or a tuple of floats
The MNI coordinates of the point where the cut is performed, in
MNI coordinates and order.
If slicer is 'ortho', this should be a 3-tuple: (x, y, z)
For slicer == 'x', 'y', or 'z', then these are the
coordinates of each cut in the corresponding direction.
If None or an int is given, then a maximally separated sequence (
with exactly cut_coords elements if cut_coords is not None) of
cut coordinates along the slicer axis is computed automatically
anat : 3D ndarray or False, optional
The anatomical image to be used as a background. If None, the
MNI152 T1 1mm template is used. If False, no anat is displayed.
anat_affine : 4x4 ndarray, optional
The affine matrix going from the anatomical image voxel space to
MNI space. This parameter is not used when the default
anatomical is used, but it is compulsory when using an
explicite anatomical image.
slicer: {'ortho', 'x', 'y', 'z'}
Choose the direction of the cuts. With 'ortho' three cuts are
performed in orthogonal directions
figure : integer or matplotlib figure, optional
Matplotlib figure used or its number. If None is given, a
new figure is created.
axes : matplotlib axes or 4 tuple of float: (xmin, ymin, width, height), optional
The axes, or the coordinates, in matplotlib figure space,
of the axes used to display the plot. If None, the complete
figure is used.
title : string, optional
The title dispayed on the figure.
threshold : a number, None, or 'auto'
If None is given, the maps are not thresholded.
If a number is given, it is used to threshold the maps:
values below the threshold are plotted as transparent. If
auto is given, the threshold is determined magically by
analysis of the map.
annotate: boolean, optional
If annotate is True, positions and left/right annotation
are added to the plot.
draw_cross: boolean, optional
If draw_cross is True, a cross is drawn on the plot to
indicate the cut plosition.
do3d: {True, False or 'interactive'}, optional
If True, Mayavi is used to plot a 3D view of the
map in addition to the slicing. If 'interactive', the
3D visualization is displayed in an additional interactive
window.
threshold_3d:
The threshold to use for the 3D view (if any). Defaults to
the same threshold as that used for the 2D view.
view_3d: tuple,
The view used to take the screenshot: azimuth, elevation,
distance and focalpoint, see the docstring of mlab.view.
black_bg: boolean, optional
If True, the background of the image is set to be black. If
you whish to save figures with a black background, you
will need to pass "facecolor='k', edgecolor='k'" to pylab's
savefig.
imshow_kwargs: extra keyword arguments, optional
Extra keyword arguments passed to pylab.imshow
Notes
-----
Arrays should be passed in numpy convention: (x, y, z)
ordered.
Use masked arrays to create transparency:
import numpy as np
map = np.ma.masked_less(map, 0.5)
plot_map(map, affine)
"""
map, affine = _xyz_order(map, affine)
nan_mask = np.isnan(np.asarray(map))
if np.any(nan_mask):
map = map.copy()
map[nan_mask] = 0
del nan_mask
# Deal with automatic settings of plot parameters
if threshold == 'auto':
# Threshold epsilon above a percentile value, to be sure that some
# voxels are indeed threshold
threshold = _fast_abs_percentile(map) + 1e-5
if do3d:
try:
try:
from mayavi import version
except ImportError:
from enthought.mayavi import version
if not int(version.version[0]) > 2:
raise ImportError
except ImportError:
warnings.warn('Mayavi > 3.x not installed, plotting only 2D')
do3d = False
if (cut_coords is None or isinstance(cut_coords, numbers.Number)
) and slicer in ['x', 'y', 'z']:
cut_coords = find_maxsep_cut_coords(map, affine, slicer=slicer,
threshold=threshold,
n_cuts=cut_coords)
slicer = SLICERS[slicer].init_with_figure(data=map, affine=affine,
threshold=threshold,
cut_coords=cut_coords,
figure=figure, axes=axes,
black_bg=black_bg,
leave_space=do3d)
# Use Mayavi for the 3D plotting
if do3d:
from .maps_3d import plot_map_3d, m2screenshot
try:
from tvtk.api import tvtk
except ImportError:
from enthought.tvtk.api import tvtk
version = tvtk.Version()
offscreen = True
if (version.vtk_major_version, version.vtk_minor_version) < (5, 2):
offscreen = False
if do3d == 'interactive':
offscreen = False
cmap = kwargs.get('cmap', pl.cm.cmap_d[pl.rcParams['image.cmap']])
# Computing vmin and vmax is costly in time, and is needed
# later, so we compute them now, and store them for future
# use
vmin = kwargs.get('vmin', map.min())
kwargs['vmin'] = vmin
vmax = kwargs.get('vmax', map.max())
kwargs['vmax'] = vmax
try:
from mayavi import mlab
except ImportError:
from enthought.mayavi import mlab
if threshold_3d is None:
threshold_3d = threshold
plot_map_3d(np.asarray(map), affine, cut_coords=cut_coords,
anat=anat, anat_affine=anat_affine,
offscreen=offscreen, cmap=cmap,
threshold=threshold_3d,
view=view_3d,
vmin=vmin, vmax=vmax)
ax = list(slicer.axes.values())[0].ax.figure.add_axes((0.001, 0, 0.29, 1))
ax.axis('off')
m2screenshot(mpl_axes=ax)
if offscreen:
# Clean up, so that the offscreen engine doesn't become the
# default
mlab.clf()
engine = mlab.get_engine()
try:
from mayavi.core.registry import registry
except:
from enthought.mayavi.core.registry import registry
for key, value in registry.engines.items():
if value is engine:
registry.engines.pop(key)
break
if threshold:
map = np.ma.masked_inside(map, -threshold, threshold, copy=False)
_plot_anat(slicer, anat, anat_affine, title=title,
annotate=annotate, draw_cross=draw_cross)
slicer.plot_map(map, affine, **imshow_kwargs)
return slicer
0
Example 144
def plot_events(events, sfreq=None, first_samp=0, color=None, event_id=None,
axes=None, equal_spacing=True, show=True):
"""Plot events to get a visual display of the paradigm.
Parameters
----------
events : array, shape (n_events, 3)
The events.
sfreq : float | None
The sample frequency. If None, data will be displayed in samples (not
seconds).
first_samp : int
The index of the first sample. Typically the raw.first_samp
attribute. It is needed for recordings on a Neuromag
system as the events are defined relative to the system
start and not to the beginning of the recording.
color : dict | None
Dictionary of event_id value and its associated color. If None,
colors are automatically drawn from a default list (cycled through if
number of events longer than list of default colors).
event_id : dict | None
Dictionary of event label (e.g. 'aud_l') and its associated
event_id value. Label used to plot a legend. If None, no legend is
drawn.
axes : instance of matplotlib.axes.AxesSubplot
The subplot handle.
equal_spacing : bool
Use equal spacing between events in y-axis.
show : bool
Show figure if True.
Returns
-------
fig : matplotlib.figure.Figure
The figure object containing the plot.
Notes
-----
.. versionadded:: 0.9.0
"""
if sfreq is None:
sfreq = 1.0
xlabel = 'samples'
else:
xlabel = 'Time (s)'
events = np.asarray(events)
unique_events = np.unique(events[:, 2])
if event_id is not None:
# get labels and unique event ids from event_id dict,
# sorted by value
event_id_rev = dict((v, k) for k, v in event_id.items())
conditions, unique_events_id = zip(*sorted(event_id.items(),
key=lambda x: x[1]))
for this_event in unique_events_id:
if this_event not in unique_events:
raise ValueError('%s from event_id is not present in events.'
% this_event)
for this_event in unique_events:
if this_event not in unique_events_id:
warn('event %s missing from event_id will be ignored'
% this_event)
else:
unique_events_id = unique_events
if color is None:
if len(unique_events) > len(COLORS):
warn('More events than colors available. You should pass a list '
'of unique colors.')
colors = cycle(COLORS)
color = dict()
for this_event, this_color in zip(unique_events_id, colors):
color[this_event] = this_color
else:
for this_event in color:
if this_event not in unique_events_id:
raise ValueError('%s from color is not present in events '
'or event_id.' % this_event)
for this_event in unique_events_id:
if this_event not in color:
warn('Color is not available for event %d. Default colors '
'will be used.' % this_event)
import matplotlib.pyplot as plt
fig = None
if axes is None:
fig = plt.figure()
ax = axes if axes else plt.gca()
unique_events_id = np.array(unique_events_id)
min_event = np.min(unique_events_id)
max_event = np.max(unique_events_id)
for idx, ev in enumerate(unique_events_id):
ev_mask = events[:, 2] == ev
kwargs = {}
if event_id is not None:
event_label = '{0} ({1})'.format(event_id_rev[ev],
np.sum(ev_mask))
kwargs['label'] = event_label
if ev in color:
kwargs['color'] = color[ev]
if equal_spacing:
ax.plot((events[ev_mask, 0] - first_samp) / sfreq,
(idx + 1) * np.ones(ev_mask.sum()), '.', **kwargs)
else:
ax.plot((events[ev_mask, 0] - first_samp) / sfreq,
events[ev_mask, 2], '.', **kwargs)
if equal_spacing:
ax.set_ylim(0, unique_events_id.size + 1)
ax.set_yticks(1 + np.arange(unique_events_id.size))
ax.set_yticklabels(unique_events_id)
else:
ax.set_ylim([min_event - 1, max_event + 1])
ax.set_xlabel(xlabel)
ax.set_ylabel('Events id')
ax.grid('on')
fig = fig if fig is not None else plt.gcf()
if event_id is not None:
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
fig.canvas.draw()
plt_show(show)
return fig
0
Example 145
Project: supersmoother Source File: windowed_sum.py
def windowed_sum(arrays, span, t=None, indices=None, tpowers=0,
period=None, subtract_mid=False):
"""Compute the windowed sum of the given arrays.
Parameters
----------
arrays : tuple of arrays
arrays to window
span : int or array of ints
The span to use for the sum at each point. If array is provided,
it must be broadcastable with ``indices``
indices : array
the indices of the center of the desired windows. If ``None``,
the indices are assumed to be ``range(len(arrays[0]))`` though
these are not actually instantiated.
t : array (optional)
Times associated with the arrays
tpowers : list (optional)
Powers of t for each array sum
period : float (optional)
Period to use, if times are periodic. If supplied, input times
must be arranged such that (t % period) is sorted!
subtract_mid : boolean
If true, then subtract the middle value from each sum
Returns
-------
arrays : tuple of ndarrays
arrays containing the windowed sum of each input array
"""
span = np.asarray(span, dtype=int)
if not np.all(span > 0):
raise ValueError("span values must be positive")
arrays = tuple(map(np.asarray, arrays))
N = arrays[0].size
if not all(a.shape == (N,) for a in arrays):
raise ValueError("sizes of provided arrays must match")
t_input = t
if t is not None:
t = np.asarray(t)
if t.shape != (N,):
raise ValueError("shape of t must match shape of arrays "
"t -> {0} arr -> {1}".format(t.shape,
arrays[0].shape))
else:
# XXX: special-case no t?
t = np.ones(N)
tpowers = np.asarray(tpowers) + np.zeros(len(arrays))
if indices is not None:
span, indices = np.broadcast_arrays(span, indices)
# For the periodic case, re-call the function with padded arrays
if period:
if t_input is None:
raise ValueError("periodic requires t to be provided")
t = t % period
t, arrays, sl = _pad_arrays(t, arrays, indices, span, period)
if len(t) > N:
# arrays are padded. Recursively call windowed_sum() and return.
if span.ndim == 0 and indices is None:
# fixed-span/no index case is done faster this way
arrs = windowed_sum(arrays, span, t=t, indices=indices,
tpowers=tpowers, period=None,
subtract_mid=subtract_mid)
return tuple([a[sl] for a in arrs])
else:
# this works for variable span and general indices
if indices is None:
indices = np.arange(N)
indices = indices + sl.start
return windowed_sum(arrays, span, t=t, indices=indices,
tpowers=tpowers, period=None,
subtract_mid=subtract_mid)
else:
# No padding needed! We can carry-on as if it's a non-periodic case
period = None
# The rest of the algorithm now proceeds without reference to the period
# just as a sanity check...
assert not period
if span.ndim == 0:
# fixed-span case. Because of the checks & manipulations above
# we know here that indices=None
assert indices is None
window = np.ones(span)
def convolve_same(a, window):
if len(window) <= len(a):
res = np.convolve(a, window, mode='same')
else:
res = np.convolve(a, window, mode='full')
start = (len(window) - 1) // 2
res = res[start:start + len(a)]
return res
results = [convolve_same(a * t ** tp, window)
for a, tp in zip(arrays, tpowers)]
indices = slice(None)
else:
# variable-span case. Use reduceat() in a clever way for speed.
if indices is None:
indices = np.arange(len(span))
# we checked this above, but just as a sanity check assert it here...
assert span.shape == indices.shape
mins = np.asarray(indices) - span // 2
results = []
for a, tp in zip(arrays, tpowers):
ranges = np.vstack([np.maximum(0, mins),
np.minimum(len(a), mins+span)]).ravel('F')
results.append(np.add.reduceat(np.append(a * t ** tp, 0),
ranges)[::2])
# Subtract the midpoint if required: this is used in cross-validation
if subtract_mid:
results = [r - a[indices] * t[indices] ** tp
for r, a, tp in zip(results, arrays, tpowers)]
return tuple(results)
0
Example 146
Project: mayavi Source File: array_handler.py
def array2vtkCellArray(num_array, vtk_array=None):
"""Given a nested Python list or a numpy array, this method
creates a vtkCellArray instance and returns it.
A variety of input arguments are supported as described in the
Parameter docuementation. If numpy arrays are given, this method
is highly efficient. This function is most efficient if the
passed numpy arrays have a typecode `ID_TYPE_CODE`. Otherwise a
typecast is necessary and this involves an extra copy. This
method *always copies* the input data.
An alternative and more efficient way to build the connectivity
list is to create a vtkIdTypeArray having data of the form
(npts,p0,p1,...p(npts-1), repeated for each cell) and then call
<vtkCellArray_instance>.SetCells(n_cell, id_list).
Parameters
----------
- num_array : numpy array or Python list/tuple
Valid values are:
1. A Python list of 1D lists. Each 1D list can contain one
cell connectivity list. This is very slow and is to be
used only when efficiency is of no consequence.
2. A 2D numpy array with the cell connectivity list.
3. A Python list of 2D numpy arrays. Each numeric array can
have a different shape. This makes it easy to generate a
cell array having cells of different kinds.
- vtk_array : `vtkCellArray` (default: `None`)
If an optional `vtkCellArray` instance, is passed as an argument
then a new array is not created and returned. The passed array
is itself modified and returned.
Example
-------
>>> a = [[0], [1, 2], [3, 4, 5], [6, 7, 8, 9]]
>>> cells = array_handler.array2vtkCellArray(a)
>>> a = numpy.array([[0,1,2], [3,4,5], [6,7,8]], 'l')
>>> cells = array_handler.array2vtkCellArray(a)
>>> l_a = [a[:,:1], a[:2,:2], a]
>>> cells = array_handler.array2vtkCellArray(l_a)
"""
if vtk_array:
cells = vtk_array
else:
cells = vtk.vtkCellArray()
assert cells.GetClassName() == 'vtkCellArray', \
'Second argument must be a `vtkCellArray` instance.'
if len(num_array) == 0:
return cells
########################################
# Internal functions.
def _slow_array2cells(z, cells):
cells.Reset()
vtk_ids = vtk.vtkIdList()
for i in z:
vtk_ids.Reset()
for j in i:
vtk_ids.InsertNextId(j)
cells.InsertNextCell(vtk_ids)
def _get_tmp_array(arr):
try:
tmp_arr = numpy.asarray(arr, ID_TYPE_CODE)
except TypeError:
tmp_arr = arr.astype(ID_TYPE_CODE)
return tmp_arr
def _set_cells(cells, n_cells, id_typ_arr):
vtk_arr = vtk.vtkIdTypeArray()
array2vtk(id_typ_arr, vtk_arr)
cells.SetCells(n_cells, vtk_arr)
########################################
msg = "Invalid argument. Valid types are a Python list of lists,"\
" a Python list of numpy arrays, or a numpy array."
if issubclass(type(num_array), (list, tuple)):
assert len(num_array[0]) > 0, "Input array must be 2D."
tp = type(num_array[0])
if issubclass(tp, list): # Pure Python list.
_slow_array2cells(num_array, cells)
return cells
elif issubclass(tp, numpy.ndarray): # List of arrays.
# Check shape of array and find total size.
tot_size = 0
n_cells = 0
for arr in num_array:
assert len(arr.shape) == 2, "Each array must be 2D"
shp = arr.shape
tot_size += shp[0]*(shp[1] + 1)
n_cells += shp[0]
# Create an empty array.
id_typ_arr = numpy.empty((tot_size,), ID_TYPE_CODE)
# Now populate it with the ids.
count = 0
for arr in num_array:
tmp_arr = _get_tmp_array(arr)
shp = arr.shape
sz = shp[0]*(shp[1] + 1)
set_id_type_array(tmp_arr, id_typ_arr[count:count+sz])
count += sz
# Now set them cells.
_set_cells(cells, n_cells, id_typ_arr)
return cells
else:
raise TypeError(msg)
elif issubclass(type(num_array), numpy.ndarray):
assert len(num_array.shape) == 2, "Input array must be 2D."
tmp_arr = _get_tmp_array(num_array)
shp = tmp_arr.shape
id_typ_arr = numpy.empty((shp[0]*(shp[1] + 1),), ID_TYPE_CODE)
set_id_type_array(tmp_arr, id_typ_arr)
_set_cells(cells, shp[0], id_typ_arr)
return cells
else:
raise TypeError(msg)
0
Example 147
Project: Ragout Source File: nx_pylab.py
def draw_networkx_nodes(G, pos,
nodelist=None,
node_size=300,
node_color='r',
node_shape='o',
alpha=1.0,
cmap=None,
vmin=None,
vmax=None,
ax=None,
linewidths=None,
label = None,
**kwds):
"""Draw the nodes of the graph G.
This draws only the nodes of the graph G.
Parameters
----------
G : graph
A networkx graph
pos : dictionary
A dictionary with nodes as keys and positions as values.
If not specified a spring layout positioning will be computed.
See networkx.layout for functions that compute node positions.
ax : Matplotlib Axes object, optional
Draw the graph in the specified Matplotlib axes.
nodelist : list, optional
Draw only specified nodes (default G.nodes())
node_size : scalar or array
Size of nodes (default=300). If an array is specified it must be the
same length as nodelist.
node_color : color string, or array of floats
Node color. Can be a single color format string (default='r'),
or a sequence of colors with the same length as nodelist.
If numeric values are specified they will be mapped to
colors using the cmap and vmin,vmax parameters. See
matplotlib.scatter for more details.
node_shape : string
The shape of the node. Specification is as matplotlib.scatter
marker, one of 'so^>v<dph8' (default='o').
alpha : float
The node transparency (default=1.0)
cmap : Matplotlib colormap
Colormap for mapping intensities of nodes (default=None)
vmin,vmax : floats
Minimum and maximum for node colormap scaling (default=None)
linewidths : [None | scalar | sequence]
Line width of symbol border (default =1.0)
label : [None| string]
Label for legend
Examples
--------
>>> G=nx.dodecahedral_graph()
>>> nodes=nx.draw_networkx_nodes(G,pos=nx.spring_layout(G))
Also see the NetworkX drawing examples at
http://networkx.lanl.gov/gallery.html
See Also
--------
draw()
draw_networkx()
draw_networkx_edges()
draw_networkx_labels()
draw_networkx_edge_labels()
"""
try:
import matplotlib.pyplot as plt
import numpy
except ImportError:
raise ImportError("Matplotlib required for draw()")
except RuntimeError:
print("Matplotlib unable to open display")
raise
if ax is None:
ax=plt.gca()
if nodelist is None:
nodelist=G.nodes()
if not nodelist or len(nodelist)==0: # empty nodelist, no drawing
return None
try:
xy=numpy.asarray([pos[v] for v in nodelist])
except KeyError as e:
raise nx.NetworkXError('Node %s has no position.'%e)
except ValueError:
raise nx.NetworkXError('Bad value in node positions.')
node_collection=ax.scatter(xy[:,0], xy[:,1],
s=node_size,
c=node_color,
marker=node_shape,
cmap=cmap,
vmin=vmin,
vmax=vmax,
alpha=alpha,
linewidths=linewidths,
label=label)
node_collection.set_zorder(2)
return node_collection
0
Example 148
Project: mtpy Source File: decimation.py
def run():
print
if len(sys.argv) < 4:
sys.exit('\nNeed 3 arguments: \n\n '
'<path to files> <output directory> <integer downsampling factor>\n \n')
inpath = sys.argv[1]
outpath = sys.argv[2]
inpath = op.abspath(op.realpath(inpath))
if not op.isdir(inpath):
sys.exit('\nData file(s) path not existing: {0}\n'.format(inpath))
try:
outpath = op.abspath(op.join(os.curdir,outpath))
if inpath == outpath:
print 'Output directory cannot be the same as the input file location'
raise
if not op.exists(outpath):
try:
os.makedirs(outpath)
except:
raise
if not os.access(outpath, os.W_OK):
raise
except:
print 'Cannot generate writable output directory {0} - using'\
' generic location "decimated" instead'.format(outpath)
outpath = os.path.join(inpath,'decimated')
if not op.exists(outpath):
os.makedirs(outpath)
try:
decimation_factor = float(sys.argv[3])
except:
sys.exit('\n\tERROR - 3rd argument must be an integer decimation factor\n')
if decimation_factor < 1 or decimation_factor%1 != 0:
sys.exit('\n\tERROR - 3rd argument must be an integer >= 1\n')
decimation_factor = int(decimation_factor)
lo_files = os.listdir(inpath)
lo_files = [i for i in lo_files if op.isfile(op.join(inpath,i))]
if len(lo_files) == 0:
sys.exit('\n\tERROR - no data files in directory {0} \n'.format(inpath))
for fn in lo_files:
infile = os.path.join(inpath,fn)
outfile = os.path.join(outpath,fn)
try:
header = MTfh.read_ts_header(infile)
except MTex.MTpyError_ts_data:
#no TS data file
print '\n\tWARNING - not a valid MTpy TS data file: {0} '.format(infile)
header = None
#continue
if header is not None:
old_sampling = header['samplingrate']
new_sampling = old_sampling/decimation_factor
try:
data = []
F = open(infile)
for i in F:
try:
if i.startswith('#'):
continue
data.append(float(i.strip()))
except:
continue
data = np.asarray(data)
N = len(data)
except:
print '\tERROR - file does not contain single column data: {0} - SKIPPED'.format(infile)
continue
if header is not None:
n_samples = header['nsamples']
print 'Decimating file {0} by factor {1} '.format(infile, decimation_factor)
if n_samples%decimation_factor != 0 :
print '\tWarning - decimation of file not continuous due to mismatching decimation factor'
# to avoid ringing in the downsampled data: use de-meaning, padding, tapering:
padlength = int(0.1*N)
if padlength%decimation_factor != 0:
padlength = int(padlength/decimation_factor)*decimation_factor
meanvalue = np.mean(data)
data -= meanvalue
prepad = np.ones((padlength)) * data[0]
postpad = np.ones((padlength)) * data[-1]
padded_data = np.append(prepad,data)
padded_data = np.append(padded_data,postpad)
tapered = taper_data(padded_data)
filtered = ss.resample(tapered,int(len(tapered)/decimation_factor),window='blackman')
new_padding = padlength/decimation_factor
padding_cut = filtered[new_padding:-new_padding]
new_data = padding_cut - np.mean(padding_cut) + meanvalue
Fout = open(outfile,'w')
if header is not None:
header['nsamples'] = len(new_data)
header['samplingrate'] = new_sampling
new_header_line = MTfh.get_ts_header_string(header)
Fout.write(new_header_line)
for i in new_data:
if i%1==0:
Fout.write('{0:d}\n'.format(int(i)))
else:
Fout.write('{0:.8}\n'.format(i))
#np.savetxt(Fout,new_data)
Fout.close()
print '\nOutput files written to {0}'.format(outpath)
print '\n...Done\n'
0
Example 149
Project: rapprentice Source File: do_task.py
def main():
demofile = h5py.File(args.h5file, 'r')
trajoptpy.SetInteractive(args.interactive)
if args.log:
LOG_DIR = osp.join(osp.expanduser("~/Data/do_task_logs"), datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
os.mkdir(LOG_DIR)
LOG_COUNT = 0
if args.execution:
rospy.init_node("exec_task",disable_signals=True)
Globals.pr2 = PR2.PR2()
Globals.env = Globals.pr2.env
Globals.robot = Globals.pr2.robot
else:
Globals.env = openravepy.Environment()
Globals.env.StopSimulation()
Globals.env.Load("robots/pr2-beta-static.zae")
Globals.robot = Globals.env.GetRobots()[0]
if not args.fake_data_segment:
grabber = cloudprocpy.CloudGrabber()
grabber.startRGBD()
Globals.viewer = trajoptpy.GetViewer(Globals.env)
#####################
while True:
redprint("Acquire point cloud")
if args.fake_data_segment:
fake_seg = demofile[args.fake_data_segment]
new_xyz = np.squeeze(fake_seg["cloud_xyz"])
hmat = openravepy.matrixFromAxisAngle(args.fake_data_transform[3:6])
hmat[:3,3] = args.fake_data_transform[0:3]
new_xyz = new_xyz.dot(hmat[:3,:3].T) + hmat[:3,3][None,:]
r2r = ros2rave.RosToRave(Globals.robot, asarray(fake_seg["joint_states"]["name"]))
r2r.set_values(Globals.robot, asarray(fake_seg["joint_states"]["position"][0]))
else:
Globals.pr2.head.set_pan_tilt(0,1.2)
Globals.pr2.rarm.goto_posture('side')
Globals.pr2.larm.goto_posture('side')
Globals.pr2.join_all()
time.sleep(.5)
Globals.pr2.update_rave()
rgb, depth = grabber.getRGBD()
T_w_k = berkeley_pr2.get_kinect_transform(Globals.robot)
new_xyz = cloud_proc_func(rgb, depth, T_w_k)
#grab_end(new_xyz)
if args.log:
LOG_COUNT += 1
import cv2
cv2.imwrite(osp.join(LOG_DIR,"rgb%i.png"%LOG_COUNT), rgb)
cv2.imwrite(osp.join(LOG_DIR,"depth%i.png"%LOG_COUNT), depth)
np.save(osp.join(LOG_DIR,"xyz%i.npy"%LOG_COUNT), new_xyz)
################################
redprint("Finding closest demonstration")
if args.select_manual:
seg_name = find_closest_manual(demofile, new_xyz)
else:
seg_name = find_closest_auto(demofile, new_xyz)
seg_info = demofile[seg_name]
redprint("closest demo: %s"%(seg_name))
if "done" in seg_name:
redprint("DONE!")
break
if args.log:
with open(osp.join(LOG_DIR,"neighbor%i.txt"%LOG_COUNT),"w") as fh: fh.write(seg_name)
################################
redprint("Generating end-effector trajectory")
handles = []
old_xyz = np.squeeze(seg_info["cloud_xyz"])
handles.append(Globals.env.plot3(old_xyz,5, (1,0,0)))
handles.append(Globals.env.plot3(new_xyz,5, (0,0,1)))
scaled_old_xyz, src_params = registration.unit_boxify(old_xyz)
scaled_new_xyz, targ_params = registration.unit_boxify(new_xyz)
f,_ = registration.tps_rpm_bij(scaled_old_xyz, scaled_new_xyz, plot_cb = tpsrpm_plot_cb,
plotting=5 if args.animation else 0,rot_reg=np.r_[1e-4,1e-4,1e-1], n_iter=50, reg_init=10, reg_final=.1)
f = registration.unscale_tps(f, src_params, targ_params)
handles.extend(plotting_openrave.draw_grid(Globals.env, f.transform_points, old_xyz.min(axis=0)-np.r_[0,0,.1], old_xyz.max(axis=0)+np.r_[0,0,.1], xres = .1, yres = .1, zres = .04))
link2eetraj = {}
for lr in 'lr':
link_name = "%s_gripper_tool_frame"%lr
old_ee_traj = asarray(seg_info[link_name]["hmat"])
new_ee_traj = f.transform_hmats(old_ee_traj)
link2eetraj[link_name] = new_ee_traj
handles.append(Globals.env.drawlinestrip(old_ee_traj[:,:3,3], 2, (1,0,0,1)))
handles.append(Globals.env.drawlinestrip(new_ee_traj[:,:3,3], 2, (0,1,0,1)))
miniseg_starts, miniseg_ends = split_trajectory_by_gripper(seg_info)
success = True
print colorize.colorize("mini segments:", "red"), miniseg_starts, miniseg_ends
for (i_miniseg, (i_start, i_end)) in enumerate(zip(miniseg_starts, miniseg_ends)):
if args.execution=="real": Globals.pr2.update_rave()
################################
redprint("Generating joint trajectory for segment %s, part %i"%(seg_name, i_miniseg))
# figure out how we're gonna resample stuff
lr2oldtraj = {}
for lr in 'lr':
manip_name = {"l":"leftarm", "r":"rightarm"}[lr]
old_joint_traj = asarray(seg_info[manip_name][i_start:i_end+1])
#print (old_joint_traj[1:] - old_joint_traj[:-1]).ptp(axis=0), i_start, i_end
if arm_moved(old_joint_traj):
lr2oldtraj[lr] = old_joint_traj
if len(lr2oldtraj) > 0:
old_total_traj = np.concatenate(lr2oldtraj.values(), 1)
JOINT_LENGTH_PER_STEP = .1
_, timesteps_rs = unif_resample(old_total_traj, JOINT_LENGTH_PER_STEP)
####
### Generate fullbody traj
bodypart2traj = {}
for (lr,old_joint_traj) in lr2oldtraj.items():
manip_name = {"l":"leftarm", "r":"rightarm"}[lr]
old_joint_traj_rs = mu.interp2d(timesteps_rs, np.arange(len(old_joint_traj)), old_joint_traj)
ee_link_name = "%s_gripper_tool_frame"%lr
new_ee_traj = link2eetraj[ee_link_name][i_start:i_end+1]
new_ee_traj_rs = resampling.interp_hmats(timesteps_rs, np.arange(len(new_ee_traj)), new_ee_traj)
if args.execution: Globals.pr2.update_rave()
new_joint_traj = planning.plan_follow_traj(Globals.robot, manip_name,
Globals.robot.GetLink(ee_link_name), new_ee_traj_rs,old_joint_traj_rs)
part_name = {"l":"larm", "r":"rarm"}[lr]
bodypart2traj[part_name] = new_joint_traj
################################
redprint("Executing joint trajectory for segment %s, part %i using arms '%s'"%(seg_name, i_miniseg, bodypart2traj.keys()))
for lr in 'lr':
success &= set_gripper_maybesim(lr, binarize_gripper(seg_info["%s_gripper_joint"%lr][i_start]))
# Doesn't actually check if grab occurred, unfortunately
if not success: break
if len(bodypart2traj) > 0:
success &= exec_traj_maybesim(bodypart2traj)
if not success: break
redprint("Segment %s result: %s"%(seg_name, success))
if args.fake_data_segment: break
0
Example 150
def doit(device_num=0,
mode_num=None,
num_buffers=30,
save=False,
max_frames=None,
trigger_mode=None,
roi=None,
):
num_modes = cam_iface.get_num_modes(device_num)
for this_mode_num in range(num_modes):
mode_str = cam_iface.get_mode_string(device_num,this_mode_num)
print 'mode %d: %s'%(this_mode_num,mode_str)
if mode_num is None:
if 'DC1394_VIDEO_MODE_FORMAT7_0' in mode_str and 'MONO8' in mode_str:
mode_num=this_mode_num
if mode_num is None:
mode_num=0
print 'choosing mode %d'%(mode_num,)
cam = cam_iface.Camera(device_num,num_buffers,mode_num)
if save:
format = cam.get_pixel_coding()
depth = cam.get_pixel_depth()
filename = time.strftime( 'simple%Y%m%d_%H%M%S.fmf' )
fly_movie = FlyMovieFormat.FlyMovieSaver(filename,
version=3,
format=format,
bits_per_pixel=depth,
)
save_queue = Queue.Queue()
save_thread = threading.Thread( target=save_func, args=(fly_movie,save_queue))
save_thread.setDaemon(True)
save_thread.start()
num_props = cam.get_num_camera_properties()
#for i in range(num_props):
# print "property %d: %s"%(i,str(cam.get_camera_property_info(i)))
n_trigger_modes = cam.get_num_trigger_modes()
print "Trigger modes:"
for i in range(n_trigger_modes):
print ' %d: %s'%(i,cam.get_trigger_mode_string(i))
if trigger_mode is not None:
cam.set_trigger_mode_number( trigger_mode )
print 'Using trigger mode %d'%(cam.get_trigger_mode_number())
cam.start_camera()
if roi is not None:
cam.set_frame_roi( *roi )
actual_roi = cam.get_frame_roi()
if roi != actual_roi:
raise ValueError("could not set ROI. Actual ROI is %s."%(actual_roi,))
frametick = 0
framecount = 0
last_fps_print = time.time()
last_fno = None
while 1:
try:
buf = nx.asarray(cam.grab_next_frame_blocking())
except cam_iface.FrameDataMissing:
sys.stdout.write('M')
sys.stdout.flush()
continue
except cam_iface.FrameSystemCallInterruption:
sys.stdout.write('I')
sys.stdout.flush()
continue
timestamp = cam.get_last_timestamp()
fno = cam.get_last_framenumber()
if last_fno is not None:
skip = (fno-last_fno)-1
if skip != 0:
print 'WARNING: skipped %d frames'%skip
## if frametick==50:
## print 'sleeping'
## time.sleep(10.0)
## print 'wake'
last_fno=fno
now = time.time()
sys.stdout.write('.')
sys.stdout.flush()
frametick += 1
framecount += 1
t_diff = now-last_fps_print
if t_diff > 5.0:
fps = frametick/t_diff
print "%.1f fps"%fps
last_fps_print = now
frametick = 0
if save:
save_queue.put( (buf,timestamp) )
if max_frames:
if framecount >= max_frames:
break