def iflatten(arrays):
"""
flatten the arrays in a stream into a single, 1D array. Note that
the order of flattening is not guaranteed.
Parameters
----------
arrays : iterable
Stream of NumPy arrays. Contrary to convention, these
arrays do not need to be of the same shape.
Yields
------
online_flatten : ndarray
Cumulative flattened array.
"""
arrays = map(np.ravel, arrays)
yield from istack(arrays, axis = 0)
python类ravel()的实例源码
def learn(self, features, labels):
""" Fits the classifier
If it's state is empty, the classifier is fitted, if not
the classifier is partially fitted.
See sklearn's SGDClassifier fit and partial_fit methods.
Args:
features (:obj:`list` of :obj:`list` of :obj:`float`)
labels (:obj:`list` of :obj:`str`): Labels for each set of features.
New features are learnt.
"""
labels = np.ravel(labels)
self.__learn_labels(labels)
if len(labels) == 0:
return
labels = self.labels.transform(labels)
if self.feature_length > 0 and hasattr(self.clf, 'partial_fit'):
# FIXME? check docs, may need to pass class=[...]
self.clf = self.clf.partial_fit(features, labels)
else:
self.clf = self.clf.fit(features, labels)
self.feature_length = len(features[0])
def glove(data_fname='glove.840B.300d.txt', out_fname='glove.pkl'):
"""Process raw dependency GloVe data from Socher '13"""
words, U, dim = [], [], None
with open(DATA_DIR + data_fname, 'rb') as f:
for j, line in enumerate(f):
x = line.strip().split()
word, vector, d = x[0], np.ravel(x[1:]), len(x) - 1
if dim is None: dim = d
elif d != dim: raise Exception('{0}: {1}!={2}'.format(j, dim, d))
U.append(vector)
words.append(word)
U = np.array(U)
print "Found {0} words".format(len(words))
print "Found {0}x{1} embedding matrix".format(*U.shape)
with open(DATA_DIR + out_fname, 'wb') as f:
cPickle.dump((words, U), f)
def test_minmax_func(self):
# Tests minimum and maximum.
(x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
# max doesn't work if shaped
xr = np.ravel(x)
xmr = ravel(xm)
# following are true because of careful selection of data
assert_equal(max(xr), maximum(xmr))
assert_equal(min(xr), minimum(xmr))
assert_equal(minimum([1, 2, 3], [4, 0, 9]), [1, 0, 3])
assert_equal(maximum([1, 2, 3], [4, 0, 9]), [4, 2, 9])
x = arange(5)
y = arange(5) - 2
x[3] = masked
y[0] = masked
assert_equal(minimum(x, y), where(less(x, y), x, y))
assert_equal(maximum(x, y), where(greater(x, y), x, y))
assert_(minimum(x) == 0)
assert_(maximum(x) == 4)
x = arange(4).reshape(2, 2)
x[-1, -1] = masked
assert_equal(maximum(x), 2)
def test_ravel(self):
# Tests ravel
a = array([[1, 2, 3, 4, 5]], mask=[[0, 1, 0, 0, 0]])
aravel = a.ravel()
assert_equal(aravel._mask.shape, aravel.shape)
a = array([0, 0], mask=[1, 1])
aravel = a.ravel()
assert_equal(aravel._mask.shape, a.shape)
a = array(np.matrix([1, 2, 3, 4, 5]), mask=[[0, 1, 0, 0, 0]])
aravel = a.ravel()
assert_equal(aravel.shape, (1, 5))
assert_equal(aravel._mask.shape, a.shape)
# Checks that small_mask is preserved
a = array([1, 2, 3, 4], mask=[0, 0, 0, 0], shrink=False)
assert_equal(a.ravel()._mask, [0, 0, 0, 0])
# Test that the fill_value is preserved
a.fill_value = -99
a.shape = (2, 2)
ar = a.ravel()
assert_equal(ar._mask, [0, 0, 0, 0])
assert_equal(ar._data, [1, 2, 3, 4])
assert_equal(ar.fill_value, -99)
# Test index ordering
assert_equal(a.ravel(order='C'), [1, 2, 3, 4])
assert_equal(a.ravel(order='F'), [1, 3, 2, 4])
def test_view(self):
# Test view w/ flexible dtype
iterator = list(zip(np.arange(10), np.random.rand(10)))
data = np.array(iterator)
a = array(iterator, dtype=[('a', float), ('b', float)])
a.mask[0] = (1, 0)
controlmask = np.array([1] + 19 * [0], dtype=bool)
# Transform globally to simple dtype
test = a.view(float)
assert_equal(test, data.ravel())
assert_equal(test.mask, controlmask)
# Transform globally to dty
test = a.view((float, 2))
assert_equal(test, data)
assert_equal(test.mask, controlmask.reshape(-1, 2))
test = a.view((float, 2), np.matrix)
assert_equal(test, data)
self.assertTrue(isinstance(test, np.matrix))
def hex2vec(h, ell):
"""hex2vec(h, ell) generates sign vector of length ell from the hex string h.
ell must be <= 4*len(h) (excluding the optional leading "0x")
"""
if h[0:2] in ['0x', '0X']:
h = h[2:]
nybble = numpy.array([
[0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 1, 1],
[0, 1, 0, 0], [0, 1, 0, 1], [
0, 1, 1, 0], [0, 1, 1, 1],
[1, 0, 0, 0], [1, 0, 0, 1], [
1, 0, 1, 0], [1, 0, 1, 1],
[1, 1, 0, 0], [1, 1, 0, 1], [1, 1, 1, 0], [1, 1, 1, 1]])
vec = numpy.ravel(numpy.array([nybble[int(x, 16)] for x in h]))
if len(vec) < ell:
raise ValueError('hex string too short')
return vec[len(vec) - ell:]
def test_multicollinearity(df, target_name, r2_threshold = 0.89):
'''Tests if any of the features could be predicted from others with R2 >= 0.89
input: dataframe, name of target (to exclude)
'''
r2s = pd.DataFrame()
for feature in df.columns.difference([target_name]):
model = sk.linear_model.Ridge()
model.fit(df[df.columns.difference([target_name,feature])], df[feature])
pos = np.in1d(model.coef_, np.sort(model.coef_)[-5:])
r2s = r2s.append(pd.DataFrame({'r2':sk.metrics.r2_score(df[feature],\
model.predict(df[df.columns.difference([target_name, feature])])),\
'predictors' : str(df.columns.difference([target_name, feature])[np.ravel(np.argwhere(pos == True))].tolist())}, index = [feature]))
print('Testing', feature)
print('-----------------')
if len(r2s[r2s['r2'] >= r2_threshold]) > 0:
print('Multicollinearity detected')
print(r2s[r2s['r2'] >= r2_threshold])
else:
print('No multicollinearity')
def __init__(self, y, nsuj, pout=1, clf='lda', **clfArg):
self._y = y
self._ry = np.ravel(np.concatenate(y))
self._nsuj = nsuj
self._pout = pout
# Manage cross-validation:
self._cv = LeavePGroupsOut(pout)
self._cv.shStr = 'Leave '+str(pout)+' subjects out'
self._cv.lgStr = self._cv.shStr
self._cv.rep = 1
self._cv.y = y[0]
# Manage classifier :
if isinstance(clf, (int, str)):
clf = defClf(self._ry, clf=clf, **clfArg)
self._clf = clf
# Manage info:
self._updatestring()
# Stat tools:
self.stat = clfstat()
def _fit(x, y, train, test, self, n_jobs):
"""Sub fit function
"""
nsuj, nfeat = x.shape
iteract = product(range(nfeat), zip(train, test))
ya = Parallel(n_jobs=n_jobs)(delayed(_subfit)(
np.concatenate(tuple(x[i].iloc[k[0]])),
np.concatenate(tuple(x[i].iloc[k[1]])),
np.concatenate(tuple(y[0].iloc[k[0]])),
np.concatenate(tuple(y[0].iloc[k[1]])),
self) for i, k in iteract)
# Re-arrange ypred and ytrue:
ypred, ytrue = zip(*ya)
ypred = [np.concatenate(tuple(k)) for k in np.split(np.array(ypred), nfeat)]
ytrue = [np.concatenate(tuple(k)) for k in np.split(np.array(ytrue), nfeat)]
da = np.ravel([100*accuracy_score(ytrue[k], ypred[k]) for k in range(nfeat)])
return da, ytrue, ypred
def _featinfo(self, clf, cv, da, grp=None, pbino=None, pperm=None):
# Manage input arguments :
dastd = np.round(100*da.std(axis=1))/100
dam = da.mean(axis=1)
if grp is None:
grp = np.array([str(k) for k in range(len(dam))])
if pbino is None:
pbino = bino_da2p(self.y, dam)
if pperm is None:
pperm = np.ones((len(dam),))
array = np.array([np.ravel(dam), np.ravel(dastd), np.ravel(pbino), np.ravel(pperm), np.ravel(grp)]).T
# Create the dataframe:
subcol = ['DA (%)', 'STD (+/-)', 'p-values (Binomial)', 'p-values (Permutations)', 'Group']
str2repeat = clf.shStr+' / '+cv.shStr
idxtuple = list(zip(*[[str2repeat]*len(subcol), subcol]))
index = pd.MultiIndex.from_tuples(idxtuple, names=['Settings', 'Results'])
return pd.DataFrame(array, columns=index)
def bonferroni(p, axis=-1):
"""Bonferroni correction
Args:
p: array
Array of p-values
Kargs:
axis: int, optional, [def: -1]
Axis to apply the Bonferroni correction. If axis is -1,
the correction is applied through all dimensions.
Return:
Corrected pvalues
"""
if axis == -1:
fact = len(np.ravel(p))
else:
fact = p.shape[axis]
return fact*p
def bino_da2p(y, da):
"""For a given vector label, get p-values of a decoding accuracy
using the binomial law.
Args:
y : array
The vector label
da: int / float / list /array [0 <= da <= 100]
The decoding accuracy array. Ex : da = [75, 33, 25, 17].
Return:
p: ndarray
The p-value associate to each decoding accuracy
"""
y = np.ravel(y)
nbepoch = len(y)
nbclass = len(np.unique(y))
if not isinstance(da, np.ndarray):
da = np.array(da)
if (da.max() > 100) or (da.min() < 0):
raise ValueError('Consider 0<=da<=100')
return 1 - binom.cdf(nbepoch * da / 100, nbepoch, 1 / nbclass)
def perm_array(x, n_perm=200, rndstate=0):
"""Generate n_perm permutations of a ndarray
Args:
x: array
Data to repeat of shape (d1, d2, ..., d3)
n_perm: int
Number of permutations
rndstate: int
Fix the random state of the machine
Returns:
perm: array
Repeated data of shape (n_perm, d1, d2, ..., d3)
idx: array
Index of permutations of shape (n_perm, d1, d2, ..., d3)
"""
dim = tuple([n_perm] + list(x.shape))
xrep = perm_rep(np.ravel(x), n_perm)
xrep, idx = _scramble2D(xrep, rndstate=rndstate)
return xrep.reshape(dim), idx.reshape(dim)
def __new__(self, ax, y, x=None, color=None, cmap='inferno', pltargs={}, **kwargs):
# Check inputs :
y = np.ravel(y)
if x is None:
x = np.arange(len(y))
else:
x = np.ravel(x)
if len(y) != len(x):
raise ValueError('x and y must have the same length')
if color is None:
color = np.arange(len(y))
# Create segments:
xy = np.array([x, y]).T[..., np.newaxis].reshape(-1, 1, 2)
segments = np.concatenate((xy[0:-1, :], xy[1::]), axis=1)
lc = LineCollection(segments, cmap=cmap, **pltargs)
lc.set_array(color)
# Plot managment:
ax.add_collection(lc)
plt.axis('tight')
_pltutils.__init__(self, ax, **kwargs)
return plt.gca()
def setUp(self):
self.betas = numpy.linspace(1e-5, 1., 10)
self.n = n = 1000
gaussian = FunnyGaussian(10, 100.)
self.samples = []
self.raw_energies = []
for beta in self.betas:
self.samples.append(gaussian.sample(n, beta))
self.raw_energies.append(gaussian.energy(self.samples[-1]))
self.raw_energies = numpy.array(self.raw_energies)
self.ensembles = [BoltzmannEnsemble(beta=beta) for beta in self.betas]
self.log_z = gaussian.log_Z()
self.log_g = gaussian.log_g(numpy.ravel(self.raw_energies))
def testTrapezoidal2D(self):
from csb.numeric import trapezoidal_2d, exp
from numpy import pi
xx = np.linspace(-10., 10, 500)
yy = np.linspace(-10., 10, 500)
X, Y = np.meshgrid(xx, yy)
x = np.array(list(zip(np.ravel(X), np.ravel(Y))))
# mean = np.zeros((2,))
cov = np.eye(2)
mu = np.ones(2)
# D = 2
q = np.sqrt(np.clip(np.sum((x - mu) * np.dot(x - mu, np.linalg.inv(cov).T), -1), 0., 1e308))
f = exp(-0.5 * q ** 2) / ((2 * pi) * np.sqrt(np.abs(np.linalg.det(cov))))
f = f.reshape((len(xx), len(yy)))
I = trapezoidal_2d(f) * (xx[1] - xx[0]) * (yy[1] - yy[0])
self.assertTrue(abs(I - 1.) <= 1e-8)
def testLogTrapezoidal2D(self):
from csb.numeric import log_trapezoidal_2d, log
from numpy import pi
xx = np.linspace(-10., 10, 500)
yy = np.linspace(-10., 10, 500)
X, Y = np.meshgrid(xx, yy)
x = np.array(list(zip(np.ravel(X), np.ravel(Y))))
# mean = np.zeros((2,))
cov = np.eye(2)
mu = np.ones(2)
# D = 2
q = np.sqrt(np.clip(np.sum((x - mu) * np.dot(x - mu, np.linalg.inv(cov).T), -1), 0., 1e308))
f = -0.5 * q ** 2 - log((2 * pi) * np.sqrt(np.abs(np.linalg.det(cov))))
f = f.reshape((len(xx), len(yy)))
logI = log_trapezoidal_2d(f, xx, yy)
self.assertTrue(abs(logI) <= 1e-8)
def load_board(string):
reverse_map = {
'X': go.BLACK,
'O': go.WHITE,
'.': go.EMPTY,
'#': go.FILL,
'*': go.KO,
'?': go.UNKNOWN
}
string = re.sub(r'[^XO\.#]+', '', string)
assert len(string) == go.N ** 2, "Board to load didn't have right dimensions"
board = np.zeros([go.N, go.N], dtype=np.int8)
for i, char in enumerate(string):
np.ravel(board)[i] = reverse_map[char]
return board
def sanitize_array(array):
"""
Replace NaN and Inf (there should not be any!)
:param array:
:return:
"""
a = np.ravel(array)
#maxi = np.nanmax((filter(lambda x: x != float('inf'), a))
# ) # Max except NaN and Inf
#mini = np.nanmin((filter(lambda x: x != float('-inf'), a))
# ) # Mini except NaN and Inf
maxi = np.nanmax(a[np.isfinite(a)])
mini = np.nanmin(a[np.isfinite(a)])
array[array == float('inf')] = maxi
array[array == float('-inf')] = mini
mid = (maxi + mini) / 2
array[np.isnan(array)] = mid
return array
def diff_approx(self, fields, pars, eps=1E-8):
nvar, N = len(fields.dependent_variables), fields.size
fpars = {key: pars[key] for key in self.pars}
fpars['dx'] = (fields['x'][-1] - fields['x'][0]) / fields['x'].size
J = np.zeros((N * nvar, N * nvar))
indices = np.indices(fields.uarray.shape)
for i, (var_index, node_index) in enumerate(zip(*map(np.ravel,
indices))):
fields_plus = fields.copy()
fields_plus.uarray[var_index, node_index] += eps
fields_moins = fields.copy()
fields_moins.uarray[var_index, node_index] -= eps
Fplus = self(fields_plus, pars)
Fmoins = self(fields_moins, pars)
J[i] = (Fplus - Fmoins) / (2 * eps)
return J.T
def bm25_weight(X, K1=100, B=0.8):
""" Weighs each row of a sparse matrix X by BM25 weighting """
# calculate idf per term (user)
X = coo_matrix(X)
N = float(X.shape[0])
idf = log(N / (1 + bincount(X.col)))
# calculate length_norm per document (artist)
row_sums = numpy.ravel(X.sum(axis=1))
average_length = row_sums.mean()
length_norm = (1.0 - B) + B * row_sums / average_length
# weight matrix rows by bm25
X.data = X.data * (K1 + 1.0) / (K1 * length_norm[X.row] + X.data) * idf[X.col]
return X
def sym2bi(x, m):
"""Convert symbols to bits.
:param x: symbol array
:param m: symbol alphabet size (must be a power of 2)
:returns: bit array
>>> import arlpy
>>> arlpy.comms.sym2bi([1, 2, 7], 8)
array([0, 0, 1, 0, 1, 0, 1, 1, 1])
"""
n = int(_np.log2(m))
if 2**n != m:
raise ValueError('m must be a power of 2')
x = _np.asarray(x, dtype=_np.int)
if _np.any(x < 0) or _np.any(x >= m):
raise ValueError('Invalid data for specified m')
y = _np.zeros((len(x), n), dtype=_np.int)
for i in range(n):
y[:, n-i-1] = (x >> i) & 1
return _np.ravel(y)
def _check_transformer_output(transformer, dataset, expected):
"""
Given a transformer and a spark dataset, check if the transformer
produces the expected results.
"""
analyzed_df = tfs.analyze(dataset)
out_df = transformer.transform(analyzed_df)
# Collect transformed values
out_colnames = list(_output_mapping.values())
_results = []
for row in out_df.select(out_colnames).collect():
curr_res = [row[colname] for colname in out_colnames]
_results.append(np.ravel(curr_res))
out_tgt = np.hstack(_results)
_err_msg = 'not close => shape {} != {}, max_diff {} > {}'
max_diff = np.max(np.abs(expected - out_tgt))
err_msg = _err_msg.format(expected.shape, out_tgt.shape,
max_diff, _all_close_tolerance)
assert np.allclose(expected, out_tgt, atol=_all_close_tolerance), err_msg
def hyperball(ndim, radius):
"""Return a binary morphological filter containing pixels within `radius`.
Parameters
----------
ndim : int
The number of dimensions of the filter.
radius : int
The radius of the filter.
Returns
-------
ball : array of bool, shape [2 * radius + 1,] * ndim
The required structural element
"""
size = 2 * radius + 1
center = [(radius,) * ndim]
coords = np.mgrid[[slice(None, size),] * ndim].reshape(ndim, -1).T
distances = np.ravel(spatial.distance_matrix(coords, center))
selector = distances <= radius
ball = np.zeros((size,) * ndim, dtype=bool)
ball.ravel()[selector] = True
return ball
def get_timepixel_image( x,y,t, det_shape = [256, 256], delta_time = None ):
'''give x,y, t data to get image in a period of delta_time (in second)'''
t0 = t.min() *6.1
tm = t.max() *6.1
if delta_time is not None:
delta_time *=1e12
if delta_time > tm:
delta_time = tm
else:
delta_time = tm
#print( delta_time)
t_ = t[t<delta_time]
x_ = x[:len(t_)]
y_ = y[:len(t_)]
img = np.zeros( det_shape, dtype= np.int32 )
pixlist = x_*det_shape[0] + y_
his = np.histogram( pixlist, bins= np.arange( det_shape[0]*det_shape[1] +1) )[0]
np.ravel( img )[:] = his
print( 'The max photon count is %d.'%img.max())
return img
def check_normalization( frame_num, q_list, imgsa, data_pixel ):
'''check the ROI intensity before and after normalization
Input:
frame_num: integer, the number of frame to be checked
q_list: list of integer, the list of q to be checked
imgsa: the raw data
data_pixel: the normalized data, caculated by fucntion Get_Pixel_Arrayc
Plot the intensities
'''
fig,ax=plt.subplots(2)
n=0
for q in q_list:
norm_data = data_pixel[frame_num][qind==q]
raw_data = np.ravel( np.array(imgsa[frame_num]) )[pixelist[qind==q]]
#print(raw_data.mean())
plot1D( raw_data,ax=ax[0], legend='q=%s'%(q), m=markers[n],
title='fra=%s_raw_data'%(frame_num))
#plot1D( raw_data/mean_int_sets_[frame_num][q-1], ax=ax[1], legend='q=%s'%(q), m=markers[n],
# xlabel='pixel',title='fra=%s_norm_data'%(frame_num))
#print( mean_int_sets_[frame_num][q-1] )
plot1D( norm_data, ax=ax[1], legend='q=%s'%(q), m=markers[n],
xlabel='pixel',title='fra=%s_norm_data'%(frame_num))
n +=1
def periodogram(self, attr):
is_equispaced = self.data.time_delta is not None
if is_equispaced:
x = np.ravel(self.data.interp(attr))
periods, pgram = periodogram_equispaced(x)
# TODO: convert periods into time_values-relative values, i.e.
# periods *= self.data.time_delta; like lombscargle already does
# periods *= self.data.time_delta
else:
times = np.asanyarray(self.data.time_values, dtype=float)
x = np.ravel(self.data[:, attr])
# Since lombscargle works with explicit times,
# we can skip any nan values
nonnan = ~np.isnan(x)
if not nonnan.all():
x, times = x[nonnan], times[nonnan]
periods, pgram = periodogram_nonequispaced(times, x)
return periods, pgram
def test_minmax_func(self):
# Tests minimum and maximum.
(x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
# max doesn't work if shaped
xr = np.ravel(x)
xmr = ravel(xm)
# following are true because of careful selection of data
assert_equal(max(xr), maximum(xmr))
assert_equal(min(xr), minimum(xmr))
assert_equal(minimum([1, 2, 3], [4, 0, 9]), [1, 0, 3])
assert_equal(maximum([1, 2, 3], [4, 0, 9]), [4, 2, 9])
x = arange(5)
y = arange(5) - 2
x[3] = masked
y[0] = masked
assert_equal(minimum(x, y), where(less(x, y), x, y))
assert_equal(maximum(x, y), where(greater(x, y), x, y))
assert_(minimum(x) == 0)
assert_(maximum(x) == 4)
x = arange(4).reshape(2, 2)
x[-1, -1] = masked
assert_equal(maximum(x), 2)
def test_ravel(self):
# Tests ravel
a = array([[1, 2, 3, 4, 5]], mask=[[0, 1, 0, 0, 0]])
aravel = a.ravel()
assert_equal(aravel._mask.shape, aravel.shape)
a = array([0, 0], mask=[1, 1])
aravel = a.ravel()
assert_equal(aravel._mask.shape, a.shape)
a = array(np.matrix([1, 2, 3, 4, 5]), mask=[[0, 1, 0, 0, 0]])
aravel = a.ravel()
assert_equal(aravel.shape, (1, 5))
assert_equal(aravel._mask.shape, a.shape)
# Checks that small_mask is preserved
a = array([1, 2, 3, 4], mask=[0, 0, 0, 0], shrink=False)
assert_equal(a.ravel()._mask, [0, 0, 0, 0])
# Test that the fill_value is preserved
a.fill_value = -99
a.shape = (2, 2)
ar = a.ravel()
assert_equal(ar._mask, [0, 0, 0, 0])
assert_equal(ar._data, [1, 2, 3, 4])
assert_equal(ar.fill_value, -99)
# Test index ordering
assert_equal(a.ravel(order='C'), [1, 2, 3, 4])
assert_equal(a.ravel(order='F'), [1, 3, 2, 4])
