def iteratorFn(self, data):
## Return 1) a function that will provide an iterator for data and 2) a list of header strings
if isinstance(data, list) or isinstance(data, tuple):
return lambda d: d.__iter__(), None
elif isinstance(data, dict):
return lambda d: iter(d.values()), list(map(asUnicode, data.keys()))
elif (hasattr(data, 'implements') and data.implements('MetaArray')):
if data.axisHasColumns(0):
header = [asUnicode(data.columnName(0, i)) for i in range(data.shape[0])]
elif data.axisHasValues(0):
header = list(map(asUnicode, data.xvals(0)))
else:
header = None
return self.iterFirstAxis, header
elif isinstance(data, np.ndarray):
return self.iterFirstAxis, None
elif isinstance(data, np.void):
return self.iterate, list(map(asUnicode, data.dtype.names))
elif data is None:
return (None,None)
else:
msg = "Don't know how to iterate over data type: {!s}".format(type(data))
raise TypeError(msg)
python类void()的实例源码
def iteratorFn(self, data):
## Return 1) a function that will provide an iterator for data and 2) a list of header strings
if isinstance(data, list) or isinstance(data, tuple):
return lambda d: d.__iter__(), None
elif isinstance(data, dict):
return lambda d: iter(d.values()), list(map(asUnicode, data.keys()))
elif (hasattr(data, 'implements') and data.implements('MetaArray')):
if data.axisHasColumns(0):
header = [asUnicode(data.columnName(0, i)) for i in range(data.shape[0])]
elif data.axisHasValues(0):
header = list(map(asUnicode, data.xvals(0)))
else:
header = None
return self.iterFirstAxis, header
elif isinstance(data, np.ndarray):
return self.iterFirstAxis, None
elif isinstance(data, np.void):
return self.iterate, list(map(asUnicode, data.dtype.names))
elif data is None:
return (None,None)
else:
msg = "Don't know how to iterate over data type: {!s}".format(type(data))
raise TypeError(msg)
def filter_sort_unique(self, max_objval=float('Inf')):
# filter
if max_objval < float('inf'):
good_idx = self.objvals <= max_objval
self.objvals = self.objvals[good_idx]
self.solutions = self.solutions[good_idx]
if len(self.objvals) > 0:
sort_idx = np.argsort(self.objvals)
self.objvals = self.objvals[sort_idx]
self.solutions = self.solutions[sort_idx]
# unique
b = np.ascontiguousarray(self.solutions).view(
np.dtype((np.void, self.solutions.dtype.itemsize * self.P)))
_, unique_idx = np.unique(b, return_index=True)
self.objvals = self.objvals[unique_idx]
self.solutions = self.solutions[unique_idx]
def unique(eq):
eq = eqsize(eq)
c1 = [None] * eq.shape
for i in range(0, eq.size):
c1.append[i] = hash(eq[i])
c1 = np.asarray(c1)
if c1.ndim == 1:
_, ia, ic = np.unique(c1, return_index=True, return_inverse=True)
ia = (ia[:, ]).conj().T
ic = (ic[:, ]).conj().T
u = eq[ia]
else:
a = c1
b = np.ascontiguousarray(a).view(
np.dtype((np.void, a.dtype.itemsize * a.shape[1])))
_, ia, ic = np.unique(b, return_index=True, return_inverse=True)
return u, ia, ic
def test_roundtrip_single_types(self):
for typ in np.typeDict.values():
dtype = np.dtype(typ)
if dtype.char in 'Mm':
# datetimes cannot be used in buffers
continue
if dtype.char == 'V':
# skip void
continue
x = np.zeros(4, dtype=dtype)
self._check_roundtrip(x)
if dtype.char not in 'qQgG':
dt = dtype.newbyteorder('<')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
dt = dtype.newbyteorder('>')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
def __new__(self, data, mask=nomask, dtype=None, fill_value=None,
hardmask=False, copy=False, subok=True):
_data = np.array(data, copy=copy, subok=subok, dtype=dtype)
_data = _data.view(self)
_data._hardmask = hardmask
if mask is not nomask:
if isinstance(mask, np.void):
_data._mask = mask
else:
try:
# Mask is already a 0D array
_data._mask = np.void(mask)
except TypeError:
# Transform the mask to a void
mdtype = make_mask_descr(dtype)
_data._mask = np.array(mask, dtype=mdtype)[()]
if fill_value is not None:
_data.fill_value = fill_value
return _data
def filled(self, fill_value=None):
"""
Return a copy with masked fields filled with a given value.
Parameters
----------
fill_value : scalar, optional
The value to use for invalid entries (None by default).
If None, the `fill_value` attribute is used instead.
Returns
-------
filled_void
A `np.void` object
See Also
--------
MaskedArray.filled
"""
return asarray(self).filled(fill_value)[()]
def stern(self,g,nsym,symop):
''' Compute star function for a specific g vector
Input:
g: G vector in real space
nsym: number of symmetries
symop: matrixes of the symmetry operations
Output:
nst: number of vectors in the star function calculated for the G vector
stg: star vectors
'''
trial = symop[:nsym].dot(g)
stg = np.unique(trial.view(np.dtype((np.void, trial.dtype.itemsize*trial.shape[1])))).view(trial.dtype).reshape(-1, trial.shape[1])
nst = len(stg)
stg = np.concatenate((stg,np.zeros((nsym-nst,3))))
return nst, stg
def find_boundary(mesh,vals,threshold=0.5):
""" Find boundary points on the phase diagram where the switching probability = threshold """
boundary_points = []
durs = mesh.points[:,0]
volts = mesh.points[:,1]
indices, indptr = mesh.vertex_neighbor_vertices
for k in range(len(vals)):
for k_nb in indptr[indices[k]:indices[k+1]]:
if (vals[k]-threshold)*(vals[k_nb]-threshold)<0:
x0 = find_cross([durs[k],vals[k]],[durs[k_nb],vals[k_nb]],cut=threshold)
y0 = find_cross([volts[k],vals[k]],[volts[k_nb],vals[k_nb]],cut=threshold)
boundary_points.append([x0,y0])
boundary_points = np.array(boundary_points)
if len(boundary_points) > 0:
b = np.ascontiguousarray(boundary_points).view(np.dtype((np.void,
boundary_points.dtype.itemsize * boundary_points.shape[1])))
_, idx = np.unique(b, return_index=True)
boundary_points = boundary_points[idx]
# Sort the boundary_points by x-axis
boundary_points = sorted(boundary_points, key=itemgetter(0))
return np.array(boundary_points)
def calc_information_sampling(data, bins, pys1, pxs, label, b, b1, len_unique_a, p_YgX, unique_inverse_x,
unique_inverse_y, calc_DKL=False):
bins = bins.astype(np.float32)
num_of_bins = bins.shape[0]
# bins = stats.mstats.mquantiles(np.squeeze(data.reshape(1, -1)), np.linspace(0,1, num=num_of_bins))
# hist, bin_edges = np.histogram(np.squeeze(data.reshape(1, -1)), normed=True)
digitized = bins[np.digitize(np.squeeze(data.reshape(1, -1)), bins) - 1].reshape(len(data), -1)
b2 = np.ascontiguousarray(digitized).view(
np.dtype((np.void, digitized.dtype.itemsize * digitized.shape[1])))
unique_array, unique_inverse_t, unique_counts = \
np.unique(b2, return_index=False, return_inverse=True, return_counts=True)
p_ts = unique_counts / float(sum(unique_counts))
PXs, PYs = np.asarray(pxs).T, np.asarray(pys1).T
if calc_DKL:
pxy_given_T = np.array(
[calc_probs(i, unique_inverse_t, label, b, b1, len_unique_a) for i in range(0, len(unique_array))]
)
p_XgT = np.vstack(pxy_given_T[:, 0])
p_YgT = pxy_given_T[:, 1]
p_YgT = np.vstack(p_YgT).T
DKL_YgX_YgT = np.sum([inf_ut.KL(c_p_YgX, p_YgT.T) for c_p_YgX in p_YgX.T], axis=0)
H_Xgt = np.nansum(p_XgT * np.log2(p_XgT), axis=1)
local_IXT, local_ITY = calc_information_from_mat(PXs, PYs, p_ts, digitized, unique_inverse_x, unique_inverse_y,
unique_array)
return local_IXT, local_ITY
def calc_by_sampling_neurons(ws_iter_index, num_of_samples, label, sigma, bins, pxs):
iter_infomration = []
for j in range(len(ws_iter_index)):
data = ws_iter_index[j]
new_data = np.zeros((num_of_samples * data.shape[0], data.shape[1]))
labels = np.zeros((num_of_samples * label.shape[0], label.shape[1]))
x = np.zeros((num_of_samples * data.shape[0], 2))
for i in range(data.shape[0]):
cov_matrix = np.eye(data[i, :].shape[0]) * sigma
t_i = np.random.multivariate_normal(data[i, :], cov_matrix, num_of_samples)
new_data[num_of_samples * i:(num_of_samples * (i + 1)), :] = t_i
labels[num_of_samples * i:(num_of_samples * (i + 1)), :] = label[i, :]
x[num_of_samples * i:(num_of_samples * (i + 1)), 0] = i
b = np.ascontiguousarray(x).view(np.dtype((np.void, x.dtype.itemsize * x.shape[1])))
unique_array, unique_indices, unique_inverse_x, unique_counts = \
np.unique(b, return_index=True, return_inverse=True, return_counts=True)
b_y = np.ascontiguousarray(labels).view(np.dtype((np.void, labels.dtype.itemsize * labels.shape[1])))
unique_array_y, unique_indices_y, unique_inverse_y, unique_counts_y = \
np.unique(b_y, return_index=True, return_inverse=True, return_counts=True)
pys1 = unique_counts_y / float(np.sum(unique_counts_y))
iter_infomration.append(
calc_information_for_layer(data=new_data, bins=bins, unique_inverse_x=unique_inverse_x,
unique_inverse_y=unique_inverse_y, pxs=pxs, pys1=pys1))
params = np.array(iter_infomration)
return params
def extract_probs(label, x):
"""calculate the probabilities of the given data and labels p(x), p(y) and (y|x)"""
pys = np.sum(label, axis=0) / float(label.shape[0])
b = np.ascontiguousarray(x).view(np.dtype((np.void, x.dtype.itemsize * x.shape[1])))
unique_array, unique_indices, unique_inverse_x, unique_counts = \
np.unique(b, return_index=True, return_inverse=True, return_counts=True)
unique_a = x[unique_indices]
b1 = np.ascontiguousarray(unique_a).view(np.dtype((np.void, unique_a.dtype.itemsize * unique_a.shape[1])))
pxs = unique_counts / float(np.sum(unique_counts))
p_y_given_x = []
for i in range(0, len(unique_array)):
indexs = unique_inverse_x == i
py_x_current = np.mean(label[indexs, :], axis=0)
p_y_given_x.append(py_x_current)
p_y_given_x = np.array(p_y_given_x).T
b_y = np.ascontiguousarray(label).view(np.dtype((np.void, label.dtype.itemsize * label.shape[1])))
unique_array_y, unique_indices_y, unique_inverse_y, unique_counts_y = \
np.unique(b_y, return_index=True, return_inverse=True, return_counts=True)
pys1 = unique_counts_y / float(np.sum(unique_counts_y))
return pys, pys1, p_y_given_x, b1, b, unique_a, unique_inverse_x, unique_inverse_y, pxs
def from_codes_and_metadata(cls,
codes,
categories,
reverse_categories,
missing_value):
"""
Rehydrate a LabelArray from the codes and metadata.
Parameters
----------
codes : np.ndarray[integral]
The codes for the label array.
categories : np.ndarray[object]
The unique string categories.
reverse_categories : dict[str, int]
The mapping from category to its code-index.
missing_value : any
The value used to represent missing data.
"""
ret = codes.view(type=cls, dtype=np.void)
ret._categories = categories
ret._reverse_categories = reverse_categories
ret._missing_value = missing_value
return ret
def test_roundtrip_single_types(self):
for typ in np.typeDict.values():
dtype = np.dtype(typ)
if dtype.char in 'Mm':
# datetimes cannot be used in buffers
continue
if dtype.char == 'V':
# skip void
continue
x = np.zeros(4, dtype=dtype)
self._check_roundtrip(x)
if dtype.char not in 'qQgG':
dt = dtype.newbyteorder('<')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
dt = dtype.newbyteorder('>')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
def __new__(self, data, mask=nomask, dtype=None, fill_value=None,
hardmask=False, copy=False, subok=True):
_data = np.array(data, copy=copy, subok=subok, dtype=dtype)
_data = _data.view(self)
_data._hardmask = hardmask
if mask is not nomask:
if isinstance(mask, np.void):
_data._mask = mask
else:
try:
# Mask is already a 0D array
_data._mask = np.void(mask)
except TypeError:
# Transform the mask to a void
mdtype = make_mask_descr(dtype)
_data._mask = np.array(mask, dtype=mdtype)[()]
if fill_value is not None:
_data.fill_value = fill_value
return _data
def filled(self, fill_value=None):
"""
Return a copy with masked fields filled with a given value.
Parameters
----------
fill_value : scalar, optional
The value to use for invalid entries (None by default).
If None, the `fill_value` attribute is used instead.
Returns
-------
filled_void
A `np.void` object
See Also
--------
MaskedArray.filled
"""
return asarray(self).filled(fill_value)[()]
test_multiarray.py 文件源码
项目：PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 42
收藏 0
点赞 0
评论 0
def test_roundtrip_single_types(self):
for typ in np.typeDict.values():
dtype = np.dtype(typ)
if dtype.char in 'Mm':
# datetimes cannot be used in buffers
continue
if dtype.char == 'V':
# skip void
continue
x = np.zeros(4, dtype=dtype)
self._check_roundtrip(x)
if dtype.char not in 'qQgG':
dt = dtype.newbyteorder('<')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
dt = dtype.newbyteorder('>')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
core.py 文件源码
项目：PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 39
收藏 0
点赞 0
评论 0
def __new__(self, data, mask=nomask, dtype=None, fill_value=None,
hardmask=False, copy=False, subok=True):
_data = np.array(data, copy=copy, subok=subok, dtype=dtype)
_data = _data.view(self)
_data._hardmask = hardmask
if mask is not nomask:
if isinstance(mask, np.void):
_data._mask = mask
else:
try:
# Mask is already a 0D array
_data._mask = np.void(mask)
except TypeError:
# Transform the mask to a void
mdtype = make_mask_descr(dtype)
_data._mask = np.array(mask, dtype=mdtype)[()]
if fill_value is not None:
_data.fill_value = fill_value
return _data
def test_roundtrip_single_types(self):
for typ in np.typeDict.values():
dtype = np.dtype(typ)
if dtype.char in 'Mm':
# datetimes cannot be used in buffers
continue
if dtype.char == 'V':
# skip void
continue
x = np.zeros(4, dtype=dtype)
self._check_roundtrip(x)
if dtype.char not in 'qQgG':
dt = dtype.newbyteorder('<')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
dt = dtype.newbyteorder('>')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
def filled(self, fill_value=None):
"""
Return a copy with masked fields filled with a given value.
Parameters
----------
fill_value : scalar, optional
The value to use for invalid entries (None by default).
If None, the `fill_value` attribute is used instead.
Returns
-------
filled_void
A `np.void` object
See Also
--------
MaskedArray.filled
"""
return asarray(self).filled(fill_value)[()]
def hashable_rows(data, digits=None):
'''
We turn our array into integers, based on the precision
given by digits, and then put them in a hashable format.
Arguments
---------
data: (n,m) input array
digits: how many digits to add to hash, if data is floating point
If none, TOL_MERGE will be turned into a digit count and used.
Returns
---------
hashable: (n) length array of custom data which can be sorted
or used as hash keys
'''
as_int = float_to_int(data, digits)
dtype = np.dtype((np.void, as_int.dtype.itemsize * as_int.shape[1]))
hashable = np.ascontiguousarray(as_int).view(dtype).reshape(-1)
return hashable
def test_roundtrip_single_types(self):
for typ in np.typeDict.values():
dtype = np.dtype(typ)
if dtype.char in 'Mm':
# datetimes cannot be used in buffers
continue
if dtype.char == 'V':
# skip void
continue
x = np.zeros(4, dtype=dtype)
self._check_roundtrip(x)
if dtype.char not in 'qQgG':
dt = dtype.newbyteorder('<')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
dt = dtype.newbyteorder('>')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
def __new__(self, data, mask=nomask, dtype=None, fill_value=None,
hardmask=False, copy=False, subok=True):
_data = np.array(data, copy=copy, subok=subok, dtype=dtype)
_data = _data.view(self)
_data._hardmask = hardmask
if mask is not nomask:
if isinstance(mask, np.void):
_data._mask = mask
else:
try:
# Mask is already a 0D array
_data._mask = np.void(mask)
except TypeError:
# Transform the mask to a void
mdtype = make_mask_descr(dtype)
_data._mask = np.array(mask, dtype=mdtype)[()]
if fill_value is not None:
_data.fill_value = fill_value
return _data
def filled(self, fill_value=None):
"""
Return a copy with masked fields filled with a given value.
Parameters
----------
fill_value : scalar, optional
The value to use for invalid entries (None by default).
If None, the `fill_value` attribute is used instead.
Returns
-------
filled_void
A `np.void` object
See Also
--------
MaskedArray.filled
"""
return asarray(self).filled(fill_value)[()]
def mask_roi_digi(self):
"""
Get the index of the unique magnitude tuple for each pixel in the ROI.
"""
# http://stackoverflow.com/q/24205045/#24206440
A = np.vstack([self.mask_1.mask_roi_sparse,self.mask_2.mask_roi_sparse]).T
B = self.mask_roi_unique
AA = np.ascontiguousarray(A)
BB = np.ascontiguousarray(B)
dt = np.dtype((np.void, AA.dtype.itemsize * AA.shape[1]))
a = AA.view(dt).ravel()
b = BB.view(dt).ravel()
idx = np.argsort(b)
indices = np.searchsorted(b[idx],a)
return idx[indices]
def test_roundtrip_single_types(self):
for typ in np.typeDict.values():
dtype = np.dtype(typ)
if dtype.char in 'Mm':
# datetimes cannot be used in buffers
continue
if dtype.char == 'V':
# skip void
continue
x = np.zeros(4, dtype=dtype)
self._check_roundtrip(x)
if dtype.char not in 'qQgG':
dt = dtype.newbyteorder('<')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
dt = dtype.newbyteorder('>')
x = np.zeros(4, dtype=dt)
self._check_roundtrip(x)
def __new__(self, data, mask=nomask, dtype=None, fill_value=None,
hardmask=False, copy=False, subok=True):
_data = np.array(data, copy=copy, subok=subok, dtype=dtype)
_data = _data.view(self)
_data._hardmask = hardmask
if mask is not nomask:
if isinstance(mask, np.void):
_data._mask = mask
else:
try:
# Mask is already a 0D array
_data._mask = np.void(mask)
except TypeError:
# Transform the mask to a void
mdtype = make_mask_descr(dtype)
_data._mask = np.array(mask, dtype=mdtype)[()]
if fill_value is not None:
_data.fill_value = fill_value
return _data
def filled(self, fill_value=None):
"""
Return a copy with masked fields filled with a given value.
Parameters
----------
fill_value : scalar, optional
The value to use for invalid entries (None by default).
If None, the `fill_value` attribute is used instead.
Returns
-------
filled_void
A `np.void` object
See Also
--------
MaskedArray.filled
"""
return asarray(self).filled(fill_value)[()]
def get_unique_rows(arr, return_indices=False):
'''
Returns the unique rows of the supplied array
this code was originally proposed at stackoverflow
http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array
Args:
arr (2d-ndarray): the array from which to extract the unique rows
return_indices (bool): if true, the indices corresponding to the unique rows in arr are
returned as well
'''
b = np.ascontiguousarray(arr).view(np.dtype((np.void, arr.dtype.itemsize * arr.shape[1])))
_, idx = np.unique(b, return_index=True)
# return the result
if return_indices:
return arr[idx], idx
else:
return arr[idx]
def find_rows(location_arr):
iterable = zip(location_arr[:,0], location_arr[:,1])
result_list = []
for thing in combinations(iterable, 2):
print(thing[0][1], thing[1][1])
if float(thing[0][1]) == float(thing[1][1]):
result_list.append(thing)
return result_list
# a = location_arr
# b = np.copy(location_arr)
# dt = np.dtype((np.void, a.dtype.itemsize * a.shape[1]))
#
# a_view = np.ascontiguousarray(a).view(dt).ravel()
# b_view = np.ascontiguousarray(b).view(dt).ravel()
#
# sort_b = np.argsort(b_view)
# where_in_b = np.searchsorted(b_view, a_view,
# sorter=sort_b)
# where_in_b = np.take(sort_b, where_in_b)
# which_in_a = np.take(b_view, where_in_b) == a_view
# where_in_b = where_in_b[which_in_a]
# which_in_a = np.nonzero(which_in_a)[0]
# return np.column_stack((which_in_a, where_in_b))
