def test_view(self):
# Test view w/ flexible dtype
iterator = list(zip(np.arange(10), np.random.rand(10)))
data = np.array(iterator)
a = array(iterator, dtype=[('a', float), ('b', float)])
a.mask[0] = (1, 0)
controlmask = np.array([1] + 19 * [0], dtype=bool)
# Transform globally to simple dtype
test = a.view(float)
assert_equal(test, data.ravel())
assert_equal(test.mask, controlmask)
# Transform globally to dty
test = a.view((float, 2))
assert_equal(test, data)
assert_equal(test.mask, controlmask.reshape(-1, 2))
test = a.view((float, 2), np.matrix)
assert_equal(test, data)
self.assertTrue(isinstance(test, np.matrix))
python类ravel()的实例源码
def get_mode(values, axis=0):
"""
Adapted from
https://github.com/scipy/scipy/blob/master/scipy/stats/stats.py#L568
"""
a, axis = _chk_asarray(values, axis)
scores = np.unique(np.ravel(a)) # get ALL unique values
testshape = list(a.shape)
testshape[axis] = 1
oldmostfreq = np.zeros(testshape)
oldcounts = np.zeros(testshape)
for score in scores:
template = (a == score)
counts = np.expand_dims(np.sum(template, axis), axis)
mostfrequent = np.where(counts > oldcounts, score, oldmostfreq)
oldcounts = np.maximum(counts, oldcounts)
oldmostfreq = mostfrequent
return mostfrequent, oldcounts
def cT_helper(x, y, z, in_srs, out_srs):
"""Helper function that wraps osr CoordinatTransformation
"""
x, y, z = np.atleast_1d(x), np.atleast_1d(y), np.atleast_1d(z)
#Handle cases where z is 0 - probably a better way to use broadcasting for this
if x.shape[0] != z.shape[0]:
#Watch out for masked array input here
orig_z = z[0]
z = np.zeros_like(x)
z[:] = orig_z
orig_shape = x.shape
cT = osr.CoordinateTransformation(in_srs, out_srs)
#x2, y2, z2 = zip(*[cT.TransformPoint(*xyz) for xyz in zip(x, y, z)])
x2, y2, z2 = list(zip(*[cT.TransformPoint(*xyz) for xyz in zip(np.ravel(x),np.ravel(y),np.ravel(z))]))
if len(x2) == 1:
x2, y2, z2 = x2[0], y2[0], z2[0]
else:
x2 = np.array(x2).reshape(orig_shape)
y2 = np.array(y2).reshape(orig_shape)
z2 = np.array(z2).reshape(orig_shape)
return x2, y2, z2
def drawGraphAllStations(self):
rows, cols = 4, 4
maeRmse = np.zeros((rows*cols,4))
fig, ax_array = plt.subplots(rows, cols, sharex=True, sharey=True )
staInd = 0
for ax in np.ravel(ax_array):
maeRmse[staInd] = self.drawGraphStation(staInd, visualise=1, ax=ax)
staInd += 1
plt.xticks([0, 100, 200, 300])#, rotation=45)
errMean = maeRmse.mean(axis=0)
print maeRmse.mean(axis=0)
filename = 'pgf/finalEpoch'
plt.savefig('{}.pgf'.format(filename))
plt.savefig('{}.pdf'.format(filename))
plt.show()
return
def _one_hot_(label, num_classes=36):
num_labels = label.shape[0]
index_offset = np.arange(num_labels) * num_classes
labels_one_hot = np.zeros((num_labels, num_classes))
labelNum = []
# ???1???????
for i in label:
# ???
if ord(i) <= 57:
chrvalue = ord(i) - 48
else:
# ???????????????10??????10
chrvalue = ord(str(i).upper()) - 65 + 10
labelNum.append(chrvalue)
newlabel = np.array(labelNum)
labels_one_hot = labels_one_hot.astype(np.float32)
labels_one_hot.flat[index_offset + newlabel.ravel()] = 1.
return labels_one_hot
def fit(self, X, y=None):
self._colmask = [True] * X.shape[1]
self._colnames = X.columns.ravel().tolist()
# Identify batches
groups = X[[self.by]].values.ravel().tolist()
self._colmask[X.columns.get_loc(self.by)] = False
# Convert groups to IDs
glist = list(set(groups))
self._groups = np.array([glist.index(group)
for group in groups])
for gid, batch in enumerate(list(set(groups))):
scaler = clone(self._base_scaler)
mask = self._groups == gid
if not np.any(mask):
continue
self._scalers[batch] = scaler.fit(
X.ix[mask, self._colmask], y)
return self
def __init__(self, basename, input_dir, verbose=False, replace_missing=True, filter_features=False):
'''Constructor'''
self.use_pickle = False # Turn this to true to save data as pickle (inefficient)
self.basename = basename
if basename in input_dir:
self.input_dir = input_dir
else:
self.input_dir = input_dir + "/" + basename + "/"
if self.use_pickle:
if os.path.exists ("tmp"):
self.tmp_dir = "tmp"
elif os.path.exists ("../tmp"):
self.tmp_dir = "../tmp"
else:
os.makedirs("tmp")
self.tmp_dir = "tmp"
info_file = os.path.join (self.input_dir, basename + '_public.info')
self.info = {}
self.getInfo (info_file)
self.feat_type = self.loadType (os.path.join(self.input_dir, basename + '_feat.type'), verbose=verbose)
self.data = {}
Xtr = self.loadData (os.path.join(self.input_dir, basename + '_train.data'), verbose=verbose, replace_missing=replace_missing)
Ytr = self.loadLabel (os.path.join(self.input_dir, basename + '_train.solution'), verbose=verbose)
Xva = self.loadData (os.path.join(self.input_dir, basename + '_valid.data'), verbose=verbose, replace_missing=replace_missing)
Xte = self.loadData (os.path.join(self.input_dir, basename + '_test.data'), verbose=verbose, replace_missing=replace_missing)
# Normally, feature selection should be done as part of a pipeline.
# However, here we do it as a preprocessing for efficiency reason
idx=[]
if filter_features: # add hoc feature selection, for the example...
fn = min(Xtr.shape[1], 1000)
idx = data_converter.tp_filter(Xtr, Ytr, feat_num=fn, verbose=verbose)
Xtr = Xtr[:,idx]
Xva = Xva[:,idx]
Xte = Xte[:,idx]
self.feat_idx = np.array(idx).ravel()
self.data['X_train'] = Xtr
self.data['Y_train'] = Ytr
self.data['X_valid'] = Xva
self.data['X_test'] = Xte
def loadLabel (self, filename, verbose=True):
''' Get the solution/truth values'''
if verbose: print("========= Reading " + filename)
start = time.time()
if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
vprint (verbose, "Loading pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
return pickle.load(pickle_file)
if 'task' not in self.info.keys():
self.getTypeProblem(filename)
# IG: Here change to accommodate the new multiclass label format
if self.info['task'] == 'multilabel.classification':
label = data_io.data(filename)
elif self.info['task'] == 'multiclass.classification':
label = data_converter.convert_to_num(data_io.data(filename))
else:
label = np.ravel(data_io.data(filename)) # get a column vector
#label = np.array([np.ravel(data_io.data(filename))]).transpose() # get a column vector
if self.use_pickle:
with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
p = pickle.Pickler(pickle_file)
p.fast = True
p.dump(label)
end = time.time()
if verbose: print( "[+] Success in %5.2f sec" % (end - start))
return label
def loadType (self, filename, verbose=True):
''' Get the variable types'''
if verbose: print("========= Reading " + filename)
start = time.time()
type_list = []
if os.path.isfile(filename):
type_list = data_converter.file_to_array (filename, verbose=False)
else:
n=self.info['feat_num']
type_list = [self.info['feat_type']]*n
type_list = np.array(type_list).ravel()
end = time.time()
if verbose: print( "[+] Success in %5.2f sec" % (end - start))
return type_list
def getTypeProblem (self, solution_filename):
''' Get the type of problem directly from the solution file (in case we do not have an info file)'''
if 'task' not in self.info.keys():
solution = np.array(data_converter.file_to_array(solution_filename))
target_num = solution.shape[1]
self.info['target_num']=target_num
if target_num == 1: # if we have only one column
solution = np.ravel(solution) # flatten
nbr_unique_values = len(np.unique(solution))
if nbr_unique_values < len(solution)/8:
# Classification
self.info['label_num'] = nbr_unique_values
if nbr_unique_values == 2:
self.info['task'] = 'binary.classification'
self.info['target_type'] = 'Binary'
else:
self.info['task'] = 'multiclass.classification'
self.info['target_type'] = 'Categorical'
else:
# Regression
self.info['label_num'] = 0
self.info['task'] = 'regression'
self.info['target_type'] = 'Numerical'
else:
# Multilabel or multiclass
self.info['label_num'] = target_num
self.info['target_type'] = 'Binary'
if any(item > 1 for item in map(np.sum,solution.astype(int))):
self.info['task'] = 'multilabel.classification'
else:
self.info['task'] = 'multiclass.classification'
return self.info['task']
def sanitize_array(array):
''' Replace NaN and Inf (there should not be any!)'''
a=np.ravel(array)
maxi = np.nanmax((filter(lambda x: x != float('inf'), a))) # Max except NaN and Inf
mini = np.nanmin((filter(lambda x: x != float('-inf'), a))) # Mini except NaN and Inf
array[array==float('inf')]=maxi
array[array==float('-inf')]=mini
mid = (maxi + mini)/2
array[np.isnan(array)]=mid
return array
def tp_filter(X, Y, feat_num=1000, verbose=True):
''' TP feature selection in the spirit of the winners of the KDD cup 2001
Only for binary classification and sparse matrices'''
if issparse(X) and len(Y.shape)==1 and len(set(Y))==2 and (sum(Y)/Y.shape[0])<0.1:
if verbose: print("========= Filtering features...")
Posidx=Y>0
#npos = sum(Posidx)
#Negidx=Y<=0
#nneg = sum(Negidx)
nz=X.nonzero()
mx=X[nz].max()
if X[nz].min()==mx: # sparse binary
if mx!=1: X[nz]=1
tp=csr_matrix.sum(X[Posidx,:], axis=0)
#fn=npos-tp
#fp=csr_matrix.sum(X[Negidx,:], axis=0)
#tn=nneg-fp
else:
tp=np.sum(X[Posidx,:]>0, axis=0)
#tn=np.sum(X[Negidx,:]<=0, axis=0)
#fn=np.sum(X[Posidx,:]<=0, axis=0)
#fp=np.sum(X[Negidx,:]>0, axis=0)
tp=np.ravel(tp)
idx=sorted(range(len(tp)), key=tp.__getitem__, reverse=True)
return idx[0:feat_num]
else:
feat_num = X.shape[1]
return range(feat_num)
def __init__(self, terrain):
self.terrain = terrain
self.x_grid, self.y_grid = np.meshgrid(range(self.terrain.width),
range(self.terrain.length))
z_vals = np.array([self.terrain[x, y] for x, y in zip(np.ravel(self.x_grid), np.ravel(self.y_grid))])
self.z_grid = z_vals.reshape(self.x_grid.shape)
def residual_multigauss(param, dataimage, nonfinite = 0.0, ravelresidual=True, showimages=False, verbose=False):
"""
Calculating the residual bestween the multigaussian model with the paramters 'param' and the data.
--- INPUT ---
param Parameters of multi-gaussian model to generate. See modelimage_multigauss() header for details
dataimage Data image to take residual
nonfinite Value to replace non-finite entries in residual with
ravelresidual To np.ravel() the residual image set this to True. Needed by scipy.optimize.leastsq()
optimizer function
showimages To show model and residiual images set to True
verbose Toggle verbosity
--- EXAMPLE OF USE ---
import tdose_model_FoV as tmf
param = [18,31,1*0.3,2.1*0.3,1.2*0.3,30*0.3, 110,90,200*0.5,20.1*0.5,15.2*0.5,0*0.5]
dataimg = pyfits.open('/Users/kschmidt/work/TDOSE/mock_cube_sourcecat161213_tdose_mock_cube.fits')[0].data[0,:,:]
residual = tmf.residual_multigauss(param, dataimg, showimages=True)
"""
if verbose: ' - Estimating residual (= model - data) between model and data image'
imgsize = dataimage.shape
xgrid, ygrid = tu.gen_gridcomponents(imgsize)
modelimg = tmf.modelimage_multigauss((xgrid, ygrid),param,imgsize,showmodelimg=showimages, verbose=verbose)
residualimg = modelimg - dataimage
if showimages:
plt.imshow(residualimg,interpolation='none', vmin=1e-5, vmax=np.max(residualimg), norm=mpl.colors.LogNorm())
plt.title('Resdiaul (= model - data) image')
plt.show()
if nonfinite is not None:
residualimg[~np.isfinite(residualimg)] = 0.0
if ravelresidual:
residualimg = np.ravel(residualimg)
return residualimg
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
def handle_multi_range_message(self, multi_range_msg):
"""Handle a ROS multi-range message by updating and publishing the state.
Args:
multi_range_msg (uwb.msg.UWBMultiRangeWithOffsets): ROS multi-range message.
"""
# Update tracker position based on time-of-flight measurements
new_estimate = self.update_estimate(multi_range_msg)
if new_estimate is None:
rospy.logwarn('Could not compute initial estimate: address={}, remote_address={}'.format(
multi_range_msg.address, multi_range_msg.remote_address))
else:
# Publish tracker message
ros_msg = uwb.msg.UWBTracker()
ros_msg.header.stamp = rospy.get_rostime()
ros_msg.address = multi_range_msg.address
ros_msg.remote_address = multi_range_msg.remote_address
ros_msg.state = new_estimate.state
ros_msg.covariance = np.ravel(new_estimate.covariance)
self.uwb_pub.publish(ros_msg)
# Publish target transform (rotation is identity)
self.tf_broadcaster.sendTransform(
(new_estimate.state[0], new_estimate.state[1], new_estimate.state[2]),
tf.transformations.quaternion_from_euler(0, 0, 0),
rospy.get_rostime(),
self.target_frame,
self.tracker_frame
)
def flatten_vars(xs, n):
ret = np.empty(n)
ind = 0
for x in xs:
size = x.size[0]*x.size[1]
ret[ind:ind+size] = np.ravel(x.value, order='F')
return ret
def sample(self, size=1):
pvals = [e.weight for e in self.__elements]
u = self.__randomstate.multinomial(1, pvals, size)
result = []
for sample in u:
elementidx = np.ravel(np.where(sample))[0]
result.append(self.__elements[elementidx].distribution.sample()[0])
return np.array(result)
def get_data_from_file(fname):
labels, sentences = [], []
with open(fname, 'rb') as f:
for line in f:
label, text = line.strip().split(' ', 1)
text = text.split(' ')
labels.append((int(label) + 1) / 2)
sentences.append(text)
labels = np.ravel(labels)
return sentences, labels
def get_data_from_file_polarity(fname):
labels, sentences = [], []
with open(fname, 'rb') as f:
for line in f:
label, text = line.strip().split(' ', 1)
text = text.split(' ')
labels.append((int(label) + 1) / 2)
sentences.append(text)
labels = np.ravel(labels)
return sentences, labels
def get_data_from_file(fname):
labels, sentences = [], []
with open(fname, 'rb') as f:
for line in f:
label, text = line.strip().split(' ', 1)
text = text.split(' ')
labels.append((int(label) + 1) / 2)
sentences.append(text)
labels = np.ravel(labels)
return sentences, labels
