def iterate_minibatches(self, batchsize, shuffle=True, train=True):
indices = []
if train:
indices = np.argwhere(np.in1d(data.labels, data.train_classes))
else:
indices = np.argwhere(np.logical_not(np.in1d(data.labels, data.train_classes)))
if shuffle:
np.random.shuffle(indices)
for start_idx in range(0, len(self.img_paths) - batchsize + 1, batchsize):
excerpt = indices[start_idx:start_idx + batchsize]
images = [self._load_preprocess_img(self.img_paths[int(i)]) for i in excerpt]
if len(images) == batchsize:
yield np.concatenate(images), np.array(self.labels[excerpt]).astype(np.int32).T
else:
raise StopIteration
python类in1d()的实例源码
def GetEdgeMask(self, angle):
"""
Returns a mask of the points of a surface mesh that have a surface
angle greater than angle
Parameters
----------
angle : float
Angle to consider an edge.
"""
featureEdges = vtk.vtkFeatureEdges()
featureEdges.SetInputData(self)
featureEdges.FeatureEdgesOn()
featureEdges.BoundaryEdgesOff()
featureEdges.NonManifoldEdgesOff()
featureEdges.ManifoldEdgesOff()
featureEdges.SetFeatureAngle(angle)
featureEdges.Update()
edges = featureEdges.GetOutput()
origID = vtkInterface.GetPointScalars(edges, 'vtkOriginalPointIds')
return np.in1d(self.GetPointScalars('vtkOriginalPointIds'),
origID,
assume_unique=True)
def RaDec2region(ra, dec, nside):
SCP_indx, NES_indx, GP_indx, WFD_indx = mutually_exclusive_regions(nside)
indices = _raDec2Hpid(nside, np.radians(ra), np.radians(dec))
result = np.empty(np.size(indices), dtype = object)
SCP = np.in1d(indices, SCP_indx)
NES = np.in1d(indices,NES_indx)
GP = np.in1d(indices,GP_indx)
WFD = np.in1d(indices,WFD_indx)
result[SCP] = 'SCP'
result[NES] = 'NES'
result[GP] = 'GP'
result[WFD] = 'WFD'
return result
def __getitem__(self, thing: Any) -> sparse.coo_matrix:
if type(thing) is slice or type(thing) is np.ndarray or type(thing) is int:
gm = GraphManager(None, axis=self.axis)
for key, g in self.items():
# Slice the graph matrix properly without making it dense
(a, b, w) = (g.row, g.col, g.data)
indices = np.arange(g.shape[0])[thing]
mask = np.logical_and(np.in1d(a, indices), np.in1d(b, indices))
a = a[mask]
b = b[mask]
w = w[mask]
d = dict(zip(np.sort(indices), np.arange(indices.shape[0])))
a = np.array([d[x] for x in a])
b = np.array([d[x] for x in b])
gm[key] = sparse.coo_matrix((w, (a, b)), shape=(len(indices), len(indices)))
return gm
else:
return self.__getattr__(thing)
def get_data_by_id(self, ids):
""" Helper for getting current data values from stored identifiers
:param float|list ids: ids for which data are requested
:return: the stored ids
:rtype: np.ndarray
"""
if self.ids is None:
raise ValueError("IDs not stored in node {}".format(self.name))
if self.data is None:
raise ValueError("No data in node {}".format(self.name))
ids = np.array(ids, ndmin=1, copy=False)
found_items = np.in1d(ids, self.ids)
if not np.all(found_items):
raise ValueError("Cannot find {} among {}".format(ids[np.logical_not(found_items)],
self.name))
idx = np.empty(len(ids), dtype='int')
for k, this_id in enumerate(ids):
if self.ids.ndim > 1:
idx[k] = np.flatnonzero(np.all(self.ids == this_id, axis=1))[0]
else:
idx[k] = np.flatnonzero(self.ids == this_id)[0]
return np.array(self.data, ndmin=1)[idx]
def split_data(data, num_folds, seed=0):
""" Split all interactions into K-fold sets of training and test dataframes. Splitting is done
by assigning student ids to the training or test sets.
:param pd.DataFrame data: all interactions
:param int num_folds: number of folds
:param int seed: seed for the splitting
:return: a generator over (train dataframe, test dataframe) tuples
:rtype: generator[(pd.DataFrame, pd.DataFrame)]
"""
# break up students into folds
fold_student_idx = _get_fold_student_idx(np.unique(data[USER_IDX_KEY]), num_folds=num_folds,
seed=seed)
for fold_test_student_idx in fold_student_idx:
test_idx = np.in1d(data[USER_IDX_KEY], fold_test_student_idx)
train_idx = np.logical_not(test_idx)
yield (data[train_idx].copy(), data[test_idx].copy())
def eval_loop(data_loader, model, base_classes, novel_classes):
model = model.eval()
top1 = None
top5 = None
all_labels = None
for i, (x,y) in enumerate(data_loader):
x = Variable(x.cuda())
scores = model(x)
top1_this, top5_this = perelement_accuracy(scores.data, y)
top1 = top1_this if top1 is None else np.concatenate((top1, top1_this))
top5 = top5_this if top5 is None else np.concatenate((top5, top5_this))
all_labels = y.numpy() if all_labels is None else np.concatenate((all_labels, y.numpy()))
is_novel = np.in1d(all_labels, novel_classes)
is_base = np.in1d(all_labels, base_classes)
is_either = is_novel | is_base
top1_novel = np.mean(top1[is_novel])
top1_base = np.mean(top1[is_base])
top1_all = np.mean(top1[is_either])
top5_novel = np.mean(top5[is_novel])
top5_base = np.mean(top5[is_base])
top5_all = np.mean(top5[is_either])
return np.array([top1_novel, top5_novel, top1_base, top5_base, top1_all, top5_all])
def _mask_edges_weights(mask, edges, weights=None):
"""Apply a mask to edges (weighted or not)"""
inds = np.arange(mask.size)
inds = inds[mask.ravel()]
ind_mask = np.logical_and(np.in1d(edges[0], inds),
np.in1d(edges[1], inds))
edges = edges[:, ind_mask]
if weights is not None:
weights = weights[ind_mask]
if len(edges.ravel()):
maxval = edges.max()
else:
maxval = 0
order = np.searchsorted(np.unique(edges.ravel()), np.arange(maxval + 1))
edges = order[edges]
if weights is None:
return edges
else:
return edges, weights
def map_2D_hist_to_ima(imaSlc2volHistMap, volHistMask):
"""Volume histogram to image mapping for slices (uses np.ind1).
Parameters
----------
imaSlc2volHistMap : TODO
volHistMask : TODO
Returns
-------
imaSlcMask : TODO
"""
imaSlcMask = np.zeros(imaSlc2volHistMap.flatten().shape)
idxUnique = np.unique(volHistMask)
for idx in idxUnique:
linIndices = np.where(volHistMask.flatten() == idx)[0]
# return logical array with length equal to nr of voxels
voxMask = np.in1d(imaSlc2volHistMap.flatten(), linIndices)
# reset mask and apply logical indexing
imaSlcMask[voxMask] = idx
imaSlcMask = imaSlcMask.reshape(imaSlc2volHistMap.shape)
return imaSlcMask
def reconstruct_goal(world):
# pdb.set_trace()
world = world.copy()
## indices for grass and puddle
background_inds = [obj['index'] for (name, obj) in library.objects.iteritems() if obj['background']]
## background mask
background = np.in1d(world, background_inds)
background = background.reshape( (world.shape) )
## set backgronud to 0
world[background] = 0
## subtract largest background ind
## so indices of objects begin at 1
world[~background] -= max(background_inds)
world = np.expand_dims(np.expand_dims(world, 0), 0)
# pdb.set_trace()
return world
def detect_input(cls, values, sample_size=200):
"""
Return first "from_" method that in more than 50% matches values,
or None.
"""
assert isinstance(values, pd.Series)
values = values.drop_duplicates().dropna()
if len(values) > sample_size:
values = values.sample(sample_size)
strlen = values.str.len().dropna().unique()
for method, *cond in ((cls.from_cc2, len(strlen) == 1 and strlen[0] == 2),
(cls.from_cc3, len(strlen) == 1 and strlen[0] == 3),
(cls.from_cc_name,),
(cls.from_us_state,),
(cls.from_city_eu,),
(cls.from_city_us,),
(cls.from_city_world,),
(cls.from_region,),
(cls.from_fips,),
(cls.from_hasc, np.in1d(strlen, [2, 5, 8]).all())):
if cond and not cond[0]:
continue
if sum(map(bool, method(values))) >= len(values) / 2:
return method
return None
def init_snapshots(self):
"""Initialize snapshots for model variables given in attributes of
Dataset.
"""
self.snapshot_vars = self.dataset.xsimlab.snapshot_vars
self.snapshot_values = {}
for vars in self.snapshot_vars.values():
self.snapshot_values.update({v: [] for v in vars})
self.snapshot_save = {
clock: np.in1d(self.dataset[self.master_clock_dim].values,
self.dataset[clock].values)
for clock in self.snapshot_vars if clock is not None
}
def crossGenotypeWindows(commonSNPsCHR, commonSNPsPOS, snpsP1, snpsP2, inFile, binLen, outFile, logDebug = True):
## inFile are the SNPs of the sample
(snpCHR, snpPOS, snpGT, snpWEI, DPmean) = snpmatch.parseInput(inFile = inFile, logDebug = logDebug)
# identifying the segregating SNPs between the accessions
# only selecting 0 or 1
segSNPsind = np.where((snpsP1 != snpsP2) & (snpsP1 >= 0) & (snpsP2 >= 0) & (snpsP1 < 2) & (snpsP2 < 2))[0]
log.info("number of segregating snps between parents: %s", len(segSNPsind))
(ChrBins, PosBins) = getBinsSNPs(commonSNPsCHR, commonSNPsPOS, binLen)
log.info("number of bins: %s", len(ChrBins))
outfile = open(outFile, 'w')
for i in range(len(PosBins)):
start = np.sum(PosBins[0:i])
end = start + PosBins[i]
# first snp positions which are segregating and are in this window
reqPOSind = segSNPsind[np.where((segSNPsind < end) & (segSNPsind >= start))[0]]
reqPOS = commonSNPsPOS[reqPOSind]
perchrTarPosind = np.where(snpCHR == ChrBins[i])[0]
perchrTarPos = snpPOS[perchrTarPosind]
matchedAccInd = reqPOSind[np.where(np.in1d(reqPOS, perchrTarPos))[0]]
matchedTarInd = perchrTarPosind[np.where(np.in1d(perchrTarPos, reqPOS))[0]]
matchedTarGTs = snpGT[matchedTarInd]
try:
TarGTBinary = snpmatch.parseGT(matchedTarGTs)
TarGTBinary[np.where(TarGTBinary == 2)[0]] = 4
genP1 = np.subtract(TarGTBinary, snpsP1[matchedAccInd])
genP1no = len(np.where(genP1 == 0)[0])
(geno, pval) = getWindowGenotype(genP1no, len(genP1))
outfile.write("%s\t%s\t%s\t%s\t%s\n" % (i+1, genP1no, len(genP1), geno, pval))
except:
outfile.write("%s\tNA\tNA\tNA\tNA\n" % (i+1))
if i % 40 == 0:
log.info("progress: %s windows", i+10)
log.info("done!")
outfile.close()
def intersect_and_sort_samples(sample_metadata, feature_table):
'''Return input tables retaining only shared samples, row order equivalent.
Parameters
----------
sample_metadata : pd.DataFrame
Contingency table with rows, columns = samples, metadata.
feature_table : pd.DataFrame
Contingency table with rows, columns = samples, features.
Returns
-------
sample_metadata, feature_table : pd.DataFrame, pd.DataFrame
Input tables with unshared samples removed and ordered equivalently.
Raises
------
ValueError
If no shared samples are found.
'''
shared_samples = np.intersect1d(sample_metadata.index, feature_table.index)
if shared_samples.size == 0:
raise ValueError('There are no shared samples between the feature '
'table and the sample metadata. Ensure that you have '
'passed the correct files.')
elif (shared_samples.size == sample_metadata.shape[0] ==
feature_table.shape[0]):
s_metadata = sample_metadata.copy()
s_features = feature_table.copy()
else:
s_metadata = sample_metadata.loc[np.in1d(sample_metadata.index,
shared_samples), :].copy()
s_features = feature_table.loc[np.in1d(feature_table.index,
shared_samples), :].copy()
return s_metadata, s_features.loc[s_metadata.index, :]
def prepare_input(d,q):
f = np.zeros(d.shape[:2]).astype('int32')
for i in range(d.shape[0]):
f[i,:] = np.in1d(d[i,:,0],q[i,:,0])
return f
def get_piece_bool(num, dict):
'''Uses a vertex number to find the right bool array
as created by divide_garment()'''
count = 0
nums = dict['garment_pieces']['numbers_array']
for i in nums:
if np.in1d(num, i):
return count
count += 1
def find_linked(ob, vert, per_face='empty'):
'''Takes a vert and returns an array of linked face indices'''
the_coffee_is_hot = True
fidx = np.arange(len(ob.data.polygons))
eidx = np.arange(len(ob.data.edges))
f_set = np.array([])
e_set = np.array([])
verts = ob.data.vertices
verts[vert].select = True
v_p_f_count = [len(p.vertices) for p in ob.data.polygons]
max_count = np.max(v_p_f_count)
if per_face == 'empty':
per_face = [[i for i in poly.vertices] for poly in ob.data.polygons]
for i in per_face:
for j in range(max_count-len(i)):
i.append(i[0])
verts_per_face = np.array(per_face)
vert=np.array([vert])
while the_coffee_is_hot:
booly = np.any(np.in1d(verts_per_face, vert).reshape(verts_per_face.shape), axis=1)
f_set = np.append(f_set, fidx[booly])
new_verts = verts_per_face[booly].ravel()
if len(new_verts) == 0:
return np.array(f_set, dtype=np.int64)
cull = np.in1d(new_verts, vert)
vert = new_verts[-cull]
verts_per_face = verts_per_face[-booly]
fidx = fidx[-booly]
def divide_garment(ob, dict):
'''Creates a set of bool arrays and a set of number arrays
for indexing a sub set of the uv coords. The nuber arrays can
be used to look up wich bool array to use based on a vertex number'''
if ob == 'empty':
ob = bpy.context.object
#-----------------------------------
v_count = len(ob.data.vertices)
idx = np.arange(v_count)
full_set = np.array([])
dict['islands'] = []
v_list = [[i for i in poly.vertices] for poly in ob.data.polygons]
v_in_faces = np.hstack(v_list)
dict['v_in_faces'] = v_in_faces
remaining = [1]
vert = 0
while len(remaining) > 0:
linked = find_linked(ob, vert, v_list)
selected = np.unique(np.hstack(np.array(v_list)[linked]).ravel())
dict['islands'].append(selected)
full_set = np.append(full_set, selected)
remain_bool = np.in1d(idx, full_set, invert=True)
remaining = idx[remain_bool]
if len(remaining) == 0:
break
vert = remaining[0]
#################################
def setdiff1d(ar1, ar2, assume_unique=False):
"""
Find the set difference of two arrays.
Return the sorted, unique values in `ar1` that are not in `ar2`.
Parameters
----------
ar1 : array_like
Input array.
ar2 : array_like
Input comparison array.
assume_unique : bool
If True, the input arrays are both assumed to be unique, which
can speed up the calculation. Default is False.
Returns
-------
setdiff1d : ndarray
Sorted 1D array of values in `ar1` that are not in `ar2`.
See Also
--------
numpy.lib.arraysetops : Module with a number of other functions for
performing set operations on arrays.
Examples
--------
>>> a = np.array([1, 2, 3, 2, 4, 1])
>>> b = np.array([3, 4, 5, 6])
>>> np.setdiff1d(a, b)
array([1, 2])
"""
if assume_unique:
ar1 = np.asarray(ar1).ravel()
else:
ar1 = unique(ar1)
ar2 = unique(ar2)
return ar1[in1d(ar1, ar2, assume_unique=True, invert=True)]
def set_snapshot_weights(ratio_dict,
orig_rng,
eg_range):
'''Determine the job distribution ratios to carry forward during
the ratio condition application period using actual jobs held ratios.
likely called at implementation month by main job assignment function
Count the number of jobs held by each of the ratio groups for each of the
affected job level numbers. Set the weightings in the distribute function
accordingly.
inputs
ratio_dict (dictionary)
dictionary containing job levels as keys and ratio groups,
weightings, month_start and month end as values.
orig_rng (numpy array)
month slice of original job array
eg_range (numpy array)
month slice of employee group code array
'''
ratio_dict = copy.deepcopy(ratio_dict)
job_nums = list(ratio_dict.keys())
for job in job_nums:
wgt_list = []
for ratio_group in ratio_dict[job][0]:
wgt_list.append(np.count_nonzero((orig_rng == job) &
(np.in1d(eg_range, ratio_group))))
ratio_dict[job][1] = tuple(wgt_list)
return ratio_dict
# ASSIGN JOBS BY RATIO CONDITION
