def getTypeProblem (self, solution_filename):
''' Get the type of problem directly from the solution file (in case we do not have an info file)'''
if 'task' not in self.info.keys():
solution = np.array(data_converter.file_to_array(solution_filename))
target_num = solution.shape[1]
self.info['target_num']=target_num
if target_num == 1: # if we have only one column
solution = np.ravel(solution) # flatten
nbr_unique_values = len(np.unique(solution))
if nbr_unique_values < len(solution)/8:
# Classification
self.info['label_num'] = nbr_unique_values
if nbr_unique_values == 2:
self.info['task'] = 'binary.classification'
self.info['target_type'] = 'Binary'
else:
self.info['task'] = 'multiclass.classification'
self.info['target_type'] = 'Categorical'
else:
# Regression
self.info['label_num'] = 0
self.info['task'] = 'regression'
self.info['target_type'] = 'Numerical'
else:
# Multilabel or multiclass
self.info['label_num'] = target_num
self.info['target_type'] = 'Binary'
if any(item > 1 for item in map(np.sum,solution.astype(int))):
self.info['task'] = 'multilabel.classification'
else:
self.info['task'] = 'multiclass.classification'
return self.info['task']
python类any()的实例源码
def conv2d(x,W,strides=[1,1,1,1],name=None):
# return an op that convolves x with W
strides = np.array(strides)
if strides.size == 1:
strides = np.array([1,strides,strides,1])
elif strides.size == 2:
strides = np.array([1,strides[0],strides[1],1])
if np.any(strides < 1):
strides = np.around(1./strides).astype(np.uint8)
return tf.nn.conv2d_transpose(x,W,strides=strides.tolist(),padding='SAME',name=name)
else:
return tf.nn.conv2d(x,W,strides=strides.tolist(),padding='SAME',name=name)
def conv3d(x,W,strides=1,name=None):
# return an op that convolves x with W
strides = np.array(strides)
if strides.size == 1:
strides = np.array([1,strides,strides,strides[0],1])
elif strides.size == 3:
strides = np.array([1,strides[0],strides[1],strides[2],1])
if np.any(strides < 1):
strides = np.around(1./strides).astype(np.uint8)
return tf.nn.conv3d_transpose(x,W,strides=strides.tolist(),padding='SAME',name=name)
else:
return tf.nn.conv3d(x,W,strides=strides.tolist(),padding='SAME',name=name)
def initial_guess(self, ranges):
"""Computes an initial position guess based on range measurements.
The initial position is computed using Gauss-Newton method.
The behavior can be modified with some parameters: `self.initial_guess_...`.
Args:
ranges (list of floats): Range measurements.
Returns:
initial_state (numpy.ndarray): Initial state vector (velocity components are zero).
"""
num_of_units = len(ranges)
position = self.initial_guess_position
H = np.zeros((num_of_units, position.size))
z = np.zeros((num_of_units, 1))
h = np.zeros((num_of_units, 1))
residuals = np.zeros((num_of_units, 1))
for i in xrange(self.initial_guess_iterations):
self._compute_measurements_and_jacobians(ranges, position, h, H, z)
new_residuals = z - h
position = position + np.dot(self._solve_equation_least_squares(np.dot(H.T, H), H.T), new_residuals)
if np.sum((new_residuals - residuals) ** 2) < self.initial_guess_tolerance:
break
residuals = new_residuals
rospy.loginfo('initial guess residuals: {}'.format(residuals))
if np.any(np.abs(residuals) > self.initial_guess_residuals_threshold):
# This initial guess is not good enough
return None
initial_state = np.zeros((6, 1))
initial_state[0:3] = position
return initial_state
def overwrite_text(cursor, text):
text_length = len(text)
cursor.clearSelection()
# Select the text after the current position (if any)
current_position = cursor.position()
cursor.movePosition(QTextCursor.Right,
mode=QTextCursor.MoveAnchor,
n=text_length)
cursor.movePosition(QTextCursor.Left,
mode=QTextCursor.KeepAnchor,
n=cursor.position()-current_position)
# Insert the text (will overwrite the selected text)
cursor.insertText(text)
def store_tasks(self):
self.stored_tasks = [cb.isChecked() for cb in self.task_comboboxes]
if not numpy.any(self.stored_tasks):
self.ui.btn_run.setEnabled(False)
elif str(self.ui.edit_file.text()) != '':
self.ui.btn_run.setEnabled(True)
self.ui.btn_plots.setEnabled(True)
def _scale_data_to_float32(self, data):
'''
This function will convert data from local data dtype into float32, the default format of the algorithm
'''
if self.data_dtype != numpy.float32:
data = data.astype(numpy.float32)
if self.dtype_offset != 0:
data -= self.dtype_offset
if numpy.any(self.gain != 1):
data *= self.gain
return numpy.ascontiguousarray(data)
def _unscale_data_from_float32(self, data):
'''
This function will convert data from float32 back to the original format of the file
'''
if numpy.any(self.gain != 1):
data /= self.gain
if self.dtype_offset != 0:
data += self.dtype_offset
if (data.dtype != self.data_dtype) and (self.data_dtype != numpy.float32):
data = data.astype(self.data_dtype)
return data
def test_ignore_nan(self):
""" Test that ignore_nan is working """
for axis in (0, 1, 2, 3, None):
with self.subTest('axis = {}'.format(axis)):
out = last(ireduce_ufunc(self.source, np.add, axis = axis, ignore_nan = True))
self.assertFalse(np.any(np.isnan(out)))
# Dynamics generation of tests on binary ufuncs
def ffill_buffer_from_prior_values(freq,
field,
buffer_frame,
digest_frame,
pv_frame,
raw=False):
"""
Forward-fill a buffer frame, falling back to the end-of-period values of a
digest frame if the buffer frame has leading NaNs.
"""
# convert to ndarray if necessary
digest_values = digest_frame
if raw and isinstance(digest_frame, pd.DataFrame):
digest_values = digest_frame.values
buffer_values = buffer_frame
if raw and isinstance(buffer_frame, pd.DataFrame):
buffer_values = buffer_frame.values
nan_sids = pd.isnull(buffer_values[0])
if np.any(nan_sids) and len(digest_values):
# If we have any leading nans in the buffer and we have a non-empty
# digest frame, use the oldest digest values as the initial buffer
# values.
buffer_values[0, nan_sids] = digest_values[-1, nan_sids]
nan_sids = pd.isnull(buffer_values[0])
if np.any(nan_sids):
# If we still have leading nans, fall back to the last known values
# from before the digest.
key_loc = pv_frame.index.get_loc((freq.freq_str, field))
filler = pv_frame.values[key_loc, nan_sids]
buffer_values[0, nan_sids] = filler
if raw:
filled = ffill(buffer_values)
return filled
return buffer_frame.ffill()
def ffill_digest_frame_from_prior_values(freq,
field,
digest_frame,
pv_frame,
raw=False):
"""
Forward-fill a digest frame, falling back to the last known prior values if
necessary.
"""
# convert to ndarray if necessary
values = digest_frame
if raw and isinstance(digest_frame, pd.DataFrame):
values = digest_frame.values
nan_sids = pd.isnull(values[0])
if np.any(nan_sids):
# If we have any leading nans in the frame, use values from pv_frame to
# seed values for those sids.
key_loc = pv_frame.index.get_loc((freq.freq_str, field))
filler = pv_frame.values[key_loc, nan_sids]
values[0, nan_sids] = filler
if raw:
filled = ffill(values)
return filled
return digest_frame.ffill()
def __new__(cls, field=None, frequency_delta=None, length_delta=None):
"""
field is a new field that was added.
frequency is a FrequencyDelta representing a new frequency was added.
length is a bar LengthDelta which is a frequency and a bar_count.
If any field is None, then no change occurred of that type.
"""
return super(HistoryContainerDelta, cls).__new__(
cls, field, frequency_delta, length_delta,
)
def create_digest_panels(self,
initial_sids,
initial_dt):
"""
Initialize a RollingPanel for each unique panel frequency being stored
by this container. Each RollingPanel pre-allocates enough storage
space to service the highest bar-count of any history call that it
serves.
"""
# Map from frequency -> first/last minute of the next digest to be
# rolled for that frequency.
first_window_starts = {}
first_window_closes = {}
# Map from frequency -> digest_panels.
panels = {}
for freq, largest_spec in iteritems(self.largest_specs):
if largest_spec.bar_count == 1:
# No need to allocate a digest panel; this frequency will only
# ever use data drawn from self.buffer_panel.
first_window_starts[freq] = freq.normalize(initial_dt)
first_window_closes[freq] = freq.window_close(
first_window_starts[freq]
)
continue
dt = initial_dt
rp = self._create_digest_panel(
dt,
spec=largest_spec,
window_starts=first_window_starts,
window_closes=first_window_closes,
)
panels[freq] = rp
return panels, first_window_starts, first_window_closes
def update(self, data, algo_dt):
"""
Takes the bar at @algo_dt's @data, checks to see if we need to roll any
new digests, then adds new data to the buffer panel.
"""
frame = self.frame_from_bardata(data, algo_dt)
self.update_last_known_values()
self.update_digest_panels(algo_dt, self.buffer_panel)
self.buffer_panel.add_frame(algo_dt, frame)
def _set_mapping_metrics(self, read_type, read_type_set):
for genome, region in itertools.product(self.genomes, self.regions):
is_read_type = (genome, region) in read_type_set
reads_frac = self._get_metric_attr('reads_frac', genome, region, read_type)
reads_frac.add(1, filter=is_read_type)
for region in self.regions:
is_read_type = any([(genome, region) in read_type_set for genome in self.genomes])
multi_reads_frac = self._get_metric_attr('reads_frac', cr_constants.MULTI_REFS_PREFIX, region,
read_type)
multi_reads_frac.add(1, filter=is_read_type)
def mark_dupes_group_cb(self, gene_id, umis, dupe_type):
total_counts = sum(umis.values())
total_umis = len(umis)
if any([count > 1 for count in umis.itervalues()]):
umi_hamming_distance = 0
else:
umi_hamming_distance = cr_utils.get_kmers_hamming_distance(umis.keys())
for reference in [cr_utils.get_genome_from_str(gene_id, self.genomes), cr_constants.MULTI_REFS_PREFIX]:
if total_counts > 0:
reads_per_dupe_group_histogram = self._get_metric_attr(
'reads_per_dupe_group_histogram', reference, dupe_type)
reads_per_dupe_group_histogram.add(total_counts)
if total_umis > 0:
umis_per_dupe_group_histogram = self._get_metric_attr(
'umis_per_dupe_group_histogram', reference, dupe_type)
umis_per_dupe_group_histogram.add(total_umis)
reads_per_molecule_histogram = self._get_metric_attr(
'reads_per_molecule_histogram', reference, dupe_type)
for count in umis.itervalues():
reads_per_molecule_histogram.add(count)
if umi_hamming_distance is not None:
umi_hamming_distance_per_dupe_group_histogram = self._get_metric_attr(
'umi_hamming_distance_per_dupe_group_histogram', reference, dupe_type)
umi_hamming_distance_per_dupe_group_histogram.add(umi_hamming_distance)
def merge_h5(in_filenames, out_filename):
""" Merge a list of h5 files """
out_h5 = h5.File(out_filename, 'a')
for filename in in_filenames:
if filename is None:
continue
in_h5 = h5.File(filename, 'r')
for name in in_h5.keys():
# If the dataset already exists,
# They must be equal or one must be all-zero.
if name in out_h5.keys():
src_data, dest_data = in_h5[name][()], out_h5[name][()]
if src_data.dtype.kind != 'S' and dest_data.dtype.kind != 'S':
# Both numeric
if not np.any(src_data):
# Source is all zero. Do nothing.
continue
elif not np.any(dest_data):
# Dest is all zero. Overwrite.
del out_h5[name]
h5.h5o.copy(in_h5.id, name, out_h5.id, name)
else:
# Both non-zero. Assert equality and do nothing.
assert np.array_equal(src_data, dest_data)
else:
# Either are non-numeric. Assert equality and do nothing.
assert np.array_equal(src_data, dest_data)
else:
# Only exists in src. Copy to dest.
h5.h5o.copy(in_h5.id, name, out_h5.id, name)
out_h5.flush()
out_h5.close()
def assert_allclose(x, y, rtol=1e-10, atol=1e-8):
"""Drop in replacement for `numpy.testing.assert_allclose` that shows the nonmatching elements"""
if np.isscalar(x) and np.isscalar(y) == 1:
return np.testing.assert_allclose(x, y, rtol=rtol, atol=atol)
if x.shape != y.shape:
raise AssertionError("Shape mismatch: %s vs %s" % (str(x.shape), str(y.shape)))
d = ~np.isclose(x, y, rtol, atol)
if np.any(d):
miss = np.where(d)[0]
raise AssertionError("""Mismatch of %d elements (%g %%) at the level of rtol=%g, atol=%g
%s
%s
%s""" % (len(miss), len(miss)/x.size, rtol, atol, repr(miss), str(x[d]), str(y[d])))
def _nn_pose_fill(valid):
"""
Looks up closest True for each False and returns
indices for fill-in-lookup
In: [True, False, True, ... , False, True]
Out: [0, 0, 2, ..., 212, 212]
"""
valid_inds, = np.where(valid)
invalid_inds, = np.where(~valid)
all_inds = np.arange(len(valid))
all_inds[invalid_inds] = -1
for j in range(10):
fwd_inds = valid_inds + j
bwd_inds = valid_inds - j
# Forward fill
invalid_inds, = np.where(all_inds < 0)
fwd_fill_inds = np.intersect1d(fwd_inds, invalid_inds)
all_inds[fwd_fill_inds] = all_inds[fwd_fill_inds-j]
# Backward fill
invalid_inds, = np.where(all_inds < 0)
if not len(invalid_inds): break
bwd_fill_inds = np.intersect1d(bwd_inds, invalid_inds)
all_inds[bwd_fill_inds] = all_inds[bwd_fill_inds+j]
# Check if any missing
invalid_inds, = np.where(all_inds < 0)
if not len(invalid_inds): break
# np.set_printoptions(threshold=np.nan)
# print valid.astype(np.int)
# print np.array_str(all_inds)
# print np.where(all_inds < 0)
return all_inds
def roidb(self, target_hash, targets=[], every_k_frames=1, verbose=True, skip_empty=True):
"""
@param target_hash: target hash map (name -> unique id)
@param targets: return only provided target names
Returns (img, bbox, targets [hashed with target_hash (int32)])
"""
self.check_ground_truth_availability()
if every_k_frames > 1 and skip_empty:
raise RuntimeError('roidb not meant for skipping frames,'
'and skipping empty simultaneously ')
# Iterate through all images
for idx, (t,ch,data) in enumerate(self.iterframes()):
# Skip every k frames, if requested
if idx % every_k_frames != 0:
continue
# Annotations may be empty, if
# unlabeled, however we can request
# to yield if its empty or not
bboxes = data.annotation.bboxes
if not len(bboxes) and skip_empty:
continue
target_names = data.annotation.pretty_names
if len(targets):
inds, = np.where([np.any([t in name for t in targets]) for name in target_names])
target_names = [target_names[ind] for ind in inds]
bboxes = bboxes[inds]
yield (data.img, bboxes, np.int32(map(lambda key: target_hash.get(key, -1), target_names)))
