def get_index_first_non_zero_slice(self, dimension):
"""Get the index of the first non zero slice in this map.
Args:
dimension (int): the dimension to search in
Returns:
int: the slice index with the first non zero values.
"""
slice_index = [slice(None)] * (self.max_dimension() + 1)
if dimension > len(slice_index) - 1:
raise ValueError('The given dimension {} is not supported.'.format(dimension))
for index in range(self.shape[dimension]):
slice_index[dimension] = index
if np.count_nonzero(self.data[slice_index]) > 0:
return index
return 0
python类count_nonzero()的实例源码
def test_get_mask():
chunk = test_get_chunks(n_chunks=1)[0]
distance = 3
n_side = 32
mask = get_mask(distance, chunk.shape, dims=(2, 1, 0))
n_side_shell = n_side - 2*distance
count_exp = 2*n_side_shell**2 + (n_side_shell - 1)*4*(n_side_shell - 2)
count_got = np.count_nonzero(mask)
print('DEBUG: non-zeros exp: {} | got: {}'.format(count_exp, count_got))
assert count_exp == count_got
distance = 5
n_side_shell = n_side - 2*distance
mask = get_mask(distance, chunk.shape, dims=(2, 1))
count_exp = (n_side_shell - 1)*4*n_side
count_got = np.count_nonzero(mask)
print('DEBUG: non-zeros exp: {} | got: {}'.format(count_exp, count_got))
assert count_exp == count_got
def get_padded_seq_lengths(padded):
"""Returns the number of (seq_len) non-nan elements per sequence.
:param padded: 2d or 3d tensor with dim 2 the time dimension
"""
if len(padded.shape) == 2:
# (n_seqs,n_timesteps)
seq_lengths = np.count_nonzero(~np.isnan(padded), axis=1)
elif len(padded.shape) == 3:
# (n_seqs,n_timesteps,n_features,..)
seq_lengths = np.count_nonzero(~np.isnan(padded[:, :, 0]), axis=1)
else:
print('not yet implemented')
# TODO
return seq_lengths
def import_data(data_csvs_in,
types_csv_in,
values_csv_in,
groups_csv_in,
dataset_out,
encoding='utf-8'):
"""Import a comma-delimited list of csv files into internal treecat format.
Common encodings include: utf-8, cp1252.
"""
schema = load_schema(types_csv_in, values_csv_in, groups_csv_in, encoding)
data = np.concatenate([
load_data(schema, data_csv_in, encoding)
for data_csv_in in data_csvs_in.split(',')
])
data.flags.writeable = False
print('Imported data shape: [{}, {}]'.format(data.shape[0], data.shape[1]))
ragged_index = schema['ragged_index']
for v, name in enumerate(schema['feature_names']):
beg, end = ragged_index[v:v + 2]
count = np.count_nonzero(data[:, beg:end].max(1))
if count == 0:
print('WARNING: No values found for feature {}'.format(name))
feature_types = [TY_MULTINOMIAL] * len(schema['feature_names'])
table = Table(feature_types, ragged_index, data)
dataset = {
'schema': schema,
'table': table,
}
pickle_dump(dataset, dataset_out)
def build_feature_files(base_directory,
new_directory,
data_loader,
n=None,
negative_example_keep_prob=1.0):
os.makedirs(new_directory, exist_ok=False)
episode_paths = frame.episode_paths(base_directory)
label_counts = [0, 0]
if n is not None:
np.random.shuffle(episode_paths)
episode_paths = episode_paths[:n]
for episode_path in tqdm.tqdm(episode_paths):
try:
features, labels = data_loader.load_features_and_labels([episode_path])
except:
traceback.print_exc()
else:
keep = np.logical_or(labels, (np.less(
np.random.rand(len(labels)), negative_example_keep_prob)))
labels = labels[keep]
for i in range(len(label_counts)):
label_counts[i] += np.count_nonzero(labels == i)
features = {k: v[keep] for k, v in features.items()}
new_path = path_relative_to_new_directory(base_directory, new_directory, episode_path,
".features")
os.makedirs(os.path.dirname(new_path), exist_ok=True)
with open(new_path, 'wb') as f:
pickle.dump((features, labels), f)
return label_counts
def threshold_from_data(self, X, y):
y_bool = y == 1. ## true if x is a catast
y_pred = self.predict_proba(X)
if np.count_nonzero(y) == 0:
return np.max(y_pred)
return np.min(y_pred[y_bool][:,1]) # TODO CHANGED FROM WILL CODE
def threshold_from_predictions(y, y_pred, false_positive_margin=0, recall=1):
"""Determines a threshold for classifying examples as positive
Args:
y: labels
y_pred: scores from the classifier
recall: Threshold is set to classify at least this fraction of positive
labelled examples as positive
false_positive_margin: Threshold is set to acheive desired recall, and
then is extended to include an additional fraction of negative
labelled examples equal to false_positive_margin (This allows adding
a buffer to the threshold while maintaining a constant "cost")
"""
n_positive = np.count_nonzero(y)
n_negative = len(y) - n_positive
if n_positive == 0:
return np.max(y_pred)
if false_positive_margin == 0 and recall == 1:
return np.min(y_pred[y])
ind = np.argsort(y_pred)
y_pred_sorted = y_pred[ind]
y_sorted = y[ind]
so_far = [0, 0]
j = 0
for i in reversed(range(len(y_sorted))):
so_far[y_sorted[i]] += 1
if so_far[1] >= int(np.floor(recall * n_positive)):
j = i
break
so_far = [0, 0]
if false_positive_margin == 0:
return y_pred_sorted[j]
k = 0
for i in reversed(range(j)):
so_far[y_sorted[i]] += 1
if so_far[0] >= false_positive_margin * n_negative:
k = i
break
return y_pred_sorted[k]
def predict_proba(self, features):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(features))
return np.count_nonzero(predictions)
def predict_proba_raw(self, obs=None, action=None):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(obs, action))
return np.count_nonzero(predictions)
def predict_raw(self, obs=None, action=None):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(obs, action))
return self.apply_threshold(np.count_nonzero(predictions))
def predict_raw_with_score(self, obs=None, action=None):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(obs, action))
return self.apply_threshold(np.count_nonzero(predictions)), np.count_nonzero(predictions)
def predict(self, features):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict(features))
return self.apply_threshold(np.count_nonzero(predictions))
def threshold_from_predictions(y, y_pred, false_positive_margin=0, recall=1):
"""Determines a threshold for classifying examples as positive
Args:
y: labels
y_pred: scores from the classifier
recall: Threshold is set to classify at least this fraction of positive
labelled examples as positive
false_positive_margin: Threshold is set to acheive desired recall, and
then is extended to include an additional fraction of negative
labelled examples equal to false_positive_margin (This allows adding
a buffer to the threshold while maintaining a constant "cost")
"""
n_positive = np.count_nonzero(y)
n_negative = len(y) - n_positive
if n_positive == 0:
return np.max(y_pred)
if false_positive_margin == 0 and recall == 1:
return np.min(y_pred[y])
ind = np.argsort(y_pred)
y_pred_sorted = y_pred[ind]
y_sorted = y[ind]
so_far = [0, 0]
j = 0
for i in reversed(range(len(y_sorted))):
so_far[y_sorted[i]] += 1
if so_far[1] >= int(np.floor(recall * n_positive)):
j = i
break
so_far = [0, 0]
if false_positive_margin == 0:
return y_pred_sorted[j]
k = 0
for i in reversed(range(j)):
so_far[y_sorted[i]] += 1
if so_far[0] >= false_positive_margin * n_negative:
k = i
break
return y_pred_sorted[k]
def predict_proba(self, features):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(features))
return np.count_nonzero(predictions)
def predict_proba_raw(self, obs=None, action=None):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(obs, action))
return np.count_nonzero(predictions)
def predict_raw_with_score(self, obs=None, action=None):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(obs, action))
return self.apply_threshold(np.count_nonzero(predictions)), np.count_nonzero(predictions)
def predict(self, features):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict(features))
return self.apply_threshold(np.count_nonzero(predictions))
def threshold_from_data(self, X, y):
y_bool = y == 1. ## true if x is a catast
y_pred = self.predict_proba(X)
if np.count_nonzero(y) == 0:
return np.max(y_pred)
return np.min(y_pred[y_bool][:,1]) # TODO CHANGED FROM WILL CODE
def threshold_from_predictions(y, y_pred, false_positive_margin=0, recall=1):
"""Determines a threshold for classifying examples as positive
Args:
y: labels
y_pred: scores from the classifier
recall: Threshold is set to classify at least this fraction of positive
labelled examples as positive
false_positive_margin: Threshold is set to acheive desired recall, and
then is extended to include an additional fraction of negative
labelled examples equal to false_positive_margin (This allows adding
a buffer to the threshold while maintaining a constant "cost")
"""
n_positive = np.count_nonzero(y)
n_negative = len(y) - n_positive
if n_positive == 0:
return np.max(y_pred)
if false_positive_margin == 0 and recall == 1:
return np.min(y_pred[y])
ind = np.argsort(y_pred)
y_pred_sorted = y_pred[ind]
y_sorted = y[ind]
so_far = [0, 0]
j = 0
for i in reversed(range(len(y_sorted))):
so_far[y_sorted[i]] += 1
if so_far[1] >= int(np.floor(recall * n_positive)):
j = i
break
so_far = [0, 0]
if false_positive_margin == 0:
return y_pred_sorted[j]
k = 0
for i in reversed(range(j)):
so_far[y_sorted[i]] += 1
if so_far[0] >= false_positive_margin * n_negative:
k = i
break
return y_pred_sorted[k]
def predict_proba(self, features):
predictions = []
for classifier in self.classifiers:
predictions.append(classifier.predict_raw(features))
return np.count_nonzero(predictions)
