def write_predictions(self, inputs):
'''
Outputs predictions in a file named <model_name_prefix>.predictions.
'''
predictions = numpy.argmax(self.model.predict(inputs), axis=1)
test_output_file = open("%s.predictions" % self.model_name_prefix, "w")
for input_indices, prediction in zip(inputs, predictions):
# The predictions are indices of words in padded sentences. We need to readjust them.
padding_length = 0
for index in input_indices:
if numpy.all(index == 0):
padding_length += 1
else:
break
prediction = prediction - padding_length + 1 # +1 because the indices start at 1.
print >>test_output_file, prediction
python类all()的实例源码
def guess_feature_type(count, values):
"""Guess the type of a feature, given statistics about the feature.
Args:
count: Total number of observations of the feature.
values: A list of uniqe observed values of the feature.
Returns:
One of: 'ordinal', 'categorical', or ''
"""
if len(values) <= 1:
return '' # Feature is useless.
if len(values) <= MAX_CATEGORIES:
if all(is_small_int(v) for (v, c) in values):
return ORDINAL
if len(values) <= min(count / 2, MAX_CATEGORIES):
return CATEGORICAL
return ''
def test_quantize_from_probs2(size, resolution):
set_random_seed(make_seed(size, resolution))
probs = np.exp(np.random.random(size)).astype(np.float32)
probs2 = probs.reshape((1, size))
quantized = quantize_from_probs2(probs2, resolution)
assert quantized.shape == probs2.shape
assert quantized.dtype == np.int8
assert np.all(quantized.sum(axis=1) == resolution)
# Check that quantized result is closer to target than any other value.
quantized = quantized.reshape((size, ))
target = resolution * probs / probs.sum()
distance = np.abs(quantized - target).sum()
for combo in itertools.combinations(range(size), resolution):
other = np.zeros(size, np.int8)
for i in combo:
other[i] += 1
assert other.sum() == resolution
other_distance = np.abs(other - target).sum()
assert distance <= other_distance
def test_server_median(N, V, C, M):
model = generate_fake_model(N, V, C, M)
config = TINY_CONFIG.copy()
config['model_num_clusters'] = M
model['config'] = config
server = TreeCatServer(model)
# Evaluate on random data.
counts = np.random.randint(10, size=[V], dtype=np.int8)
table = generate_dataset(N, V, C)['table']
median = server.median(counts, table.data)
assert median.shape == table.data.shape
assert median.dtype == np.int8
for v in range(V):
beg, end = table.ragged_index[v:v + 2]
totals = median[:, beg:end].sum(axis=1)
assert np.all(totals == counts[v])
def test_observed_perplexity(N, V, C, M):
set_random_seed(make_seed(N, V, C, M))
model = generate_fake_model(N, V, C, M)
config = TINY_CONFIG.copy()
config['model_num_clusters'] = M
model['config'] = config
server = TreeCatServer(model)
for count in [1, 2, 3]:
if count > 1 and C > 2:
continue # NotImplementedError.
counts = 1
perplexity = server.observed_perplexity(counts)
print(perplexity)
assert perplexity.shape == (V, )
assert np.all(1 <= perplexity)
assert np.all(perplexity <= count * C)
def observed_perplexity(self, counts):
"""Compute perplexity = exp(entropy) of observed variables.
Perplexity is an information theoretic measure of the number of
clusters or latent classes. Perplexity is a real number in the range
[1, M], where M is model_num_clusters.
Args:
counts: A [V]-shaped array of multinomial counts.
Returns:
A [V]-shaped numpy array of perplexity.
"""
V, E, M, R = self._VEMR
if counts is not None:
counts = np.ones(V, dtype=np.int8)
assert counts.shape == (V, )
assert counts.dtype == np.int8
assert np.all(counts > 0)
observed_entropy = np.empty(V, dtype=np.float32)
for v in range(V):
beg, end = self._ragged_index[v:v + 2]
probs = np.dot(self._feat_cond[beg:end, :], self._vert_probs[v, :])
observed_entropy[v] = multinomial_entropy(probs, counts[v])
return np.exp(observed_entropy)
def quantize_from_probs2(probs, resolution):
"""Quantize multiple non-normalized probs to given resolution.
Args:
probs: An [N, M]-shaped numpy array of non-normalized probabilities.
Returns:
An [N, M]-shaped array of quantized probabilities such that
np.all(result.sum(axis=1) == resolution).
"""
assert len(probs.shape) == 2
N, M = probs.shape
probs = probs / probs.sum(axis=1, keepdims=True)
result = np.zeros(probs.shape, np.int8)
range_N = np.arange(N, dtype=np.int32)
for _ in range(resolution):
sample = probs.argmax(axis=1)
result[range_N, sample] += 1
probs[range_N, sample] -= 1.0 / resolution
return result
def read_chunk(self, idx, chunk_size, padding=(0, 0), nodes=None):
self._open()
t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)
if self.time_axis == 0:
local_chunk = self.data[t_start:t_stop, :]
elif self.time_axis == 1:
local_chunk = self.data[:, t_start:t_stop].T
self._close()
if nodes is not None:
if not numpy.all(nodes == numpy.arange(self.nb_channels)):
local_chunk = numpy.take(local_chunk, nodes, axis=1)
return self._scale_data_to_float32(local_chunk)
def read_chunk(self, idx, chunk_size, padding=(0, 0), nodes=None):
t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)
local_shape = t_stop - t_start
local_chunk = numpy.zeros((self.nb_channels, local_shape), dtype=self.data_dtype)
data_slice = self._get_slice_(t_start, t_stop)
self._open()
count = 0
for s in data_slice:
t_slice = len(s)//self.nb_channels
local_chunk[:, count:count + t_slice] = self.data[s].reshape(self.nb_channels, len(s)//self.nb_channels)
count += t_slice
local_chunk = local_chunk.T
self._close()
if nodes is not None:
if not numpy.all(nodes == numpy.arange(self.nb_channels)):
local_chunk = numpy.take(local_chunk, nodes, axis=1)
return self._scale_data_to_float32(local_chunk)
def transform(self, img, lbl):
img = img[:, :, ::-1]
img = img.astype(np.float64)
img -= self.mean
img = m.imresize(img, (self.img_size[0], self.img_size[1]))
# Resize scales images from 0 to 255, thus we need
# to divide by 255.0
img = img.astype(float) / 255.0
# NHWC -> NCWH
img = img.transpose(2, 0, 1)
lbl = self.encode_segmap(lbl)
classes = np.unique(lbl)
lbl = lbl.astype(float)
lbl = m.imresize(lbl, (self.img_size[0], self.img_size[1]), 'nearest', mode='F')
lbl = lbl.astype(int)
assert(np.all(classes == np.unique(lbl)))
img = torch.from_numpy(img).float()
lbl = torch.from_numpy(lbl).long()
return img, lbl
def test_against_numpy(self):
""" Test iall against numpy.all """
stream = [np.zeros((8, 16, 2)) for _ in range(11)]
stream[3][3,0,1] = 1 # so that np.all(axis = None) evaluates to False
stack = np.stack(stream, axis = -1)
with self.subTest('axis = None'):
from_numpy = np.all(stack, axis = None)
from_stream = last(iall(stream, axis = None))
self.assertEqual(from_numpy, from_stream)
for axis in range(stack.ndim):
with self.subTest('axis = {}'.format(axis)):
from_numpy = np.all(stack, axis = axis)
from_stream = last(iall(stream, axis = axis))
self.assertTrue(np.allclose(from_numpy, from_stream))
def test_against_numpy(self):
""" Test iany against numpy.any """
stream = [np.zeros((8, 16, 2)) for _ in range(11)]
stream[3][3,0,1] = 1 # so that np.all(axis = None) evaluates to False
stack = np.stack(stream, axis = -1)
with self.subTest('axis = None'):
from_numpy = np.any(stack, axis = None)
from_stream = last(iany(stream, axis = None))
self.assertEqual(from_numpy, from_stream)
for axis in range(stack.ndim):
with self.subTest('axis = {}'.format(axis)):
from_numpy = np.any(stack, axis = axis)
from_stream = last(iany(stream, axis = axis))
self.assertTrue(np.allclose(from_numpy, from_stream))
def create_buffer_panel(self, initial_dt, bar_data):
"""
Initialize a RollingPanel containing enough minutes to service all our
frequencies.
"""
max_bars_needed = max(
freq.max_bars for freq in self.unique_frequencies
)
freq = '1m' if self.data_frequency == 'minute' else '1d'
spec = HistorySpec(
max_bars_needed + 1, freq, None, None, self.env,
self.data_frequency,
)
rp = self._create_panel(
initial_dt, spec,
)
self.buffer_spec = spec
if bar_data is not None:
frame = self.frame_from_bardata(bar_data, initial_dt)
rp.add_frame(initial_dt, frame)
return rp
def load(group):
gene_ids = list(getattr(group, cr_constants.H5_GENE_IDS_ATTR).read())
if hasattr(group, cr_constants.H5_GENE_NAMES_ATTR):
gene_names = list(getattr(group, cr_constants.H5_GENE_NAMES_ATTR).read())
else:
gene_names = gene_ids
assert len(gene_ids) == len(gene_names)
genes = [cr_constants.Gene(id, name, None, None, None) for id, name in itertools.izip(gene_ids, gene_names)]
bcs = list(getattr(group, cr_constants.H5_BCS_ATTR).read())
matrix = GeneBCMatrix(genes, bcs)
shape = getattr(group, cr_constants.H5_MATRIX_SHAPE_ATTR).read()
data = getattr(group, cr_constants.H5_MATRIX_DATA_ATTR).read()
indices = getattr(group, cr_constants.H5_MATRIX_INDICES_ATTR).read()
indptr = getattr(group, cr_constants.H5_MATRIX_INDPTR_ATTR).read()
# quick check to make sure indptr increases monotonically (to catch overflow bugs)
assert np.all(np.diff(indptr)>=0)
matrix.m = sp_sparse.csc_matrix((data, indices, indptr), shape=shape)
return matrix
def save_barcode_summary_h5(self, filename):
""" Generate a minimal barcode summary h5 without going through the reporter.
NOTE: only use this if all genomes have the same set of barcodes, i.e. a raw matrix.
"""
bc_sequences = None
bc_table_cols = {}
for (genome, matrix) in self.matrices.iteritems():
if bc_sequences is None:
bc_sequences = np.array(matrix.bcs)
bc_table_cols[cr_constants.H5_BC_SEQUENCE_COL] = bc_sequences
conf_mapped_deduped_reads_key = cr_utils.format_barcode_summary_h5_key(genome,
cr_constants.TRANSCRIPTOME_REGION, cr_constants.CONF_MAPPED_DEDUPED_READ_TYPE)
conf_mapped_deduped_reads = matrix.get_reads_per_bc()
if len(bc_sequences) != len(conf_mapped_deduped_reads):
raise ValueError('Cannot write barcode summary since different genomes have different number of barcodes!')
bc_table_cols[conf_mapped_deduped_reads_key] = conf_mapped_deduped_reads
cr_utils.write_h5(filename, bc_table_cols)
def test_aligned_read():
delete_layer()
cv, data = create_layer(size=(50,50,50,1), offset=(0,0,0))
# the last dimension is the number of channels
assert cv[0:50,0:50,0:50].shape == (50,50,50,1)
assert np.all(cv[0:50,0:50,0:50] == data)
delete_layer()
cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))
# the last dimension is the number of channels
assert cv[0:64,0:64,0:64].shape == (64,64,64,1)
assert np.all(cv[0:64,0:64,0:64] == data[:64,:64,:64,:])
delete_layer()
cv, data = create_layer(size=(128,64,64,1), offset=(10,20,0))
cutout = cv[10:74,20:84,0:64]
# the last dimension is the number of channels
assert cutout.shape == (64,64,64,1)
assert np.all(cutout == data[:64,:64,:64,:])
# get the second chunk
cutout2 = cv[74:138,20:84,0:64]
assert cutout2.shape == (64,64,64,1)
assert np.all(cutout2 == data[64:128,:64,:64,:])
def test_non_aligned_read():
delete_layer()
cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))
# the last dimension is the number of channels
assert cv[31:65,0:64,0:64].shape == (34,64,64,1)
assert np.all(cv[31:65,0:64,0:64] == data[31:65,:64,:64,:])
# read a single pixel
delete_layer()
cv, data = create_layer(size=(64,64,64,1), offset=(0,0,0))
# the last dimension is the number of channels
assert cv[22:23,22:23,22:23].shape == (1,1,1,1)
assert np.all(cv[22:23,22:23,22:23] == data[22:23,22:23,22:23,:])
# Test steps (negative steps are not supported)
img1 = cv[::2, ::2, ::2, :]
img2 = cv[:, :, :, :][::2, ::2, ::2, :]
assert np.array_equal(img1, img2)
def test_write():
delete_layer()
cv, data = create_layer(size=(50,50,50,1), offset=(0,0,0))
replacement_data = np.zeros(shape=(50,50,50,1), dtype=np.uint8)
cv[0:50,0:50,0:50] = replacement_data
assert np.all(cv[0:50,0:50,0:50] == replacement_data)
replacement_data = np.random.randint(255, size=(50,50,50,1), dtype=np.uint8)
cv[0:50,0:50,0:50] = replacement_data
assert np.all(cv[0:50,0:50,0:50] == replacement_data)
# out of bounds
delete_layer()
cv, data = create_layer(size=(128,64,64,1), offset=(10,20,0))
with pytest.raises(ValueError):
cv[74:150,20:84,0:64] = np.ones(shape=(64,64,64,1), dtype=np.uint8)
# non-aligned writes
delete_layer()
cv, data = create_layer(size=(128,64,64,1), offset=(10,20,0))
with pytest.raises(ValueError):
cv[21:85,0:64,0:64] = np.ones(shape=(64,64,64,1), dtype=np.uint8)
def exists(self, bbox_or_slices):
"""
Produce a summary of whether all the requested chunks exist.
bbox_or_slices: accepts either a Bbox or a tuple of slices representing
the requested volume.
Returns: { chunk_file_name: boolean, ... }
"""
if type(bbox_or_slices) is Bbox:
requested_bbox = bbox_or_slices
else:
(requested_bbox, steps, channel_slice) = self.__interpret_slices(bbox_or_slices)
realized_bbox = self.__realized_bbox(requested_bbox)
cloudpaths = self.__chunknames(realized_bbox, self.bounds, self.key, self.underlying)
with Storage(self.layer_cloudpath, progress=self.progress) as storage:
existence_report = storage.files_exist(cloudpaths)
return existence_report
def test_repeat(self):
""" Test if `repeat` works the same as np.repeat."""
with tf.Session().as_default():
# try different tensor types
for npdtype, tfdtype in [(np.int32, tf.int32), (np.float32, tf.float32)]:
for init_value in [np.array([0, 1, 2, 3], dtype=npdtype),
np.array([[0, 1], [2, 3], [4, 5]], dtype=npdtype)]:
# and all their axes
for axis in range(len(init_value.shape)):
for repeats in [1, 2, 3, 11]:
tensor = tf.constant(init_value, dtype=tfdtype)
repeated_value = repeat(tensor, repeats=repeats, axis=axis).eval()
expected_value = np.repeat(init_value, repeats=repeats, axis=axis)
self.assertTrue(np.all(repeated_value == expected_value))
def _import_module(module_name, warn=True, prefix='_py_', ignore='_'):
"""Try import all public attributes from module into global namespace.
Existing attributes with name clashes are renamed with prefix.
Attributes starting with underscore are ignored by default.
Return True on successful import.
"""
try:
module = __import__(module_name)
except ImportError:
if warn:
warnings.warn("Failed to import module " + module_name)
else:
for attr in dir(module):
if ignore and attr.startswith(ignore):
continue
if prefix:
if attr in globals():
globals()[prefix + attr] = globals()[attr]
elif warn:
warnings.warn("No Python implementation of " + attr)
globals()[attr] = getattr(module, attr)
return True
def __call__(self, x, inverse=False):
"""Rotates the input array `x` with a fixed rotation matrix
(``self.dicMatrices[len(x)]``)
"""
x = np.array(x, copy=False)
N = x.shape[0] # can be an array or matrix, TODO: accept also a list of arrays?
if N not in self.dicMatrices: # create new N-basis once and for all
rstate = np.random.get_state()
np.random.seed(self.seed) if self.seed else np.random.seed()
self.state = np.random.get_state() # only keep last state
B = np.random.randn(N, N)
np.random.set_state(rstate) # keep untouched/good sequence from outside view
for i in range(N):
for j in range(0, i):
B[i] -= np.dot(B[i], B[j]) * B[j]
B[i] /= sum(B[i]**2)**0.5
self.dicMatrices[N] = B
if inverse:
return np.dot(self.dicMatrices[N].T, x) # compute rotation
else:
return np.dot(self.dicMatrices[N], x) # compute rotation
def __imul__(self, factor):
"""define ``self *= factor``.
As a shortcut for::
self = self.__imul__(factor)
"""
try:
if factor == 1:
return self
except: pass
try:
if (np.size(factor) == np.size(self.scaling) and
all(factor == 1)):
return self
except: pass
if self.is_identity and np.size(self.scaling) == 1:
self.scaling = np.ones(np.size(factor))
self.is_identity = False
self.scaling *= factor
self.dim = np.size(self.scaling)
return self
def get_bareground_fn(datadir=None):
"""Calls external shell script `get_bareground.sh` to fetch:
~2010 global bare ground, 30 m
Note: unzipped file size is 64 GB! Original products are uncompressed, and tiles are available globally (including empty data over ocean)
The shell script will compress all downloaded tiles using lossless LZW compression.
http://landcover.usgs.gov/glc/BareGroundDescriptionAndDownloads.php
"""
if datadir is None:
datadir = iolib.get_datadir()
bg_fn = os.path.join(datadir, 'bare2010/bare2010.vrt')
if not os.path.exists(bg_fn):
cmd = ['get_bareground.sh',]
subprocess.call(cmd)
return bg_fn
#Download latest global RGI glacier db
def lpf(x, cutoff, fs, order=5):
"""
low pass filters signal with Butterworth digital
filter according to cutoff frequency
filter uses Gustafsson’s method to make sure
forward-backward filt == backward-forward filt
Note that edge effects are expected
Args:
x (array): signal data (numpy array)
cutoff (float): cutoff frequency (Hz)
fs (int): sample rate (Hz)
order (int): order of filter (default 5)
Returns:
filtered (array): low pass filtered data
"""
nyquist = fs / 2
b, a = butter(order, cutoff / nyquist)
if not np.all(np.abs(np.roots(a)) < 1):
raise PsolaError('Filter with cutoff at {} Hz is unstable given '
'sample frequency {} Hz'.format(cutoff, fs))
filtered = filtfilt(b, a, x, method='gust')
return filtered
def test_pitch_estimation(self):
"""
test pitch estimation algo with contrived small example
if pitch is within 5 Hz, then say its good (for this small example,
since the algorithm wasn't made for this type of synthesized signal)
"""
cfg = ExperimentConfig(pitch_strength_thresh=-np.inf)
# the next 3 variables are in Hz
tolerance = 5
fs = 48000
f = 150
# create a sine wave of f Hz freq sampled at fs Hz
x = np.sin(2*np.pi * f/fs * np.arange(2**10))
# estimate the pitch, it should be close to f
p, t, s = pest.pitch_estimation(x, fs, cfg)
self.assertTrue(np.all(np.abs(p - f) < tolerance))
def validate_dict(self,a_dict):
#Check keys
for key,val in self.dict.items():
if not key in a_dict.keys():
raise ValueError('key:',key,'was not in a_dict.keys()')
for key,val in a_dict.items():
#Check same keys
if not key in self.dict.keys():
raise ValueError('argument key:',key,'was not in self.dict')
if isinstance(val,np.ndarray):
#print('ndarray')
my_val=self.dict[key]
if not np.all(val.shape[1:]==my_val.shape[1:]):
raise ValueError('key:',key,'value shape',val.shape,'does\
not match existing shape',my_val.shape)
else: #scalar
a_val=np.array([[val]])#[1,1]shape array
my_val=self.dict[key]
if not np.all(my_val.shape[1:]==a_val.shape[1:]):
raise ValueError('key:',key,'value shape',val.shape,'does\
not match existing shape',my_val.shape)
def getColors(self, mode=None):
"""Return list of all color stops converted to the specified mode.
If mode is None, then no conversion is done."""
if isinstance(mode, basestring):
mode = self.enumMap[mode.lower()]
color = self.color
if mode in [self.BYTE, self.QCOLOR] and color.dtype.kind == 'f':
color = (color * 255).astype(np.ubyte)
elif mode == self.FLOAT and color.dtype.kind != 'f':
color = color.astype(float) / 255.
if mode == self.QCOLOR:
color = [QtGui.QColor(*x) for x in color]
return color
def register(self, name):
"""
Add this ViewBox to the registered list of views.
This allows users to manually link the axes of any other ViewBox to
this one. The specified *name* will appear in the drop-down lists for
axis linking in the context menus of all other views.
The same can be accomplished by initializing the ViewBox with the *name* attribute.
"""
ViewBox.AllViews[self] = None
if self.name is not None:
del ViewBox.NamedViews[self.name]
self.name = name
if name is not None:
ViewBox.NamedViews[name] = self
ViewBox.updateAllViewLists()
sid = id(self)
self.destroyed.connect(lambda: ViewBox.forgetView(sid, name) if (ViewBox is not None and 'sid' in locals() and 'name' in locals()) else None)
#self.destroyed.connect(self.unregister)
def autoRange(self, padding=None, items=None, item=None):
"""
Set the range of the view box to make all children visible.
Note that this is not the same as enableAutoRange, which causes the view to
automatically auto-range whenever its contents are changed.
============== ============================================================
**Arguments:**
padding The fraction of the total data range to add on to the final
visible range. By default, this value is set between 0.02
and 0.1 depending on the size of the ViewBox.
items If specified, this is a list of items to consider when
determining the visible range.
============== ============================================================
"""
if item is None:
bounds = self.childrenBoundingRect(items=items)
else:
print("Warning: ViewBox.autoRange(item=__) is deprecated. Use 'items' argument instead.")
bounds = self.mapFromItemToView(item, item.boundingRect()).boundingRect()
if bounds is not None:
self.setRange(bounds, padding=padding)
