def encode(self, input, sampling):
output = input
for layer in self.D_layers:
output = layer.encode(output)
# now compute mu and sigma
Mu = self.Mu_layer.encode(output)
if sampling:
shape = Mu.get_shape()
eps = tf.random_uniform(shape)
output = tf.select(eps - Mu <= 0, tf.ones(shape), tf.zeros(shape))
else:
output = Mu
return output, Mu
python类select()的实例源码
def encode(self, input, sampling):
output = input
for layer in self.D_layers:
output = layer.encode(output)
# now compute mu
Mu = self.Mu_layer.encode(output)
if sampling:
shape = Mu.get_shape()
eps = tf.random_uniform(shape)
output = tf.select(eps - Mu <= 0, tf.ones(shape), tf.zeros(shape))
else:
output = Mu
output = output * 2.0 - 1.0
return output, Mu
def clipped_error(x):
# Huber loss
try:
return tf.select(tf.abs(x) < 1.0, 0.5 * tf.square(x), tf.abs(x) - 0.5)
except:
return tf.where(tf.abs(x) < 1.0, 0.5 * tf.square(x), tf.abs(x) - 0.5)
def beam_setup(self, time):
emit_output = None
next_cell_state = self.initial_state
next_input = self.initial_input
# Set up the beam search tracking state
cand_symbols = tf.fill([self.batch_size_times_beam_size, 0], tf.constant(self.stop_token, dtype=tf.int32))
cand_logprobs = tf.ones((self.batch_size_times_beam_size,), dtype=tf.float32) * -float('inf')
first_in_beam_mask = tf.equal(tf.range(self.batch_size_times_beam_size) % self.beam_size, 0)
beam_symbols = tf.fill([self.batch_size_times_beam_size, 0], tf.constant(self.stop_token, dtype=tf.int32))
beam_logprobs = tf.select(
first_in_beam_mask,
tf.fill([self.batch_size_times_beam_size], 0.0),
tf.fill([self.batch_size_times_beam_size], self.INVALID_SCORE)
)
# Set up correct dimensions for maintaining loop invariants.
# Note that the last dimension (initialized to zero) is not a loop invariant,
# so we need to clear it.
# inference so that _shape is not necessary?
cand_symbols._shape = tf.TensorShape((self.inferred_batch_size_times_beam_size, None))
cand_logprobs._shape = tf.TensorShape((self.inferred_batch_size_times_beam_size,))
beam_symbols._shape = tf.TensorShape((self.inferred_batch_size_times_beam_size, None))
beam_logprobs._shape = tf.TensorShape((self.inferred_batch_size_times_beam_size,))
next_loop_state = (
cand_symbols,
cand_logprobs,
beam_symbols,
beam_logprobs,
)
emit_output = tf.zeros(self.cell.output_size)
elements_finished = tf.zeros([self.batch_size], dtype=tf.bool)
return elements_finished, next_input, next_cell_state, emit_output, next_loop_state
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10):
correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))
correctly_recognised_indices = tf.squeeze(tf.where(correct_prediction), [1]) # indices of correctly recognised images
incorrectly_recognised_indices = tf.squeeze(tf.where(tf.logical_not(correct_prediction)), [1]) # indices of incorrectly recognised images
everything_incorrect_first = tf.concat(0, [incorrectly_recognised_indices, correctly_recognised_indices]) # images reordered with indeces of unrecognised images first
everything_incorrect_first = tf.slice(everything_incorrect_first, [0], [n]) # compute first 100 only - no space to display more anyway
# compute n=100 digits to display only
Xs = tf.gather(X, everything_incorrect_first)
Ys = tf.gather(Y, everything_incorrect_first)
Ys_ = tf.gather(Y_, everything_incorrect_first)
correct_prediction_s = tf.gather(correct_prediction, everything_incorrect_first)
digits_left = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_left())
correct_tags = tf.gather(digits_left, tf.argmax(Ys_, 1)) # correct digits to be printed on the images
digits_right = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_right())
computed_tags = tf.gather(digits_right, tf.argmax(Ys, 1)) # computed digits to be printed on the images
#superimposed_digits = correct_tags+computed_tags
superimposed_digits = tf.select(correct_prediction_s, tf.zeros_like(correct_tags),correct_tags+computed_tags) # only pring the correct and computed digits on unrecognised images
correct_bkg = tf.reshape(tf.tile([1.3,1.3,1.3], [28*28]), [1, 28,28,3]) # white background
incorrect_bkg = tf.reshape(tf.tile([1.3,1.0,1.0], [28*28]), [1, 28,28,3]) # red background
recognised_bkg = tf.gather(tf.concat(0, [incorrect_bkg, correct_bkg]), tf.cast(correct_prediction_s, tf.int32)) # pick either the red or the white background depending on recognised status
I = tf.image.grayscale_to_rgb(Xs)
I = ((1-(I+superimposed_digits))*recognised_bkg)/1.3 # stencil extra data on top of images and reorder them unrecognised first
I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True)
Islices = [] # 100 images => 10x10 image block
for imslice in range(lines):
Islices.append(tf.concat(1, tf.unpack(tf.slice(I, [imslice*n//lines,0,0,0], [n//lines,28,28,3]))))
I = tf.concat(0, Islices)
return I
# n = HISTOGRAM_BUCKETS (global)
# Buckets the data into n buckets so that there are an equal number of data points in
# each bucket. Returns n+1 bucket boundaries. Spreads the reaminder data.size % n more
# or less evenly among the central buckets.
# data: 1-D ndarray containing float data, MUST BE SORTED in ascending order
# n: integer, the number of desired output buckets
# return value: ndarray, 1-D vector of size n+1 containing the bucket boundaries
# the first value is the min of the data, the last value is the max
def _get_sharding_func(size, num_shards):
"""Create sharding function for scatter update."""
def func(ids):
if num_shards == 1:
return None, ids
else:
ids_per_shard = size // num_shards
extras = size % num_shards
assignments = tf.maximum(ids // (ids_per_shard + 1),
(ids - extras) // ids_per_shard)
new_ids = tf.select(assignments < extras,
ids % (ids_per_shard + 1),
(ids - extras) % ids_per_shard)
return assignments, new_ids
return func
def l1_robust_loss(predictions, targets, name=None):
with tf.name_scope(name, 'HuberLoss', [predictions, targets]):
delta = predictions - targets
return tf.select(tf.abs(delta) < 1,
0.5 * tf.square(delta),
tf.abs(delta) - 0.5)
def huber_loss(y_true, y_pred, clip_value):
# Huber loss, see https://en.wikipedia.org/wiki/Huber_loss and
# https://medium.com/@karpathy/yes-you-should-understand-backprop-e2f06eab496b
# for details.
assert clip_value > 0.
x = y_true - y_pred
if np.isinf(clip_value):
# Spacial case for infinity since Tensorflow does have problems
# if we compare `K.abs(x) < np.inf`.
return .5 * K.square(x)
condition = K.abs(x) < clip_value
squared_loss = .5 * K.square(x)
linear_loss = clip_value * (K.abs(x) - .5 * clip_value)
if K.backend() == 'tensorflow':
import tensorflow as tf
if hasattr(tf, 'select'):
return tf.select(condition, squared_loss, linear_loss) # condition, true, false
else:
return tf.where(condition, squared_loss, linear_loss) # condition, true, false
elif K.backend() == 'theano':
from theano import tensor as T
return T.switch(condition, squared_loss, linear_loss)
else:
raise RuntimeError('Unknown backend "{}".'.format(K.backend()))
def elu(x, alpha=1.):
'''Exponential linear unit.
# Arguments
x: Tensor to compute the activation function for.
alpha: scalar
'''
res = tf.nn.elu(x)
if alpha == 1:
return res
else:
return tf.select(x > 0, res, alpha * res)
def random_binomial(shape, p=0.0, dtype=None, seed=None):
if dtype is None:
dtype = floatx()
if seed is None:
seed = np.random.randint(10e6)
return tf.select(tf.random_uniform(shape, dtype=dtype, seed=seed) <= p,
tf.ones(shape, dtype=dtype),
tf.zeros(shape, dtype=dtype))
# CTC
# tensorflow has a native implemenation, but it uses sparse tensors
# and therefore requires a wrapper for Keras. The functions below convert
# dense to sparse tensors and also wraps up the beam search code that is
# in tensorflow's CTC implementation
def pseudo_predict(self, predictions, targets):
""""""
random_flip = tf.random_uniform(tf.shape(predictions))
return tf.select(tf.greater(random_flip, self.global_sigmoid), predictions, targets)
#=============================================================
def reduce_mean(seq_batch, allow_empty=False):
"""Compute the mean of each sequence in a SequenceBatch.
Args:
seq_batch (SequenceBatch): a SequenceBatch with the following attributes:
values (Tensor): a Tensor of shape (batch_size, seq_length, :, ..., :)
mask (Tensor): if the mask values are arbitrary floats (rather than binary), the mean will be
a weighted average.
allow_empty (bool): allow computing the average of an empty sequence. In this case, we assume 0/0 == 0, rather
than NaN. Default is False, causing an error to be thrown.
Returns:
Tensor: of shape (batch_size, :, ..., :)
"""
values, mask = seq_batch.values, seq_batch.mask
# compute weights for the average
sums = tf.reduce_sum(mask, 1, keep_dims=True) # (batch_size, 1)
if allow_empty:
asserts = [] # no assertion
sums = tf.select(tf.equal(sums, 0), tf.ones(tf.shape(sums)), sums) # replace 0's with 1's
else:
asserts = [tf.assert_positive(sums)] # throw error if 0's exist
with tf.control_dependencies(asserts):
weights = mask / sums # (batch_size, seq_length)
return weighted_sum(seq_batch, weights)
def _train(self, avg_loss):
lr = tf.select(tf.less(self._global_step, 32000), 0.1, tf.select(tf.less(self._global_step, 48000), 0.01, 0.001))
return tf.train.MomentumOptimizer(learning_rate=lr, momentum=0.9).minimize(avg_loss, global_step=self._global_step)
def _train(self, avg_loss):
lr = tf.select(tf.less(self._global_step, 32000), 0.1, tf.select(tf.less(self._global_step, 48000), 0.01, 0.001))
return tf.train.MomentumOptimizer(learning_rate=lr, momentum=0.9).minimize(avg_loss, global_step=self._global_step)
def set_logp_to_neg_inf(X, logp, bounds):
"""Set `logp` to negative infinity when `X` is outside the allowed bounds.
# Arguments
X: tensorflow.Tensor
The variable to apply the bounds to
logp: tensorflow.Tensor
The log probability corrosponding to `X`
bounds: list of `Region` objects
The regions corrosponding to allowed regions of `X`
# Returns
logp: tensorflow.Tensor
The newly bounded log probability
"""
conditions = []
for l, u in bounds:
lower_is_neg_inf = not isinstance(l, tf.Tensor) and np.isneginf(l)
upper_is_pos_inf = not isinstance(u, tf.Tensor) and np.isposinf(u)
if not lower_is_neg_inf and upper_is_pos_inf:
conditions.append(tf.greater(X, l))
elif lower_is_neg_inf and not upper_is_pos_inf:
conditions.append(tf.less(X, u))
elif not (lower_is_neg_inf or upper_is_pos_inf):
conditions.append(tf.logical_and(tf.greater(X, l), tf.less(X, u)))
if len(conditions) > 0:
is_inside_bounds = conditions[0]
for condition in conditions[1:]:
is_inside_bounds = tf.logical_or(is_inside_bounds, condition)
logp = tf.select(
is_inside_bounds,
logp,
tf.fill(tf.shape(X), config.dtype(-np.inf))
)
return logp
def _create_state(self, batch_size, dtype, cell_state=None):
cand_symbols = tf.fill([batch_size, self.max_len], tf.constant(self.stop_token,
dtype=tf.int32))
cand_logprobs = tf.ones((batch_size,), dtype=tf.float32) * -float('inf')
if cell_state is None:
cell_state = self.cell.zero_state(batch_size*self.beam_size, dtype=dtype)
else:
cell_state = BeamDecoder._tile_along_beam(self.beam_size, cell_state)
full_size = batch_size * self.beam_size
first_in_beam_mask = tf.equal(tf.range(full_size) % self.beam_size, 0)
beam_symbols = tf.fill([full_size, self.max_len], tf.constant(self.stop_token,
dtype=tf.int32))
beam_logprobs = tf.select(
first_in_beam_mask,
tf.fill([full_size], 0.0),
tf.fill([full_size], -1e18), # top_k does not play well with -inf
# TODO: dtype-dependent value here
)
return (
cand_symbols,
cand_logprobs,
beam_symbols,
beam_logprobs,
cell_state,
)
def binomial_draw(shape=[1], p=0.5, dtype='float32'):
return tf.select(tf.less(tf.random_uniform(shape=shape, minval=0, maxval=1, dtype='float32'), tf.fill(shape, p)), tf.ones(shape, dtype=dtype), tf.zeros(shape, dtype=dtype))
def salt_and_pepper(X, rate=0.3):
a = binomial_draw(shape=tf.shape(X), p=1-rate, dtype='float32')
b = binomial_draw(shape=tf.shape(X), p=0.5, dtype='float32')
z = tf.zeros(tf.shape(X), dtype='float32')
c = tf.select(tf.equal(a, z), b, z)
return tf.add(tf.mul(X, a), c)
# Xavier Initializers
def binomial_draw(shape=[1], p=0.5, dtype='float32'):
return tf.select(tf.less(tf.random_uniform(shape=shape, minval=0, maxval=1, dtype='float32'), tf.fill(shape, p)), tf.ones(shape, dtype=dtype), tf.zeros(shape, dtype=dtype))
def binomial_draw_vec(p_vec, dtype='float32'):
shape = tf.shape(p_vec)
return tf.select(tf.less(tf.random_uniform(shape=shape, minval=0, maxval=1, dtype='float32'), p_vec), tf.ones(shape, dtype=dtype), tf.zeros(shape, dtype=dtype))