def make_test_node(self, hypers_name):
outputs = self.tf_nodes[hypers_name]["outputs"]
deltas = []
for var_name, output_node in outputs.iteritems():
data_node = self.tf_nodes[hypers_name]["placeholders"][var_name]
output_rank = output_node.get_shape().ndims
if output_rank == 1:
output_node = tf.tile(tf.expand_dims(output_node, 0), [tf.shape(data_node)[0], 1])
deltas.append(
tf.to_int32(tf.argmax(output_node, dimension=1)) - data_node)
zero_if_correct = tf.reduce_sum(tf.pack(deltas), reduction_indices=0)
zero_elements = tf.equal(zero_if_correct, tf.zeros_like(zero_if_correct))
n_correct = tf.reduce_sum(tf.to_int32(zero_elements))
n_total = tf.shape(zero_if_correct)[0]
accuracy = tf.truediv(n_correct, n_total)
self.summary_nodes["test"] = tf.scalar_summary('test_accuracy', accuracy)
self.tf_nodes[hypers_name]["accuracy"] = accuracy
python类zeros_like()的实例源码
def retrieve_seq_length_op2(data):
"""An op to compute the length of a sequence, from input shape of [batch_size, n_step(max)],
it can be used when the features of padding (on right hand side) are all zeros.
Parameters
-----------
data : tensor
[batch_size, n_step(max)] with zero padding on right hand side.
Examples
--------
>>> data = [[1,2,0,0,0],
... [1,2,3,0,0],
... [1,2,6,1,0]]
>>> o = retrieve_seq_length_op2(data)
>>> sess = tf.InteractiveSession()
>>> tl.layers.initialize_global_variables(sess)
>>> print(o.eval())
... [2 3 4]
"""
return tf.reduce_sum(tf.cast(tf.greater(data, tf.zeros_like(data)), tf.int32), 1)
def _define_loss(self):
"""Define loss function that will be used to optimize model params"""
# define generator loss
with tf.variable_scope('generator'):
self.loss_gen = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.disc_gen,
labels=tf.ones_like(self.disc_gen)))
# define discriminator loss
with tf.variable_scope('discriminator'):
self.loss_disc = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.disc_real,
labels=tf.ones_like(self.disc_real)) +
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.disc_gen,
labels=tf.zeros_like(self.disc_gen)))
# save summaries of losses
tf.summary.scalar('loss_gen', self.loss_gen)
tf.summary.scalar('loss_disc', self.loss_disc)
def _gan_loss(self, logits_real, logits_fake, feature_real, feature_fake, use_features=False):
discriminator_loss_real = self._cross_entropy_loss(logits_real, tf.ones_like(logits_real),
name="disc_real_loss")
discriminator_loss_fake = self._cross_entropy_loss(logits_fake, tf.zeros_like(logits_fake),
name="disc_fake_loss")
self.discriminator_loss = discriminator_loss_fake + discriminator_loss_real
gen_loss_disc = self._cross_entropy_loss(logits_fake, tf.ones_like(logits_fake), name="gen_disc_loss")
if use_features:
gen_loss_features = tf.reduce_mean(tf.nn.l2_loss(feature_real - feature_fake)) / (self.crop_image_size ** 2)
else:
gen_loss_features = 0
self.gen_loss = gen_loss_disc + 0.1 * gen_loss_features
tf.summary.scalar("Discriminator_loss", self.discriminator_loss)
tf.summary.scalar("Generator_loss", self.gen_loss)
def resize_axis(tensor, axis, new_size, fill_value=0):
"""Truncates or pads a tensor to new_size on on a given axis.
Truncate or extend tensor such that tensor.shape[axis] == new_size. If the
size increases, the padding will be performed at the end, using fill_value.
Args:
tensor: The tensor to be resized.
axis: An integer representing the dimension to be sliced.
new_size: An integer or 0d tensor representing the new value for
tensor.shape[axis].
fill_value: Value to use to fill any new entries in the tensor. Will be
cast to the type of tensor.
Returns:
The resized tensor.
"""
tensor = tf.convert_to_tensor(tensor)
shape = tf.unstack(tf.shape(tensor))
pad_shape = shape[:]
pad_shape[axis] = tf.maximum(0, new_size - shape[axis])
shape[axis] = tf.minimum(shape[axis], new_size)
shape = tf.stack(shape)
resized = tf.concat([
tf.slice(tensor, tf.zeros_like(shape), shape),
tf.fill(tf.stack(pad_shape), tf.cast(fill_value, tensor.dtype))
], axis)
# Update shape.
new_shape = tensor.get_shape().as_list() # A copy is being made.
new_shape[axis] = new_size
resized.set_shape(new_shape)
return resized
def resize_axis(tensor, axis, new_size, fill_value=0):
"""Truncates or pads a tensor to new_size on on a given axis.
Truncate or extend tensor such that tensor.shape[axis] == new_size. If the
size increases, the padding will be performed at the end, using fill_value.
Args:
tensor: The tensor to be resized.
axis: An integer representing the dimension to be sliced.
new_size: An integer or 0d tensor representing the new value for
tensor.shape[axis].
fill_value: Value to use to fill any new entries in the tensor. Will be
cast to the type of tensor.
Returns:
The resized tensor.
"""
tensor = tf.convert_to_tensor(tensor)
shape = tf.unstack(tf.shape(tensor))
pad_shape = shape[:]
pad_shape[axis] = tf.maximum(0, new_size - shape[axis])
shape[axis] = tf.minimum(shape[axis], new_size)
shape = tf.stack(shape)
resized = tf.concat([
tf.slice(tensor, tf.zeros_like(shape), shape),
tf.fill(tf.stack(pad_shape), tf.cast(fill_value, tensor.dtype))
], axis)
# Update shape.
new_shape = tensor.get_shape().as_list() # A copy is being made.
new_shape[axis] = new_size
resized.set_shape(new_shape)
return resized
def get_optimizers(self, learning_rate=0.002, smooth=0.1):
#-----------------------------------------------------------------------
# Define loss functions
#-----------------------------------------------------------------------
with tf.variable_scope('loses'):
dsc_real_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.dsc_real_logits,
labels=tf.ones_like(self.dsc_real_logits) * (1 - smooth)))
dsc_fake_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.dsc_fake_logits,
labels=tf.zeros_like(self.dsc_fake_logits)))
dsc_loss = (dsc_real_loss + dsc_fake_loss)/2
gen_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.dsc_fake_logits,
labels=tf.ones_like(self.dsc_fake_logits)))
#-----------------------------------------------------------------------
# Optimizers
#-----------------------------------------------------------------------
trainable_vars = tf.trainable_variables()
gen_vars = [var for var in trainable_vars \
if var.name.startswith('generator')]
dsc_vars = [var for var in trainable_vars \
if var.name.startswith('discriminator')]
with tf.variable_scope('optimizers'):
with tf.variable_scope('deiscriminator_optimizer'):
dsc_train_opt = tf.train.AdamOptimizer(learning_rate) \
.minimize(dsc_loss, var_list=dsc_vars)
with tf.variable_scope('generator_optimizer'):
gen_train_opt = tf.train.AdamOptimizer(learning_rate) \
.minimize(gen_loss, var_list=gen_vars)
return dsc_train_opt, gen_train_opt, dsc_loss, gen_loss
def loss(self, x, y):
with tf.name_scope('loss'):
z_mu, z_lv = self._encode(x)
z = GaussianSampleLayer(z_mu, z_lv)
xh = self._generate(z, y)
D_KL = tf.reduce_mean(
GaussianKLD(
slim.flatten(z_mu),
slim.flatten(z_lv),
slim.flatten(tf.zeros_like(z_mu)),
slim.flatten(tf.zeros_like(z_lv)),
)
)
logPx = tf.reduce_mean(
GaussianLogDensity(
slim.flatten(x),
slim.flatten(xh),
tf.zeros_like(slim.flatten(xh))),
)
loss = dict()
loss['G'] = - logPx + D_KL
loss['D_KL'] = D_KL
loss['logP'] = logPx
tf.summary.scalar('KL-div', D_KL)
tf.summary.scalar('logPx', logPx)
tf.summary.histogram('xh', xh)
tf.summary.histogram('x', x)
return loss
def loss(self, x, y):
'''
Args:
x: shape=[s, b, c]
y: shape=[s, b]
Returns:
a `dict` of losses
'''
z_mu, z_lv = self._encode(x, is_training=self.is_training)
z = GaussianSampleLayer(z_mu, z_lv)
xh = self._decode(z, y, is_training=self.is_training)
with tf.name_scope('loss'):
with tf.name_scope('E_log_p_x_zy'):
L_x = -1.0 * tf.reduce_mean(
GaussianLogDensity(x, xh, tf.zeros_like(x)),
)
with tf.name_scope('D_KL_z'):
L_z = tf.reduce_mean(
GaussianKLD(
z_mu, z_lv,
tf.zeros_like(z_mu), tf.zeros_like(z_lv)
)
)
loss = {
'L_x': L_x,
'L_z': L_z,
}
tf.summary.scalar('L_x', L_x)
tf.summary.scalar('L_z', L_z)
return loss
def dice_whole_mod(y_true, y_pred):
"""
Computes the Sorensen-Dice metric, where P come from class 1,2,3,4,0
TP
Dice = 2 -------
T + P
Parameters
----------
y_true : keras.placeholder
Placeholder that contains the ground truth labels of the classes
y_pred : keras.placeholder
Placeholder that contains the class prediction
Returns
-------
scalar
Dice metric
"""
# mask = K.expand_dims(K.sum(y_true,axis=4),axis=4)
# cmp_mask = K.concatenate([K.ones_like(mask) - mask,K.zeros_like(mask), K.zeros_like(mask)],axis=4)
# y_pred = y_pred + cmp_mask
y_true = y_true[:,:,:,:,:3]
y_pred_decision = tf.floor((y_pred + K.epsilon()) / K.max(y_pred, axis=4, keepdims=True))
mask_true = K.sum(y_true, axis=4)
mask_pred = K.sum(y_pred_decision, axis=4) * K.sum(y_true, axis=4)
y_sum = K.sum(mask_true * mask_pred)
return (2. * y_sum + K.epsilon()) / (K.sum(mask_true) + K.sum(mask_pred) + K.epsilon())
def dice_core_mod(y_true, y_pred):
"""
Computes the Sorensen-Dice metric, where P come from class 1,2,3,4,5
TP
Dice = 2 -------
T + P
Parameters
----------
y_true : keras.placeholder
Placeholder that contains the ground truth labels of the classes
y_pred : keras.placeholder
Placeholder that contains the class prediction
Returns
-------
scalar
Dice metric
"""
y_true = y_true[:,:,:,:,:3]
y_pred_decision = tf.floor((y_pred + K.epsilon()) / K.max(y_pred, axis=4, keepdims=True))
y_pred_decision = tf.where(tf.is_nan(y_pred_decision), tf.zeros_like(y_pred_decision), y_pred_decision)
mask_true1 = K.expand_dims(y_true[:, :, :, :, 2],axis=4)
mask_true2 = K.expand_dims(y_true[:, :, :, :, 0],axis=4)
mask_true = K.sum(K.concatenate([mask_true1, mask_true2], axis=4), axis=4)
mask_pred1 = K.expand_dims(y_pred_decision[:, :, :, :, 2],axis=4)
mask_pred2 = K.expand_dims(y_pred_decision[:, :, :, :, 0],axis=4)
mask_pred = K.sum(K.concatenate([mask_pred1, mask_pred2], axis=4), axis=4) * K.sum(y_true, axis=4)
y_sum = K.sum(mask_true * mask_pred)
return (2. * y_sum + K.epsilon()) / (K.sum(mask_true) + K.sum(mask_pred) + K.epsilon())
def dice_enhance_mod(y_true, y_pred):
"""
Computes the Sorensen-Dice metric, where P come from class 1,2,3,4,5
TP
Dice = 2 -------
T + P
Parameters
----------
y_true : keras.placeholder
Placeholder that contains the ground truth labels of the classes
y_pred : keras.placeholder
Placeholder that contains the class prediction
Returns
-------
scalar
Dice metric
"""
y_true = y_true[:,:,:,:,:3]
y_pred_decision = tf.floor((y_pred + K.epsilon()) / K.max(y_pred, axis=4, keepdims=True))
# y_pred_decision = tf.where(tf.is_nan(y_pred_decision), tf.zeros_like(y_pred_decision), y_pred_decision)
mask_true = y_true[:, :, :, :, 2]
mask_pred = y_pred_decision[:, :, :, :, 2] * K.sum(y_true, axis=4)
y_sum = K.sum(mask_true * mask_pred)
return (2. * y_sum + K.epsilon()) / (K.sum(mask_true) + K.sum(mask_pred) + K.epsilon())
def accumulate_gradients(self, minibatch_grads, num_minibatches=1):
"""Accumulate gradients for `num_minibatches` minibatches."""
if self.var_list is None:
self.var_list = tf.trainable_variables()
if self.grads_and_vars is None:
self.grads_and_vars = [(
tf.Variable(tf.zeros_like(var.initialized_value()),
dtype=tf.float32,
trainable=False),
var) for var in self.var_list]
# Add 1/num_minibatches * minibatch_grads to current gradients.
def _add_op(gv_tmp, mgv_tmp):
return tf.add(gv_tmp, tf.divide(mgv_tmp, num_minibatches))
def _set_op(gv_tmp, mgv_tmp):
return tf.assign(gv_tmp, tf.divide(mgv_tmp, num_minibatches))
#grads = [(gv[0].assign_add(tf.divide(mgv[0], num_minibatches)), gv[1])
# for (gv, mgv) in zip(self.grads_and_vars, minibatch_grads)]
#grads = tf.cond(tf.less(self.mini_flag[0], 0.5), fn1 = lambda: _add_op(), fn2 = lambda: _set_op())
grads = [tf.cond(tf.less(self.mini_flag[0], 0.5), fn1 = lambda: _set_op(gv[0], mgv[0]), fn2 = lambda: _add_op(gv[0], mgv[0]))
for (gv, mgv) in zip(self.grads_and_vars, minibatch_grads)]
with tf.control_dependencies(grads):
self.mini_flag = tf.assign(self.mini_flag, tf.constant([1], dtype = tf.float32))
grads = [(only_grad, gv[1])
for (gv, only_grad) in zip(self.grads_and_vars, grads)]
return self.mini_flag, grads
def zero_grad(self):
if self.grads_and_vars is None:
self.grads_and_vars = [(
tf.Variable(tf.zeros_like(var), dtype=tf.float32, trainable=False),
var) for var in self.var_list]
return [tf.assign(gv[0], tf.zeros_like(gv[0]))
for gv in self.grads_and_vars]
def compute_losses(self, images, wrong_images, fake_images, embeddings):
real_logit = self.model.get_discriminator(images, embeddings)
wrong_logit = self.model.get_discriminator(wrong_images, embeddings)
fake_logit = self.model.get_discriminator(fake_images, embeddings)
real_d_loss =\
tf.nn.sigmoid_cross_entropy_with_logits(real_logit,
tf.ones_like(real_logit))
real_d_loss = tf.reduce_mean(real_d_loss)
wrong_d_loss =\
tf.nn.sigmoid_cross_entropy_with_logits(wrong_logit,
tf.zeros_like(wrong_logit))
wrong_d_loss = tf.reduce_mean(wrong_d_loss)
fake_d_loss =\
tf.nn.sigmoid_cross_entropy_with_logits(fake_logit,
tf.zeros_like(fake_logit))
fake_d_loss = tf.reduce_mean(fake_d_loss)
if cfg.TRAIN.B_WRONG:
discriminator_loss =\
real_d_loss + (wrong_d_loss + fake_d_loss) / 2.
self.log_vars.append(("d_loss_wrong", wrong_d_loss))
else:
discriminator_loss = real_d_loss + fake_d_loss
self.log_vars.append(("d_loss_real", real_d_loss))
self.log_vars.append(("d_loss_fake", fake_d_loss))
generator_loss = \
tf.nn.sigmoid_cross_entropy_with_logits(fake_logit,
tf.ones_like(fake_logit))
generator_loss = tf.reduce_mean(generator_loss)
return discriminator_loss, generator_loss
def create_fisher_ops(self):
self.fisher_diagonal = self.bias_shaped_variables(name='bias_grads2', c=0.0, trainable=False) +\
self.weight_shaped_variables(name='weight_grads2', c=0.0, trainable=False)
self.fisher_accumulate_op = [tf.assign_add(f1, f2) for f1, f2 in zip(self.fisher_diagonal, self.fisher_minibatch)]
scale = 1 / float(self.ewc_batches * self.ewc_batch_size)
self.fisher_full_batch_average_op = [tf.assign(var, scale * var) for var in self.fisher_diagonal]
self.fisher_zero_op = [tf.assign(tensor, tf.zeros_like(tensor)) for tensor in self.fisher_diagonal]
def resize_axis(tensor, axis, new_size, fill_value=0):
"""Truncates or pads a tensor to new_size on on a given axis.
Truncate or extend tensor such that tensor.shape[axis] == new_size. If the
size increases, the padding will be performed at the end, using fill_value.
Args:
tensor: The tensor to be resized.
axis: An integer representing the dimension to be sliced.
new_size: An integer or 0d tensor representing the new value for
tensor.shape[axis].
fill_value: Value to use to fill any new entries in the tensor. Will be
cast to the type of tensor.
Returns:
The resized tensor.
"""
tensor = tf.convert_to_tensor(tensor)
shape = tf.unstack(tf.shape(tensor))
pad_shape = shape[:]
pad_shape[axis] = tf.maximum(0, new_size - shape[axis])
shape[axis] = tf.minimum(shape[axis], new_size)
shape = tf.stack(shape)
resized = tf.concat([
tf.slice(tensor, tf.zeros_like(shape), shape),
tf.fill(tf.stack(pad_shape), tf.cast(fill_value, tensor.dtype))
], axis)
# Update shape.
new_shape = tensor.get_shape().as_list() # A copy is being made.
new_shape[axis] = new_size
resized.set_shape(new_shape)
return resized
def _instantiate_subnet(self, batch, block_idx, seq_prefix):
def zeros_fn():
return tf.zeros_like(batch)
def base_case_fn():
return self._children[block_idx, seq_prefix](batch)
def recursive_case_fn():
first_subnet = self._instantiate_subnet(
batch, block_idx, seq_prefix + (0,))
return self._instantiate_subnet(
first_subnet, block_idx, seq_prefix + (1,))
if len(seq_prefix) == self._fractal_block_depth:
return base_case_fn()
else:
choice = self._drop_path_choices[self._choice_id[(block_idx, seq_prefix)]]
base_case = tf.cond(
tf.not_equal(choice, self._JUST_RECURSE), base_case_fn, zeros_fn)
base_case.set_shape(batch.get_shape())
recursive_case = tf.cond(
tf.not_equal(choice, self._JUST_BASE), recursive_case_fn, zeros_fn)
recursive_case.set_shape(batch.get_shape())
cases = [
(tf.equal(choice, self._BOTH),
lambda: self._mixer(base_case, recursive_case)),
(tf.equal(choice, self._JUST_BASE), lambda: base_case),
(tf.equal(choice, self._JUST_RECURSE), lambda: recursive_case)]
result = tf.case(cases, lambda: base_case)
result.set_shape(batch.get_shape())
return result
def create_discriminator_loss(disc_real_output, disc_fake_output):
# I.e. did we correctly identify the input as real or not?
cross_entropy_real = tf.nn.sigmoid_cross_entropy_with_logits(labels = disc_real_output, logits = tf.ones_like(disc_real_output))
disc_real_loss = tf.reduce_mean(cross_entropy_real, name='disc_real_loss')
cross_entropy_fake = tf.nn.sigmoid_cross_entropy_with_logits(labels = disc_fake_output, logits = tf.zeros_like(disc_fake_output))
disc_fake_loss = tf.reduce_mean(cross_entropy_fake, name='disc_fake_loss')
return disc_real_loss, disc_fake_loss
def zeros_like(x, name=None):
'''Instantiates an all-zeros tensor
of the same shape as another tensor.
'''
return tf.zeros_like(x, name=name)
