def gradient_penalty(self):
config = self.config
gan = self.gan
gradient_penalty = config.gradient_penalty
if has_attr(gan.inputs, 'gradient_penalty_label'):
x = gan.inputs.gradient_penalty_label
else:
x = gan.inputs.x
generator = self.generator or gan.generator
g = generator.sample
discriminator = self.discriminator or gan.discriminator
shape = [1 for t in g.get_shape()]
shape[0] = gan.batch_size()
uniform_noise = tf.random_uniform(shape=shape,minval=0.,maxval=1.)
print("[gradient penalty] applying x:", x, "g:", g, "noise:", uniform_noise)
interpolates = x + uniform_noise * (g - x)
reused_d = discriminator.reuse(interpolates)
gradients = tf.gradients(reused_d, [interpolates])[0]
penalty = tf.sqrt(tf.reduce_sum(tf.square(gradients), axis=1))
penalty = tf.reduce_mean(tf.square(penalty - 1.))
return float(gradient_penalty) * penalty
python类sqrt()的实例源码
def setup_param_noise(self, normalized_obs0):
assert self.param_noise is not None
# Configure perturbed actor.
param_noise_actor = copy(self.actor)
param_noise_actor.name = 'param_noise_actor'
self.perturbed_actor_tf = param_noise_actor(normalized_obs0)
logger.info('setting up param noise')
self.perturb_policy_ops = get_perturbed_actor_updates(self.actor, param_noise_actor, self.param_noise_stddev)
# Configure separate copy for stddev adoption.
adaptive_param_noise_actor = copy(self.actor)
adaptive_param_noise_actor.name = 'adaptive_param_noise_actor'
adaptive_actor_tf = adaptive_param_noise_actor(normalized_obs0)
self.perturb_adaptive_policy_ops = get_perturbed_actor_updates(self.actor, adaptive_param_noise_actor, self.param_noise_stddev)
self.adaptive_policy_distance = tf.sqrt(tf.reduce_mean(tf.square(self.actor_tf - adaptive_actor_tf)))
multimodal_autoencoder.py 文件源码
项目:MultimodalAutoencoder
作者: natashamjaques
项目源码
文件源码
阅读 26
收藏 0
点赞 0
评论 0
def weight_variable(shape, name, var_type='normal', const=1):
"""Initializes a tensorflow weight variable.
Args:
shape: An array representing shape of the weight variable
name: A string name given to the variable.
var_type: can be either 'normal', for weights following a Gaussian
distribution around 0, or 'xavier', for the Xavier method
const: Numeric value that controls the range of the weights within
the Xavier method.
Returns: Tensor variable for the weights
"""
if var_type == 'xavier':
""" Xavier initialization of network weights.
Taken from: https://gist.github.com/blackecho/3a6e4d512d3aa8aa6cf9
https://stackoverflow.com/questions/33640581/how-to-do-xavier-initialization-on-tensorflow
"""
assert len(shape) == 2
low = -const * np.sqrt(6.0 / (shape[0] + shape[1]))
high = const * np.sqrt(6.0 / (shape[0] + shape[1]))
initial = tf.random_uniform((shape[0], shape[1]), minval=low, maxval=high)
else:
initial = tf.truncated_normal(shape, stddev=1.0 / math.sqrt(float(shape[0])), dtype=tf.float32)
return tf.Variable(initial, name=name)
def testGetBackwardOpsChain(self):
# a -> b -> c
a = tf.placeholder(tf.float32)
b = tf.sqrt(a)
c = tf.square(b)
for n in range(4):
for seed_tensors in permutations([a, b, c], n):
if c in seed_tensors:
truth = [a.op, b.op, c.op]
elif b in seed_tensors:
truth = [a.op, b.op]
elif a in seed_tensors:
truth = [a.op]
else:
truth = []
self.assertEqual(get_backward_ops(seed_tensors), truth)
self.assertEqual(get_backward_ops([c], treat_as_inputs=[b]), [c.op])
self.assertEqual(
get_backward_ops([b, c], treat_as_inputs=[b]), [c.op])
self.assertEqual(
get_backward_ops([a, c], treat_as_inputs=[b]), [a.op, c.op])
def tune(self, acceptance_rate, fresh_start):
def adapt_stepsize():
new_step = tf.assign(self.step, (1 - fresh_start) * self.step + 1)
rate1 = tf.div(1.0, new_step + self.t0)
new_h_bar = tf.assign(
self.h_bar, (1 - fresh_start) * (1 - rate1) * self.h_bar +
rate1 * (self.delta - acceptance_rate))
log_epsilon = self.mu - tf.sqrt(new_step) / self.gamma * new_h_bar
rate = tf.pow(new_step, -self.kappa)
new_log_epsilon_bar = tf.assign(
self.log_epsilon_bar,
rate * log_epsilon + (1 - fresh_start) * (1 - rate) *
self.log_epsilon_bar)
with tf.control_dependencies([new_log_epsilon_bar]):
new_log_epsilon = tf.identity(log_epsilon)
return tf.exp(new_log_epsilon)
c = tf.cond(self.adapt_step_size,
adapt_stepsize,
lambda: tf.exp(self.log_epsilon_bar))
return c
def variable_summaries(var, name, collections=None):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization).
Args:
- var: Tensor for variable from which we want to log.
- name: Variable name.
- collections: List of collections to save the summary to.
"""
with tf.name_scope(name):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean, collections)
num_params = tf.reduce_prod(tf.shape(var))
tf.summary.scalar('num_params', num_params, collections)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev, collections)
tf.summary.scalar('max', tf.reduce_max(var), collections)
tf.summary.scalar('min', tf.reduce_min(var), collections)
tf.summary.histogram('histogram', var, collections)
tf.summary.scalar('sparsity', tf.nn.zero_fraction(var), collections)
def normal_map(tensor, shape):
"""
Generate a tangent-space normal map.
:param Tensor tensor:
:param list[int] shape:
:return: Tensor
"""
height, width, channels = shape
reference = value_map(tensor, shape, keep_dims=True)
x = normalize(1 - convolve(ConvKernel.sobel_x, reference, [height, width, 1]))
y = normalize(convolve(ConvKernel.sobel_y, reference, [height, width, 1]))
z = 1 - tf.abs(normalize(tf.sqrt(x * x + y * y)) * 2 - 1) * .5 + .5
return tf.stack([x[:, :, 0], y[:, :, 0], z[:, :, 0]], 2)
def euclidean_distance(self):
x = tf.argmax(tf.reduce_max(self.smoothed_sigm_network, 1), 1)
y = tf.argmax(tf.reduce_max(self.smoothed_sigm_network, 2), 1)
x = tf.cast(x, tf.float32)
y = tf.cast(y, tf.float32)
dy = tf.squeeze(self.desired_points[:, 0, :])
dx = tf.squeeze(self.desired_points[:, 1, :])
sx = tf.squared_difference(x, dx)
sy = tf.squared_difference(y, dy)
l2_dist = tf.sqrt(sx + sy)
return l2_dist
def _decoder(self, z):
"""Define p(x|z) network"""
if z is None:
mean = None
stddev = None
logits = None
class_predictions = None
input_sample = self.epsilon
else:
z = tf.reshape(z, [-1, self.flags['hidden_size'] * 2])
mean, stddev = tf.split(1, 2, z) # Compute latent variables (z) by calculating mean, stddev
stddev = tf.sqrt(tf.exp(stddev))
mlp = Layers(mean)
mlp.fc(self.flags['num_classes'])
class_predictions = mlp.get_output()
logits = tf.nn.softmax(class_predictions)
input_sample = mean + self.epsilon * stddev
decoder = Layers(tf.expand_dims(tf.expand_dims(input_sample, 1), 1))
decoder.deconv2d(3, 128, padding='VALID')
decoder.deconv2d(3, 64, padding='VALID', stride=2)
decoder.deconv2d(3, 64, stride=2)
decoder.deconv2d(5, 32, stride=2)
decoder.deconv2d(7, 1, activation_fn=tf.nn.tanh, s_value=None)
return decoder.get_output(), mean, stddev, class_predictions, logits
def _decoder(self, z):
""" Define p(x|z) network"""
if z is None:
mean = None
stddev = None
input_sample = self.epsilon
else:
z = tf.reshape(z, [-1, self.flags['hidden_size'] * 2])
print(z.get_shape())
mean, stddev = tf.split(1, 2, z)
stddev = tf.sqrt(tf.exp(stddev))
input_sample = mean + self.epsilon * stddev
decoder = Layers(tf.expand_dims(tf.expand_dims(input_sample, 1), 1))
decoder.deconv2d(3, 128, padding='VALID')
decoder.deconv2d(3, 128, padding='VALID', stride=2)
decoder.deconv2d(3, 64, stride=2)
decoder.deconv2d(3, 64, stride=2)
decoder.deconv2d(5, 1, activation_fn=tf.nn.tanh, s_value=None)
return decoder.get_output(), mean, stddev
ln_lstm2.py 文件源码
项目:Multi-channel-speech-extraction-using-DNN
作者: zhr1201
项目源码
文件源码
阅读 24
收藏 0
点赞 0
评论 0
def ln(tensor, scope=None, epsilon=1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
ln_lstm.py 文件源码
项目:Multi-channel-speech-extraction-using-DNN
作者: zhr1201
项目源码
文件源码
阅读 25
收藏 0
点赞 0
评论 0
def ln(tensor, scope=None, epsilon=1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
def build_model(user_indices, item_indices, rank, ratings, user_cnt, item_cnt, lr, lamb, mu, init_value):
W_user = tf.Variable(tf.truncated_normal([user_cnt, rank], stddev=init_value/math.sqrt(float(rank)), mean=0), name = 'user_embedding', dtype=tf.float32)
W_item = tf.Variable(tf.truncated_normal([item_cnt, rank], stddev=init_value/math.sqrt(float(rank)), mean=0), name = 'item_embedding', dtype=tf.float32)
W_user_bias = tf.concat([W_user, tf.ones((user_cnt,1), dtype=tf.float32)], 1, name='user_embedding_bias')
W_item_bias = tf.concat([tf.ones((item_cnt,1), dtype=tf.float32), W_item], 1, name='item_embedding_bias')
user_feature = tf.nn.embedding_lookup(W_user_bias, user_indices, name = 'user_feature')
item_feature = tf.nn.embedding_lookup(W_item_bias, item_indices, name = 'item_feature')
preds = tf.add(tf.reduce_sum( tf.multiply(user_feature , item_feature) , 1), mu)
square_error = tf.sqrt(tf.reduce_mean( tf.squared_difference(preds, ratings)))
loss = square_error + lamb*(tf.reduce_mean(tf.nn.l2_loss(W_user)) + tf.reduce_mean(tf.nn.l2_loss(W_item)))
tf.summary.scalar('square_error', square_error)
tf.summary.scalar('loss', loss)
merged_summary = tf.summary.merge_all()
#tf.global_variables_initializer()
train_step = tf.train.GradientDescentOptimizer(lr).minimize(loss) # tf.train.AdadeltaOptimizer(learning_rate=lr).minimize(loss) #
return train_step, square_error, loss, merged_summary
def gaussian_stochastic(self, input_tensor, num_maps, scope):
"""
:param inputs_list: list of Tensors to be added and input into the block
:return: random variable single draw, mean, standard deviation, and intermediate representation
"""
with tf.variable_scope(scope):
input_tensor = tf.expand_dims(tf.expand_dims(input_tensor, 1), 1) if len(input_tensor.get_shape()) != 4 \
else input_tensor
intermediate = slim.conv2d(input_tensor, self._hidden_size, [1, 1], weights_initializer=self._initializer,
scope='conv1')
mean = slim.conv2d(intermediate, num_maps, [1, 1], weights_initializer=self._initializer,
activation_fn=None, scope='mean')
sigma2 = tf.nn.softplus(
slim.conv2d(intermediate, num_maps, [1, 1], weights_initializer=self._initializer,
activation_fn=None, scope='sigma2'))
rv_single_draw = mean + tf.sqrt(sigma2) * tf.random_normal(tf.shape(mean))
self.split_labeled_unlabeled(mean, '{}_mu'.format(scope))
self.split_labeled_unlabeled(sigma2, '{}_sigma2'.format(scope))
self.split_labeled_unlabeled(rv_single_draw, '{}_sample'.format(scope))
return rv_single_draw
def gaussian_stochastic(self, input_tensor, num_maps, scope):
"""
:param inputs_list: list of Tensors to be added and input into the block
:return: random variable single draw, mean, standard deviation, and intermediate representation
"""
with tf.variable_scope(scope):
input_tensor = tf.expand_dims(tf.expand_dims(input_tensor, 1), 1) if len(input_tensor.get_shape()) != 4 \
else input_tensor
intermediate = slim.conv2d(input_tensor, self._hidden_size, [1, 1], weights_initializer=self._initializer,
scope='conv1')
mean = slim.conv2d(intermediate, num_maps, [1, 1], weights_initializer=self._initializer,
activation_fn=None, scope='mean')
sigma2 = tf.nn.softplus(
slim.conv2d(intermediate, num_maps, [1, 1], weights_initializer=self._initializer,
activation_fn=None, scope='sigma2'))
rv_single_draw = mean + tf.sqrt(sigma2) * tf.random_normal(tf.shape(mean))
self.split_labeled_unlabeled(mean, '{}_mu'.format(scope))
self.split_labeled_unlabeled(sigma2, '{}_sigma2'.format(scope))
self.split_labeled_unlabeled(rv_single_draw, '{}_sample'.format(scope))
return rv_single_draw
def linear_mapping_stupid(inputs, out_dim, in_dim=None, dropout=1.0, var_scope_name="linear_mapping"):
with tf.variable_scope(var_scope_name):
print('name', tf.get_variable_scope().name)
input_shape_tensor = tf.shape(inputs) # dynamic shape, no None
input_shape = inputs.get_shape().as_list() # static shape. may has None
print('input_shape', input_shape)
assert len(input_shape) == 3
inputs = tf.reshape(inputs, [-1, input_shape_tensor[-1]])
linear_mapping_w = tf.get_variable("linear_mapping_w", [input_shape[-1], out_dim], initializer=tf.random_normal_initializer(mean=0, stddev=tf.sqrt(dropout*1.0/input_shape[-1])))
linear_mapping_b = tf.get_variable("linear_mapping_b", [out_dim], initializer=tf.zeros_initializer())
output = tf.matmul(inputs, linear_mapping_w) + linear_mapping_b
print('xxxxx_params', input_shape, out_dim)
#output = tf.reshape(output, [input_shape[0], -1, out_dim])
output = tf.reshape(output, [input_shape_tensor[0], -1, out_dim])
return output
def linear_mapping_weightnorm(inputs, out_dim, in_dim=None, dropout=1.0, var_scope_name="linear_mapping"):
with tf.variable_scope(var_scope_name):
input_shape = inputs.get_shape().as_list() # static shape. may has None
input_shape_tensor = tf.shape(inputs)
# use weight normalization (Salimans & Kingma, 2016) w = g* v/2-norm(v)
V = tf.get_variable('V', shape=[int(input_shape[-1]), out_dim], dtype=tf.float32, initializer=tf.random_normal_initializer(mean=0, stddev=tf.sqrt(dropout*1.0/int(input_shape[-1]))), trainable=True)
V_norm = tf.norm(V.initialized_value(), axis=0) # V shape is M*N, V_norm shape is N
g = tf.get_variable('g', dtype=tf.float32, initializer=V_norm, trainable=True)
b = tf.get_variable('b', shape=[out_dim], dtype=tf.float32, initializer=tf.zeros_initializer(), trainable=True) # weightnorm bias is init zero
assert len(input_shape) == 3
inputs = tf.reshape(inputs, [-1, input_shape[-1]])
inputs = tf.matmul(inputs, V)
inputs = tf.reshape(inputs, [input_shape_tensor[0], -1, out_dim])
#inputs = tf.matmul(inputs, V) # x*v
scaler = tf.div(g, tf.norm(V, axis=0)) # g/2-norm(v)
inputs = tf.reshape(scaler,[1, out_dim])*inputs + tf.reshape(b,[1, out_dim]) # x*v g/2-norm(v) + b
return inputs
def make_attention(target_embed, encoder_output, decoder_hidden, layer_idx):
with tf.variable_scope("attention_layer_" + str(layer_idx)):
embed_size = target_embed.get_shape().as_list()[-1] #k
dec_hidden_proj = linear_mapping_weightnorm(decoder_hidden, embed_size, var_scope_name="linear_mapping_att_query") # M*N1*k1 --> M*N1*k
dec_rep = (dec_hidden_proj + target_embed) * tf.sqrt(0.5)
encoder_output_a = encoder_output.outputs
encoder_output_c = encoder_output.attention_values # M*N2*K
att_score = tf.matmul(dec_rep, encoder_output_a, transpose_b=True) #M*N1*K ** M*N2*K --> M*N1*N2
att_score = tf.nn.softmax(att_score)
length = tf.cast(tf.shape(encoder_output_c), tf.float32)
att_out = tf.matmul(att_score, encoder_output_c) * length[1] * tf.sqrt(1.0/length[1]) #M*N1*N2 ** M*N2*K --> M*N1*k
att_out = linear_mapping_weightnorm(att_out, decoder_hidden.get_shape().as_list()[-1], var_scope_name="linear_mapping_att_out")
return att_out
def __init__(self, input_dim, hidden_dim, epoch=250, learning_rate = 0.001):
self.epoch = epoch
self.learning_rate = learning_rate
x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim])
with tf.name_scope("encode"):
weights = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32), name="weights")
biases = tf.Variable(tf.zeros([hidden_dim]), name="biases")
encoded = tf.nn.tanh(tf.matmul(x, weights) + biases)
with tf.name_scope("decode"):
weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32), name="weights")
biases = tf.Variable(tf.zeros([input_dim]), name="biases")
decoded = tf.matmul(encoded, weights) + biases
self.x = x
self.encoded = encoded
self.decoded = decoded
self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(self.x, self.decoded))))
self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)
self.saver = tf.train.Saver()
def _norm(input, is_train, reuse=True, norm=None):
assert norm in ['instance', 'batch', None]
if norm == 'instance':
with tf.variable_scope('instance_norm', reuse=reuse):
eps = 1e-5
mean, sigma = tf.nn.moments(input, [1, 2], keep_dims=True)
normalized = (input - mean) / (tf.sqrt(sigma) + eps)
out = normalized
# Apply momentum (not mendatory)
#c = input.get_shape()[-1]
#shift = tf.get_variable('shift', shape=[c],
# initializer=tf.zeros_initializer())
#scale = tf.get_variable('scale', shape=[c],
# initializer=tf.random_normal_initializer(1.0, 0.02))
#out = scale * normalized + shift
elif norm == 'batch':
with tf.variable_scope('batch_norm', reuse=reuse):
out = tf.contrib.layers.batch_norm(input,
decay=0.99, center=True,
scale=True, is_training=is_train,
updates_collections=None)
else:
out = input
return out