def bow_loss_by_example(logits,
targets,
weights,
average_across_timesteps=False):
"""Loss for a bow of logits (per example).
As opposed to sequence loss this is supposed to ignore the order.
Does not seem to work yet.
Args:
logits: List of 2D Tensors of shape [batch_size x num_decoder_symbols].
targets: List of 1D batch-sized int32 Tensors of the same length as
logits.
weights: List of 1D batch-sized float-Tensors of the same length as
logits.
average_across_timesteps: If set, divide the returned cost by the total
label weight.
Returns:
1D batch-sized float Tensor: The loss for each bow.
Raises:
ValueError: If len(logits) is different from len(targets) or len(weights).
"""
if len(targets) != len(logits) or len(weights) != len(logits):
raise ValueError('Lengths of logits, weights, and targets must be the same '
'%d, %d, %d.' % (len(logits), len(weights), len(targets)))
batch_size = logits[0].shape[0]
vocab_size = logits[0].shape[1]
logitssum = tf.zeros((batch_size, vocab_size), tf.float32)
targetset = tf.zeros((batch_size, vocab_size), tf.float32)
for target, weight in zip(targets, weights):
targetset += (tf.one_hot(target, vocab_size) * weight[:, None])
weight = tf.ones((batch_size), tf.float32)
for logit in logits:
softmax = tf.nn.softmax(logit)
logitssum += (logitssum * weight[:, None])
weight = tf.maximum(0.0, weight - softmax[:, 3])
# logitssum = tf.minimum(logitssum, 1.0)
# targetset = tf.minimum(targetset, 1.0)
# loss = tf.nn.sigmoid_cross_entropy_with_logits(
# labels=targetset, logits=logitssum)
loss = tf.reduce_sum(tf.squared_difference(logitssum, targetset), axis=1)
# crossent = tf.maximum(logitssum, 0.0) - (
# logitssum * targetset) + tf.log(1.0 + tf.exp(-1.0 * tf.abs(logitssum)))
# log_perps = tf.reduce_logsumexp(crossent, axis=1)
if average_across_timesteps:
total_size = tf.add_n(weights)
total_size += 1e-12 # Just to avoid division by 0 for all-0 weights.
loss /= total_size
return loss
评论列表
文章目录
