def distance_biases(time_steps, window_size=10, reuse=False):
"""
Return a 2-d tensor with the values of the distance biases to be applied
on the intra-attention matrix of size sentence_size
Args:
time_steps: tensor scalar
window_size: window size
reuse: reuse variables
Returns:
2-d tensor (time_steps, time_steps)
"""
with tf.variable_scope('distance-bias', reuse=reuse):
# this is d_{i-j}
distance_bias = tf.get_variable('dist_bias', [window_size], initializer=tf.zeros_initializer())
r = tf.range(0, time_steps)
r_matrix = tf.tile(tf.reshape(r, [1, -1]), tf.stack([time_steps, 1]))
raw_idxs = r_matrix - tf.reshape(r, [-1, 1])
clipped_idxs = tf.clip_by_value(raw_idxs, 0, window_size - 1)
values = tf.nn.embedding_lookup(distance_bias, clipped_idxs)
return values
评论列表
文章目录