data_loader.py 文件源码

def preprocess(data):
    # PaddingFIFOQueue pads to the max size seen in the data (instead of the minibatch)
    # by chopping off the ends, this limits redundant computations in the output layer
    sequence_length = tf.reduce_sum(tf.cast(tf.not_equal(data, 0), dtype=tf.int32), axis=1)
    maximum_sequence_length = tf.reduce_max(sequence_length)
    data = data[:, :maximum_sequence_length] 

    source = data[:, :-1]
    target = data[:, 1:]
    sequence_length -= 1
    return source, target, sequence_length