model.py 文件源码

def layer_normalization(inputs,epsilon = 1e-5,scope=None):
    mean,var=tf.nn.moments(inputs,[1],keep_dims=True)
    with tf.variable_scope(scope+"LN",reuse=None):
          scale=tf.get_variable(name="scale",shape=[inputs.get_shape()[1]],initializer=tf.constant_initializer(1))
          shift=tf.get_variable(name="shift",shape=[inputs.get_shape()[1]],initializer=tf.constant_initializer(0))
    LN_output=scale*(inputs-mean)/tf.sqrt(var + epsilon) + shift
    return LN_output