rnn_ops.py 文件源码

def layer_norm(x, axes=1, initial_bias_value=0.0, epsilon=1e-3, name="var"):
    """
    Apply layer normalization to x
    Args:
        x: input variable.
        initial_bias_value: initial value for the LN bias.
        epsilon: small constant value to avoid division by zero.
        scope: scope or name for the LN op.
    Returns:
        LN(x) with same shape as x
    """
    if not isinstance(axes, list):
        axes = [axes]

    scope = tf.get_variable_scope()
    with tf.variable_scope(scope):
        with tf.variable_scope(name):
            mean = tf.reduce_mean(x, axes, keep_dims=True)
            variance = tf.sqrt(tf.reduce_mean(tf.square(x - mean), axes, keep_dims=True))

            with tf.device('/cpu:0'):
                gain = tf.get_variable('gain', x.get_shape().as_list()[1:],
                                       initializer=tf.constant_initializer(1.0))
                bias = tf.get_variable('bias', x.get_shape().as_list()[1:],
                                       initializer=tf.constant_initializer(initial_bias_value))

            return gain * (x - mean) / (variance + epsilon) + bias