base.py 文件源码

def _adaptive_max_norm(self, norm, std_factor, decay, global_step, epsilon, name):
        """Find max_norm given norm and previous average."""
        with tf.variable_scope(name, "AdaptiveMaxNorm", [norm]):
            log_norm = tf.log(norm + epsilon)

            def moving_average(name, value, decay):
                moving_average_variable = tf.get_variable(name,
                                                          shape=value.get_shape(),
                                                          dtype=value.dtype,
                                                          initializer=tf.zeros_initializer(),
                                                          trainable=False)
                return moving_averages.assign_moving_average(moving_average_variable, value, decay, zero_debias=False)

            # quicker adaptation at the beginning
            if global_step is not None:
                n = tf.to_float(global_step)
                decay = tf.minimum(decay, n / (n + 1.))

            # update averages
            mean = moving_average("mean", log_norm, decay)
            sq_mean = moving_average(
                "sq_mean", tf.square(log_norm), decay)

            variance = sq_mean - tf.square(mean)
            std = tf.sqrt(tf.maximum(epsilon, variance))
            max_norms = tf.exp(mean + std_factor * std)
            return max_norms, mean