rnn_cell_modern.py 文件源码

def _inner_function(self, inputs, past_hidden_state, activation=tf.nn.tanh):
    """second order function as described equation 11 in delta rnn paper
    The main goal is to produce z_t of this function
    """
    V_x_d = linear(past_hidden_state, self._num_units, True)

    # We make this a private variable to be reused in the _outer_function
    self._W_x_inputs = linear(inputs, self._num_units, True)

    alpha = tf.get_variable("alpha", [self._num_units], dtype=tf.float32, initializer=tf.ones_initializer)

    beta_one = tf.get_variable("beta_one", [self._num_units], dtype=tf.float32, initializer=tf.ones_initializer)

    beta_two = tf.get_variable("beta_two", [self._num_units], dtype=tf.float32, initializer=tf.ones_initializer)

    z_t_bias = tf.get_variable("z_t_bias", [self._num_units], dtype=tf.float32, initializer=tf.zeros_initializer)

    # Second Order Cell Calculations
    d_1_t = alpha * V_x_d * self._W_x_inputs
    d_2_t = beta_one * V_x_d + beta_two * self._W_x_inputs

    z_t = activation(d_1_t + d_2_t + z_t_bias)

    return z_t