python类orthogonal_initializer()的实例源码

def __init__(self, config={}, device="/gpu:0"):
        config = hc.Config(config)
        dtype = config.dtype or "float32"
        initializer = config.initializer or 'orthogonal'
        orthogonal_gain = config.orthogonal_gain or 1.0
        random_stddev = config.random_stddev or 0.02

        self.dtype = self.parse_dtype(dtype)
        self.scope_count = 0
        self.description = ''
        self.weights = []
        self.biases = []
        self.device = config.device
        self.initialized = False
        self._reuse = False
        if initializer == 'orthogonal':
            self.initializer = self.orthogonal_initializer(orthogonal_gain)
        else:
            self.initializer = self.random_initializer(random_stddev)

def recurrent_layer(tensor, cell=None, hidden_dims=128, sequence_length=None, decoder_fn=None, 
                    activation=tf.nn.tanh, initializer=tf.orthogonal_initializer(), initial_state=None, 
                    keep_prob=1.0,
                    return_final_state=False, return_next_cell_input=True, **opts):
    if cell is None:
        cell = tf.contrib.rnn.BasicRNNCell(hidden_dims, activation=activation)
        # cell = tf.contrib.rnn.LSTMCell(hidden_dims, activation=activation)

    if keep_prob < 1.0:
        keep_prob = _global_keep_prob(keep_prob)
        cell = tf.contrib.rnn.DropoutWrapper(cell, keep_prob, keep_prob)

    if opts.get("name"):
        tf.add_to_collection(opts.get("name"), cell)

    if decoder_fn is None:
        outputs, final_state = tf.nn.dynamic_rnn(cell, tensor, 
            sequence_length=sequence_length, initial_state=initial_state, dtype=tf.float32)
        final_context_state = None
    else:
        # TODO: turn off sequence_length?
        outputs, final_state, final_context_state = seq2seq.dynamic_rnn_decoder(
            cell, decoder_fn, inputs=None, sequence_length=sequence_length)

    if return_final_state:
        return final_state
    else:
        return outputs

def __init__(self, n_documents, n_topics, n_dim, temperature=1.0,
                 W_in=None, factors_in=None):
        self.n_documents = n_documents
        # self.n_topics = n_topics
        # self.n_dim = n_dim
        self.temperature = temperature
        self.dropout = tf.placeholder_with_default(1., shape=[], name="dropout")

        scalar = 1 / np.sqrt(n_documents + n_topics)

        self.W = (tf.Variable( # unnormalized embedding weights
            tf.random_normal([n_documents, n_topics], mean=0, stddev=50*scalar),
                name="doc_embeddings") if W_in is None else W_in)

        # factors = (tf.Variable( # topic vectors
        #       _orthogonal_matrix((n_topics, n_dim)).astype("float32") * scalar,
        #       name="topics") if factors_in is None else factors_in)

        # tf 0.12.0 only
        factors = (tf.get_variable("topics", shape=(n_topics, n_dim),
                                   dtype=tf.float32, initializer=
                                   tf.orthogonal_initializer(gain=scalar))
                   if factors_in is None else factors_in)
        self.factors = tf.nn.dropout(factors, self.dropout)

def embed_inputs(raw_inputs,
                 vocab_size,
                 embed_size,
                 reserve_zero=True,
                 name='embed',
                 embeddings=None):
    with tf.variable_scope(name):
        if embeddings is None:
            shape = (vocab_size, embed_size)
            embeddings = tf.get_variable(
                'embeddings',
                shape=(vocab_size, embed_size),
                initializer=tf.orthogonal_initializer(),
                dtype=tf.float32)

            # If reserve_zero, make sure first row is always zeros
            if reserve_zero:
                zeros = tf.zeros((1, embed_size), dtype=tf.float32)
                embeddings = tf.concat([zeros, embeddings], axis=0)

        inputs = tf.nn.embedding_lookup(embeddings, raw_inputs)
        return inputs

def head(endpoints, embedding_dim, is_training):
    endpoints['head_output'] = slim.fully_connected(
        endpoints['model_output'], 1024, normalizer_fn=slim.batch_norm,
        normalizer_params={
            'decay': 0.9,
            'epsilon': 1e-5,
            'scale': True,
            'is_training': is_training,
            'updates_collections': tf.GraphKeys.UPDATE_OPS,
        })

    endpoints['emb'] = endpoints['emb_raw'] = slim.fully_connected(
        endpoints['head_output'], embedding_dim, activation_fn=None,
        weights_initializer=tf.orthogonal_initializer(), scope='emb')

    return endpoints

def head(endpoints, embedding_dim, is_training):
    endpoints['head_output'] = slim.fully_connected(
        endpoints['model_output'], 1024, normalizer_fn=slim.batch_norm,
        normalizer_params={
            'decay': 0.9,
            'epsilon': 1e-5,
            'scale': True,
            'is_training': is_training,
            'updates_collections': tf.GraphKeys.UPDATE_OPS,
        })

    endpoints['emb_raw'] = slim.fully_connected(
        endpoints['head_output'], embedding_dim, activation_fn=None,
        weights_initializer=tf.orthogonal_initializer(), scope='emb')
    endpoints['emb'] = tf.nn.l2_normalize(endpoints['emb_raw'], -1)

    return endpoints

def random_orthogonal_initializer(use_gpu=True):
    return tf.orthogonal_initializer()

def __init__(self, cell_size):
        self.cell_size = cell_size
        self.default_initializer = tf.get_variable_scope().initializer or init_ops.glorot_uniform_initializer()
        self.initializer = tf.orthogonal_initializer()

def orthogonal_initializer(self, gain):
        def _build():
            return tf.orthogonal_initializer(gain)
        return _build

def Weight_variable(self, shape, weight_decay=0.000004):
        with tf.name_scope('Weight') as scope:
            #weights = tf.get_variable(name='Weight', initializer=tf.truncated_normal(shape, stddev=0.1), trainable=True, regularizer=self.Regloss_l2)
            #initi = tf.contrib.layers.xavier_initializer()
            #initi = tf.orthogonal_initializer()
            initi = tf.random_uniform_initializer(minval=-0.08, maxval=0.08)
            weights = tf.Variable(initi(shape))
            tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, tf.nn.l2_loss(weights)* weight_decay)
            return weights

def get_initializer(name):
  if name == 'identity':
    return identity_initializer()
  elif name == 'orthogonal':
    return tf.orthogonal_initializer(gain=1.0)
  elif name == 'truncated_normal':
    return tf.truncated_normal_initializer()

def orthogonal_initializer():
    """Return an orthogonal initializer.

    Random orthogonal matrix is byproduct of singular value decomposition
    applied on a matrix initialized with normal distribution.

    The initializer works with 2D square matrices and matrices that can be
    splitted along axis 1 to several 2D matrices. In the latter case, each
    submatrix is initialized independently and the resulting orthogonal
    matrices are concatenated along axis 1.

    Note this is a higher order function in order to mimic the tensorflow
    initializer API.
    """

    # pylint: disable=unused-argument
    def func(shape, dtype, partition_info=None):
        if len(shape) != 2:
            raise ValueError(
                "Orthogonal initializer only works with 2D matrices.")

        if shape[1] % shape[0] != 0:
            raise ValueError("Shape {} is not compatible with orthogonal "
                             "initializer.".format(str(shape)))

        mult = int(shape[1] / shape[0])
        dim = shape[0]

        orthogonals = []
        for _ in range(mult):
            matrix = tf.random_normal([dim, dim], dtype=dtype)
            orthogonals.append(tf.svd(matrix)[1])

        return tf.concat(orthogonals, 1)
    # pylint: enable=unused-argument

    return func


# pylint: disable=too-few-public-methods

def __init__(self, num_units, activation=None, reuse=None):
        tf.contrib.rnn.GRUCell.__init__(
            self, num_units, activation, reuse,
            kernel_initializer=tf.orthogonal_initializer())

def call(self, inputs, state):
        """Gated recurrent unit (GRU) with nunits cells."""
        with tf.variable_scope("gates"):
            input_to_gates = tf.layers.dense(
                inputs, 2 * self._num_units, name="input_proj",
                kernel_initializer=tf.glorot_normal_initializer(),
                use_bias=self.use_input_bias)

            # Nematus does the orthogonal initialization probably differently
            state_to_gates = tf.layers.dense(
                state, 2 * self._num_units,
                use_bias=self.use_state_bias,
                kernel_initializer=orthogonal_initializer(),
                name="state_proj")

            gates_input = state_to_gates + input_to_gates
            reset, update = tf.split(
                tf.sigmoid(gates_input), num_or_size_splits=2, axis=1)

        with tf.variable_scope("candidate"):
            input_to_candidate = tf.layers.dense(
                inputs, self._num_units, use_bias=self.use_input_bias,
                kernel_initializer=tf.glorot_normal_initializer(),
                name="input_proj")

            state_to_candidate = tf.layers.dense(
                state, self._num_units, use_bias=self.use_state_bias,
                kernel_initializer=orthogonal_initializer(),
                name="state_proj")

            candidate = self._activation(
                state_to_candidate * reset + input_to_candidate)

        new_state = update * state + (1 - update) * candidate
        return new_state, new_state

def orthogonal_initializer():
    """Return an orthogonal initializer.

    Random orthogonal matrix is byproduct of singular value decomposition
    applied on a matrix initialized with normal distribution.

    The initializer works with 2D square matrices and matrices that can be
    splitted along axis 1 to several 2D matrices. In the latter case, each
    submatrix is initialized independently and the resulting orthogonal
    matrices are concatenated along axis 1.

    Note this is a higher order function in order to mimic the tensorflow
    initializer API.
    """

    # pylint: disable=unused-argument
    def func(shape, dtype, partition_info=None):
        if len(shape) != 2:
            raise ValueError(
                "Orthogonal initializer only works with 2D matrices.")

        if shape[1] % shape[0] != 0:
            raise ValueError("Shape {} is not compatible with orthogonal "
                             "initializer.".format(str(shape)))

        mult = int(shape[1] / shape[0])
        dim = shape[0]

        orthogonals = []
        for _ in range(mult):
            matrix = tf.random_normal([dim, dim], dtype=dtype)
            orthogonals.append(tf.svd(matrix)[1])

        return tf.concat(orthogonals, 1)
    # pylint: enable=unused-argument

    return func


# pylint: disable=too-few-public-methods

def __init__(self, num_units, activation=None, reuse=None):
        tf.contrib.rnn.GRUCell.__init__(
            self, num_units, activation, reuse,
            kernel_initializer=tf.orthogonal_initializer())

def orthogonal_initializer():
    """Return an orthogonal initializer.

    Random orthogonal matrix is byproduct of singular value decomposition
    applied on a matrix initialized with normal distribution.

    The initializer works with 2D square matrices and matrices that can be
    splitted along axis 1 to several 2D matrices. In the latter case, each
    submatrix is initialized independently and the resulting orthogonal
    matrices are concatenated along axis 1.

    Note this is a higher order function in order to mimic the tensorflow
    initializer API.
    """

    # pylint: disable=unused-argument
    def func(shape, dtype, partition_info=None):
        if len(shape) != 2:
            raise ValueError(
                "Orthogonal initializer only works with 2D matrices.")

        if shape[1] % shape[0] != 0:
            raise ValueError("Shape {} is not compatible with orthogonal "
                             "initializer.".format(str(shape)))

        mult = int(shape[1] / shape[0])
        dim = shape[0]

        orthogonals = []
        for _ in range(mult):
            matrix = tf.random_normal([dim, dim], dtype=dtype)
            orthogonals.append(tf.svd(matrix)[1])

        return tf.concat(orthogonals, 1)
    # pylint: enable=unused-argument

    return func


# pylint: disable=too-few-public-methods

def __init__(self, num_units, activation=None, reuse=None):
        tf.contrib.rnn.GRUCell.__init__(
            self, num_units, activation, reuse,
            kernel_initializer=tf.orthogonal_initializer())

def call(self, inputs, state):
        """Gated recurrent unit (GRU) with nunits cells."""
        with tf.variable_scope("gates"):
            input_to_gates = tf.layers.dense(
                inputs, 2 * self._num_units, name="input_proj",
                kernel_initializer=tf.glorot_normal_initializer(),
                use_bias=self.use_input_bias)

            # Nematus does the orthogonal initialization probably differently
            state_to_gates = tf.layers.dense(
                state, 2 * self._num_units,
                use_bias=self.use_state_bias,
                kernel_initializer=orthogonal_initializer(),
                name="state_proj")

            gates_input = state_to_gates + input_to_gates
            reset, update = tf.split(
                tf.sigmoid(gates_input), num_or_size_splits=2, axis=1)

        with tf.variable_scope("candidate"):
            input_to_candidate = tf.layers.dense(
                inputs, self._num_units, use_bias=self.use_input_bias,
                kernel_initializer=tf.glorot_normal_initializer(),
                name="input_proj")

            state_to_candidate = tf.layers.dense(
                state, self._num_units, use_bias=self.use_state_bias,
                kernel_initializer=orthogonal_initializer(),
                name="state_proj")

            candidate = self._activation(
                state_to_candidate * reset + input_to_candidate)

        new_state = update * state + (1 - update) * candidate
        return new_state, new_state

def get_variable_initializer(hparams):
  """Get variable initializer from hparams."""
  if hparams.initializer == "orthogonal":
    return tf.orthogonal_initializer(gain=hparams.initializer_gain)
  elif hparams.initializer == "uniform":
    max_val = 0.1 * hparams.initializer_gain
    return tf.random_uniform_initializer(-max_val, max_val)
  elif hparams.initializer == "normal_unit_scaling":
    return tf.variance_scaling_initializer(
        hparams.initializer_gain, mode="fan_avg", distribution="normal")
  elif hparams.initializer == "uniform_unit_scaling":
    return tf.variance_scaling_initializer(
        hparams.initializer_gain, mode="fan_avg", distribution="uniform")
  else:
    raise ValueError("Unrecognized initializer: %s" % hparams.initializer)

def head(endpoints, embedding_dim, is_training):
    endpoints['emb'] = endpoints['emb_raw'] = slim.fully_connected(
        endpoints['model_output'], embedding_dim, activation_fn=None,
        weights_initializer=tf.orthogonal_initializer(), scope='emb')

    return endpoints

def head(endpoints, embedding_dim, is_training):
    endpoints['emb_raw'] = slim.fully_connected(
        endpoints['model_output'], embedding_dim, activation_fn=None,
        weights_initializer=tf.orthogonal_initializer(), scope='emb')
    endpoints['emb'] = tf.nn.l2_normalize(endpoints['emb_raw'], -1)

    return endpoints

def create_model(session, config, forward_only):
  """Create translation model and initialize or load parameters in session."""
  dtype = tf.float32
  optimizer = None
  if not forward_only:
    optimizer = tf.train.AdamOptimizer(config.learning_rate)
  if config.activation == "elu":
    activation = tf.nn.elu
  elif config.activation == "prelu":
    activation = prelu
  else:
    activation = tf.identity

  weight_initializer = tf.orthogonal_initializer if config.orthogonal_initializer else tf.uniform_unit_scaling_initializer
  bias_initializer = tf.zeros_initializer

  model = seq2seq_model.Seq2SeqModel(
      config.en_vocab_size,
      config.fr_vocab_size,
      config.buckets,
      config.size,
      config.num_layers,
      config.latent_dim,
      config.max_gradient_norm,
      config.batch_size,
      config.learning_rate,
      config.kl_min,
      config.word_dropout_keep_prob,
      config.anneal,
      config.use_lstm,
      optimizer=optimizer,
      activation=activation,
      forward_only=forward_only,
      feed_previous=config.feed_previous,
      bidirectional=config.bidirectional,
      weight_initializer=weight_initializer,
      bias_initializer=bias_initializer,
      iaf=config.iaf,
      dtype=dtype)
  ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir)
  if not FLAGS.new and ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
    print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
    model.saver.restore(session, ckpt.model_checkpoint_path)
  else:
    print("Created model with fresh parameters.")
    session.run(tf.global_variables_initializer())
  return model