python类dropout()的实例源码

def q_net(x, n_xl, n_z, n_particles, is_training):
    with zs.BayesianNet() as variational:
        normalizer_params = {'is_training': is_training,
                             'updates_collections': None}
        lz_x = tf.reshape(tf.to_float(x), [-1, n_xl, n_xl, 1])
        lz_x = layers.conv2d(
            lz_x, 32, kernel_size=5, stride=2,
            normalizer_fn=layers.batch_norm,
            normalizer_params=normalizer_params)
        lz_x = layers.conv2d(
            lz_x, 64, kernel_size=5, stride=2,
            normalizer_fn=layers.batch_norm,
            normalizer_params=normalizer_params)
        lz_x = layers.conv2d(
            lz_x, 128, kernel_size=5, padding='VALID',
            normalizer_fn=layers.batch_norm,
            normalizer_params=normalizer_params)
        lz_x = layers.dropout(lz_x, keep_prob=0.9, is_training=is_training)
        lz_x = tf.reshape(lz_x, [-1, 128 * 3 * 3])
        lz_mean = layers.fully_connected(lz_x, n_z, activation_fn=None)
        lz_logstd = layers.fully_connected(lz_x, n_z, activation_fn=None)
        z = zs.Normal('z', lz_mean, logstd=lz_logstd, n_samples=n_particles,
                      group_ndims=1)
    return variational

def doc2vec_prediction_model(input_vectors, input_gene, input_variation, output_label, batch_size,
                             is_training, embedding_size, output_classes):
    # inputs/outputs
    input_vectors = tf.reshape(input_vectors, [batch_size, embedding_size])
    input_gene = tf.reshape(input_gene, [batch_size, embedding_size])
    input_variation = tf.reshape(input_variation, [batch_size, embedding_size])
    targets = None
    if output_label is not None:
        output_label = tf.reshape(output_label, [batch_size, 1])
        targets = tf.one_hot(output_label, axis=-1, depth=output_classes, on_value=1.0,
                             off_value=0.0)
        targets = tf.squeeze(targets, axis=1)

    net = tf.concat([input_vectors, input_gene, input_variation], axis=1)
    net = layers.fully_connected(net, embedding_size * 2, activation_fn=tf.nn.relu)
    net = layers.dropout(net, keep_prob=0.85, is_training=is_training)
    net = layers.fully_connected(net, embedding_size, activation_fn=tf.nn.relu)
    net = layers.dropout(net, keep_prob=0.85, is_training=is_training)
    net = layers.fully_connected(net, embedding_size // 4, activation_fn=tf.nn.relu)
    logits = layers.fully_connected(net, output_classes, activation_fn=None)

    return logits, targets

def rnn(self, sequence, sequence_length, max_length, dropout, batch_size, training,
            num_hidden=TC_MODEL_HIDDEN, num_layers=TC_MODEL_LAYERS):
        # Recurrent network.
        cells = []
        for _ in range(num_layers):
            cell = tf.nn.rnn_cell.GRUCell(num_hidden)
            if training:
                cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=dropout)
            cells.append(cell)
        network = tf.nn.rnn_cell.MultiRNNCell(cells)
        type = sequence.dtype

        sequence_output, _ = tf.nn.dynamic_rnn(network, sequence, dtype=tf.float32,
                                               sequence_length=sequence_length,
                                               initial_state=network.zero_state(batch_size, type))
        # get last output of the dynamic_rnn
        sequence_output = tf.reshape(sequence_output, [batch_size * max_length, num_hidden])
        indexes = tf.range(batch_size) * max_length + (sequence_length - 1)
        output = tf.gather(sequence_output, indexes)
        return output

def get_arg_scope(is_training):
        weight_decay_l2 = 0.1
        batch_norm_decay = 0.999
        batch_norm_epsilon = 0.0001

        with slim.arg_scope([slim.conv2d, slim.fully_connected, layers.separable_convolution2d],
                            weights_regularizer = slim.l2_regularizer(weight_decay_l2),
                            biases_regularizer = slim.l2_regularizer(weight_decay_l2),
                            weights_initializer = layers.variance_scaling_initializer(),
                            ):
            batch_norm_params = {
                'decay': batch_norm_decay,
                'epsilon': batch_norm_epsilon
            }
            with slim.arg_scope([slim.batch_norm, slim.dropout],
                                is_training = is_training):
                with slim.arg_scope([slim.batch_norm],
                                    **batch_norm_params):
                    with slim.arg_scope([slim.conv2d, layers.separable_convolution2d, layers.fully_connected],
                                        activation_fn = tf.nn.elu,
                                        normalizer_fn = slim.batch_norm,
                                        normalizer_params = batch_norm_params) as scope:
                        return scope

def aux_logit_layer( inputs, num_classes, is_training ):
    with tf.variable_scope("pool2d"):
        pooled = layers.avg_pool2d(inputs, [ 5, 5 ], stride = 3 )
    with tf.variable_scope("conv11"):
        conv11 = layers.conv2d( pooled, 128, [1, 1] )
    with tf.variable_scope("flatten"):
        flat = tf.reshape( conv11, [-1, 2048] )
    with tf.variable_scope("fc"):
        fc = layers.fully_connected( flat, 1024, activation_fn=None )
    with tf.variable_scope("drop"):
        drop = layers.dropout( fc, 0.3, is_training = is_training )
    with tf.variable_scope( "linear" ):
        linear = layers.fully_connected( drop, num_classes, activation_fn=None )
    with tf.variable_scope("soft"):
        soft = tf.nn.softmax( linear )
    return soft

def _block_output(net, endpoints, num_classes, dropout_keep_prob=0.5):
    with tf.variable_scope('Output'):
        net = layers.flatten(net, scope='Flatten')

        # 7 x 7 x 512
        net = layers.fully_connected(net, 4096, scope='Fc1')
        net = endpoints['Output/Fc1'] = layers.dropout(net, dropout_keep_prob, scope='Dropout1')

        # 1 x 1 x 4096
        net = layers.fully_connected(net, 4096, scope='Fc2')
        net = endpoints['Output/Fc2'] = layers.dropout(net, dropout_keep_prob, scope='Dropout2')

        logits = layers.fully_connected(net, num_classes, activation_fn=None, scope='Logits')
        # 1 x 1 x num_classes
        endpoints['Logits'] = logits
    return logits

def model(H, x, training):
  net = dropout(x, 0.5, is_training = training)
  # net = conv2d(net, 64, [3, 3], activation_fn = tf.nn.relu)
  # net = conv2d(net, 64, [3, 3], activation_fn = tf.nn.relu)
  # net = max_pool2d(net, [2, 2], padding = 'VALID')
  # net = conv2d(net, 128, [3, 3], activation_fn = tf.nn.relu)
  # net = conv2d(net, 128, [3, 3], activation_fn = tf.nn.relu)
  # net = max_pool2d(net, [2, 2], padding = 'VALID')
  # ksize = net.get_shape().as_list()
  # net = max_pool2d(net, [ksize[1], ksize[2]])
  net = fully_connected(flatten(net), 256, activation_fn = tf.nn.relu)
  net = dropout(net, 0.5, is_training = training)
  logits = fully_connected(net, 1, activation_fn = tf.nn.sigmoid)
  preds = tf.cast(tf.greater(logits, 0.5), tf.int64)
  return logits, preds

def model_fully_connected(self, output, gene, variation, num_output_classes, dropout, training):
        output = layers.dropout(output, keep_prob=dropout, is_training=training)
        net = tf.concat([output, gene, variation], axis=1)
        net = layers.fully_connected(net, 128, activation_fn=tf.nn.relu)
        net = layers.dropout(net, keep_prob=dropout, is_training=training)
        logits = layers.fully_connected(net, num_output_classes, activation_fn=None)
        return logits

def _han(self, input_words, embeddings, gene, variation, batch_size, embeddings_size,
             num_hidden, dropout, word_output_size, sentence_output_size, training=True):

        input_words = tf.reshape(input_words, [batch_size, MAX_SENTENCES, MAX_WORDS_IN_SENTENCE])
        embedded_sequence, sentences_length, words_length = \
            self._embed_sequence_with_length(embeddings, input_words)
        _, sentence_size, word_size, _ = tf.unstack(tf.shape(embedded_sequence))

        # RNN word level
        with tf.variable_scope('word_level'):
            word_level_inputs = tf.reshape(embedded_sequence,
                                           [batch_size * sentence_size, word_size, embeddings_size])
            word_level_lengths = tf.reshape(words_length, [batch_size * sentence_size])

            word_level_output = self._bidirectional_rnn(word_level_inputs, word_level_lengths,
                                                        num_hidden)
            word_level_output = tf.reshape(word_level_output, [batch_size, sentence_size, word_size,
                                                               num_hidden * 2])
            word_level_output = self._attention(word_level_output, word_output_size, gene,
                                                variation)
            word_level_output = layers.dropout(word_level_output, keep_prob=dropout,
                                               is_training=training)
        # RNN sentence level
        with tf.variable_scope('sentence_level'):
            sentence_level_inputs = tf.reshape(word_level_output,
                                               [batch_size, sentence_size, word_output_size])
            sentence_level_output = self._bidirectional_rnn(sentence_level_inputs, sentences_length,
                                                            num_hidden)
            sentence_level_output = self._attention(sentence_level_output, sentence_output_size,
                                                    gene, variation)
            sentence_level_output = layers.dropout(sentence_level_output, keep_prob=dropout,
                                                   is_training=training)

        return sentence_level_output

def __call__(self, inputs, state, scope=None):
    if isinstance(self.state_size, tuple) != isinstance(self._zoneout_prob, tuple):
      raise TypeError("Subdivided states need subdivided zoneouts.")
    if isinstance(self.state_size, tuple) and len(tuple(self.state_size)) != len(tuple(self._zoneout_prob)):
      raise ValueError("State and zoneout need equally many parts.")
    output, new_state = self._cell(inputs, state, scope)
    if isinstance(self.state_size, tuple):
      if self.is_training:
        new_state = self._tuple([(1 - state_part_zoneout_prob) * dropout(
          new_state_part - state_part, (1 - state_part_zoneout_prob)) + state_part
                          for new_state_part, state_part, state_part_zoneout_prob in
                          zip(new_state, state, self._zoneout_prob)])
      else:
        new_state = self._tuple([state_part_zoneout_prob * state_part + (1 - state_part_zoneout_prob) * new_state_part
                          for new_state_part, state_part, state_part_zoneout_prob in
                          zip(new_state, state, self._zoneout_prob)])
    else:
      if self.is_training:
        new_state = (1 - state_part_zoneout_prob) * dropout(
          new_state_part - state_part, (1 - state_part_zoneout_prob)) + state_part
      else:
        new_state = state_part_zoneout_prob * state_part + (1 - state_part_zoneout_prob) * new_state_part
    return output, new_state

# # Wrap your cells like this
# cell = ZoneoutWrapper(tf.nn.rnn_cell.LSTMCell(hidden_units, initializer=random_uniform(), state_is_tuple=True),
# zoneout_prob=(z_prob_cells, z_prob_states))

def __call__(self, inputs, state, scope=None):

        return tf.cond(self.is_training,\
                    lambda: DropoutWrapper(self._cell,self._input_keep_prob,self._output_keep_prob).__call__(inputs,state,scope=None),\
                    lambda: DropoutWrapper(self._cell,1.0,1.0).__call__(inputs,state,scope=None))
           #return self._cell(dropout(inputs,self._input_keep_prob,is_training=self.is_training,scope=None),state,scope=None)

def __init__(self,
               num_label_columns,
               hidden_units,
               optimizer=None,
               activation_fn=nn.relu,
               dropout=None,
               gradient_clip_norm=None,
               num_ps_replicas=0,
               scope=None):
    """Initializes DNNComposableModel objects.

    Args:
      num_label_columns: The number of label/target columns.
      hidden_units: List of hidden units per layer. All layers are fully
        connected.
      optimizer: An instance of `tf.Optimizer` used to apply gradients to
        the model. If `None`, will use a FTRL optimizer.
      activation_fn: Activation function applied to each layer. If `None`,
        will use `tf.nn.relu`.
      dropout: When not None, the probability we will drop out
        a given coordinate.
      gradient_clip_norm: A float > 0. If provided, gradients are clipped
        to their global norm with this clipping ratio. See
        tf.clip_by_global_norm for more details.
      num_ps_replicas: The number of parameter server replicas.
      scope: Optional scope for variables created in this model. If not scope
        is supplied, one is generated.
    """
    scope = "dnn" if not scope else scope
    super(DNNComposableModel, self).__init__(
        num_label_columns=num_label_columns,
        optimizer=optimizer,
        gradient_clip_norm=gradient_clip_norm,
        num_ps_replicas=num_ps_replicas,
        scope=scope)
    self._hidden_units = hidden_units
    self._activation_fn = activation_fn
    self._dropout = dropout

def __init__(self,
               num_label_columns,
               hidden_units,
               optimizer=None,
               activation_fn=nn.relu,
               dropout=None,
               gradient_clip_norm=None,
               num_ps_replicas=0,
               scope=None):
    """Initializes DNNComposableModel objects.

    Args:
      num_label_columns: The number of label columns.
      hidden_units: List of hidden units per layer. All layers are fully
        connected.
      optimizer: An instance of `tf.Optimizer` used to apply gradients to
        the model. If `None`, will use a FTRL optimizer.
      activation_fn: Activation function applied to each layer. If `None`,
        will use `tf.nn.relu`.
      dropout: When not None, the probability we will drop out
        a given coordinate.
      gradient_clip_norm: A float > 0. If provided, gradients are clipped
        to their global norm with this clipping ratio. See
        tf.clip_by_global_norm for more details.
      num_ps_replicas: The number of parameter server replicas.
      scope: Optional scope for variables created in this model. If not scope
        is supplied, one is generated.
    """
    scope = "dnn" if not scope else scope
    super(DNNComposableModel, self).__init__(
        num_label_columns=num_label_columns,
        optimizer=optimizer,
        gradient_clip_norm=gradient_clip_norm,
        num_ps_replicas=num_ps_replicas,
        scope=scope)
    self._hidden_units = hidden_units
    self._activation_fn = activation_fn
    self._dropout = dropout

def encZ(x, ACTIVATION):

   conv1 = tcl.conv2d(x, 32, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv1')
   conv1 = activate(conv1, ACTIVATION)

   conv2 = tcl.conv2d(conv1, 64, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv2')
   conv2 = activate(conv2, ACTIVATION)

   conv3 = tcl.conv2d(conv2, 128, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv3')
   conv3 = activate(conv3, ACTIVATION)

   conv4 = tcl.conv2d(conv3, 256, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv4')
   conv4 = activate(conv4, ACTIVATION)

   conv4_flat = tcl.flatten(conv4)

   fc1 = tcl.fully_connected(conv4_flat, 4096, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='fc1')
   fc1 = activate(fc1, ACTIVATION)
   #fc1 = tcl.dropout(fc1, 0.5)

   fc2 = tcl.fully_connected(fc1, 100, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='fc2')

   print 'input:',x
   print 'conv1:',conv1
   print 'conv2:',conv2
   print 'conv3:',conv3
   print 'conv4:',conv4
   print 'fc1:',fc1
   print 'fc2:',fc2
   print 'END ENCODER\n'

   tf.add_to_collection('vars', conv1)
   tf.add_to_collection('vars', conv2)
   tf.add_to_collection('vars', conv3)
   tf.add_to_collection('vars', conv4)
   tf.add_to_collection('vars', fc1)
   tf.add_to_collection('vars', fc2)

   return fc2

def conv_learn(X, y, mode):
    # Ensure our images are 2d 
    X = tf.reshape(X, [-1, 36, 36, 1])
    # We'll need these in one-hot format
    y = tf.one_hot(tf.cast(y, tf.int32), 5, 1, 0)

    # conv layer will compute 4 kernels for each 5x5 patch
    with tf.variable_scope('conv_layer'):
        # 5x5 convolution, pad with zeros on edges
        h1 = layers.convolution2d(X, num_outputs=4,
                kernel_size=[5, 5], 
                activation_fn=tf.nn.relu)
        # 2x2 Max pooling, no padding on edges
        p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
                strides=[1, 2, 2, 1], padding='VALID')

        # Need to flatten conv output for use in dense layer
    p1_size = np.product(
              [s.value for s in p1.get_shape()[1:]])
    p1f = tf.reshape(p1, [-1, p1_size ])

    # densely connected layer with 32 neurons and dropout
    h_fc1 = layers.fully_connected(p1f,
             5,
             activation_fn=tf.nn.relu)
    drop = layers.dropout(h_fc1, keep_prob=0.5, is_training=mode == tf.contrib.learn.ModeKeys.TRAIN)

    logits = layers.fully_connected(drop, 5, activation_fn=None)
    loss = tf.losses.softmax_cross_entropy(y, logits)
    # Setup the training function manually
    train_op = layers.optimize_loss(
        loss,
        tf.contrib.framework.get_global_step(),
        optimizer='Adam',
        learning_rate=0.01)
    return tf.argmax(logits, 1), loss, train_op
# Use generic estimator with our function

def conv_learn(X, y, mode):
    # Ensure our images are 2d 
    X = tf.reshape(X, [-1, 36, 36, 1])
    # We'll need these in one-hot format
    y = tf.one_hot(tf.cast(y, tf.int32), 5, 1, 0)

    # conv layer will compute 4 kernels for each 5x5 patch
    with tf.variable_scope('conv_layer'):
        # 5x5 convolution, pad with zeros on edges
        h1 = layers.convolution2d(X, num_outputs=4,
                kernel_size=[5, 5], 
                activation_fn=tf.nn.relu)
        # 2x2 Max pooling, no padding on edges
        p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
                strides=[1, 2, 2, 1], padding='VALID')

        # Need to flatten conv output for use in dense layer
    p1_size = np.product(
              [s.value for s in p1.get_shape()[1:]])
    p1f = tf.reshape(p1, [-1, p1_size ])

    # densely connected layer with 32 neurons and dropout
    h_fc1 = layers.fully_connected(p1f,
             5,
             activation_fn=tf.nn.relu)
    drop = layers.dropout(h_fc1, keep_prob=0.5, is_training=mode == tf.contrib.learn.ModeKeys.TRAIN)

    logits = layers.fully_connected(drop, 5, activation_fn=None)
    loss = tf.losses.softmax_cross_entropy(y, logits)
    # Setup the training function manually
    train_op = layers.optimize_loss(
        loss,
        tf.contrib.framework.get_global_step(),
        optimizer='Adam',
        learning_rate=0.01)
    return tf.argmax(logits, 1), loss, train_op
# Use generic estimator with our function

def _build_vgg16(
        inputs,
        num_classes=1000,
        dropout_keep_prob=0.5,
        is_training=True,
        scope=''):
    """Blah"""

    endpoints = {}
    with tf.name_scope(scope, 'vgg16', [inputs]):
        with arg_scope(
                [layers.batch_norm, layers.dropout], is_training=is_training):
            with arg_scope(
                    [layers.conv2d, layers.max_pool2d], 
                    stride=1,
                    padding='SAME'):

                net = _block_a(inputs, endpoints, d=64, scope='Scale1')
                net = _block_a(net, endpoints, d=128, scope='Scale2')
                net = _block_b(net, endpoints, d=256, scope='Scale3')
                net = _block_b(net, endpoints, d=512, scope='Scale4')
                net = _block_b(net, endpoints, d=512, scope='Scale5')
                logits = _block_output(net, endpoints, num_classes, dropout_keep_prob)

                endpoints['Predictions'] = tf.nn.softmax(logits, name='Predictions')
                return logits, endpoints

def _build_vgg19(
        inputs,
        num_classes=1000,
        dropout_keep_prob=0.5,
        is_training=True,
        scope=''):
    """Blah"""

    endpoints = {}
    with tf.name_scope(scope, 'vgg19', [inputs]):
        with arg_scope(
                [layers.batch_norm, layers.dropout], is_training=is_training):
            with arg_scope(
                    [layers.conv2d, layers.max_pool2d],
                    stride=1,
                    padding='SAME'):

                net = _block_a(inputs, endpoints, d=64, scope='Scale1')
                net = _block_a(net, endpoints, d=128, scope='Scale2')
                net = _block_c(net, endpoints, d=256, scope='Scale3')
                net = _block_c(net, endpoints, d=512, scope='Scale4')
                net = _block_c(net, endpoints, d=512, scope='Scale5')
                logits = _block_output(net, endpoints, num_classes, dropout_keep_prob)

                endpoints['Predictions'] = tf.nn.softmax(logits, name='Predictions')
                return logits, endpoints

def _block_output(net, endpoints, num_classes=1000, dropout_keep_prob=0.5, scope='Output'):
    with tf.variable_scope(scope):
        # 8 x 8 x 1536
        shape = net.get_shape()
        net = layers.avg_pool2d(net, shape[1:3], padding='VALID', scope='Pool1_Global')
        endpoints['Output/Pool1'] = net
        # 1 x 1 x 1536
        net = layers.dropout(net, dropout_keep_prob)
        net = layers.flatten(net)
        # 1536
        net = layers.fully_connected(net, num_classes, activation_fn=None, scope='Logits')
        # num classes
        endpoints['Logits'] = net
    return net

def discriminator(inputs, reuse=False):
    with tf.variable_scope('discriminator'):
        if reuse:
            tf.get_variable_scope().reuse_variables()
        net = lays.conv2d_transpose(inputs, 64, 3, stride=1, scope='conv1', padding='SAME', activation_fn=leaky_relu)
        net = lays.max_pool2d(net, 2, 2, 'SAME', scope='max1')
        net = lays.conv2d_transpose(net, 128, 3, stride=1, scope='conv2', padding='SAME', activation_fn=leaky_relu)
        net = lays.max_pool2d(net, 2, 2, 'SAME', scope='max2')
        net = lays.conv2d_transpose(net, 256, 3, stride=1, scope='conv3', padding='SAME', activation_fn=leaky_relu)
        net = lays.max_pool2d(net, 2, 2, 'SAME', scope='max3')
        net = tf.reshape(net, (batch_size, 4 * 4 * 256))
        net = lays.fully_connected(net, 128, scope='fc1', activation_fn=leaky_relu)
        net = lays.dropout(net, 0.5)
        net = lays.fully_connected(net, 1, scope='fc2', activation_fn=None)
        return net

def __call__(self, inputs, state, scope=None):
    if isinstance(self.state_size, tuple) != isinstance(self._zoneout_prob, tuple):
      raise TypeError("Subdivided states need subdivided zoneouts.")
    if isinstance(self.state_size, tuple) and len(tuple(self.state_size)) != len(tuple(self._zoneout_prob)):
      raise ValueError("State and zoneout need equally many parts.")
    output, new_state = self._cell(inputs, state, scope)
    if isinstance(self.state_size, tuple):
      if self.is_training:
        new_state = self._tuple([(1 - state_part_zoneout_prob) * dropout(
          new_state_part - state_part, (1 - state_part_zoneout_prob)) + state_part
                          for new_state_part, state_part, state_part_zoneout_prob in
                          zip(new_state, state, self._zoneout_prob)])
      else:
        new_state = self._tuple([state_part_zoneout_prob * state_part + (1 - state_part_zoneout_prob) * new_state_part
                          for new_state_part, state_part, state_part_zoneout_prob in
                          zip(new_state, state, self._zoneout_prob)])
    else:
      if self.is_training:
        new_state = (1 - state_part_zoneout_prob) * dropout(
          new_state_part - state_part, (1 - state_part_zoneout_prob)) + state_part
      else:
        new_state = state_part_zoneout_prob * state_part + (1 - state_part_zoneout_prob) * new_state_part
    return output, new_state

# # Wrap your cells like this
# cell = ZoneoutWrapper(tf.nn.rnn_cell.LSTMCell(hidden_units, initializer=random_uniform(), state_is_tuple=True),
# zoneout_prob=(z_prob_cells, z_prob_states))

def model(self, input_text_begin, input_text_end, gene, variation, num_output_classes,
              batch_size, embeddings, training=True, dropout=TC_MODEL_DROPOUT):

        """
        Creates a model for text classification
        :param tf.Tensor input_text: the input data, the text as
        [batch_size, text_vector_max_length, embeddings_size]
        :param int num_output_classes: the number of output classes for the classifier
        :param int batch_size: batch size, the same used in the dataset
        :param List[List[float]] embeddings: a matrix with the embeddings for the embedding lookup
        :param int num_hidden: number of hidden GRU cells in every layer
        :param int num_layers: number of layers of the model
        :param float dropout: dropout value between layers
        :param boolean training: whether the model is built for training or not
        :return Dict[str,tf.Tensor]: a dict with logits and prediction tensors
        """
        input_text_begin = tf.reshape(input_text_begin, [batch_size, MAX_WORDS])
        if input_text_end is not None:
            input_text_end = tf.reshape(input_text_end, [batch_size, MAX_WORDS])
        embedded_sequence_begin, sequence_length_begin, \
        embedded_sequence_end, sequence_length_end, \
        gene, variation = \
            self.model_embedded_sequence(embeddings, input_text_begin, input_text_end, gene,
                                         variation)
        _, max_length, _ = tf.unstack(tf.shape(embedded_sequence_begin))

        with tf.variable_scope('text_begin'):
            output_begin = self.rnn(embedded_sequence_begin, sequence_length_begin, max_length,
                                    dropout, batch_size, training)
        if input_text_end is not None:
            with tf.variable_scope('text_end'):
                output_end = self.rnn(embedded_sequence_end, sequence_length_end, max_length,
                                      dropout, batch_size, training)
            output = tf.concat([output_begin, output_end], axis=1)
        else:
            output = output_begin

        # full connected layer
        logits = self.model_fully_connected(output, gene, variation, num_output_classes, dropout,
                                            training)

        prediction = tf.nn.softmax(logits)

        return {
            'logits'    : logits,
            'prediction': prediction,
            }

def get_network(self, input_tensor, is_training, reuse = False):
        net = input_tensor

        with tf.variable_scope('GaitNN', reuse = reuse):
            with slim.arg_scope(self.get_arg_scope(is_training)):
                with tf.variable_scope('DownSampling'):
                    with tf.variable_scope('17x17'):
                        net = layers.convolution2d(net, num_outputs = 256, kernel_size = 1)
                        slim.repeat(net, 3, self.residual_block, ch = 256, ch_inner = 64)

                    with tf.variable_scope('8x8'):
                        net = self.residual_block(net, ch = 512, ch_inner = 64, stride = 2)
                        slim.repeat(net, 2, self.residual_block, ch = 512, ch_inner = 128)

                    with tf.variable_scope('4x4'):
                        net = self.residual_block(net, ch = 512, ch_inner = 128, stride = 2)
                        slim.repeat(net, 1, self.residual_block, ch = 512, ch_inner = 256)

                        net = layers.convolution2d(net, num_outputs = 256, kernel_size = 1)
                        net = layers.convolution2d(net, num_outputs = 256, kernel_size = 3)

                with tf.variable_scope('FullyConnected'):
                    # net = tf.reduce_mean(net, [1, 2], name = 'GlobalPool')
                    net = layers.flatten(net)
                    net = layers.fully_connected(net, 512, activation_fn = None, normalizer_fn = None)

                with tf.variable_scope('Recurrent', initializer = tf.contrib.layers.xavier_initializer()):
                    cell_type = {
                        'GRU': tf.nn.rnn_cell.GRUCell,
                        'LSTM': tf.nn.rnn_cell.LSTMCell
                    }

                    cell = cell_type[self.recurrent_unit](self.FEATURES)
                    cell = tf.nn.rnn_cell.MultiRNNCell([cell] * self.rnn_layers, state_is_tuple = True)

                    net = tf.expand_dims(net, 0)
                    net, state = tf.nn.dynamic_rnn(cell, net, initial_state = cell.zero_state(1, dtype = tf.float32))
                    net = tf.reshape(net, [-1, self.FEATURES])

                    # Temporal Avg-Pooling
                    gait_signature = tf.reduce_mean(net, 0)

                if is_training:
                    net = tf.expand_dims(gait_signature, 0)
                    net = layers.dropout(net, 0.7)

                    with tf.variable_scope('Logits'):
                        net = layers.fully_connected(net, self.num_of_persons, activation_fn = None,
                                                     normalizer_fn = None)

                return net, gait_signature, state

def _init_body(self, scope):
    with tf.variable_scope(scope):

      word_level_inputs = tf.reshape(self.inputs_embedded, [
        self.document_size * self.sentence_size,
        self.word_size,
        self.embedding_size
      ])
      word_level_lengths = tf.reshape(
        self.word_lengths, [self.document_size * self.sentence_size])

      with tf.variable_scope('word') as scope:
        word_encoder_output, _ = bidirectional_rnn(
          self.word_cell, self.word_cell,
          word_level_inputs, word_level_lengths,
          scope=scope)

        with tf.variable_scope('attention') as scope:
          word_level_output = task_specific_attention(
            word_encoder_output,
            self.word_output_size,
            scope=scope)

        with tf.variable_scope('dropout'):
          word_level_output = layers.dropout(
            word_level_output, keep_prob=self.dropout_keep_proba,
            is_training=self.is_training,
          )

      # sentence_level

      sentence_inputs = tf.reshape(
        word_level_output, [self.document_size, self.sentence_size, self.word_output_size])

      with tf.variable_scope('sentence') as scope:
        sentence_encoder_output, _ = bidirectional_rnn(
          self.sentence_cell, self.sentence_cell, sentence_inputs, self.sentence_lengths, scope=scope)

        with tf.variable_scope('attention') as scope:
          sentence_level_output = task_specific_attention(
            sentence_encoder_output, self.sentence_output_size, scope=scope)

        with tf.variable_scope('dropout'):
          sentence_level_output = layers.dropout(
            sentence_level_output, keep_prob=self.dropout_keep_proba,
            is_training=self.is_training,
          )

      with tf.variable_scope('classifier'):
        self.logits = layers.fully_connected(
          sentence_level_output, self.classes, activation_fn=None)

        self.prediction = tf.argmax(self.logits, axis=-1)

def define_feedforward_model(self):
      layer_list=[]
      with self.graph.as_default() as g:
          is_training_batch=tf.placeholder(tf.bool,shape=(),name="is_training_batch")
          bn_params={"is_training":is_training_batch,"decay":0.99,"updates_collections":None}
          g.add_to_collection("is_training_batch",is_training_batch)
          with tf.name_scope("input"):
              input_layer=tf.placeholder(dtype=tf.float32,shape=(None,self.n_in),name="input_layer")
              if self.dropout_rate!=0.0:
                 print "Using dropout to avoid overfitting and the dropout rate is",self.dropout_rate
                 is_training_drop=tf.placeholder(dtype=tf.bool,shape=(),name="is_training_drop")
                 input_layer_drop=dropout(input_layer,self.dropout_rate,is_training=is_training_drop)
                 layer_list.append(input_layer_drop)
                 g.add_to_collection(name="is_training_drop",value=is_training_drop)
              else:
                 layer_list.append(input_layer)
          g.add_to_collection("input_layer",layer_list[0])
          for i in xrange(len(self.hidden_layer_size)):
              with tf.name_scope("hidden_layer_"+str(i+1)):
                if self.dropout_rate!=0.0:
                    last_layer=layer_list[-1]
                    if self.hidden_layer_type[i]=="tanh":
                       new_layer=fully_connected(last_layer,self.hidden_layer_size[i],activation_fn=tf.nn.tanh,normalizer_fn=batch_norm,\
                                  normalizer_params=bn_params)
                    if self.hidden_layer_type[i]=="sigmoid":
                        new_layer=fully_connected(last_layer,self.hidden_layer_size[i],activation_fn=tf.nn.sigmoid,normalizer_fn=batch_norm,\
                                  normalizer_params=bn_params)
                    new_layer_drop=dropout(new_layer,self.dropout_rate,is_training=is_training_drop)
                    layer_list.append(new_layer_drop)
                else:
                    last_layer=layer_list[-1]
                    if self.hidden_layer_type[i]=="tanh":
                       new_layer=fully_connected(last_layer,self.hidden_layer_size[i],activation_fn=tf.nn.tanh,normalizer_fn=batch_norm,\
                                 normalizer_params=bn_params)
                    if self.hidden_layer_type[i]=="sigmoid":
                       new_layer=fully_connected(last_layer,self.hidden_layer_size[i],activation_fn=tf.nn.sigmoid,normalizer_fn=batch_norm,\
                                 normalizer_params=bn_params)
                    layer_list.append(new_layer)
          with tf.name_scope("output_layer"):
              if self.output_type=="linear":
                 output_layer=fully_connected(layer_list[-1],self.n_out,activation_fn=None)
              if self.output_type=="tanh":
                 output_layer=fully_connected(layer_list[-1],self.n_out,activation_fn=tf.nn.tanh)
              g.add_to_collection(name="output_layer",value=output_layer)
          with tf.name_scope("training_op"):
               if self.optimizer=="adam":
                  self.training_op=tf.train.AdamOptimizer()

def define_sequence_model(self):
      seed=12345
      np.random.seed(12345)
      layer_list=[]
      with self.graph.as_default() as g:
          utt_length=tf.placeholder(tf.int32,shape=(None))
          g.add_to_collection(name="utt_length",value=utt_length)
          with tf.name_scope("input"):
               input_layer=tf.placeholder(dtype=tf.float32,shape=(None,None,self.n_in),name="input_layer")
               if self.dropout_rate!=0.0:
                  print "Using dropout to avoid overfitting and the dropout rate is",self.dropout_rate
                  is_training_drop=tf.placeholder(dtype=tf.bool,shape=(),name="is_training_drop")
                  input_layer_drop=dropout(input_layer,self.dropout_rate,is_training=is_training_drop)
                  layer_list.append(input_layer_drop)
                  g.add_to_collection(name="is_training_drop",value=is_training_drop)
               else:
                  layer_list.append(input_layer)
          g.add_to_collection("input_layer",layer_list[0])
          with tf.name_scope("hidden_layer"):
             basic_cell=[]
             if "tanh" in self.hidden_layer_type:
                 is_training_batch=tf.placeholder(dtype=tf.bool,shape=(),name="is_training_batch")
                 bn_params={"is_training":is_training_batch,"decay":0.99,"updates_collections":None}
                 g.add_to_collection("is_training_batch",is_training_batch)
             for i in xrange(len(self.hidden_layer_type)):
                 if self.dropout_rate!=0.0:
                     if self.hidden_layer_type[i]=="tanh":
                         new_layer=fully_connected(layer_list[-1],self.hidden_layer_size[i],activation_fn=tf.nn.tanh,normalizer_fn=batch_norm,normalizer_params=bn_params)
                         new_layer_drop=dropout(new_layer,self.dropout_rate,is_training=is_training_drop)
                         layer_list.append(new_layer_drop)
                     if self.hidden_layer_type[i]=="lstm":
                         basic_cell.append(MyDropoutWrapper(BasicLSTMCell(num_units=self.hidden_layer_size[i]),self.dropout_rate,self.dropout_rate,is_training=is_training_drop))
                     if self.hidden_layer_type[i]=="gru":
                         basic_cell.append(MyDropoutWrapper(GRUCell(num_units=self.hidden_layer_size[i]),self.dropout_rate,self.dropout_rate,is_training=is_training_drop))
                 else:
                     if self.hidden_layer_type[i]=="tanh":
                        new_layer=fully_connected(layer_list[-1],self.hidden_layer_size[i],activation_fn=tf.nn.tanh,normalizer_fn=batch_norm,normalizer_params=bn_params)
                        layer_list.append(new_layer)
                     if self.hidden_layer_type[i]=="lstm":
                        basic_cell.append(LayerNormBasicLSTMCell(num_units=self.hidden_layer_size[i]))
                     if self.hidden_layer_type[i]=="gru":
                        basic_cell.append(LayerNormGRUCell(num_units=self.hidden_layer_size[i]))
             multi_cell=MultiRNNCell(basic_cell)
             rnn_outputs,rnn_states=tf.nn.dynamic_rnn(multi_cell,layer_list[-1],dtype=tf.float32,sequence_length=utt_length)
             layer_list.append(rnn_outputs)
          with tf.name_scope("output_layer"):
               if self.output_type=="linear" :
                   output_layer=tf.layers.dense(rnn_outputs,self.n_out)
                #  stacked_rnn_outputs=tf.reshape(rnn_outputs,[-1,self.n_out])
                #  stacked_outputs=tf.layers.dense(stacked_rnn_outputs,self.n_out)
                #  output_layer=tf.reshape(stacked_outputs,[-1,utt_length,self.n_out])
               g.add_to_collection(name="output_layer",value=output_layer)
          with tf.name_scope("training_op"):
               if self.optimizer=="adam":
                   self.training_op=tf.train.AdamOptimizer()

def build_model(self, features, feature_columns, is_training):
    """See base class."""
    self._feature_columns = feature_columns

    input_layer_partitioner = (
        partitioned_variables.min_max_variable_partitioner(
            max_partitions=self._num_ps_replicas,
            min_slice_size=64 << 20))
    with variable_scope.variable_scope(
        self._scope + "/input_from_feature_columns",
        values=features.values(),
        partitioner=input_layer_partitioner) as scope:
      net = layers.input_from_feature_columns(
          features,
          self._get_feature_columns(),
          weight_collections=[self._scope],
          scope=scope)

    hidden_layer_partitioner = (
        partitioned_variables.min_max_variable_partitioner(
            max_partitions=self._num_ps_replicas))
    for layer_id, num_hidden_units in enumerate(self._hidden_units):
      with variable_scope.variable_scope(
          self._scope + "/hiddenlayer_%d" % layer_id,
          values=[net],
          partitioner=hidden_layer_partitioner) as scope:
        net = layers.fully_connected(
            net,
            num_hidden_units,
            activation_fn=self._activation_fn,
            variables_collections=[self._scope],
            scope=scope)
        if self._dropout is not None and is_training:
          net = layers.dropout(
              net,
              keep_prob=(1.0 - self._dropout))
      self._add_hidden_layer_summary(net, scope.name)

    with variable_scope.variable_scope(
        self._scope + "/logits",
        values=[net],
        partitioner=hidden_layer_partitioner) as scope:
      logits = layers.fully_connected(
          net,
          self._num_label_columns,
          activation_fn=None,
          variables_collections=[self._scope],
          scope=scope)
    self._add_hidden_layer_summary(logits, "logits")
    return logits

def build_model(self, features, feature_columns, is_training):
    """See base class."""
    self._feature_columns = feature_columns

    input_layer_partitioner = (
        partitioned_variables.min_max_variable_partitioner(
            max_partitions=self._num_ps_replicas,
            min_slice_size=64 << 20))
    with variable_scope.variable_scope(
        self._scope + "/input_from_feature_columns",
        values=features.values(),
        partitioner=input_layer_partitioner) as scope:
      net = layers.input_from_feature_columns(
          features,
          self._get_feature_columns(),
          weight_collections=[self._scope],
          scope=scope)

    hidden_layer_partitioner = (
        partitioned_variables.min_max_variable_partitioner(
            max_partitions=self._num_ps_replicas))
    for layer_id, num_hidden_units in enumerate(self._hidden_units):
      with variable_scope.variable_scope(
          self._scope + "/hiddenlayer_%d" % layer_id,
          values=[net],
          partitioner=hidden_layer_partitioner) as scope:
        net = layers.fully_connected(
            net,
            num_hidden_units,
            activation_fn=self._activation_fn,
            variables_collections=[self._scope],
            scope=scope)
        if self._dropout is not None and is_training:
          net = layers.dropout(
              net,
              keep_prob=(1.0 - self._dropout))
      self._add_hidden_layer_summary(net, scope.name)

    with variable_scope.variable_scope(
        self._scope + "/logits",
        values=[net],
        partitioner=hidden_layer_partitioner) as scope:
      logits = layers.fully_connected(
          net,
          self._num_label_columns,
          activation_fn=None,
          variables_collections=[self._scope],
          scope=scope)
    self._add_hidden_layer_summary(logits, "logits")
    return logits

def encZ(x, ACTIVATION):

   conv1 = tcl.conv2d(x, 32, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv1')
   conv1 = activate(conv1, ACTIVATION)

   conv2 = tcl.conv2d(conv1, 64, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv2')
   conv2 = activate(conv2, ACTIVATION)

   conv3 = tcl.conv2d(conv2, 128, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv3')
   conv3 = activate(conv3, ACTIVATION)

   conv4 = tcl.conv2d(conv3, 256, 5, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='conv4')
   conv4 = activate(conv4, ACTIVATION)

   conv4_flat = tcl.flatten(conv4)

   fc1 = tcl.fully_connected(conv4_flat, 4096, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='fc1')
   fc1 = activate(fc1, ACTIVATION)
   fc1 = tcl.dropout(fc1, 0.5)

   fc2 = tcl.fully_connected(fc1, 100, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='fc2')

   # relu to match the [0,1] range from the distribution
   fc2 = relu(fc2)

   print 'input:',x
   print 'conv1:',conv1
   print 'conv2:',conv2
   print 'conv3:',conv3
   print 'conv4:',conv4
   print 'fc1:',fc1
   print 'fc2:',fc2
   print 'END ENCODER\n'

   tf.add_to_collection('vars', conv1)
   tf.add_to_collection('vars', conv2)
   tf.add_to_collection('vars', conv3)
   tf.add_to_collection('vars', conv4)
   tf.add_to_collection('vars', fc1)
   tf.add_to_collection('vars', fc2)

   return fc2

def _build_inception_v4(
        inputs,
        stack_counts=[4, 7, 3],
        dropout_keep_prob=0.8,
        num_classes=1000,
        is_training=True,
        scope=''):
    """Inception v4 from http://arxiv.org/abs/

    Args:
      inputs: a tensor of size [batch_size, height, width, channels].
      dropout_keep_prob: dropout keep_prob.
      num_classes: number of predicted classes.
      is_training: whether is training or not.
      scope: Optional scope for op_scope.

    Returns:
      a list containing 'logits' Tensors and a dict of Endpoints.
    """
    # endpoints will collect relevant activations for external use, for example, summaries or losses.
    endpoints = {}
    name_scope_net = tf.name_scope(scope, 'Inception_v4', [inputs])
    arg_scope_train = arg_scope([layers.batch_norm, layers.dropout], is_training=is_training)
    arg_scope_conv = arg_scope([layers.conv2d, layers.max_pool2d, layers.avg_pool2d], stride=1, padding='SAME')
    with name_scope_net, arg_scope_train, arg_scope_conv:

        net = _block_stem(inputs, endpoints)
        # 35 x 35 x 384

        with tf.variable_scope('Scale1'):
            net = _stack(net, endpoints, fn=_block_a, count=stack_counts[0], scope='BlockA')
            # 35 x 35 x 384

        with tf.variable_scope('Scale2'):
            net = _block_a_reduce(net, endpoints)
            # 17 x 17 x 1024
            net = _stack(net, endpoints, fn=_block_b, count=stack_counts[1], scope='BlockB')
            # 17 x 17 x 1024

        with tf.variable_scope('Scale3'):
            net = _block_b_reduce(net, endpoints)
            # 8 x 8 x 1536
            net = _stack(net, endpoints, fn=_block_c, count=stack_counts[2], scope='BlockC')
            # 8 x 8 x 1536

        logits = _block_output(net, endpoints, num_classes, dropout_keep_prob, scope='Output')
        endpoints['Predictions'] = tf.nn.softmax(logits, name='Predictions')

        return logits, endpoints