python类unpack()的实例源码

def nonseq2seq(tensor, seq_length, length, name=None):
    '''
    Convert non sequential data to sequential data

    Args:
        tensor: non sequential data, which is a TxF tensor where T is the sum of
            all sequence lengths
        seq_length: a vector containing the sequence lengths
        length: the constant length of the output sequences
        name: [optional] the name of the operation

    Returns:
        sequential data, wich is a list containing an N x F
        tensor for each time step where N is the batch size and F is the
        input dimension
    '''

    with tf.name_scope(name or'nonseq2seq'):
        #get the cumulated sequence lengths to specify the positions in tensor
        cum_seq_length = tf.concat(0, [tf.constant([0]), tf.cumsum(seq_length)])

        #get the indices in the tensor for each sequence
        indices = [tf.range(cum_seq_length[l], cum_seq_length[l+1])
                   for l in range(int(seq_length.get_shape()[0]))]

        #create the non-padded sequences
        sequences = [tf.gather(tensor, i) for i in indices]

        #pad the sequences with zeros
        sequences = [tf.pad(sequences[s], [[0, length-seq_length[s]], [0, 0]])
                     for s in range(len(sequences))]

        #specify that the sequences have been padded to the constant length
        for seq in sequences:
            seq.set_shape([length, int(tensor.get_shape()[1])])

        #convert the list for eqch sequence to a list for eqch time step
        tensorlist = tf.unpack(tf.pack(sequences), axis=1)

    return tensorlist

def compute_states(self,emb):
        def unpack_sequence(tensor):
            return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2]))



        with tf.variable_scope("Composition",initializer=
                               tf.contrib.layers.xavier_initializer(),regularizer=
                               tf.contrib.layers.l2_regularizer(self.reg)):
            cell_fw = rnn_cell.LSTMCell(self.hidden_dim)
            cell_bw = rnn_cell.LSTMCell(self.hidden_dim)
            #tf.cond(tf.less(self.dropout
            #if tf.less(self.dropout, tf.constant(1.0)):
            cell_fw = rnn_cell.DropoutWrapper(cell_fw,
                                           output_keep_prob=self.dropout,input_keep_prob=self.dropout)
            cell_bw=rnn_cell.DropoutWrapper(cell_bw, output_keep_prob=self.dropout,input_keep_prob=self.dropout)

            #output, state = rnn.dynamic_rnn(cell,emb,sequence_length=self.lngths,dtype=tf.float32)
            outputs,_,_=rnn.bidirectional_rnn(cell_fw,cell_bw,unpack_sequence(emb),sequence_length=self.lngths,dtype=tf.float32)
            #output = pack_sequence(outputs)
        sum_out=tf.reduce_sum(tf.pack(outputs),[0])
        sent_rep = tf.div(sum_out,tf.expand_dims(tf.to_float(self.lngths),1))



        final_state=sent_rep
        return final_state

def process_example(self, tensors, mode='eval', thread_id=0):
        train = (mode == 'train')
        image, image_timestamp, camera_id = tensors[:3]

        #FIXME push single/multi image handling into image_process_sdc if we want to share random augmentations
        if self.num_input_images > 1:
            assert(len(image.get_shape()) > 0)
            print('Multi image', image.get_shape())
            split_image = tf.unpack(image)
            split_processed = []
            for i, x in enumerate(split_image):
                suffix = '%d' % i
                xp, _ = image_preprocess_sdc(
                    x, camera_id,
                    height=self.height, width=self.width, image_fmt=self.image_fmt,
                    normalize=self.standardize_input, train=train, summary_suffix=suffix, thread_id=thread_id)
                split_processed.append(xp)
            processed_image = tf.pack(split_processed)
            #FIXME need to sort out flip across mult-images
            flip_coeff = tf.constant(1.0, dtype=tf.float32)
        else:
            print('Single image')
            processed_image, flip_coeff = image_preprocess_sdc(
                image, camera_id,
                height=self.height, width=self.width, image_fmt=self.image_fmt,
                normalize=self.standardize_input, train=train, thread_id=thread_id)

        if mode != 'pred':
            steering_angle, gps_coord = tensors[-2:]
            if steering_angle is not None:
                steering_angle = tf.mul(steering_angle, flip_coeff)
                if self.standardize_labels:
                    steering_angle /= STEERING_STD
                elif self.mu_law_steering:
                    print("Encode mu-law angles")
                    steering_angle = mu_law_steering_enc(steering_angle)
            if gps_coord is not None and self.standardize_labels:
                gps_coord = (gps_coord - GPS_MEAN) / GPS_STD
            return processed_image, image_timestamp, steering_angle, gps_coord
        else:
            return processed_image, image_timestamp, tf.zeros((1,)), tf.zeros((2,))

def lstm(
        inputs,
        num_units,
        num_layers=1,
        initializer_fn=tf.truncated_normal,
        initializer_params=_default_initializer_params,
        dtype=tf.float32,
        scope=None
):
    print('input shape', inputs.get_shape())
    shape = inputs.get_shape().as_list()
    batch_size = shape[0]
    inputs_unpacked = tf.unpack(inputs, axis=1)

    cell = tf.contrib.rnn.python.ops.lstm_ops.LSTMBlockCell(num_units=num_units)
    print('cell state size', cell.state_size)

    if num_layers > 1:
        cell = tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers)

    initializer_params = initializer_params or {}
    initializer_params['dtype'] = dtype
    if isinstance(cell.state_size, tuple):
        initial_state = tuple(initializer_fn([batch_size, s]) for s in cell.state_size)
    else:
        initial_state = initializer_fn(shape=[batch_size, cell.state_size], **initializer_params)

    outputs, _, _ = tf.nn.rnn(
        cell,
        inputs_unpacked,
        initial_state=initial_state,
        dtype=dtype,
        scope=scope)

    outputs = tf.pack(outputs, axis=1)
    print('output shape', outputs.get_shape())

    return outputs

def RNN(X, weights, biases):
    # hidden layer for input to cell, we have to start by reshaping, we have to reshape the 3 dimensional x into a 2 dimensional X
    # in the hidden layer Wx + b, the W is a 2d dimension and so is the x so everything must be reshaped into a 2d dimension
    # X -> (128 batch * 28 steps, 28 inputs) is now it should be reshaped, 128 batch * 28 steps is one dimension and 28 inputs is the other dimension.
    X = tf.reshape(X, [-1, n_inputs])
    # after this, we can multiply by the weights metric, the X is now 2d and the weight['in'] is also 2d so they can be multiplied.  We calculate the x_in which is the result in the input hidden layer
    # X_in -> (128 batch * 28 steps, 128 hidden)
    X_in = tf.matmul(X, weights['in']) + biases['in']
    # once we've done that, we then reshape it back to 3 dimensions
    # X_in -> (128 batch, 28 steps, 128 hidden) - this 128 hidden is the hidden units in the cell (seen below)
    X_in = tf.reshape(X_in, [-1, n_steps, n_hidden_units])

    # cell LSTM, 128 hidden is the hidden units in the cell and we input the X_in result into the cell - this is 3d and the cell can accept 3d, these next two steps are used to define the cells.  LSTM has a forget gate and the forget_bias is the initial bias for the forget gate and it's set to 1 which is to say that the first few steps we want to open our forget gate, we don't want to forget anything in the beginning since the beginning is very important in the learning procedure.  So open the gate and don't forget anything, later on we can figure out how much we want to forget or don't want to forget.  lstm cell is divided into two states (c_state, h_state). c_state is what is happening in the main story (going back to our storyline example)or overall state, the h_state is the sub storyline/instance moment or latest state.  state_is_tuple just means that we pass a tuple of states, 2 states, for LSTM.  The basic RNN cell doesn't have 2 states, it doesn't have a c_state, just a h_state.  Tensorflow recommends state_is_tuple  = True over False, because the calculation is faster in the true state.
    lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden_units, forget_bias=1.0, state_is_tuple=True)
    # lstm cell is divided into two parts (c_state, h_state)
    _init_state = lstm_cell.zero_state(batch_size, dtype=tf.float32)  # lstm_cell.zero_state means we define the initial state as all zero.  So the state takes 1 input as the batch_size so we just pass batch_size in.  Once we define the initial state we can now use dynamic_rnn
    outputs, final_state = tf.nn.dynamic_rnn(lstm_cell, X_in, initial_state = _init_state, time_major = False)  # dynamic_rnn is the main loop/structure in the RNN, you can use just rnn but this one is considered better.  lstm_cell is just passed in from above.  X_in is from above as well, just reshaped.  initial_state is just the _init_state from above.  time_major = False, if you look at the what we have, (128 batch, 28 steps, 128 hidden), the time is 28 because we have 28 time steps and that is in the 2nd dimension which is not a major dimension so we pass the false in here.  If we have the time step in the 1st dimension then the time_major would be True.  From this loop, we output all outputs from every step, in other words, this output is the collection of all outputs for every step.  And the final_state is just one final state in the last time step


    # hidden layer for output as the final results, basically we want to calculate the outputs from the cell.  We have 2 options for this.
    results = tf.matmul(final_state[1], weights['out']) + biases['out']  # option 1, here we only care about the last output in the last step because the rest of them are meaningless we only care once we have reached through all the rows in the image and what are conclusion is at that point.  And the outputs at the last time step is also this state.  Thus we use final_state[1] with the 1 representing the h of the final state (h_state).  So use the h_state * the weights + biases give you the final result

    # # option 2 unpack to list [(batch, outputs)...] * steps, use the outputs itself and use the last output as the results once we've reach through all the rows.  Remember for each of the rows we have one output and the collection of outputs is stored in outputs so we want to unpack and transpose the outputs in order to get out the last output from the outputs
    # outputs = tf.unpack(tf.transpose(outputs, [1,0,2]))  # states is the last output
    # results = tf.matmul(outputs[-1], weights['out']) + biases['out']  # outputs[-1] is the last output in outputs.  These two steps are bascially the same as the one step we had in option 1 but there are times where that may not be the case and it's important to know both.  You can tell the options are the same when you run both and the first values are identical
    return results

def lstm_cell(i, o, state):
    m_input = tf.pack([i for _ in range(m_rows)])
    m_saved_output = tf.pack([o for _ in range(m_rows)])
    m_all = tf.batch_matmul(m_input, m_input_w) + tf.batch_matmul(m_saved_output, m_middle) + m_biases
    m_all = tf.unpack(m_all)
    input_gate = tf.sigmoid(m_all[m_input_index])
    forget_gate = tf.sigmoid(m_all[m_forget_index])
    update = m_all[m_update_index]
    state = forget_gate * state + input_gate * tf.tanh(update)
    output_gate = tf.sigmoid(m_all[m_output_index])
    return output_gate * tf.tanh(state), state

def decode_flow(img_str):
  # IMPORTANT NOTE: I am now resizing the flow frames before running through
  # the preprocessing. I was not doing that earlier (in the master). This leads
  # to the 66 number to drop to 63 on HMDB. But it should be fixable by
  # re-training with this setup
  with tf.variable_scope('decode_flow_frame'):
    img = tf.concat(2, [tf.image.decode_jpeg(el, channels=1)
      for el in tf.unpack(img_str)])
    # Always convert before resize, this is a bug in TF
    # https://github.com/tensorflow/tensorflow/issues/1763
    img = tf.image.convert_image_dtype(img, dtype=tf.float32)
    img = tf.image.resize_images(img, [IM_HT, IM_WD])
  return [img]

def get_preprocessing(name, is_training=False):
  """Returns preprocessing_fn(image, height, width, **kwargs).

  Args:
    name: The name of the preprocessing function.
    is_training: `True` if the model is being used for training and `False`
      otherwise.

  Returns:
    preprocessing_fn: A function that preprocessing a single image (pre-batch).
      It has the following signature:
        image = preprocessing_fn(image, output_height, output_width, ...).

  Raises:
    ValueError: If Preprocessing `name` is not recognized.
  """
  preprocessing_fn_map = {
      'vgg_ucf': vgg_ucf_preprocessing,
  }

  if name not in preprocessing_fn_map:
    raise ValueError('Preprocessing name [%s] was not recognized' % name)

  def preprocessing_fn(image, output_height, output_width, **kwargs):
    with tf.variable_scope('preprocess_image'):
      if len(image.get_shape()) == 3:
        return preprocessing_fn_map[name].preprocess_image(
            image, output_height, output_width, is_training=is_training, **kwargs)
      elif len(image.get_shape()) == 4:
        # preprocess all the images in one set in the same way by concat-ing
        # them in channels
        nImgs = image.get_shape().as_list()[0]
        final_img_concat = preprocessing_fn_map[name].preprocess_image(
            tf.concat(2, tf.unpack(image)),
            output_height, output_width, is_training=is_training, **kwargs)
        return tf.concat(0, tf.split(3, nImgs, final_img_concat))
      else:
        print('Incorrect dims image!')

  return preprocessing_fn

def rnn(input_data, weights, biases):
    # hidden layer for input to cell
    ########################################

    # transpose the inputs shape from
    # X?128 batch ,28 steps, 18 inputs?
    # ==> (128 batch * 28 steps, 28 inputs)
    input_data = tf.reshape(input_data, [-1, n_inputs])

    # into hidden
    # data_in = (128 batch * 28 steps, 128 hidden)
    data_in = tf.matmul(input_data, weights['in']) + biases['in']
    # data_in ==> (128 batch, 28 steps, 128 hidden_units)
    data_in = tf.reshape(data_in, [-1, n_steps, n_hidden_units])

    # cell
    ##########################################

    # basic LSTM Cell.
    lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden_units, forget_bias=1.0, state_is_tuple=True)
    # lstm cell is divided into two parts (c_state, h_state)
    _init_state = lstm_cell.zero_state(batch_size, dtype=tf.float32)

    # You have 2 options for following step.
    # 1: tf.nn.rnn(cell, inputs);
    # 2: tf.nn.dynamic_rnn(cell, inputs).
    # If use option 1, you have to modified the shape of data_in, go and check out this:
    # https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/recurrent_network.py
    # In here, we go for option 2.
    # dynamic_rnn receive Tensor (batch, steps, inputs) or (steps, batch, inputs) as data_in.
    # Make sure the time_major is changed accordingly.
    outputs, final_state = tf.nn.dynamic_rnn(lstm_cell, data_in, initial_state=_init_state, time_major=False)

    # hidden layer for output as the final results
    #############################################
    # results = tf.matmul(final_state[1], weights_1['out']) + biases_1['out']

    # # or
    # unpack to list [(batch, outputs)..] * steps
    outputs = tf.unpack(tf.transpose(outputs, [1, 0, 2]))    # states is the last outputs
    results = tf.matmul(outputs[-1], weights['out']) + biases['out']

    return results

def standard_nextstep_inference(cell, inputs, output_size, scope=None,
                                return_states=False):
    """Gets the forward pass of a standard model for next step prediction.

    Args:
        cell (tf.nn.rnn_cell.RNNCell): the cell to use at each step. For
            multiple layers, wrap them up in tf.nn.rnn_cell.MultiRNNCell.
        inputs: [sequence_length, batch_size, num_features] float input tensor.
            If the inputs are discrete, we expect this to already be embedded.
        output_size (int): the size we project the output to (ie. the number of
            symbols in the vocabulary).
        scope (Optional): variable scope, defaults to "rnn".
        return_states (Optional[bool]): whether or not to return the states
            as well as the projected logits.

    Returns:
        tuple: (initial_state, logits, final_state)
            - *initial_state* are float tensors representing the first state of
                the rnn, to pass states across batches.
            - *logits* is a list of [batch_size, vocab_size] float tensors
                representing the output of the network at each timestep.
            - *final_state* are float tensors containing the final state of
                the network, evaluate these to get a state to feed into
                initial_state next batch.
    """
    with tf.variable_scope(scope or "rnn") as scope:
        inputs = tf.unpack(inputs)
        batch_size = inputs[0].get_shape()[0].value
        initial_state = cell.zero_state(batch_size, tf.float32)

        states, final_state = tf.nn.rnn(
            cell, inputs, initial_state=initial_state, dtype=tf.float32,
            scope=scope)

        # we want to project them all at once, because it's faster
        logits = tf.concat(0, states)

        with tf.variable_scope('output_layer'):
            # output layer needs weights
            weights = tf.get_variable('weights',
                                      [cell.output_size, output_size])
            biases = tf.get_variable('bias',
                                     [output_size])
            logits = tf.nn.bias_add(tf.matmul(logits, weights), biases)
        # and split them back up to what we expect
        logits = tf.split(0, len(inputs), logits)

        if return_states:
            logits = (states, logits)

    return [initial_state], logits, [final_state]

def standard_nextstep_sample(cell, inputs, output_size, embedding, scope=None,
                             argmax=False, softmax_temperature=1):
    """Generate samples from the standard next step prediction model.
    Assumes that we are modelling sequence of discrete symbols.

    Args:
        cell (tf.nn.rnn_cell.RNNCell): a cell to reproduce the model.
        inputs: input variable, all but the first is ignored.
        output_size (int): the size of the vocabulary.
        embedding: the embedding matrix used.
        scope (Optional): variable scope, needs to line up with what was used
            to make the model for inference the first time around.
        argmax (Optional[bool]): if True, then instead of sampling we simply
            take the argmax of the logits, if False we put a softmax on
            first. Defaults to False.
        softmax_temperature (Optional[bool]): the temparature for softmax.
            The logits are divided by this before feeding into the softmax:
            a high value means higher entropy. Default 1.

    Returns:
        tuple: (initial_state, sequence, final_state) where
            - *initial_state* is a variable for the starting state of the net.
            - *sequence* is a list of len(inputs) length containing the sampled
                symbols.
            - *final_state* the finishing state of the network, to pass along.
    """
    # have to be quite careful to ensure the scopes line up
    with tf.variable_scope(scope or 'rnn') as scope:
        inputs = tf.unpack(inputs)
        batch_size = inputs[0].get_shape()[0].value
        initial_state = cell.zero_state(batch_size, tf.float32)

        # get the output weights
        with tf.variable_scope('output_layer'):
            weights = tf.get_variable('weights',
                                      [cell.output_size, output_size])
            biases = tf.get_variable('bias',
                                     [output_size])

        # choose an appropriate loop function
        sequence = []
        if argmax:
            loop_fn = argmax_and_embed(embedding, output_list=sequence,
                                       output_projection=(weights, biases))
        else:
            loop_fn = sample_and_embed(embedding, softmax_temperature,
                                       output_list=sequence,
                                       output_projection=(weights, biases))

        all_states, fstate = tf.nn.seq2seq.rnn_decoder(
            inputs, initial_state, cell, loop_function=loop_fn, scope=scope)

    return [initial_state], sequence, [fstate]

def __init__(self, input_size, target_size, num_layers, hidden_layer_size,
                 init_scale, dropout_keep_prob):
        """ A base class for LSTM models that includes predictions and loss.

        Args:
            input_size: An integer. The number of inputs per time step.
            target_size: An integer. The dimensionality of the one-hot
                encoded targets.
            num_layers: An integer. The number of hidden layers.
            hidden_layer_size: An integer. The number of hidden units per layer.
            init_scale: A float. All weights will be initialized using a
                uniform distribution over `[-init_scale, init_scale]`.
            dropout_keep_prob: A float. The fraction of inputs to keep whenever
                dropout is applied. Dropout is applied to the inputs/outputs
                of each time step, and is never applied across time steps.
        """

        self.input_size = input_size
        self.target_size = target_size
        self.num_layers = num_layers
        self.hidden_layer_size = hidden_layer_size
        self.init_scale = init_scale
        self.dropout_keep_prob = dropout_keep_prob

        self._inputs = tf.placeholder(
            tf.float32, shape=[None, None, input_size], name='inputs')
        self._resets = tf.placeholder(
            tf.bool, shape=[None, None, 1], name='resets')
        self._targets = tf.placeholder(
            tf.float32, shape=[None, None, target_size], name='targets')
        self._training = tf.placeholder(tf.bool, shape=[], name='training')

        outputs = self._compute_rnn_outputs()
        output_size = self._compute_rnn_output_size()

        initializer = tf.random_uniform_initializer(-self.init_scale,
                                                    self.init_scale)
        with tf.variable_scope('logits', initializer=initializer):
            W = tf.get_variable('W', shape=[output_size, self.target_size])
            b = tf.get_variable('b', shape=[self.target_size])
            outputs_matrix = tf.reshape(outputs, [-1, output_size])
            logits = tf.nn.xw_plus_b(outputs_matrix, W, b)
            batch_size, duration, _ = tf.unpack(tf.shape(self.inputs))
            logits_shape = tf.pack([batch_size, duration, self.target_size])
            self._logits = tf.reshape(logits, logits_shape, name='logits')

        with tf.variable_scope('loss'):
            logits = tf.reshape(self.logits, [-1, self.target_size])
            targets = tf.reshape(self.targets, [-1, self.target_size])
            cross_entropies = tf.nn.softmax_cross_entropy_with_logits(logits,
                                                                      targets)
            self._loss = tf.reduce_mean(cross_entropies, name='loss')

def create_graph(g):
    initer = tf.random_uniform_initializer(0.0,INIT_SCALE)

    with tf.variable_scope("graph", reuse=None, initializer=initer):
        g['x'] = list()
        g['y'] = list()
        g['s'] = list()
        g['seq_lengths'] = tf.placeholder(tf.int64,shape=[BATCH_SIZE]);

        for _ in range(UNROLLS):
            g['x'].append( tf.placeholder(tf.float32,shape=[BATCH_SIZE,INPUT_SIZE]) )
            g['y'].append( tf.placeholder(tf.float32,shape=[BATCH_SIZE,INPUT_SIZE]) )
            g['s'].append( tf.placeholder(tf.float32,shape=[BATCH_SIZE]) )

        num_inputs  = INPUT_SIZE * UNROLLS
        # num_outputs = OUTPUT_SIZE * UNROLLS

        g['w'] = tf.get_variable("softmax_w", [num_inputs,OUTPUT_SIZE])
        g['b'] = tf.get_variable("softmax_b", [OUTPUT_SIZE])

        g['cat_x'] = tf.concat(1, g['x'] )

        g['logits'] = tf.nn.xw_plus_b(g['cat_x'], g['w'], g['b'] )

        g['cat_y'] = tf.unpack(tf.reverse_sequence(tf.reshape( tf.concat(1, g['y'] ),
            [BATCH_SIZE,UNROLLS,OUTPUT_SIZE] ),g['seq_lengths'],1,0),axis=1)[0]

        g['loss'] = tf.nn.softmax_cross_entropy_with_logits(g['logits'], g['cat_y'])

        g['r_s'] = tf.unpack(tf.reverse_sequence(tf.transpose(
            tf.reshape( tf.concat(0, g['s'] ), [UNROLLS, BATCH_SIZE] ) ),
            g['seq_lengths'],1,0),axis=1)[0]

        g['train_loss'] = tf.mul( g['loss'], g['r_s'] )

        g['preds'] = tf.nn.softmax(g['logits'])

        g['class_preds'] =  tf.floor( g['preds'] + 0.5 )

        g['accy'] = tf.mul( g['class_preds'],  g['cat_y'] )

        g['w_accy'] = tf.mul(g['accy'], tf.reshape(g['r_s'],shape=[BATCH_SIZE,1]))

def build_rnn_layer(self, inputs_, train_phase):
    """
    Build the computation graph from inputs to outputs of the RNN layer.
    :param inputs_: [batch, t, emb], float
    :param train_phase: bool
    :return: rnn_outputs_: [batch, t, hid_dim], float
    """
    config = self.config

    def unrolled_rnn(cell, emb_inputs_, initial_state_, seq_len_):
      if not config.fix_seq_len:
        raise Exception("`config.fix_seq_len` should be set to `True` if using unrolled_rnn()")
      outputs = []
      state = initial_state_
      with tf.variable_scope("unrolled_rnn"):
        for t in range(config.max_seq_len):
          if t > 0:
            tf.get_variable_scope().reuse_variables()
          output, state = cell(emb_inputs_[:, t], state)  # [batch, hid_dim]
          outputs.append(output)
        rnn_outputs_ = tf.pack(outputs, axis=1)  # [batch, t, hid_dim]
      return rnn_outputs_
    def dynamic_rnn(cell, emb_inputs_, initial_state_, seq_len_):
      rnn_outputs_, last_states_ = tf.nn.dynamic_rnn(cell, emb_inputs_, initial_state=initial_state_,
                                                     sequence_length=seq_len_,
                                                     dtype=config.float_type)  # you should define dtype if initial_state is not provided
      return rnn_outputs_
    def bidirectional_rnn(cell, emb_inputs_, initial_state_, seq_len_):
      rnn_outputs_, output_states = tf.nn.bidirectional_dynamic_rnn(cell, cell, emb_inputs_, seq_len_,
                                                                    initial_state_, initial_state_, config.float_type)
      return tf.concat(2, rnn_outputs_)
    def rnn(cell, emb_inputs_, initial_state_, seq_len_):
      if not config.fix_seq_len:
        raise Exception("`config.fix_seq_len` should be set to `True` if using rnn()")
      inputs_ = tf.unpack(emb_inputs_, axis=1)
      outputs_, states_ = tf.nn.rnn(cell, inputs_, initial_state_, dtype=config.float_type, sequence_length=seq_len_)
      return outputs_

    if config.rnn == 'rnn':
      cell = tf.nn.rnn_cell.BasicRNNCell(config.hidden_dim)
    elif config.rnn == 'lstm':
      cell = tf.nn.rnn_cell.BasicLSTMCell(config.hidden_dim)
    elif config.rnn == 'gru':
      cell = tf.nn.rnn_cell.GRUCell(config.hidden_dim)
    else:
      raise Exception("`config.rnn` should be correctly defined.")

    if train_phase and config.keep_prob < 1:
      cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=config.keep_prob)

    if config.num_layers is not None and config.num_layers > 1:
      cell = tf.nn.rnn_cell.MultiRNNCell([cell] * config.num_layers)

    initial_state_ = cell.zero_state(config.batch_size, dtype=config.float_type)
    if config.use_seq_len_in_rnn:
      seq_len_ = self.seq_len_
    else:
      seq_len_ = None
    rnn_outputs_ = dynamic_rnn(cell, inputs_, initial_state_, seq_len_)  # [batch, time, hid_dim]
    return rnn_outputs_

def create_recurrent_unit(self, params):
        # Weights and Bias for input and hidden tensor
        self.Wi = tf.Variable(self.params[1])
        self.Ui = tf.Variable(self.params[2])
        self.bi = tf.Variable(self.params[3])

        self.Wf = tf.Variable(self.params[4])
        self.Uf = tf.Variable(self.params[5])
        self.bf = tf.Variable(self.params[6])

        self.Wog = tf.Variable(self.params[7])
        self.Uog = tf.Variable(self.params[8])
        self.bog = tf.Variable(self.params[9])

        self.Wc = tf.Variable(self.params[10])
        self.Uc = tf.Variable(self.params[11])
        self.bc = tf.Variable(self.params[12])
        params.extend([
            self.Wi, self.Ui, self.bi,
            self.Wf, self.Uf, self.bf,
            self.Wog, self.Uog, self.bog,
            self.Wc, self.Uc, self.bc])

        def unit(x, hidden_memory_tm1):
            previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1)

            # Input Gate
            i = tf.sigmoid(
                tf.matmul(x, self.Wi) +
                tf.matmul(previous_hidden_state, self.Ui) + self.bi
            )

            # Forget Gate
            f = tf.sigmoid(
                tf.matmul(x, self.Wf) +
                tf.matmul(previous_hidden_state, self.Uf) + self.bf
            )

            # Output Gate
            o = tf.sigmoid(
                tf.matmul(x, self.Wog) +
                tf.matmul(previous_hidden_state, self.Uog) + self.bog
            )

            # New Memory Cell
            c_ = tf.nn.tanh(
                tf.matmul(x, self.Wc) +
                tf.matmul(previous_hidden_state, self.Uc) + self.bc
            )

            # Final Memory cell
            c = f * c_prev + i * c_

            # Current Hidden state
            current_hidden_state = o * tf.nn.tanh(c)

            return tf.pack([current_hidden_state, c])

        return unit

def create_recurrent_unit(self):
        # Weights and Bias for input and hidden tensor
        self.Wi = tf.identity(self.lstm.Wi)
        self.Ui = tf.identity(self.lstm.Ui)
        self.bi = tf.identity(self.lstm.bi)

        self.Wf = tf.identity(self.lstm.Wf)
        self.Uf = tf.identity(self.lstm.Uf)
        self.bf = tf.identity(self.lstm.bf)

        self.Wog = tf.identity(self.lstm.Wog)
        self.Uog = tf.identity(self.lstm.Uog)
        self.bog = tf.identity(self.lstm.bog)

        self.Wc = tf.identity(self.lstm.Wc)
        self.Uc = tf.identity(self.lstm.Uc)
        self.bc = tf.identity(self.lstm.bc)

        def unit(x, hidden_memory_tm1):
            previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1)

            # Input Gate
            i = tf.sigmoid(
                tf.matmul(x, self.Wi) +
                tf.matmul(previous_hidden_state, self.Ui) + self.bi
            )

            # Forget Gate
            f = tf.sigmoid(
                tf.matmul(x, self.Wf) +
                tf.matmul(previous_hidden_state, self.Uf) + self.bf
            )

            # Output Gate
            o = tf.sigmoid(
                tf.matmul(x, self.Wog) +
                tf.matmul(previous_hidden_state, self.Uog) + self.bog
            )

            # New Memory Cell
            c_ = tf.nn.tanh(
                tf.matmul(x, self.Wc) +
                tf.matmul(previous_hidden_state, self.Uc) + self.bc
            )

            # Final Memory cell
            c = f * c_prev + i * c_

            # Current Hidden state
            current_hidden_state = o * tf.nn.tanh(c)

            return tf.pack([current_hidden_state, c])

        return unit

def update_recurrent_unit(self):
        # Weights and Bias for input and hidden tensor
        self.Wi = self.update_rate * self.Wi + (1 - self.update_rate) * tf.identity(self.lstm.Wi)
        self.Ui = self.update_rate * self.Ui + (1 - self.update_rate) * tf.identity(self.lstm.Ui)
        self.bi = self.update_rate * self.bi + (1 - self.update_rate) * tf.identity(self.lstm.bi)

        self.Wf = self.update_rate * self.Wf + (1 - self.update_rate) * tf.identity(self.lstm.Wf)
        self.Uf = self.update_rate * self.Uf + (1 - self.update_rate) * tf.identity(self.lstm.Uf)
        self.bf = self.update_rate * self.bf + (1 - self.update_rate) * tf.identity(self.lstm.bf)

        self.Wog = self.update_rate * self.Wog + (1 - self.update_rate) * tf.identity(self.lstm.Wog)
        self.Uog = self.update_rate * self.Uog + (1 - self.update_rate) * tf.identity(self.lstm.Uog)
        self.bog = self.update_rate * self.bog + (1 - self.update_rate) * tf.identity(self.lstm.bog)

        self.Wc = self.update_rate * self.Wc + (1 - self.update_rate) * tf.identity(self.lstm.Wc)
        self.Uc = self.update_rate * self.Uc + (1 - self.update_rate) * tf.identity(self.lstm.Uc)
        self.bc = self.update_rate * self.bc + (1 - self.update_rate) * tf.identity(self.lstm.bc)

        def unit(x, hidden_memory_tm1):
            previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1)

            # Input Gate
            i = tf.sigmoid(
                tf.matmul(x, self.Wi) +
                tf.matmul(previous_hidden_state, self.Ui) + self.bi
            )

            # Forget Gate
            f = tf.sigmoid(
                tf.matmul(x, self.Wf) +
                tf.matmul(previous_hidden_state, self.Uf) + self.bf
            )

            # Output Gate
            o = tf.sigmoid(
                tf.matmul(x, self.Wog) +
                tf.matmul(previous_hidden_state, self.Uog) + self.bog
            )

            # New Memory Cell
            c_ = tf.nn.tanh(
                tf.matmul(x, self.Wc) +
                tf.matmul(previous_hidden_state, self.Uc) + self.bc
            )

            # Final Memory cell
            c = f * c_prev + i * c_

            # Current Hidden state
            current_hidden_state = o * tf.nn.tanh(c)

            return tf.pack([current_hidden_state, c])

        return unit

def create_recurrent_unit(self, params):
        # Weights and Bias for input and hidden tensor
        self.Wi = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim]))
        self.Ui = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim]))
        self.bi = tf.Variable(self.init_matrix([self.hidden_dim]))

        self.Wf = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim]))
        self.Uf = tf.Variable(self.init_matrix([self.hidden_dim, self.hidden_dim]))
        self.bf = tf.Variable(self.init_matrix([self.hidden_dim]))

        self.Wog = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim]))
        self.Uog = tf.Variable(self.init_matrix([self.hidden_dim, self.hidden_dim]))
        self.bog = tf.Variable(self.init_matrix([self.hidden_dim]))

        self.Wc = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim]))
        self.Uc = tf.Variable(self.init_matrix([self.hidden_dim, self.hidden_dim]))
        self.bc = tf.Variable(self.init_matrix([self.hidden_dim]))
        params.extend([
            self.Wi, self.Ui, self.bi,
            self.Wf, self.Uf, self.bf,
            self.Wog, self.Uog, self.bog,
            self.Wc, self.Uc, self.bc])

        def unit(x, hidden_memory_tm1):
            previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1)

            # Input Gate
            i = tf.sigmoid(
                tf.matmul(x, self.Wi) +
                tf.matmul(previous_hidden_state, self.Ui) + self.bi
            )

            # Forget Gate
            f = tf.sigmoid(
                tf.matmul(x, self.Wf) +
                tf.matmul(previous_hidden_state, self.Uf) + self.bf
            )

            # Output Gate
            o = tf.sigmoid(
                tf.matmul(x, self.Wog) +
                tf.matmul(previous_hidden_state, self.Uog) + self.bog
            )

            # New Memory Cell
            c_ = tf.nn.tanh(
                tf.matmul(x, self.Wc) +
                tf.matmul(previous_hidden_state, self.Uc) + self.bc
            )

            # Final Memory cell
            c = f * c_prev + i * c_

            # Current Hidden state
            current_hidden_state = o * tf.nn.tanh(c)

            return tf.pack([current_hidden_state, c])

        return unit

def build_model(self):
        with tf.variable_scope("encoder"):
            source_xs = tf.nn.embedding_lookup(self.s_emb, self.source)
            source_xs = tf.split(1, self.max_size, source_xs)
        with tf.variable_scope("decoder"):
            target_xs = tf.nn.embedding_lookup(self.t_emb, self.target)
            target_xs = tf.split(1, self.max_size, target_xs)

        s = self.encoder.zero_state(self.batch_size, tf.float32)
        encoder_hs = []
        with tf.variable_scope("encoder"):
            for t in xrange(self.max_size):
                if t > 0: tf.get_variable_scope().reuse_variables()
                x = tf.squeeze(source_xs[t], [1])
                x = tf.matmul(x, self.s_proj_W) + self.s_proj_b
                h, s = self.encoder(x, s)
                encoder_hs.append(h)
        encoder_hs = tf.pack(encoder_hs)

        s = self.decoder.zero_state(self.batch_size, tf.float32)
        logits = []
        probs  = []
        with tf.variable_scope("decoder"):
            for t in xrange(self.max_size):
                if t > 0: tf.get_variable_scope().reuse_variables()
                if not self.is_test or t == 0:
                    x = tf.squeeze(target_xs[t], [1])
                x = tf.matmul(x, self.t_proj_W) + self.t_proj_b
                h_t, s = self.decoder(x, s)
                h_tld = self.attention(h_t, encoder_hs)

                oemb  = tf.matmul(h_tld, self.proj_W) + self.proj_b
                logit = tf.matmul(oemb, self.proj_Wo) + self.proj_bo
                prob  = tf.nn.softmax(logit)
                logits.append(logit)
                probs.append(prob)
                if self.is_test:
                    x = tf.cast(tf.argmax(prob, 1), tf.int32)
                    x = tf.nn.embedding_lookup(self.t_emb, x)

        logits     = logits[:-1]
        targets    = tf.split(1, self.max_size, self.target)[1:]
        weights    = tf.unpack(tf.sequence_mask(self.target_len - 1, self.max_size - 1,
                                                dtype=tf.float32), None, 1)
        self.loss  = tf.nn.seq2seq.sequence_loss(logits, targets, weights)
        self.probs = tf.transpose(tf.pack(probs), [1, 0, 2])

        self.optim = tf.contrib.layers.optimize_loss(self.loss, None,
                self.lr_init, "SGD", clip_gradients=5.,
                summaries=["learning_rate", "loss", "gradient_norm"])

        tf.initialize_all_variables().run()
        self.saver = tf.train.Saver()

def _reverse_seq(input_seq, lengths):
    """Reverse a list of Tensors up to specified lengths.
    Args:
        input_seq: Sequence of seq_len tensors of dimension (batch_size, n_features)
                   or nested tuples of tensors.
        lengths:   A `Tensor` of dimension batch_size, containing lengths for each
                   sequence in the batch. If "None" is specified, simply reverses
                   the list.
    Returns:
        time-reversed sequence
    """
    if lengths is None:
        return list(reversed(input_seq))

    flat_input_seq = tuple(nest.flatten(input_) for input_ in input_seq)

    flat_results = [[] for _ in range(len(input_seq))]
    for sequence in zip(*flat_input_seq):
        input_shape = tensor_shape.unknown_shape(
                ndims=sequence[0].get_shape().ndims)
        for input_ in sequence:
            input_shape.merge_with(input_.get_shape())
            input_.set_shape(input_shape)

        # Join into (time, batch_size, depth)
        s_joined = array_ops.pack(sequence)

        # TODO(schuster, ebrevdo): Remove cast when reverse_sequence takes int32
        if lengths is not None:
            lengths = math_ops.to_int64(lengths)

        # Reverse along dimension 0
        s_reversed = array_ops.reverse_sequence(s_joined, lengths, 0, 1)
        # Split again into list
        result = array_ops.unpack(s_reversed)
        for r, flat_result in zip(result, flat_results):
            r.set_shape(input_shape)
            flat_result.append(r)

    results = [nest.pack_sequence_as(structure=input_, flat_sequence=flat_result)
               for input_, flat_result in zip(input_seq, flat_results)]
    return results