python类map_fn()的实例源码

def load_images(image_files, resize=True):
  """Load images from files and optionally resize it."""

  images = []
  for image_file in image_files:
    with file_io.FileIO(image_file, 'r') as ff:
      images.append(ff.read())
  if resize is False:
    return images

  # To resize, run a tf session so we can reuse 'decode_and_resize()'
  # which is used in prediction graph. This makes sure we don't lose
  # any quality in prediction, while decreasing the size of the images
  # submitted to the model over network.
  image_str_tensor = tf.placeholder(tf.string, shape=[None])
  image = tf.map_fn(resize_image, image_str_tensor, back_prop=False)
  feed_dict = collections.defaultdict(list)
  feed_dict[image_str_tensor.name] = images
  with tf.Session() as sess:
    images_resized = sess.run(image, feed_dict=feed_dict)
  return images_resized

def build_inputs_and_outputs(self):
    if self.frame_features:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      fn = lambda x: self.build_prediction_graph(x)
      video_id_output, top_indices_output, top_predictions_output = (
          tf.map_fn(fn, serialized_examples,
                    dtype=(tf.string, tf.int32, tf.float32)))

    else:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      video_id_output, top_indices_output, top_predictions_output = (
          self.build_prediction_graph(serialized_examples))

    inputs = {"example_bytes":
              saved_model_utils.build_tensor_info(serialized_examples)}

    outputs = {
        "video_id": saved_model_utils.build_tensor_info(video_id_output),
        "class_indexes": saved_model_utils.build_tensor_info(top_indices_output),
        "predictions": saved_model_utils.build_tensor_info(top_predictions_output)}

    return inputs, outputs

def decode_jpeg_original(self, image_buffer, scope=None):
        with tf.op_scope([image_buffer], scope, 'decode_jpeg'):
            # decode jpeg
            fn = lambda encoded: tf.image.decode_jpeg(encoded,
                                                      channels=3,
                                                      ratio=FLAGS.decode_downsample_factor)
            # TODO: change parallel iterations
            decoded = tf.map_fn(fn,
                                image_buffer,
                                dtype=tf.uint8,
                                parallel_iterations=1,
                                back_prop=False,
                                swap_memory=False,
                                infer_shape=True,
                                name="map_decode_jpeg")

            # move the float convertion to GPU, to save bandwidth
            # to float, to the range 0.0 - 255.0
            #images = tf.cast(decoded, dtype=tf.float32, name="image_to_float")
            decoded.set_shape([FLAGS.FRAMES_IN_SEG // FLAGS.temporal_downsample_factor,
                              FLAGS.IM_HEIGHT / FLAGS.decode_downsample_factor,
                              FLAGS.IM_WIDTH / FLAGS.decode_downsample_factor, 3])
            return decoded

def decode_png(self, image_buffer, scope=None):
        with tf.op_scope([image_buffer], scope, 'decode_png'):
            # decode PNG
            fn = lambda encoded: tf.image.decode_png(encoded,
                                                      channels=1)

            decoded = tf.map_fn(fn,
                                image_buffer,
                                dtype=tf.uint8,
                                parallel_iterations=10,
                                back_prop=False,
                                swap_memory=False,
                                infer_shape=True,
                                name="map_decode_png")

            # to float, to the range 0.0 - 255.0
            images = tf.cast(decoded, dtype=tf.int32, name="image_to_float")
            tf.image.resize_nearest_neighbor(images, [FLAGS.IM_HEIGHT, FLAGS.IM_WIDTH], align_corners=None, name=None)
            images.set_shape([FLAGS.FRAMES_IN_SEG // FLAGS.temporal_downsample_factor,
                              FLAGS.IM_HEIGHT / FLAGS.decode_downsample_factor,
                              FLAGS.IM_WIDTH / FLAGS.decode_downsample_factor, 1])
            return images

def random_crop(images, height, width):
    """Randomly crops an image/images to a given size.

    Args:
        images: 4-D Tensor of shape `[batch, height, width, channels]` or
            3-D Tensor of shape `[height, width, channels]`.
        height: `float`. The height to crop to.
        width: `float`. The width to crop to.

    Returns:
        If `images` was 4-D, a 4-D float Tensor of shape
        `[batch, new_height, new_width, channels]`.
        If `images` was 3-D, a 3-D float Tensor of shape
        `[new_height, new_width, channels]`.
    """
    images_shape = get_shape(images)
    if len(images_shape) > 4:
        ValueError("'image' must have either 3 or 4 dimensions, "
                   "received `{}`.".format(images_shape))

    if len(images_shape) == 4:
        return tf.map_fn(lambda img: tf.random_crop(img, [height, width, images_shape[-1]]), images)

    return tf.random_crop(images, [height, width, images_shape[-1]])

def _sum_attentions(attentions, document):
    assert static_rank(attentions) == 2 and static_rank(document) == 2

    num_entities = tf.reduce_max(document) + 1

    @func_scope()
    def _sum_attention(args):
        attentions, document = args
        assert static_rank(attentions) == 1 and static_rank(document) == 1
        return tf.unsorted_segment_sum(attentions, document, num_entities)

    attentions = tf.map_fn(_sum_attention,
                           [attentions, document],
                           dtype=FLAGS.float_type)

    return attentions[:, FLAGS.first_entity_index:FLAGS.last_entity_index + 1]

def build_inputs_and_outputs(self):
    if self.frame_features:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      fn = lambda x: self.build_prediction_graph(x)
      video_id_output, top_indices_output, top_predictions_output = (
          tf.map_fn(fn, serialized_examples,
                    dtype=(tf.string, tf.int32, tf.float32)))

    else:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      video_id_output, top_indices_output, top_predictions_output = (
          self.build_prediction_graph(serialized_examples))

    inputs = {"example_bytes":
              saved_model_utils.build_tensor_info(serialized_examples)}

    outputs = {
        "video_id": saved_model_utils.build_tensor_info(video_id_output),
        "class_indexes": saved_model_utils.build_tensor_info(top_indices_output),
        "predictions": saved_model_utils.build_tensor_info(top_predictions_output)}

    return inputs, outputs

def build_inputs_and_outputs(self):
    if self.frame_features:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      fn = lambda x: self.build_prediction_graph(x)
      video_id_output, top_indices_output, top_predictions_output = (
          tf.map_fn(fn, serialized_examples,
                    dtype=(tf.string, tf.int32, tf.float32)))

    else:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      video_id_output, top_indices_output, top_predictions_output = (
          self.build_prediction_graph(serialized_examples))

    inputs = {"example_bytes":
              saved_model_utils.build_tensor_info(serialized_examples)}

    outputs = {
        "video_id": saved_model_utils.build_tensor_info(video_id_output),
        "class_indexes": saved_model_utils.build_tensor_info(top_indices_output),
        "predictions": saved_model_utils.build_tensor_info(top_predictions_output)}

    return inputs, outputs

def build_inputs_and_outputs(self):
    if self.frame_features:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      fn = lambda x: self.build_prediction_graph(x)
      video_id_output, top_indices_output, top_predictions_output = (
          tf.map_fn(fn, serialized_examples,
                    dtype=(tf.string, tf.int32, tf.float32)))

    else:
      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      video_id_output, top_indices_output, top_predictions_output = (
          self.build_prediction_graph(serialized_examples))

    inputs = {"example_bytes":
              saved_model_utils.build_tensor_info(serialized_examples)}

    outputs = {
        "video_id": saved_model_utils.build_tensor_info(video_id_output),
        "class_indexes": saved_model_utils.build_tensor_info(top_indices_output),
        "predictions": saved_model_utils.build_tensor_info(top_predictions_output)}

    return inputs, outputs

def process_leafs(self,emb):

        with tf.variable_scope("Composition",reuse=True):
            cU = tf.get_variable("cU",[self.emb_dim,2*self.hidden_dim])
            cb = tf.get_variable("cb",[4*self.hidden_dim])
            b = tf.slice(cb,[0],[2*self.hidden_dim])
            def _recurseleaf(x):

                concat_uo = tf.matmul(tf.expand_dims(x,0),cU) + b
                u,o = tf.split(axis=1,num_or_size_splits=2,value=concat_uo)
                o=tf.nn.sigmoid(o)
                u=tf.nn.tanh(u)

                c = u#tf.squeeze(u)
                h = o * tf.nn.tanh(c)


                hc = tf.concat(axis=1,values=[h,c])
                hc=tf.squeeze(hc)
                return hc

        hc = tf.map_fn(_recurseleaf,emb)
        return hc

def process_leafs(self,emb):

        with tf.variable_scope("Composition",reuse=True):
            cUW = tf.get_variable("cUW")
            cb = tf.get_variable("cb")
            U = tf.slice(cUW,[0,0],[self.emb_dim,2*self.hidden_dim])
            b = tf.slice(cb,[0],[2*self.hidden_dim])
            def _recurseleaf(x):

                concat_uo = tf.matmul(tf.expand_dims(x,0),U) + b
                u,o = tf.split(axis=1,num_or_size_splits=2,value=concat_uo)
                o=tf.nn.sigmoid(o)
                u=tf.nn.tanh(u)

                c = u#tf.squeeze(u)
                h = o * tf.nn.tanh(c)


                hc = tf.concat(axis=1,values=[h,c])
                hc=tf.squeeze(hc)
                return hc

            hc = tf.map_fn(_recurseleaf,emb)
            return hc

def preprocess(self, inputs):
    """Feature-extractor specific preprocessing.

    See base class.

    Args:
      inputs: a [batch, height_in, width_in, channels] float tensor representing
        a batch of images with values between 0 and 255.0.

    Returns:
      preprocessed_inputs: a [batch, height_out, width_out, channels] float
        tensor representing a batch of images.
    Raises:
      ValueError: if inputs tensor does not have type tf.float32
    """
    if inputs.dtype is not tf.float32:
      raise ValueError('`preprocess` expects a tf.float32 tensor')
    with tf.name_scope('Preprocessor'):
      # TODO: revisit whether to always use batch size as  the number of
      # parallel iterations vs allow for dynamic batching.
      resized_inputs = tf.map_fn(self._image_resizer_fn,
                                 elems=inputs,
                                 dtype=tf.float32)
      return self._feature_extractor.preprocess(resized_inputs)

def _tf_example_input_placeholder():
  """Returns input that accepts a batch of strings with tf examples.

  Returns:
    a tuple of placeholder and input nodes that output decoded images.
  """
  batch_tf_example_placeholder = tf.placeholder(
      tf.string, shape=[None], name='tf_example')
  def decode(tf_example_string_tensor):
    tensor_dict = tf_example_decoder.TfExampleDecoder().decode(
        tf_example_string_tensor)
    image_tensor = tensor_dict[fields.InputDataFields.image]
    return image_tensor
  return (batch_tf_example_placeholder,
          tf.map_fn(decode,
                    elems=batch_tf_example_placeholder,
                    dtype=tf.uint8,
                    parallel_iterations=32,
                    back_prop=False))

def multisample_conditional(self, X, full_cov=False):
        if full_cov is True:
            # this is unlikely to be called in a performance critical application, so we use
            # this clear but slow implementation
            f = lambda a: self.conditional(a, full_cov=full_cov)
            mean, var = tf.map_fn(f, X, dtype=(tf.float64, tf.float64))
            return tf.stack(mean), tf.stack(var)
        else:
            # this should be faster as only computes the Z_uu once, but could be made faster
            # still perhaps by avoiding reshaping (but need to rewrite conditional)
            S, N, D = shape_as_list(X)
            X_flat = tf.reshape(X, [S*N, D])
            mean, var = self.conditional(X_flat)
            return [tf.reshape(m, [S, N, -1]) for m in [mean, var]]

def build_likelihood(self):
        Fmean, Fvar = self.build_predict(self.X, full_cov=False, S=self.num_samples)

        S, N, D = shape_as_list(Fmean)
        Y = tile_over_samples(self.Y, self.num_samples)

        f = lambda a: self.likelihood.variational_expectations(a[0], a[1], a[2])
        var_exp = tf.map_fn(f, (Fmean, Fvar, Y), dtype=float_type)
        var_exp = tf.stack(var_exp) #SN

        var_exp = tf.reduce_mean(var_exp, 0) # S,N -> N. Average over samples
        L = tf.reduce_sum(var_exp) # N -> scalar. Sum over data (minibatch)

        KL = 0.
        for layer in self.layers:
            KL += layer.KL()

        scale = tf.cast(self.num_data, float_type)
        scale /= tf.cast(tf.shape(self.X)[0], float_type)  # minibatch size
        return L * scale - KL

def random_image_mirror_left_right(input_layer):
    """
    Flip each image left-right like in a mirror, randomly, even at test-time.

    This acts as a data augmentation technique. See:
    https://stackoverflow.com/questions/39574999/tensorflow-tf-image-functions-on-an-image-batch
    """
    return keras.layers.core.Lambda(function=lambda batch_imgs: tf.map_fn(
        lambda img: tf.image.random_flip_left_right(img), batch_imgs
    )
    )(input_layer)

def postprocess_images(self, ims):
        def _postprocess_images(im):
            im = tf.decode_raw(im, np.uint8)
            im = tf.image.convert_image_dtype(im, dtype=tf.float32)
            im = tf.reshape(im, [256, 256, 3])
            im = tf.random_crop(im, [self.crop_size, self.crop_size, 3])
            return im
        return tf.map_fn(lambda im: _postprocess_images(im), ims, dtype=tf.float32)

def get_words_from_chars(characters_list: List[str], sequence_lengths: List[int], name='chars_conversion'):
    with tf.name_scope(name=name):
        def join_charcaters_fn(coords):
            return tf.reduce_join(characters_list[coords[0]:coords[1]])

        def coords_several_sequences():
            end_coords = tf.cumsum(sequence_lengths)
            start_coords = tf.concat([[0], end_coords[:-1]], axis=0)
            coords = tf.stack([start_coords, end_coords], axis=1)
            coords = tf.cast(coords, dtype=tf.int32)
            return tf.map_fn(join_charcaters_fn, coords, dtype=tf.string)

        def coords_single_sequence():
            return tf.reduce_join(characters_list, keep_dims=True)

        words = tf.cond(tf.shape(sequence_lengths)[0] > 1,
                        true_fn=lambda: coords_several_sequences(),
                        false_fn=lambda: coords_single_sequence())

        return words

def attentive_matching(input_sentence, att_matrix, weights):
    """
    Parameters
    ----------
    input_sentence: Tensor
        Tensor of shape (batch_size, num_sentence_words, rnn_hidden_dim)

    att_matrix: Tensor
        Tensor of shape (batch_size, num_sentence_words, rnn_hidden_dim)
    """
    def single_instance(inputs):
        # Shapes: (num_sentence_words, rnn_hidden_dim)
        sentence_a_single = inputs[0]
        sentence_b_single_att = inputs[1]

        # Shapes: (num_sentence_words, multiperspective_dims, rnn_hidden_dim)
        expanded_sentence_a_single = multi_perspective_expand_for_2D(
            sentence_a_single, weights)
        expanded_sentence_b_single_att = multi_perspective_expand_for_2D(
            sentence_b_single_att, weights)
        # Shape: (num_sentence_words, multiperspective_dims)
        return cosine_distance(expanded_sentence_a_single,
                               expanded_sentence_b_single_att)

    elems = (input_sentence, att_matrix)
    # Shape: (batch_size, num_sentence_words, multiperspective_dims)
    return tf.map_fn(single_instance, elems, dtype="float")

def map_fn(fn, elems, name=None):
    '''Map the function fn over the elements elems and return the outputs.

    # Arguments
        fn: Callable that will be called upon each element in elems
        elems: tensor
        name: A string name for the map node in the graph

    # Returns
        Tensor with first dimension equal to the elems and second depending on
        fn
    '''
    return tf.map_fn(fn, elems, name=name)