python类parse_example()的实例源码

def parse_examples(examples):
  feature_map = {
      'labels': tf.FixedLenFeature(
          shape=[], dtype=tf.int64, default_value=[-1]),
      'images': tf.FixedLenFeature(
          shape=[IMAGE_PIXELS], dtype=tf.float32),
  }
  return tf.parse_example(examples, features=feature_map)

def parse_examples(examples):
  feature_map = {
      'labels':
          tf.FixedLenFeature(
              shape=[], dtype=tf.int64, default_value=[-1]),
      'images':
          tf.FixedLenFeature(
              shape=[IMAGE_PIXELS], dtype=tf.float32),
  }
  return tf.parse_example(examples, features=feature_map)

def prepare_serialized_examples(self, serialized_examples, width=50, height=50):
    # set the mapping from the fields to data types in the proto
    feature_map = {
           'image': tf.FixedLenFeature((), tf.string, default_value=''),
           'label': tf.FixedLenFeature([], tf.int64)
    }
    features = tf.parse_example(serialized_examples, features=feature_map)

    def decode_and_resize(image_str_tensor):
      """Decodes png string, resizes it and returns a uint8 tensor."""

      # Output a grayscale (channels=1) image
      image = tf.image.decode_png(image_str_tensor, channels=1)

      # Note resize expects a batch_size, but tf_map supresses that index,
      # thus we have to expand then squeeze.  Resize returns float32 in the
      # range [0, uint8_max]
      image = tf.expand_dims(image, 0)
      image = tf.image.resize_bilinear(
          image, [height, width], align_corners=False)
      image = tf.squeeze(image, squeeze_dims=[0])
      image = tf.cast(image, dtype=tf.uint8)
      return image

    images_str_tensor = features["image"]
    images = tf.map_fn(
        decode_and_resize, images_str_tensor, back_prop=False, dtype=tf.uint8)
    images = tf.image.convert_image_dtype(images, dtype=tf.float32)
    images = tf.subtract(images, 0.5)
    images = tf.multiply(images, 2.0)

    def dense_to_one_hot(label_batch, num_classes):
      one_hot = tf.map_fn(lambda x : tf.cast(slim.one_hot_encoding(x, num_classes), tf.int32), label_batch)
      one_hot = tf.reshape(one_hot, [-1, num_classes])
      return one_hot

    labels = tf.cast(features['label'], tf.int32)
    labels = dense_to_one_hot(labels, 10)

    return images, labels

def prepare_serialized_examples(self, serialized_examples, width=50, height=50):
    # set the mapping from the fields to data types in the proto
    feature_map = {
           'image': tf.FixedLenFeature((), tf.string, default_value=''),
           'image_id': tf.FixedLenFeature((), tf.string, default_value=''),
    }
    features = tf.parse_example(serialized_examples, features=feature_map)

    def decode_and_resize(image_str_tensor):
      """Decodes png string, resizes it and returns a uint8 tensor."""

      # Output a grayscale (channels=1) image
      image = tf.image.decode_png(image_str_tensor, channels=1)

      # Note resize expects a batch_size, but tf_map supresses that index,
      # thus we have to expand then squeeze.  Resize returns float32 in the
      # range [0, uint8_max]
      image = tf.expand_dims(image, 0)
      image = tf.image.resize_bilinear(
          image, [height, width], align_corners=False)
      image = tf.squeeze(image, squeeze_dims=[0])
      image = tf.cast(image, dtype=tf.uint8)
      return image

    images_str_tensor = features["image"]
    images = tf.map_fn(
        decode_and_resize, images_str_tensor, back_prop=False, dtype=tf.uint8)
    images = tf.image.convert_image_dtype(images, dtype=tf.float32)
    images = tf.subtract(images, 0.5)
    images = tf.multiply(images, 2.0)

    image_id = features["image_id"]
    return image_id, images

def prepare_serialized_examples(self, serialized_examples, width=32, height=32, channels=3):
    # set the mapping from the fields to data types in the proto
    feature_map = {
           'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
           'image/filename': tf.FixedLenFeature((), tf.string, default_value='')
    }
    features = tf.parse_example(serialized_examples, features=feature_map)


    def decode_and_resize(image_str_tensor):
      """Decodes jpeg string, resizes it and returns a uint8 tensor."""

      image = tf.image.decode_jpeg(image_str_tensor, channels=channels)

      # Note resize expects a batch_size, but tf_map supresses that index,
      # thus we have to expand then squeeze.  Resize returns float32 in the
      # range [0, uint8_max]
      image = tf.expand_dims(image, 0)
      image = tf.image.resize_bilinear(
          image, [height, width], align_corners=False)
      image = tf.squeeze(image, squeeze_dims=[0])
      image = tf.cast(image, dtype=tf.uint8)
      return image

    images_str_tensor = features["image/encoded"]
    images = tf.map_fn(
        decode_and_resize, images_str_tensor, back_prop=False, dtype=tf.uint8)
    images = tf.image.convert_image_dtype(images, dtype=tf.float32)
    images = tf.subtract(images, 0.5)
    images = tf.multiply(images, 2.0)


    image_ids = features['image/filename']

    return image_ids, images

def from_feature_spec(feature_spec):
  """Convert a feature_spec to a Schema.

  Args:
    feature_spec: a features specification in the format expected by
        tf.parse_example(), i.e.
        `{name: FixedLenFeature(...), name: VarLenFeature(...), ...'

  Returns:
    A Schema representing the provided set of columns.
  """
  return Schema({
      key: _from_parse_feature(parse_feature)
      for key, parse_feature in six.iteritems(feature_spec)
  })

def build_prediction_graph(self, export_dir):
    """Builds prediction graph and registers appropriate endpoints."""
    logging.info('Exporting prediction graph to %s', export_dir)
    examples = tf.placeholder(tf.string, shape=(None,))
    features = {
        'image': tf.FixedLenFeature(
            shape=[IMAGE_PIXELS], dtype=tf.float32),
        'key': tf.FixedLenFeature(
            shape=[], dtype=tf.string),
    }

    parsed = tf.parse_example(examples, features)
    images = parsed['image']
    keys = parsed['key']

    # Build a Graph that computes predictions from the inference model.
    logits = inference(images, self.hidden1, self.hidden2)
    softmax = tf.nn.softmax(logits)
    prediction = tf.argmax(softmax, 1)

    # Mark the inputs and the outputs
    # Marking the input tensor with an alias with suffix _bytes. This is to
    # indicate that this tensor value is raw bytes and will be base64 encoded
    # over HTTP.
    # Note that any output tensor marked with an alias with suffix _bytes, shall
    # be base64 encoded in the HTTP response. To get the binary value, it
    # should be base64 decoded.
    tf.add_to_collection('inputs',
                         json.dumps({'examples_bytes': examples.name}))
    tf.add_to_collection('outputs', json.dumps({
        'key': keys.name,
        'prediction': prediction.name,
        'scores': softmax.name
    }))

def parse_examples(examples):
  feature_map = {
      'labels': tf.FixedLenFeature(
          shape=[], dtype=tf.int64, default_value=[-1]),
      'images': tf.FixedLenFeature(
          shape=[IMAGE_PIXELS], dtype=tf.float32),
  }
  return tf.parse_example(examples, features=feature_map)

def build_prediction_graph(self, export_dir):
    """Builds prediction graph and registers appropriate endpoints."""
    logging.info('Exporting prediction graph to %s', export_dir)
    examples = tf.placeholder(tf.string, shape=(None,))
    features = {
        'image': tf.FixedLenFeature(
            shape=[IMAGE_PIXELS], dtype=tf.float32),
        'key': tf.FixedLenFeature(
            shape=[], dtype=tf.string),
    }

    parsed = tf.parse_example(examples, features)
    images = parsed['image']
    keys = parsed['key']

    # Build a Graph that computes predictions from the inference model.
    logits = inference(images, self.hidden1, self.hidden2)
    softmax = tf.nn.softmax(logits)
    prediction = tf.argmax(softmax, 1)

    # Mark the inputs and the outputs
    # Marking the input tensor with an alias with suffix _bytes. This is to
    # indicate that this tensor value is raw bytes and will be base64 encoded
    # over HTTP.
    # Note that any output tensor marked with an alias with suffix _bytes, shall
    # be base64 encoded in the HTTP response. To get the binary value, it
    # should be base64 decoded.
    tf.add_to_collection('inputs',
                         json.dumps({'examples_bytes': examples.name}))
    tf.add_to_collection('outputs', json.dumps({
        'key': keys.name,
        'prediction': prediction.name,
        'scores': softmax.name
    }))

def parse_examples(examples):
  feature_map = {
      'labels': tf.FixedLenFeature(
          shape=[], dtype=tf.int64, default_value=[-1]),
      'images': tf.FixedLenFeature(
          shape=[IMAGE_PIXELS], dtype=tf.float32),
  }
  return tf.parse_example(examples, features=feature_map)

def parse_instances(self, instances, prediction=False):
    """Parses input instances according to the associated schema.

    Arguments:
      instances: The tensor containing input strings.
      prediction: Whether the instances are being parsed for producing predictions or not.
    Returns:
      A dictionary of tensors key'ed by field names.
    """
    # Convert the schema into an equivalent Example schema (expressed as features in Example
    # terminology).
    features = {}
    for field in self.schema:
      if field.type == SchemaFieldType.integer:
        dtype = tf.int64
        default_value = [0]
      elif field.type == SchemaFieldType.real:
        dtype = tf.float32
        default_value = [0.0]
      else:
        # discrete
        dtype = tf.string
        default_value = ['']

      if field.length == 0:
        feature = tf.VarLenFeature(dtype=dtype)
      else:
        if field.length != 1:
          default_value = default_value * field.length
        feature = tf.FixedLenFeature(shape=[field.length], dtype=dtype, default_value=default_value)

      features[field.name] = feature

    return tf.parse_example(instances, features, name='examples')

def prepare_serialized_examples(self, serialized_examples, width=50, height=50):
    # set the mapping from the fields to data types in the proto
    feature_map = {
           'image': tf.FixedLenFeature((), tf.string, default_value=''),
           'label': tf.FixedLenFeature([], tf.int64)
    }
    features = tf.parse_example(serialized_examples, features=feature_map)

    def decode_and_resize(image_str_tensor):
      """Decodes png string, resizes it and returns a uint8 tensor."""

      # Output a grayscale (channels=1) image
      image = tf.image.decode_png(image_str_tensor, channels=1)

      # Note resize expects a batch_size, but tf_map supresses that index,
      # thus we have to expand then squeeze.  Resize returns float32 in the
      # range [0, uint8_max]
      image = tf.expand_dims(image, 0)
      image = tf.image.resize_bilinear(
          image, [height, width], align_corners=False)
      image = tf.squeeze(image, squeeze_dims=[0])
      image = tf.cast(image, dtype=tf.uint8)
      return image

    images_str_tensor = features["image"]
    images = tf.map_fn(
        decode_and_resize, images_str_tensor, back_prop=False, dtype=tf.uint8)
    images = tf.image.convert_image_dtype(images, dtype=tf.float32)
    images = tf.subtract(images, 0.5)
    images = tf.multiply(images, 2.0)

    def dense_to_one_hot(label_batch, num_classes):
      one_hot = tf.map_fn(lambda x : tf.cast(slim.one_hot_encoding(x, num_classes), tf.int32), label_batch)
      one_hot = tf.reshape(one_hot, [-1, num_classes])
      return one_hot

    labels = tf.cast(features['label'], tf.int32)
    labels = dense_to_one_hot(labels, 10)

    return images, labels

def prepare_serialized_examples(self, serialized_examples, width=50, height=50):
    # set the mapping from the fields to data types in the proto
    feature_map = {
           'image': tf.FixedLenFeature((), tf.string, default_value=''),
           'image_id': tf.FixedLenFeature((), tf.string, default_value=''),
    }
    features = tf.parse_example(serialized_examples, features=feature_map)

    def decode_and_resize(image_str_tensor):
      """Decodes png string, resizes it and returns a uint8 tensor."""

      # Output a grayscale (channels=1) image
      image = tf.image.decode_png(image_str_tensor, channels=1)

      # Note resize expects a batch_size, but tf_map supresses that index,
      # thus we have to expand then squeeze.  Resize returns float32 in the
      # range [0, uint8_max]
      image = tf.expand_dims(image, 0)
      image = tf.image.resize_bilinear(
          image, [height, width], align_corners=False)
      image = tf.squeeze(image, squeeze_dims=[0])
      image = tf.cast(image, dtype=tf.uint8)
      return image

    images_str_tensor = features["image"]
    images = tf.map_fn(
        decode_and_resize, images_str_tensor, back_prop=False, dtype=tf.uint8)
    images = tf.image.convert_image_dtype(images, dtype=tf.float32)
    images = tf.subtract(images, 0.5)
    images = tf.multiply(images, 2.0)

    image_id = features["image_id"]
    return image_id, images

def prepare_serialized_examples(self, serialized_examples, width=32, height=32, channels=3):
    # set the mapping from the fields to data types in the proto
    feature_map = {
           'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
           'image/filename': tf.FixedLenFeature((), tf.string, default_value='')
    }
    features = tf.parse_example(serialized_examples, features=feature_map)


    def decode_and_resize(image_str_tensor):
      """Decodes jpeg string, resizes it and returns a uint8 tensor."""

      image = tf.image.decode_jpeg(image_str_tensor, channels=channels)

      # Note resize expects a batch_size, but tf_map supresses that index,
      # thus we have to expand then squeeze.  Resize returns float32 in the
      # range [0, uint8_max]
      image = tf.expand_dims(image, 0)
      image = tf.image.resize_bilinear(
          image, [height, width], align_corners=False)
      image = tf.squeeze(image, squeeze_dims=[0])
      image = tf.cast(image, dtype=tf.uint8)
      return image

    images_str_tensor = features["image/encoded"]
    images = tf.map_fn(
        decode_and_resize, images_str_tensor, back_prop=False, dtype=tf.uint8)
    images = tf.image.convert_image_dtype(images, dtype=tf.float32)
    images = tf.subtract(images, 0.5)
    images = tf.multiply(images, 2.0)


    image_ids = features['image/filename']

    return image_ids, images

def build_prediction_graph(self):
    """Builds prediction graph and registers appropriate endpoints."""
    examples = tf.placeholder(tf.string, shape=(None,))
    features = {
        'image': tf.FixedLenFeature(
            shape=[IMAGE_PIXELS], dtype=tf.float32),
        'key': tf.FixedLenFeature(
            shape=[], dtype=tf.string),
    }

    parsed = tf.parse_example(examples, features)
    images = parsed['image']
    keys = parsed['key']

    # Build a Graph that computes predictions from the inference model.
    logits = inference(images, self.hidden1, self.hidden2)
    softmax = tf.nn.softmax(logits)
    prediction = tf.argmax(softmax, 1)

    # Mark the inputs and the outputs
    # Marking the input tensor with an alias with suffix _bytes. This is to
    # indicate that this tensor value is raw bytes and will be base64 encoded
    # over HTTP.
    # Note that any output tensor marked with an alias with suffix _bytes, shall
    # be base64 encoded in the HTTP response. To get the binary value, it
    # should be base64 decoded.
    tf.add_to_collection('inputs',
                         json.dumps({'examples_bytes': examples.name}))
    tf.add_to_collection('outputs',
                         json.dumps({
                             'key': keys.name,
                             'prediction': prediction.name,
                             'scores': softmax.name
                         }))

def parse_examples(examples):
  feature_map = {
      'labels': tf.FixedLenFeature(
          shape=[], dtype=tf.int64, default_value=[-1]),
      'images': tf.FixedLenFeature(
          shape=[IMAGE_PIXELS], dtype=tf.float32),
  }
  return tf.parse_example(examples, features=feature_map)

def parse_examples(examples):
  feature_map = {
      'labels':
          tf.FixedLenFeature(
              shape=[], dtype=tf.int64, default_value=[-1]),
      'images':
          tf.FixedLenFeature(
              shape=[IMAGE_PIXELS], dtype=tf.float32),
  }
  return tf.parse_example(examples, features=feature_map)

def import_images(tfrecord_file_names, max_reads=100, batch_size=50):
    with tf.variable_scope('import'):

        training_filename_queue = tf.train.string_input_producer(tfrecord_file_names, num_epochs=None)

        reader_options = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.GZIP)
        reader = tf.TFRecordReader(options=reader_options)

        keys, values = reader.read_up_to(training_filename_queue, max_reads)
        features = tf.parse_example(
            values,
            features={
                'raw': tf.FixedLenFeature([], tf.string),
                'type': tf.FixedLenFeature([], tf.int64)
            })

        types = features['type']
        images = tf.decode_raw(features['raw'], tf.uint8)
        images = tf.reshape(images, shape=(-1, 180, 320, 3))
        images = tf.image.convert_image_dtype(images, dtype=tf.float32)

        image_batch, type_batch = tf.train.shuffle_batch(
            [images, types],
            enqueue_many=True,
            batch_size=batch_size,
            min_after_dequeue=batch_size,
            allow_smaller_final_batch=True,
            capacity=2000,
            name='shuffle_batch')

        return image_batch, type_batch

def parse_serialized_examples_batch(serialized_examples_batch, batch_size):
    feature_to_tensor = {
        'image': tf.FixedLenFeature([], tf.string),
        'height': tf.FixedLenFeature([1], tf.int64),
        'width': tf.FixedLenFeature([1], tf.int64),
        'label': tf.VarLenFeature(tf.int64),
        'label_length': tf.FixedLenFeature([1], tf.int64)
    }
    features = tf.parse_example(serialized_examples_batch, feature_to_tensor)

    class ocrRecord(object):
        pass

    result = ocrRecord()

    result.heights = tf.cast(features['height'], tf.int32)
    result.widths = tf.cast(features['width'], tf.int32)
    result.depth = 1

    # shape_1d = result.height * result.width * result.depth
    shape_1d = IMAGE_HEIGHT * IMAGE_WIDTH * IMAGE_DEPTH

    def decode_image_string(string):
        decoded_image = tf.decode_raw(string, tf.uint8)
        return tf.cast(decoded_image, tf.uint8)

    imgs_1d = tf.map_fn(decode_image_string, features['image'], dtype=tf.uint8,
                        back_prop=False, parallel_iterations=15)

    imgs_1d = tf.reshape(imgs_1d, [batch_size, shape_1d])
    imgs_1d.set_shape([batch_size, shape_1d])

    result.uint8images = tf.reshape(imgs_1d, [batch_size, IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_DEPTH])
    result.uint8images.set_shape([batch_size, IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_DEPTH])

    result.label_lengths = tf.cast(features['label_length'], tf.int32)
    result.label_lengths = tf.reshape(result.label_lengths, [batch_size])
    result.label_lengths.set_shape([batch_size])

    result.labels = tf.cast(features['label'], tf.int32)

    # Convert for timestep input
    result.uint8image = tf.transpose(result.uint8images, [0, 2, 1, 3])
    return result

def prepare_serialized_examples(self, serialized_examples, width=32, height=32, channels=3):
    # set the mapping from the fields to data types in the proto
    feature_map = {
           'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
           'image/filename': tf.FixedLenFeature((), tf.string, default_value=''),
           'image/class/label': tf.FixedLenFeature([], tf.int64)
    }
    features = tf.parse_example(serialized_examples, features=feature_map)


    def decode_and_resize(image_str_tensor):
      """Decodes jpeg string, resizes it and returns a uint8 tensor."""

      image = tf.image.decode_jpeg(image_str_tensor, channels=channels)

      # Note resize expects a batch_size, but tf_map supresses that index,
      # thus we have to expand then squeeze.  Resize returns float32 in the
      # range [0, uint8_max]
      image = tf.expand_dims(image, 0)
      image = tf.image.resize_bilinear(
          image, [height, width], align_corners=False)
      image = tf.squeeze(image, squeeze_dims=[0])
      image = tf.cast(image, dtype=tf.uint8)
      return image

    images_str_tensor = features["image/encoded"]
    images = tf.map_fn(
        decode_and_resize, images_str_tensor, back_prop=False, dtype=tf.uint8)
    images = tf.image.convert_image_dtype(images, dtype=tf.float32)
    images = tf.subtract(images, 0.5)
    images = tf.multiply(images, 2.0)

    def dense_to_one_hot(label_batch, num_classes):
      one_hot = tf.map_fn(lambda x : tf.cast(slim.one_hot_encoding(x, num_classes), tf.int32), label_batch)
      one_hot = tf.reshape(one_hot, [-1, num_classes])
      return one_hot

    labels = tf.cast(features['image/class/label'], tf.int32)
    labels = tf.reshape(labels, [-1, 1])


    image_ids = features['image/filename']

    return image_ids, images, labels