python类read_file()的实例源码

def __read_imagenet(path, shuffle=True, save_file = 'imagenet_files.csv'):
    if not os.path.exists(save_file):
        def class_index(fn):
            class_id = re.search(r'(n\d+)', fn).group(1)
            return synset_map[class_id]['index']

        file_list = glob.glob(path+'/*/*.JPEG')
        label_indexes = []
        with open(save_file, 'wb') as csv_file:
            wr = csv.writer(csv_file, quoting=csv.QUOTE_NONE)
            for f in file_list:
                idx = class_index(f)
                label_indexes.append(idx)
                wr.writerow([f, idx])

    with open(save_file, 'rb') as f:
        reader = csv.reader(f)
        file_list = list(reader)
    file_tuple, label_tuple = zip(*file_list)

    filename, labels = tf.train.slice_input_producer([list(file_tuple), list(label_tuple)], shuffle=shuffle)
    images = tf.image.decode_jpeg(tf.read_file(filename), channels=3)
    images = tf.div(tf.add(tf.to_float(images), -127), 128)
    return images, tf.string_to_number(labels, tf.int32)

def read_tensor_from_image_file(self, image_file):
        input_name = "file_reader"
        file_reader = tf.read_file(image_file, input_name)
        if image_file.endswith(".png"):
            image_reader = tf.image.decode_png(file_reader, channels=3,
                                               name='png_reader')
        elif image_file.endswith(".gif"):
            image_reader = tf.squeeze(tf.image.decode_gif(file_reader,
                                                          name='gif_reader'))
        elif image_file.endswith(".bmp"):
            image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')
        else:
            image_reader = tf.image.decode_jpeg(file_reader, channels=3,
                                                name='jpeg_reader')

        image_preprocessing_fn = preprocessing_factory.get_preprocessing(
            self.model_name, is_training=False)
        processed_image = image_preprocessing_fn(image_reader, 224, 224)
        processed_images = tf.expand_dims(processed_image, 0)
        sess = tf.Session()
        im_result = sess.run(processed_images)
        return im_result

def read_tensor_from_image_file(file_name, input_height=299, input_width=299,
                input_mean=0, input_std=255):
  input_name = "file_reader"
  output_name = "normalized"
  file_reader = tf.read_file(file_name, input_name)
  if file_name.endswith(".png"):
    image_reader = tf.image.decode_png(file_reader, channels = 3,
                                       name='png_reader')
  elif file_name.endswith(".gif"):
    image_reader = tf.squeeze(tf.image.decode_gif(file_reader,
                                                  name='gif_reader'))
  elif file_name.endswith(".bmp"):
    image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')
  else:
    image_reader = tf.image.decode_jpeg(file_reader, channels = 3,
                                        name='jpeg_reader')
  float_caster = tf.cast(image_reader, tf.float32)
  dims_expander = tf.expand_dims(float_caster, 0);
  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
  sess = tf.Session()
  result = sess.run(normalized)

  return result

def read_images_from_disk(input_queue):
    """Consumes a single filename and label as a ' '-delimited string.
    Args:
      filename_and_label_tensor: A scalar string tensor.
    Returns:
      Two tensors: the decoded image, and the string label.
    """
    label = input_queue[1]
    file_contents = tf.read_file(input_queue[0])
    example = tf.image.decode_image(file_contents, channels=3)
    return example, label

def read_bbbc006(all_files_queue):
    """Reads and parses examples from BBBC006 data files.
    Recommendation: if you want N-way read parallelism, call this function
    N times.  This will give you N independent Readers reading different
    files & positions within those files, which will give better mixing of
    examples.

    Args:
        filename_queue: A queue of strings with the filenames to read from.
    Returns:
        An object representing a single example, with the following fields:
            label: a [height, width, 2] uint8 Tensor with contours tensor in depth 0 and
                segments tensor in depth 1.
            uint8image: a [height, width, depth] uint8 Tensor with the image data
    """

    class BBBC006Record(object):
        pass

    result = BBBC006Record()

    # Read a record, getting filenames from the filename_queue.
    text_reader = tf.TextLineReader()
    _, csv_content = text_reader.read(all_files_queue)

    i_path, c_path, s_path = tf.decode_csv(csv_content,
                                           record_defaults=[[""], [""], [""]])

    result.uint8image = read_from_queue(tf.read_file(i_path))
    contour = read_from_queue(tf.read_file(c_path))
    segment = read_from_queue(tf.read_file(s_path))

    result.label = tf.concat([contour, segment], 2)
    return result

def read_images_from_disk(input_queue, input_size, random_scale): 
    """Read one image and its corresponding mask with optional pre-processing.

    Args:
      input_queue: tf queue with paths to the image and its mask.
      input_size: a tuple with (height, width) values.
                  If not given, return images of original size.
      random_scale: whether to randomly scale the images prior
                    to random crop.

    Returns:
      Two tensors: the decoded image and its mask.
    """
    img_contents = tf.read_file(input_queue[0])
    label_contents = tf.read_file(input_queue[1])
    shape = input_queue[2]

    img = tf.image.decode_jpeg(img_contents, channels=3)
    label = tf.image.decode_png(label_contents, channels=1)
    if input_size is not None:
        h, w = input_size
        if random_scale:
            scale = tf.random_uniform([1], minval=0.75, maxval=1.25, dtype=tf.float32, seed=None)
            h_new = tf.to_int32(tf.multiply(tf.to_float(tf.shape(img)[0]), scale))
            w_new = tf.to_int32(tf.multiply(tf.to_float(tf.shape(img)[1]), scale))
            new_shape = tf.squeeze(tf.stack([h_new, w_new]), axis=[1])
            img = tf.image.resize_images(img, new_shape)
            label = tf.image.resize_nearest_neighbor(tf.expand_dims(label, 0), new_shape)
            label = tf.squeeze(label, axis=[0]) # resize_image_with_crop_or_pad accepts 3D-tensor.
        img = tf.image.resize_image_with_crop_or_pad(img, h, w)
        label = tf.image.resize_image_with_crop_or_pad(label, h, w)
    # RGB -> BGR.
    img_r, img_g, img_b = tf.split(axis=2, num_or_size_splits=3, value=img)
    img = tf.cast(tf.concat(axis=2, values=[img_b, img_g, img_r]), dtype=tf.float32)
    # Extract mean.
    img -= IMG_MEAN

    return img, label, shape

def load_image(self, image_path, is_jpeg):
        # Read the file
        file_data = tf.read_file(image_path)
        # Decode the image data
        img = tf.cond(
            is_jpeg,
            lambda: tf.image.decode_jpeg(file_data, channels=self.data_spec.channels),
            lambda: tf.image.decode_png(file_data, channels=self.data_spec.channels))
        if self.data_spec.expects_bgr:
            # Convert from RGB channel ordering to BGR
            # This matches, for instance, how OpenCV orders the channels.
            img = tf.reverse(img, [False, False, True])
        return img

def process(self):
    idx, image_path = self.path_queue.dequeue()
    img = tf.image.decode_jpeg(tf.read_file(image_path), channels=3) # It is an RGB PNG
    img = tf.reverse(img, [False, False, True]) # RGB -> BGR
    return (idx, ImageReader.process_single_image(img, self.image_spec['scale_size'], 
                                               self.image_spec['crop_size'], 
                                               self.image_spec['mean']))

def read_images_and_labels_from_disk(input_queue):
    """ Reads images and labels from disk and sets their shape

      Args:
        input_queue: created by slice_input_producer
    """
    image_contents = tf.read_file(input_queue[0])
    label_contents = tf.read_file(input_queue[1])
    image = tf.image.decode_png(image_contents, IMAGE_CHANNELS)
    label = tf.image.decode_png(label_contents, LABEL_CHANNELS)
    image.set_shape([IMAGE_HEIGHT_ORIG, IMAGE_WIDTH_ORIG, IMAGE_CHANNELS])
    label.set_shape([IMAGE_HEIGHT_ORIG, IMAGE_WIDTH_ORIG, LABEL_CHANNELS])
    print("Image has dtype", image.dtype, "and shape", image.get_shape(), "after decoding from disk.")
    print("Label has dtype", label.dtype, "and shape", label.get_shape(), "after decoding from disk.")
    return image, label

def tf_ops(self, capacity=32):
        im_path, label_path = tf.train.slice_input_producer(
            [tf.constant(self.images), tf.constant(self.labels)],
            capacity=capacity,
            shuffle=self.shuffle)
        im_shape = [1024, 1024 + self.overlap, 3]
        label_shape = [1024, 1024 + self.overlap]
        queue = tf.FIFOQueue(capacity, [tf.float32, tf.int32],
                             shapes=[im_shape, label_shape])
        im = tf.read_file(im_path)
        im = tf.image.decode_image(im, channels=3)
        im = tf.cast(im, tf.float32)
        left_im = im[:, :1024 + self.overlap, :]
        right_im = im[:, 1024 - self.overlap:, :]
        left_im.set_shape(im_shape)
        right_im.set_shape(im_shape)
        label = tf.read_file(label_path)
        label = tf.image.decode_image(label, channels=1)
        label = label[:, :, 0]
        label = tf.cast(label, tf.int32)
        label_pad = tf.ones([1024, self.overlap], dtype=tf.int32) * 255
        left_label = tf.concat([label[:, :1024], label_pad], 1)
        right_label = tf.concat([label_pad, label[:, 1024:]], 1)
        left_label.set_shape(label_shape)
        right_label.set_shape(label_shape)
        ims = tf.stack([left_im, right_im], 0)
        labels = tf.stack([left_label, right_label], 0)
        enqueue_op = queue.enqueue_many([ims, labels])
        qr = tf.train.QueueRunner(queue, [enqueue_op])
        tf.train.add_queue_runner(qr)
        return queue.dequeue()

def tf_ops(self, capacity=32):
        images = ops.convert_to_tensor(self._image_fn_list, dtype=dtypes.string)
        labels = ops.convert_to_tensor(self._label_list, dtype=dtypes.int32)

        # Makes an input queue
        im_fn_q, labl_q = tf.train.slice_input_producer(
            [images, labels], capacity=capacity, shuffle=True)

        file_contents_q = tf.read_file(im_fn_q)
        im_q = self._decoder(file_contents_q, channels=3)

        return im_q, labl_q

def mp3_tensors_from_directory(directory, batch_size, channels=2, format='mp3', seconds=30, bitrate=16384):
      filenames = glob.glob(directory+"/**/*."+format)
      labels,total_labels = build_labels(sorted(glob.glob(directory+"/*")))
      num_examples_per_epoch = 10000

      # Create a queue that produces the filenames to read.
      classes = [labels[f.split('/')[-2]] for f in filenames]
      print("Found files", len(filenames))

      filenames = tf.convert_to_tensor(filenames, dtype=tf.string)
      classes = tf.convert_to_tensor(classes, dtype=tf.int32)
      print("[0]", filenames[0], classes[0])

      input_queue = tf.train.slice_input_producer([filenames, classes])

      # Read examples from files in the filename queue.
      print("INPUT_QUEUE", input_queue[0])
      value = tf.read_file(input_queue[0])
      #preprocess = tf.read_file(input_queue[0]+'.preprocess')

      print("Preloaded data", value)
      #print("Loaded data", data)

      label = input_queue[1]

      min_fraction_of_examples_in_queue = 0.4
      min_queue_examples = int(num_examples_per_epoch *
                               min_fraction_of_examples_in_queue)

      #data = tf.cast(data, tf.float32)
      data = ffmpeg.decode_audio(value, file_format=format, samples_per_second=bitrate, channel_count=channels)
      data = shared.resize_audio_patch.resize_audio_with_crop_or_pad(data, seconds*bitrate*channels, 0,True)
      #data = tf.slice(data, [0,0], [seconds*bitrate, channels])
      tf.Tensor.set_shape(data, [seconds*bitrate, channels])
      #data = tf.minimum(data, 1)
      #data = tf.maximum(data, -1)
      data = data/tf.reduce_max(tf.reshape(tf.abs(data),[-1]))
      print("DATA IS", data)
      x,y=_get_data(data, label, min_queue_examples, batch_size)

      return x, y, total_labels, num_examples_per_epoch

def read_image_jpg(image_path, depth=3, scale=True):
    """Reads the image from image_path (tf.string tensor) [jpg image].
    Cast the result to float32 and if scale=True scale it in [-1,1]
    using scale_image. Otherwise the values are in [0,1]
    Reuturn:
        the decoded jpeg image, casted to float32
    """

    image = tf.image.convert_image_dtype(
        tf.image.decode_jpeg(tf.read_file(image_path), channels=depth),
        dtype=tf.float32)
    if scale:
        image = scale_image(image)
    return image

def read_image_png(image_path, depth=3, scale=True):
    """Reads the image from image_path (tf.string tensor) [jpg image].
    Cast the result to float32 and if scale=True scale it in [-1,1]
    using scale_image. Otherwise the values are in [0,1]
    Reuturn:
        the decoded jpeg image, casted to float32
    """
    image = tf.image.convert_image_dtype(
        tf.image.decode_png(tf.read_file(image_path), channels=depth),
        dtype=tf.float32)
    if scale:
        image = scale_image(image)
    return image

def read_dataset(self, queue, input_size, data_format):
        image = tf.image.decode_jpeg(
            tf.read_file(queue[0]), channels=3, name=self.scope+'/image')
        label = tf.image.decode_png(
            tf.read_file(queue[1]), channels=1, name=self.scope+'/label')
        image = tf.image.resize_images(image, input_size)
        label = tf.image.resize_images(label, input_size, 1)
        if data_format == 'NCHW':
            self.channel_axis = 1
            image = tf.transpose(image, [2, 0, 1])
            label = tf.transpose(label, [2, 0, 1])
        image -= tf.reduce_mean(tf.cast(image, dtype=tf.float32),
                                (0, 1), name=self.scope+'/mean')
        return image, label

def read_my_file_format(self, filename_queue):
        reader = tf.TextLineReader()
        key, record_string = reader.read(filename_queue)
        # "a" means representative value to indicate type for csv cell value.
        image_file_name, depth_file_name = tf.decode_csv(record_string, [["a"], ["a"]])

        image_png_data = tf.read_file(image_file_name)
        depth_png_data = tf.read_file(depth_file_name)
        # channels=1 means image is read as gray-scale
        image_decoded = tf.image.decode_png(image_png_data, channels=1)
        image_decoded.set_shape([512, 512, 1])
        depth_decoded = tf.image.decode_png(depth_png_data, channels=1)
        depth_decoded.set_shape([512, 512, 1])
        return image_decoded, depth_decoded

def read_images_from_disk(input_queue, input_size, random_scale): 
    """Read one image and its corresponding mask with optional pre-processing.

    Args:
      input_queue: tf queue with paths to the image and its mask.
      input_size: a tuple with (height, width) values.
                  If not given, return images of original size.
      random_scale: whether to randomly scale the images prior
                    to random crop.

    Returns:
      Two tensors: the decoded image and its mask.
    """
    img_contents = tf.read_file(input_queue[0])
    label_contents = tf.read_file(input_queue[1])

    img = tf.image.decode_jpeg(img_contents, channels=3)
    label = tf.image.decode_png(label_contents, channels=1)
    if input_size is not None:
        h, w = input_size
        if random_scale:
            scale = tf.random_uniform([1], minval=0.75, maxval=1.25, dtype=tf.float32, seed=None)
            h_new = tf.to_int32(tf.mul(tf.to_float(tf.shape(img)[0]), scale))
            w_new = tf.to_int32(tf.mul(tf.to_float(tf.shape(img)[1]), scale))
            new_shape = tf.squeeze(tf.pack([h_new, w_new]), squeeze_dims=[1])
            img = tf.image.resize_images(img, new_shape)
            label = tf.image.resize_nearest_neighbor(tf.expand_dims(label, 0), new_shape)
            label = tf.squeeze(label, squeeze_dims=[0]) # resize_image_with_crop_or_pad accepts 3D-tensor.
        img = tf.image.resize_image_with_crop_or_pad(img, h, w)
        label = tf.image.resize_image_with_crop_or_pad(label, h, w)
    # RGB -> BGR.
    img_r, img_g, img_b = tf.split(split_dim=2, num_split=3, value=img)
    img = tf.cast(tf.concat(2, [img_b, img_g, img_r]), dtype=tf.float32)
    # Extract mean.
    img -= IMG_MEAN 
    return img, label

def read_dataset(self, queue, input_size, data_format):
        image = tf.image.decode_jpeg(
            tf.read_file(queue[0]), channels=3, name=self.scope+'/image')
        label = tf.image.decode_png(
            tf.read_file(queue[1]), channels=1, name=self.scope+'/label')
        image = tf.image.resize_images(image, input_size)
        label = tf.image.resize_images(label, input_size, 1)
        if data_format == 'NCHW':
            self.channel_axis = 1
            image = tf.transpose(image, [2, 0, 1])
            label = tf.transpose(label, [2, 0, 1])
        image -= tf.reduce_mean(tf.cast(image, dtype=tf.float32),
                                (0, 1), name=self.scope+'/mean')
        return image, label

def distorted_inputs():
    data = load_data(FLAGS.data_dir)

    filenames = [ d['filename'] for d in data ]
    label_indexes = [ d['label_index'] for d in data ]

    filename, label_index = tf.train.slice_input_producer([filenames, label_indexes], shuffle=True)

    num_preprocess_threads = 4
    images_and_labels = []
    for thread_id in range(num_preprocess_threads):
        image_buffer = tf.read_file(filename)

        bbox = []
        train = True
        image = image_preprocessing(image_buffer, bbox, train, thread_id)
        images_and_labels.append([image, label_index])

    images, label_index_batch = tf.train.batch_join(
        images_and_labels,
        batch_size=FLAGS.batch_size,
        capacity=2 * num_preprocess_threads * FLAGS.batch_size)

    height = FLAGS.input_size
    width = FLAGS.input_size
    depth = 3

    images = tf.cast(images, tf.float32)
    images = tf.reshape(images, shape=[FLAGS.batch_size, height, width, depth])

    return images, tf.reshape(label_index_batch, [FLAGS.batch_size])

def distorted_inputs():
    data = load_data(FLAGS.data_dir)

    filenames = [ d['filename'] for d in data ]
    label_indexes = [ d['label_index'] for d in data ]

    filename, label_index = tf.train.slice_input_producer([filenames, label_indexes], shuffle=True)

    num_preprocess_threads = 4
    images_and_labels = []
    for thread_id in range(num_preprocess_threads):
        image_buffer = tf.read_file(filename)

        bbox = []
        train = True
        image = image_preprocessing(image_buffer, bbox, train, thread_id)
        images_and_labels.append([image, label_index])

    images, label_index_batch = tf.train.batch_join(
        images_and_labels,
        batch_size=FLAGS.batch_size,
        capacity=2 * num_preprocess_threads * FLAGS.batch_size)

    height = FLAGS.input_size
    width = FLAGS.input_size
    depth = 3

    images = tf.cast(images, tf.float32)
    images = tf.reshape(images, shape=[FLAGS.batch_size, height, width, depth])

    return images, tf.reshape(label_index_batch, [FLAGS.batch_size])