def read_example(self, filename_queue):
# TFRecoard reader
reader = tf.TFRecordReader()
key, serialized_example = reader.read(filename_queue)
# read data from serialized examples
features = tf.parse_single_example(
serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string)
})
label = features['label']
image = features['image_raw']
# decode raw image data as integers
if self.image_format == 'jpeg':
decoded_image = tf.image.decode_jpeg(
image, channels=self.image_channels)
else:
decoded_image = tf.decode_raw(image, tf.uint8)
return decoded_image, label
python类TFRecordReader()的实例源码
def prepare_reader(self, filename_queue, batch_size=1024):
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return self.prepare_serialized_examples(serialized_example,
max_quantized_value, min_quantized_value)
def _tfrecord_to_graph_ops(self, num_epochs):
with tf.variable_scope('tfrec_to_graph'):
file_queue = tf.train.string_input_producer(
self.filenames_list,
name=self.name+'_file_queue',
num_epochs=num_epochs
)
reader = tf.TFRecordReader(
options=tf.python_io.TFRecordOptions(
compression_type=self.compression
), name=self.name+'_tfrec_reader'
)
_, tfrecord = reader.read(file_queue)
features, targets = parse_mnist_tfrec(
tfrecord, self.name, self.features_shape
)
return features, targets
def read_and_decode(filename_queue, batch_size):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
feature = features()
feature = tf.parse_single_example(
serialized_example,
features = feature,
)
hr_image = tf.decode_raw(feature['hr_image'], tf.uint8)
height = tf.cast(feature['height'], tf.int32)
width = tf.cast(feature['width'], tf.int32)
print(height)
image_shape = tf.stack([128, 128,3 ])
hr_image = tf.reshape(hr_image, image_shape)
hr_image = tf.image.random_flip_left_right(hr_image)
hr_image = tf.image.random_contrast(hr_image, 0.5, 1.3)
hr_images = tf.train.shuffle_batch([hr_image], batch_size = batch_size, capacity = 30,
num_threads = 2,
min_after_dequeue = 10)
return hr_images
def read(self, shuffle=True, num_epochs=None):
with tf.name_scope('input'):
reader = tf.TFRecordReader()
filename_queue = tf.train.string_input_producer([self.filename], num_epochs=num_epochs)
_, serialized_input = reader.read(filename_queue)
inputs = tf.parse_single_example(serialized_input,
features={
'inputs_seq': tf.FixedLenFeature([self.seq_len * 2 + 3], tf.int64),
'output': tf.FixedLenFeature([1], tf.int64)
})
inputs_seq = inputs['inputs_seq']
output = inputs['output']
min_after_dequeue = 100
if shuffle:
inputs_seqs, outputs = tf.train.shuffle_batch([inputs_seq, output], batch_size=self.batch_size, num_threads=2, capacity=min_after_dequeue + 3 * self.batch_size, min_after_dequeue=min_after_dequeue)
else:
inputs_seqs, outputs = tf.train.batch([inputs_seq, output], batch_size=self.batch_size)
return inputs_seqs, outputs
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return self.prepare_serialized_examples(serialized_example,
max_quantized_value, min_quantized_value)
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return self.prepare_serialized_examples(serialized_example,
max_quantized_value, min_quantized_value)
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label_raw': tf.FixedLenFeature([], tf.string),
})
image = tf.decode_raw(features['image_raw'], tf.int16)
image.set_shape([IMAGE_HEIGHT * IMAGE_WIDTH])
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
reshape_image = tf.reshape(image, [IMAGE_HEIGHT, IMAGE_WIDTH, 1])
label = tf.decode_raw(features['label_raw'], tf.uint8)
label.set_shape([CHARS_NUM * CLASSES_NUM])
reshape_label = tf.reshape(label, [CHARS_NUM, CLASSES_NUM])
return tf.cast(reshape_image, tf.float32), tf.cast(reshape_label, tf.float32)
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return self.prepare_serialized_examples(serialized_example,
max_quantized_value, min_quantized_value)
def read_and_decode_embedding(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'label': tf.FixedLenFeature(
[], tf.int64),
'sequence_raw': tf.FixedLenFeature(
[], tf.string),
})
sequence = features['sequence_raw']
# preprocess
s_decode = tf.decode_raw(sequence, tf.int32)
s_decode.set_shape([FLAGS.embed_length])
# Convert label from a scalar uint8 tensor to an int32 scalar.
label = tf.cast(features['label'], tf.int32)
return s_decode, label
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example, features={
'song_spec': tf.FixedLenFeature([], tf.string),
'voice_spec': tf.FixedLenFeature([], tf.string),
'mixed_spec': tf.FixedLenFeature([], tf.string)
})
song_spec = transform_spec_from_raw(features['song_spec'])
voice_spec = transform_spec_from_raw(features['voice_spec'])
mixed_spec = transform_spec_from_raw(features['mixed_spec'])
input_spec = stack_spectrograms(mixed_spec) # this will be the input
target_spec = tf.concat([song_spec, voice_spec], axis=1) # target spec is going to be a concatenation of song_spec and voice_spec
return input_spec, target_spec
def __init__(self, tfrecords_file, image_size=256,
min_queue_examples=1000, batch_size=1, num_threads=8, name=''):
"""
Args:
tfrecords_file: string, tfrecords file path
min_queue_examples: integer, minimum number of samples to retain in the queue that provides of batches of examples
batch_size: integer, number of images per batch
num_threads: integer, number of preprocess threads
"""
self.tfrecords_file = tfrecords_file
self.image_size = image_size
self.min_queue_examples = min_queue_examples
self.batch_size = batch_size
self.num_threads = num_threads
self.reader = tf.TFRecordReader()
self.name = name
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return self.prepare_serialized_examples(serialized_example,
max_quantized_value, min_quantized_value)
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)
if self.decode_zlib:
reader = tf.TFRecordReader(options=opts)
else:
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
tf.add_to_collection("serialized_examples", serialized_examples)
return self.prepare_serialized_examples(serialized_examples)
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return self.prepare_serialized_examples(serialized_example,
max_quantized_value, min_quantized_value)
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)
if self.decode_zlib:
reader = tf.TFRecordReader(options=opts)
else:
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
tf.add_to_collection("serialized_examples", serialized_examples)
return self.prepare_serialized_examples(serialized_examples)
def decode_from_tfrecords(filename,num_epoch=None):
filename_queue=tf.train.string_input_producer([filename],num_epochs=num_epoch)#???????????????????????????????????????
reader=tf.TFRecordReader()
_,serialized=reader.read(filename_queue)
example=tf.parse_single_example(serialized,features={
'height':tf.FixedLenFeature([],tf.int64),
'width':tf.FixedLenFeature([],tf.int64),
'nchannel':tf.FixedLenFeature([],tf.int64),
'image':tf.FixedLenFeature([],tf.string),
'label':tf.FixedLenFeature([],tf.int64)
})
label=tf.cast(example['label'], tf.int32)
image=tf.decode_raw(example['image'],tf.uint8)
image=tf.reshape(image,tf.pack([
tf.cast(example['height'], tf.int32),
tf.cast(example['width'], tf.int32),
tf.cast(example['nchannel'], tf.int32)]))
return image,label
def read_and_decode(filename, batch_size):
# ???????????
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue) # ????????
features = tf.parse_single_example(
serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'img_raw': tf.FixedLenFeature([], tf.string),
}
)
img = tf.decode_raw(features['img_raw'], tf.uint8)
print('xxxx: ', img.get_shape())
img = tf.reshape(img, [512, 144, 3])
img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
image_batch, label_batch = tf.train.batch([img, label],
batch_size=batch_size,
num_threads=64,
capacity=2000)
return image_batch, tf.reshape(label_batch, [batch_size])
ML_Final_Project.py 文件源码
项目：apparent-age-gender-classification
作者: danielyou0230
项目源码
文件源码
阅读 29
收藏 0
点赞 0
评论 0
def read_and_decode(filename, img_size=128, depth=1):
if not filename.endswith('.tfrecords'):
print "Invalid file \"{:s}\"".format(filename)
return [], []
else:
data_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(data_queue)
features = tf.parse_single_example(serialized_example,
features={
'label' : tf.FixedLenFeature([], tf.int64),
'img_raw' : tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['img_raw'], tf.uint8)
img = tf.reshape(img, [img_size, img_size, depth])
# Normalize the image
img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
label_onehot = tf.stack(tf.one_hot(label, n_classes))
return img, label_onehot
#read_and_decode('test.tfrecords')
ML_Final_Project_LBP.py 文件源码
项目：apparent-age-gender-classification
作者: danielyou0230
项目源码
文件源码
阅读 35
收藏 0
点赞 0
评论 0
def read_and_decode(filename, img_size=128, depth=1):
if not filename.endswith('.tfrecords'):
print "Invalid file \"{:s}\"".format(filename)
return [], []
else:
data_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(data_queue)
features = tf.parse_single_example(serialized_example,
features={
'label' : tf.FixedLenFeature([], tf.int64),
'img_raw' : tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['img_raw'], tf.uint8)
img = tf.reshape(img, [img_size, img_size, depth])
# Normalize the image
img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
label_onehot = tf.stack(tf.one_hot(label, n_classes))
return img, label_onehot
#read_and_decode('test.tfrecords')
def read_and_decode(filename, img_size=128, depth=1):
if not filename.endswith('.tfrecords'):
print "Invalid file \"{:s}\"".format(filename)
return [], []
else:
data_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(data_queue)
features = tf.parse_single_example(serialized_example,
features={
'label' : tf.FixedLenFeature([], tf.int64),
'img_raw' : tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['img_raw'], tf.uint8)
img = tf.reshape(img, [img_size, img_size, depth])
# Normalize the image
img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
label_onehot = tf.stack(tf.one_hot(label, n_classes))
return img, label_onehot
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return self.prepare_serialized_examples(serialized_example,
max_quantized_value, min_quantized_value)
def test_batch_randomized(self):
batch_size = 17
queue_capacity = 1234
name = "my_batch"
with tf.Graph().as_default() as g, self.test_session(graph=g) as sess:
inputs = tf.contrib.learn.io.read_batch_examples(
_VALID_FILE_PATTERN, batch_size,
reader=tf.TFRecordReader, randomize_input=True,
queue_capacity=queue_capacity, name=name)
self.assertEqual("%s:1" % name, inputs.name)
file_name_queue_name = "%s/file_name_queue" % name
file_names_name = "%s/input" % file_name_queue_name
example_queue_name = "%s/random_shuffle_queue" % name
op_nodes = test_util.assert_ops_in_graph({
file_names_name: "Const",
file_name_queue_name: "FIFOQueue",
"%s/read/TFRecordReader" % name: "TFRecordReader",
example_queue_name: "RandomShuffleQueue",
name: "QueueDequeueMany"
}, g)
self.assertEqual(
set(_FILE_NAMES), set(sess.run(["%s:0" % file_names_name])[0]))
self.assertEqual(
queue_capacity, op_nodes[example_queue_name].attr["capacity"].i)
def read_my_file_format(filename_queue, resize_shape=None):
"""Sets up part of the pipeline that takes elements from the filename queue
and turns it into a tf.Tensor of a batch of images.
:param filename_queue:
tf.train.string_input_producer object
:param resize_shape:
2 element list defining the shape to resize images to.
"""
reader = tf.TFRecordReader()
key, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example, features={
'image/encoded': tf.FixedLenFeature([], tf.string),
'image/height': tf.FixedLenFeature([], tf.int64),
'image/channels': tf.FixedLenFeature([], tf.int64),
'image/width': tf.FixedLenFeature([], tf.int64)})
example = tf.image.decode_jpeg(features['image/encoded'], 3)
processed_example = preprocessing(example, resize_shape)
return processed_example
def decode_image_objects(paths):
with tf.name_scope(inspect.stack()[0][3]):
with tf.name_scope('parse_example'):
reader = tf.TFRecordReader()
_, serialized = reader.read(tf.train.string_input_producer(paths))
example = tf.parse_single_example(serialized, features={
'imagepath': tf.FixedLenFeature([], tf.string),
'imageshape': tf.FixedLenFeature([3], tf.int64),
'objects': tf.FixedLenFeature([2], tf.string),
})
imagepath = example['imagepath']
objects = example['objects']
with tf.name_scope('decode_objects'):
objects_class = tf.decode_raw(objects[0], tf.int64, name='objects_class')
objects_coord = tf.decode_raw(objects[1], tf.float32)
objects_coord = tf.reshape(objects_coord, [-1, 4], name='objects_coord')
with tf.name_scope('load_image'):
imagefile = tf.read_file(imagepath)
image = tf.image.decode_jpeg(imagefile, channels=3)
return image, example['imageshape'], objects_class, objects_coord
retrain.py 文件源码
项目：Embarrassingly-Parallel-Image-Classification
作者: Azure
项目源码
文件源码
阅读 22
收藏 0
点赞 0
评论 0
def get_dataset(dataset_name, dataset_dir, image_count, class_count, split_name):
slim = tf.contrib.slim
items_to_descriptions = {'image': 'A color image.',
'label': 'An integer in range(0, class_count)'}
file_pattern = os.path.join(dataset_dir, '{}_{}_*.tfrecord'.format(dataset_name, split_name))
reader = tf.TFRecordReader
keys_to_features = {'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label': tf.FixedLenFeature([], tf.int64,
default_value=tf.zeros([], dtype=tf.int64))}
items_to_handlers = {'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label')}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers)
labels_to_names = read_label_file(dataset_dir)
return(slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=image_count,
items_to_descriptions=items_to_descriptions,
num_classes=class_count,
labels_to_names=labels_to_names,
shuffle=True))
def decode_record(filename_queue, patch_size,
channel_num=3):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image': tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['image'], tf.uint8)
img = tf.reshape(img, [patch_size, patch_size, channel_num])
img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
return img, label
def read(filename_queue, feature_num=2, dtypes=[list, int]):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
feature_dict={}
for i in range(feature_num):
# here, only three data types are allowed: tf.float32, tf.int64, tf.string
if dtypes[i] is int:
feature_dict['feature'+str(i+1)]=tf.FixedLenFeature([], tf.int64)
else:
feature_dict['feature'+str(i+1)]=tf.FixedLenFeature([], tf.string)
features = tf.parse_single_example(
serialized_example,
features=feature_dict)
return features
#======================================================================================
## test code
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'image_left': tf.FixedLenFeature([], tf.string),
'image_right': tf.FixedLenFeature([], tf.string),
})
image_left = tf.decode_raw(features['image_left'], tf.uint8)
image_right = tf.decode_raw(features['image_right'], tf.uint8)
width = 960
height = 540
depth = 4
image_left.set_shape([width*height*depth])
image_right.set_shape([width*height*depth])
return image_left, image_right
