def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example,features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label_raw': tf.FixedLenFeature([], tf.string)})
image = tf.cast(tf.decode_raw(features['image_raw'], tf.int16), tf.float32)
labels = tf.decode_raw(features['label_raw'], tf.int16)
#PW 2017/03/03: Zero-center data here?
image.set_shape([IMG_DIM*IMG_DIM*IMG_DIM])
image = tf.reshape(image, [IMG_DIM,IMG_DIM,IMG_DIM,1])
labels.set_shape([IMG_DIM*IMG_DIM*IMG_DIM])
labels = tf.reshape(image, [IMG_DIM,IMG_DIM,IMG_DIM])
# Dimensions (X, Y, Z, channles)
return image, labels
python类parse_single_example()的实例源码
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
"label": tf.FixedLenFeature([], tf.float32),
"categorical_features": tf.FixedLenFeature([CATEGORICAL_FEATURES_SIZE], tf.string),
"continuous_features": tf.FixedLenFeature([CONTINUOUS_FEATURES_SIZE], tf.float32),
})
label = features["label"]
continuous_features = features["continuous_features"]
categorical_features = tf.cast(tf.string_to_hash_bucket(features["categorical_features"], BUCKET_SIZE), tf.float32)
return label, tf.concat(0, [continuous_features, categorical_features])
# Read serialized examples from filename queue
def test_parse_spec():
fc = FeatureColumns(
True,
False,
VOCAB_FILE,
VOCAB_SIZE,
10,
10,
1000,
10)
parse_spec = tf.feature_column.make_parse_example_spec(fc)
print parse_spec
reader = tf.python_io.tf_record_iterator(INPUT_FILE)
sess = tf.Session()
for record in reader:
example = tf.parse_single_example(
record,
parse_spec)
print sess.run(example)
break
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'image_raw': tf.FixedLenFeature([], tf.string),
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
image = tf.reshape(image, [227, 227, 6])
# Convert from [0, 255] -> [-0.5, 0.5] floats.
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
return tf.split(image, 2, 2) # 3rd dimension two parts
def read_and_decode_aug(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'image_raw': tf.FixedLenFeature([], tf.string),
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
image = tf.image.random_flip_left_right(tf.reshape(image, [227, 227, 6]))
# Convert from [0, 255] -> [-0.5, 0.5] floats.
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
image = tf.image.random_brightness(image, 0.01)
image = tf.image.random_contrast(image, 0.95, 1.05)
return tf.split(image, 2, 2) # 3rd dimension two parts
def read_from_tfrecord(filenames):
tfrecord_file_queue = tf.train.string_input_producer(filenames,name='queue')
reader = tf.TFRecordReader()
_,tfrecord_serialized = reader.read(tfrecord_file_queue)
tfrecord_features = tf.parse_single_example(tfrecord_serialized,features={
'label':tf.FixedLenFeature([],tf.int64),
'shape':tf.FixedLenFeature([],tf.string),
'image':tf.FixedLenFeature([],tf.string),
},name='features')
image = tf.decode_raw(tfrecord_features['image'],tf.uint8)
shape = tf.decode_raw(tfrecord_features['shape'],tf.int32)
image = tf.reshape(image,shape)
label = tfrecord_features['label']
return label,shape,image
def read_decode_tfrecords(records_path, num_epochs=1020, batch_size=Flags.batch_size, num_threads=2):
if gfile.IsDirectory(records_path):
records_path = [os.path.join(records_path, i) for i in os.listdir(records_path)]
else:
records_path = [records_path]
records_path_queue = tf.train.string_input_producer(records_path, seed=123,
num_epochs=num_epochs,
name="string_input_producer")
reader = tf.TFRecordReader()
_, serialized_example = reader.read(records_path_queue, name="serialized_example")
features = tf.parse_single_example(serialized=serialized_example,
features={"img_raw": tf.FixedLenFeature([], tf.string),
"label": tf.FixedLenFeature([], tf.int64),
"height": tf.FixedLenFeature([], tf.int64),
"width": tf.FixedLenFeature([], tf.int64),
"depth": tf.FixedLenFeature([], tf.int64)},
name="parse_single_example")
image = tf.decode_raw(features["img_raw"], tf.uint8, name="decode_raw")
image.set_shape([height * width * 3])
image = tf.cast(image, tf.float32) * (1.0 / 255) - 0.5
label = tf.cast(features["label"], tf.int32)
images, labels = tf.train.shuffle_batch([image, label], batch_size=batch_size, num_threads=num_threads,
name="shuffle_bath", capacity=1020, min_after_dequeue=64)
print("images' shape is :", str(images.shape))
return images, labels
def read_decode_tfrecords(records_path, num_epochs=1, batch_size=Flags.batch_size, num_threads=1):
if gfile.IsDirectory(records_path):
records_path = [os.path.join(records_path, i) for i in os.listdir(records_path)]
else:
records_path = [records_path]
records_path_queue = tf.train.string_input_producer(records_path, seed=123,
num_epochs=None,
name="string_input_producer")
reader = tf.TFRecordReader()
_, serialized_example = reader.read(records_path_queue, name="serialized_example")
features = tf.parse_single_example(serialized=serialized_example,
features={"img_raw": tf.FixedLenFeature([], tf.string),
"label": tf.FixedLenFeature([], tf.int64),
"height": tf.FixedLenFeature([], tf.int64),
"width": tf.FixedLenFeature([], tf.int64),
"depth": tf.FixedLenFeature([], tf.int64)},
name="parse_single_example")
image = tf.decode_raw(features["img_raw"], tf.uint8, name="decode_raw")
image.set_shape([IMAGE_PIXELS])
image = tf.cast(image, tf.float32) * (1.0 / 255) - 0.5
label = tf.cast(features["label"], tf.int32)
images, labels = tf.train.shuffle_batch([image, label], batch_size=batch_size, num_threads=num_threads,
name="shuffle_bath", capacity=1020, min_after_dequeue=50)
return images, labels
def read_decode_tfrecords(records_path, num_epochs=1020, batch_size=Flags.batch_size, num_threads=2):
if gfile.IsDirectory(records_path):
records_path = [os.path.join(records_path, i) for i in os.listdir(records_path)]
else:
records_path = [records_path]
records_path_queue = tf.train.string_input_producer(records_path, seed=123,
# num_epochs=num_epochs,
name="string_input_producer")
reader = tf.TFRecordReader()
_, serialized_example = reader.read(records_path_queue, name="serialized_example")
features = tf.parse_single_example(serialized=serialized_example,
features={"img_raw": tf.FixedLenFeature([], tf.string),
"label": tf.FixedLenFeature([], tf.int64),
"height": tf.FixedLenFeature([], tf.int64),
"width": tf.FixedLenFeature([], tf.int64),
"depth": tf.FixedLenFeature([], tf.int64)},
name="parse_single_example")
image = tf.decode_raw(features["img_raw"], tf.uint8, name="decode_raw")
image.set_shape([IMAGE_PIXELS])
image = tf.cast(image, tf.float32) * (1.0 / 255) - 0.5
label = tf.cast(features["label"], tf.int32)
# images, labels = tf.train.shuffle_batch([image, label], batch_size=batch_size, num_threads=num_threads,
# name="shuffle_bath", capacity=1020, min_after_dequeue=64)
return image, label
def read_and_decode(filename):
# generate a queue with a given file name
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue) # return the file and the name of file
features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example
features={
'label': tf.FixedLenFeature([], tf.int64),
'img_raw' : tf.FixedLenFeature([], tf.string),
})
# You can do more image distortion here for training data
img = tf.decode_raw(features['img_raw'], tf.uint8)
img = tf.reshape(img, [224, 224, 3])
# img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
return img, label
def read_and_decode(filename):
# generate a queue with a given file name
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue) # return the file and the name of file
features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example
features={
'label': tf.FixedLenFeature([], tf.int64),
'img_raw' : tf.FixedLenFeature([], tf.string),
})
# You can do more image distortion here for training data
img = tf.decode_raw(features['img_raw'], tf.uint8)
img = tf.reshape(img, [224, 224, 3])
# img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
return img, label
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.VarLenFeature(tf.int64),
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
image = tf.reshape(image, [730, 38])
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
return image, label
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.VarLenFeature(tf.int64),
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
image = tf.reshape(image, [730, 38])
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
return image, label
def _read_example(filename_queue, n_labels=50, n_samples=59049):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'raw_labels': tf.FixedLenFeature([], tf.string),
'raw_segment': tf.FixedLenFeature([], tf.string)
})
segment = tf.decode_raw(features['raw_segment'], tf.float32)
segment.set_shape([n_samples])
labels = tf.decode_raw(features['raw_labels'], tf.uint8)
labels.set_shape([n_labels])
labels = tf.cast(labels, tf.float32)
return segment, labels
def read_and_decode(filename, one_hot=True, n_classes=None):
""" Return tensor to read from TFRecord """
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example,
features={
'label': tf.FixedLenFeature([],
tf.int64),
'image_raw': tf.FixedLenFeature([],
tf.string),
})
# You can do more image distortion here for training data
img = tf.decode_raw(features['image_raw'], tf.uint8)
img.set_shape([28 * 28])
img = tf.reshape(img, [28, 28, 1])
img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
if one_hot and n_classes:
label = tf.one_hot(label, n_classes)
return img, label
def parse_example_proto(example_serialized):
# Dense features in Example proto.
feature_map = {
'image/encoded': tf.FixedLenFeature([], dtype=tf.string,
default_value=''),
'image/filename': tf.FixedLenFeature([], dtype=tf.string,
default_value=''),
'image/class/label': tf.FixedLenFeature([1], dtype=tf.int64,
default_value=-1),
'image/class/text': tf.FixedLenFeature([], dtype=tf.string,
default_value=''),
'image/height': tf.FixedLenFeature([1], dtype=tf.int64,
default_value=-1),
'image/width': tf.FixedLenFeature([1], dtype=tf.int64,
default_value=-1),
}
features = tf.parse_single_example(example_serialized, feature_map)
label = tf.cast(features['image/class/label'], dtype=tf.int32)
return features['image/encoded'], label, features['image/filename']
def read_and_decode2(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'file_bytes': tf.FixedLenFeature([], tf.string),
})
# decode the png image
image = tf.image.decode_png(features['file_bytes'], channels=3)
# Convert to float image
image = tf.cast(image, tf.float32)
image.set_shape((IMAGE_SIZE, IMAGE_SIZE, CHANNELS))
# convert to grayscale if needed
if CHANNELS == 1:
image = tf.reduce_mean(image, reduction_indices=[2], keep_dims=True)
# normalize
image = image * (2. / 255) - 1
return image
def read_from_tfrecord(filenames):
tfrecord_file_queue = tf.train.string_input_producer(filenames, name='queue')
reader = tf.TFRecordReader()
_, tfrecord_serialized = reader.read(tfrecord_file_queue)
# label and image are stored as bytes but could be stored as
# int64 or float64 values in a serialized tf.Example protobuf.
tfrecord_features = tf.parse_single_example(tfrecord_serialized,
features={
'label': tf.FixedLenFeature([], tf.int64),
'shape': tf.FixedLenFeature([], tf.string),
'image': tf.FixedLenFeature([], tf.string),
}, name='features')
# image was saved as uint8, so we have to decode as uint8.
image = tf.decode_raw(tfrecord_features['image'], tf.uint8)
shape = tf.decode_raw(tfrecord_features['shape'], tf.int32)
# the image tensor is flattened out, so we have to reconstruct the shape
image = tf.reshape(image, shape)
label = tfrecord_features['label']
return label, shape, image
def read_and_decode(filename):
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'img_raw': tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['img_raw'], tf.uint8)
img = tf.reshape(img, [28, 28, 3])
img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
label = tf.cast(features['label'], tf.int32)
return img, label
def read_and_decode(record_file):
print(record_file)
# read_and_decode_test(record_file)
data_queue = tf.train.input_producer([record_file], capacity=1e5, name="string_input_producer")
reader = tf.TFRecordReader()
_, serialized_example = reader.read(data_queue)
features = tf.parse_single_example(
serialized_example,
features={'label': tf.FixedLenFeature([], tf.int64),
'target': tf.FixedLenFeature([], tf.float32),
'data': tf.FixedLenFeature([cfg.time_step * 4], tf.float32)})
data_raw = features['data']
label = features['label']
target = features['target']
data = tf.reshape(data_raw, [cfg.time_step, 4])
data.set_shape([cfg.time_step, 4])
if cfg.is_training:
data_batch, label_batch, target_batch = tf.train.batch([data, label, target],
batch_size=cfg.batch_size,
capacity=cfg.batch_size * 50,
num_threads=4)
return data_batch, label_batch, target_batch
else:
return tf.expand_dims(data, 0), tf.expand_dims(label, 0), tf.expand_dims(target, 0)
