def example_queue_shuffle(reader,
filename_queue,
is_training,
example_queue_name='example_queue',
capacity=50000,
num_reader_threads=1):
"""
This function shuffle the examples within the filename queues. Since there's no
padding option in shuffle_batch, we have to manually shuffle the example queue.
The process is given as below.
create filename queue >> read examples from filename queue >> enqueue example to example queue(RandomShuffleQueue)
However, this is not totally random shuffle since the memory limiation. Therefore,
we need to specify a capacity of the example queue.
Args:
reader: A TFRecord Reader
filename_queue: A queue generated by string_input_producer
is_traning: If not training then use FIFOqueue(No need to shuffle).
example_queue_name: Name of the example queue
capacity: Value queue capacity. Should be large enough for better mixing
num_reader_threads: Number of thread to enqueue the value queue
Returns:
example_queue: An example queue that is shuffled. Ready for parsing and batching.
"""
#Init queue
if is_training:
example_queue = tf.RandomShuffleQueue(
capacity=capacity,
min_after_dequeue=capacity % 2,
dtypes=[tf.string],
name="random_" + example_queue_name)
else:
example_queue = tf.FIFOQueue(
capacity=capacity,
dtypes=[tf.string],
name="fifo_" + example_queue_name)
#Manually create ops to enqueue
enqueue_example_ops = []
for _ in range(num_reader_threads):
_, example = reader.read(filename_queue)
enqueue_example_ops.append(example_queue.enqueue([example]))
#Add queue runner
tf.train.queue_runner.add_queue_runner(
tf.train.queue_runner.QueueRunner(example_queue, enqueue_example_ops))
tf.summary.scalar(
"queue/%s/fraction_of_%d_full" % (example_queue.name, capacity),
tf.cast(example_queue.size(), tf.float32) * (1. / capacity))
return example_queue
评论列表
文章目录