def check_video_id():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.eval_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["mean_rgb","mean_audio"], feature_sizes=[1024,128])
input_data_pattern = FLAGS.input_data_pattern
if FLAGS.eval_data_patterns is "":
raise IOError("'eval_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
all_eval_data_patterns=all_patterns,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_equal = tf.get_collection("video_id_equal")[0]
input_distance = tf.get_collection("input_distance")[0]
check_loop(video_id_equal, input_distance, all_patterns)
python类Graph()的实例源码
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def test_qrnn(self):
print("QRNN Working check")
with tf.Graph().as_default() as qrnn:
self.check_by_digits(qrnn, qrnn=5)
def test_baseline(self):
print("Baseline(LSTM) Working check")
with tf.Graph().as_default() as baseline:
self.check_by_digits(baseline, baseline=True)
def test_random(self):
print("Random Working check")
with tf.Graph().as_default() as random:
self.check_by_digits(random, random=True)
def load_metadata(model_dir):
"""Loads RunMetadata, Graph and OpLog from files
"""
# Import RunMetadata
run_meta_path = os.path.join(model_dir, "metadata/run_meta")
run_meta = tf.RunMetadata()
if gfile.Exists(run_meta_path):
with gfile.GFile(run_meta_path, "rb") as file:
run_meta.MergeFromString(file.read())
print("Loaded RunMetadata from {}".format(run_meta_path))
else:
print("RunMetadata does not exist a {}. Skipping.".format(run_meta_path))
# Import Graph
graph_def_path = os.path.join(model_dir, "graph.pbtxt")
graph = tf.Graph()
if gfile.Exists(graph_def_path):
with graph.as_default():
_register_function_ops(CUSTOM_OP_FUNCTIONS)
graph_def = tf.GraphDef()
with gfile.GFile(graph_def_path, "rb") as file:
text_format.Parse(file.read(), graph_def)
tf.import_graph_def(graph_def, name="")
print("Loaded Graph from {}".format(graph_def_path))
else:
print("Graph does not exist a {}. Skipping.".format(graph_def_path))
# Import OpLog
op_log_path = os.path.join(model_dir, "metadata/tfprof_log")
op_log = tfprof_log_pb2.OpLog()
if gfile.Exists(op_log_path):
with gfile.GFile(op_log_path, "rb") as file:
op_log.MergeFromString(file.read())
print("Loaded OpLog from {}".format(op_log_path))
else:
print("OpLog does not exist a {}. Skipping.".format(op_log_path))
return run_meta, graph, op_log
def evaluate():
"""Eval CIFAR-10 for a number of steps."""
with tf.Graph().as_default():
# Get images and labels for CIFAR-10.
eval_data = FLAGS.eval_data == 'test'
images, labels = cifar10.inputs(eval_data=eval_data)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate predictions.
top_k_op = tf.nn.in_top_k(logits, labels, 1)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
cifar10.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
graph_def = tf.get_default_graph().as_graph_def()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir,
graph_def=graph_def)
while True:
eval_once(saver, summary_writer, top_k_op, summary_op)
if FLAGS.run_once:
break
time.sleep(FLAGS.eval_interval_secs)
def evaluate():
"""Eval BBBC006 for a number of steps."""
with tf.Graph().as_default() as g:
# Get images and labels for BBBC006.
eval_data = FLAGS.eval_data == 'test'
images, labels = bbbc006.inputs(eval_data=eval_data)
# Build a Graph that computes the logits predictions from the
# inference model.
c_fuse, s_fuse = bbbc006.inference(images, train=False)
dice_op = bbbc006.dice_op(c_fuse, s_fuse, labels)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
bbbc006.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, g)
while True:
eval_once(saver, dice_op, summary_writer, summary_op)
if FLAGS.run_once:
break
time.sleep(FLAGS.eval_interval_secs)
def tower_loss(scope, images, labels):
"""Calculate the total loss on a single tower running the BBBC006 model.
Args:
scope: unique prefix string identifying the BBBC006 tower, e.g. 'tower_0'
images: Images. 4D tensor of shape [batch_size, height, width, 3].
labels: Labels. 1D tensor of shape [batch_size].
Returns:
Tensor of shape [] containing the total loss for a batch of data
"""
# Build inference Graph.
c_fuse, s_fuse = bbbc006.inference(images)
# Build the portion of the Graph calculating the losses. Note that we will
# assemble the total_loss using a custom function below.
_ = bbbc006.loss(c_fuse, s_fuse, labels)
# Assemble all of the losses for the current tower only.
losses = tf.get_collection('losses', scope)
# Calculate the total loss for the current tower.
total_loss = tf.add_n(losses, name='total_loss')
# Attach a scalar summary to all individual losses and the total loss; do the
# same for the averaged version of the losses.
for l in losses + [total_loss]:
# Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
# session. This helps the clarity of presentation on tensorboard.
loss_name = re.sub('%s_[0-9]*/' % bbbc006.TOWER_NAME, '', l.op.name)
tf.summary.scalar(loss_name, l)
return total_loss
def load_graph(frozen_graph_filename):
# We load the protobuf file from the disk and parse it to retrieve the
# unserialized graph_def
with tf.gfile.GFile(frozen_graph_filename, "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
# Then, we can use again a convenient built-in function to import a graph_def into the
# current default Graph
with tf.Graph().as_default() as graph:
tf.import_graph_def(
graph_def
)
return graph
def run(self):
"""Run the benchmark task assigned to this process.
Returns:
Dictionary of statistics for training or eval.
Raises:
ValueError: unrecognized job name.
"""
if self.params.job_name == 'ps':
log_fn('Running parameter server %s' % self.task_index)
self.cluster_manager.join_server()
return {}
# For distributed_all_reduce with multiple workers, drive
# from a separate controller process.
if self.params.variable_update == 'distributed_all_reduce':
if self.params.job_name == 'worker':
log_fn('Starting worker %s' % self.task_index)
self.cluster_manager.join_server()
return
elif self.params.job_name and self.params.job_name != 'controller':
raise ValueError('unrecognized job name: %s' % self.params.job_name)
with tf.Graph().as_default():
if self.params.eval:
return self._eval_cnn()
else:
return self._benchmark_cnn()
def graph(self) -> tf.Graph:
"""TF graph object."""
return self._graph
def __init__(self, fnames, shuffle=True, num_epochs=None):
"""Init from a list of filenames to enqueue.
Args:
fnames: list of .tfrecords filenames to enqueue.
shuffle: if true, shuffle the list at each epoch
"""
self._fnames = fnames
self._fname_queue = tf.train.string_input_producer(
self._fnames,
capacity=1000,
shuffle=shuffle,
num_epochs=num_epochs,
shared_name='input_files')
self._reader = tf.TFRecordReader()
# Read first record to initialize the shape parameters
with tf.Graph().as_default():
fname_queue = tf.train.string_input_producer(self._fnames)
reader = tf.TFRecordReader()
_, serialized = reader.read(fname_queue)
shapes = self._parse_shape(serialized)
dtypes = self._parse_dtype(serialized)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
self.shapes = sess.run(shapes)
self.shapes = {k: self.shapes[k+'_sz'].tolist() for k in self.FEATURES}
self.dtypes = sess.run(dtypes)
self.dtypes = {k: REVERSE_TYPEMAP[self.dtypes[k+'_dtype'][0]] for k in self.FEATURES}
coord.request_stop()
coord.join(threads)
def __init__(self,num_states,num_actions):
self.g=tf.Graph()
with self.g.as_default():
self.sess = tf.InteractiveSession()
#actor network model parameters:
self.W1_a, self.B1_a, self.W2_a, self.B2_a, self.W3_a, self.B3_a,\
self.actor_state_in, self.actor_model = self.create_actor_net(num_states, num_actions)
#target actor network model parameters:
self.t_W1_a, self.t_B1_a, self.t_W2_a, self.t_B2_a, self.t_W3_a, self.t_B3_a,\
self.t_actor_state_in, self.t_actor_model = self.create_actor_net(num_states, num_actions)
#cost of actor network:
self.q_gradient_input = tf.placeholder("float",[None,num_actions]) #gets input from action_gradient computed in critic network file
self.actor_parameters = [self.W1_a, self.B1_a, self.W2_a, self.B2_a, self.W3_a, self.B3_a]
self.parameters_gradients = tf.gradients(self.actor_model,self.actor_parameters,-self.q_gradient_input)#/BATCH_SIZE)
self.optimizer = tf.train.AdamOptimizer(LEARNING_RATE).apply_gradients(zip(self.parameters_gradients,self.actor_parameters))
#initialize all tensor variable parameters:
self.sess.run(tf.initialize_all_variables())
#To make sure actor and target have same intial parmameters copy the parameters:
# copy target parameters
self.sess.run([
self.t_W1_a.assign(self.W1_a),
self.t_B1_a.assign(self.B1_a),
self.t_W2_a.assign(self.W2_a),
self.t_B2_a.assign(self.B2_a),
self.t_W3_a.assign(self.W3_a),
self.t_B3_a.assign(self.B3_a)])
self.update_target_actor_op = [
self.t_W1_a.assign(TAU*self.W1_a+(1-TAU)*self.t_W1_a),
self.t_B1_a.assign(TAU*self.B1_a+(1-TAU)*self.t_B1_a),
self.t_W2_a.assign(TAU*self.W2_a+(1-TAU)*self.t_W2_a),
self.t_B2_a.assign(TAU*self.B2_a+(1-TAU)*self.t_B2_a),
self.t_W3_a.assign(TAU*self.W3_a+(1-TAU)*self.t_W3_a),
self.t_B3_a.assign(TAU*self.B3_a+(1-TAU)*self.t_B3_a)]
def predict(model_scope, result_dir, result_file, img_features, k=1):
"""
Args:
model_scope: The variable_scope used when this model was trained.
result_dir: The full path to the folder in which the result file locates.
result_file: The file that saves the training results.
img_features: A 2-D ndarray (matrix) each row of which holds the pixels as
features of one image. One or more rows (image samples) can be requested
to be predicted at once.
k: Optional. Number of elements to be predicted.
Returns:
values and indices. Refer to tf.nn.top_k for details.
"""
with tf.Session(graph=tf.Graph()) as sess:
saver = tf.train.import_meta_graph(os.path.join(result_dir, result_file + ".meta"))
saver.restore(sess, os.path.join(result_dir, result_file))
# Retrieve the Ops we 'remembered'.
logits = tf.get_collection(model_scope+"logits")[0]
images_placeholder = tf.get_collection(model_scope+"images")[0]
keep_prob_placeholder = tf.get_collection(model_scope+"keep_prob")[0]
# Add an Op that chooses the top k predictions. Apply softmax so that
# we can have the probabilities (percentage) in the output.
eval_op = tf.nn.top_k(tf.nn.softmax(logits), k=k)
values, indices = sess.run(eval_op, feed_dict={images_placeholder: img_features,
keep_prob_placeholder: 1.0})
return values, indices
def main(_):
img = load_image("data/cat.jpg")
print img
img_p = preprocess(img)
for layers in [50, 101, 152]:
g = tf.Graph()
with g.as_default():
print "CONVERT", layers
convert(g, img, img_p, layers)
