def run(self, fetches, feed_dict=None, options=None, run_metadata=None):
# Make sure there is no disagreement doing this.
if options is not None:
if options.trace_level != self.profiler_options.trace_level: # pragma: no cover
raise ValueError(
'In profiler session. Inconsistent trace '
'level from run call') # pragma: no cover
self.profiler_options.update(options) # pragma: no cover
self.local_run_metadata = tf.RunMetadata()
output = super(TracerSession, self).run(
fetches, feed_dict=feed_dict,
options=self.profiler_options,
run_metadata=self.local_run_metadata)
trace_time = timeline.Timeline(self.local_run_metadata.step_stats)
ctf = trace_time.generate_chrome_trace_format()
with open(self._trace_filename(), 'w') as trace_file:
trace_file.write(ctf)
if self.each_time:
self.counter += 1
return output
python类RunMetadata()的实例源码
def run(self, fetches, feed_dict=None):
"""like Session.run, but return a Timeline object in Chrome trace format (JSON).
Save the json to a file, go to chrome://tracing, and open the file.
Args:
fetches
feed_dict
Returns:
dict: a JSON dict
"""
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
super(ProfiledSession, self).run(fetches, feed_dict, options=options, run_metadata=run_metadata)
# Create the Timeline object, and write it to a json
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
return json.loads(ctf)
def run(self, fetches, feed_dict=None):
"""like Session.run, but return a Timeline object in Chrome trace format (JSON).
Save the json to a file, go to chrome://tracing, and open the file.
Args:
fetches
feed_dict
Returns:
dict: a JSON dict
"""
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
super(ProfiledSession, self).run(fetches, feed_dict, options=options, run_metadata=run_metadata)
# Create the Timeline object, and write it to a json
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
return json.loads(ctf)
def traced_run(fetches):
"""Runs fetches, dumps timeline files in current directory."""
global sess
assert sess
global timeline_counter
run_metadata = tf.RunMetadata()
root = os.getcwd()+"/data"
from tensorflow.python.client import timeline
results = sess.run(fetches,
options=run_options,
run_metadata=run_metadata);
tl = timeline.Timeline(step_stats=run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format(show_memory=True,
show_dataflow=False)
open(root+"/timeline_%d.json"%(timeline_counter,), "w").write(ctf)
open(root+"/stepstats_%d.pbtxt"%(timeline_counter,), "w").write(str(
run_metadata.step_stats))
timeline_counter+=1
return results
def sessrun(*args, **kwargs):
sess = u.get_default_session()
if not GLOBAL_PROFILE:
return sess.run(*args, **kwargs)
run_metadata = tf.RunMetadata()
kwargs['options'] = full_trace_options
kwargs['run_metadata'] = run_metadata
result = sess.run(*args, **kwargs)
first_entry = args[0]
if isinstance(first_entry, list):
if len(first_entry) == 0 and len(args) == 1:
return None
first_entry = first_entry[0]
name = first_entry.name
name = name.replace('/', '-')
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open('timelines/%s.json'%(name,), 'w') as f:
f.write(ctf)
with open('timelines/%s.pbtxt'%(name,), 'w') as f:
f.write(str(run_metadata))
return result
def traced_run(fetches):
"""Runs fetches, dumps timeline files in current directory."""
from tensorflow.python.client import timeline
global timeline_counter
run_metadata = tf.RunMetadata()
results = sess.run(fetches,
options=run_options,
run_metadata=run_metadata);
tl = timeline.Timeline(step_stats=run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format(show_memory=True,
show_dataflow=False)
open("timeline_%d.json"%(timeline_counter,), "w").write(ctf)
open("stepstats_%d.pbtxt"%(timeline_counter,), "w").write(str(
run_metadata.step_stats))
timeline_counter+=1
return results
def train_it(sess, step=1):
_pat_chars_i, _pat_lens = get_batch(__batch_size)
inputs = {
pat_chars_i: _pat_chars_i,
pat_lens: _pat_lens}
# Run optimization op (backprop)
#run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
#run_metadata = tf.RunMetadata()
#sess.run(optimizer, feed_dict=inputs, options=run_options, run_metadata=run_metadata)
sess.run(optimizer, feed_dict=inputs)
#with open('timeline.json', 'w') as f:
# f.write(
# timeline.Timeline(run_metadata.step_stats)
# .generate_chrome_trace_format())
if step % display_step == 0:
# Calculate batch loss
cost_f = sess.run(cost, feed_dict=inputs)
print ("Iter {}, cost= {:.6f}".format(
str(step*__batch_size), cost_f))
def optimize(self, data, with_metrics=False, with_trace=False):
""" Optimize a single batch """
run_metadata = tf.RunMetadata() if with_trace else None
trace = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) if with_trace else None
_, metrics = self.run(
self.training_operation, data,
run_options=trace, run_metadata=run_metadata)
if with_metrics:
self.timer_update()
steps, elapsed = self.elapsed()
num_devices = len(self.towers)
examples = steps * self.batch_size * num_devices
print('Step {}, examples/sec {:.3f}, ms/batch {:.1f}'.format(
self.global_step, examples / elapsed, 1000 * elapsed / num_devices))
self.output_metrics(data, metrics)
self.write_summaries(data)
if with_trace:
step = '{}/step{}'.format(self.name, self.global_step)
self.summary_writer.add_run_metadata(run_metadata, step, global_step=self.global_step)
def train(self, images, labels, summaries=False, run_metadata=False):
if (summaries or run_metadata) and not self.summary_writer:
raise ValueError("Logdir is required for summaries or run_metadata.")
args = {"feed_dict": {self.images: images, self.labels: labels}}
targets = [self.training]
if summaries:
targets.append(self.summaries["training"])
if run_metadata:
args["options"] = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
args["run_metadata"] = tf.RunMetadata()
results = self.session.run(targets, **args)
if summaries:
self.summary_writer.add_summary(results[-1], self.training_step - 1)
if run_metadata:
self.summary_writer.add_run_metadata(args["run_metadata"], "step{:05}".format(self.training_step - 1))
def train(self, images, labels, summaries=False, run_metadata=False):
if (summaries or run_metadata) and not self.summary_writer:
raise ValueError("Logdir is required for summaries or run_metadata.")
args = {"feed_dict": {self.images: images, self.labels: labels}}
targets = [self.training]
if summaries:
targets.append(self.summaries["training"])
if run_metadata:
args["options"] = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
args["run_metadata"] = tf.RunMetadata()
results = self.session.run(targets, **args)
if summaries:
self.summary_writer.add_summary(results[-1], self.training_step - 1)
if run_metadata:
self.summary_writer.add_run_metadata(args["run_metadata"], "step{:05}".format(self.training_step - 1))
def RunMetadata(self, tag):
"""Given a tag, return the associated session.run() metadata.
Args:
tag: A string tag associated with the event.
Raises:
ValueError: If the tag is not found.
Returns:
The metadata in form of `RunMetadata` proto.
"""
if tag not in self._tagged_metadata:
raise ValueError('There is no run metadata with this tag name')
run_metadata = tf.RunMetadata()
run_metadata.ParseFromString(self._tagged_metadata[tag])
return run_metadata
def load_metadata(model_dir):
"""Loads RunMetadata, Graph and OpLog from files
"""
# Import RunMetadata
run_meta_path = os.path.join(model_dir, "metadata/run_meta")
run_meta = tf.RunMetadata()
if gfile.Exists(run_meta_path):
with gfile.GFile(run_meta_path, "rb") as file:
run_meta.MergeFromString(file.read())
print("Loaded RunMetadata from {}".format(run_meta_path))
else:
print("RunMetadata does not exist a {}. Skipping.".format(run_meta_path))
# Import Graph
graph_def_path = os.path.join(model_dir, "graph.pbtxt")
graph = tf.Graph()
if gfile.Exists(graph_def_path):
with graph.as_default():
_register_function_ops(CUSTOM_OP_FUNCTIONS)
graph_def = tf.GraphDef()
with gfile.GFile(graph_def_path, "rb") as file:
text_format.Parse(file.read(), graph_def)
tf.import_graph_def(graph_def, name="")
print("Loaded Graph from {}".format(graph_def_path))
else:
print("Graph does not exist a {}. Skipping.".format(graph_def_path))
# Import OpLog
op_log_path = os.path.join(model_dir, "metadata/tfprof_log")
op_log = tfprof_log_pb2.OpLog()
if gfile.Exists(op_log_path):
with gfile.GFile(op_log_path, "rb") as file:
op_log.MergeFromString(file.read())
print("Loaded OpLog from {}".format(op_log_path))
else:
print("OpLog does not exist a {}. Skipping.".format(op_log_path))
return run_meta, graph, op_log
def train(self, nIter, machine=None, summary_op=None):
# Xh = self._validate(machine=machine, n=10)
run_metadata = tf.RunMetadata()
sv = tf.train.Supervisor(
logdir=self.dirs['logdir'],
# summary_writer=summary_writer,
# summary_op=None,
# is_chief=True,
save_model_secs=300,
global_step=self.opt['global_step'])
# sess_config = configure_gpu_settings(args.gpu_cfg)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=True))
with sv.managed_session(config=sess_config) as sess:
sv.loop(60, self._refresh_status, (sess,))
for step in range(self.arch['training']['max_iter']):
if sv.should_stop():
break
# main loop
sess.run(self.opt['g'])
# # output img
# if step % 1000 == 0:
# xh = sess.run(Xh)
# with tf.gfile.GFile(
# os.path.join(
# self.dirs['logdir'],
# 'img-anime-{:03d}k.png'.format(step // 1000),
# ),
# mode='wb',
# ) as fp:
# fp.write(xh)
def train(self, nIter, machine=None, summary_op=None):
Xh = self._validate(machine=machine, n=10)
run_metadata = tf.RunMetadata()
sv = tf.train.Supervisor(
logdir=self.dirs['logdir'],
# summary_writer=summary_writer,
# summary_op=None,
# is_chief=True,
# save_model_secs=600,
global_step=self.opt['global_step'])
# sess_config = configure_gpu_settings(args.gpu_cfg)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=True))
with sv.managed_session(config=sess_config) as sess:
sv.loop(60, self._refresh_status, (sess,))
for step in range(self.arch['training']['max_iter']):
if sv.should_stop():
break
# main loop
sess.run(self.opt['g'])
# output img
if step % 1000 == 0:
xh = sess.run(Xh)
with tf.gfile.GFile(
os.path.join(
self.dirs['logdir'],
'img-anime-{:03d}k.png'.format(step // 1000),
),
mode='wb',
) as fp:
fp.write(xh)
def load_metadata(model_dir):
"""Loads RunMetadata, Graph and OpLog from files
"""
# Import RunMetadata
run_meta_path = os.path.join(model_dir, "metadata/run_meta")
run_meta = tf.RunMetadata()
if gfile.Exists(run_meta_path):
with gfile.GFile(run_meta_path, "rb") as file:
run_meta.MergeFromString(file.read())
print("Loaded RunMetadata from {}".format(run_meta_path))
else:
print("RunMetadata does not exist a {}. Skipping.".format(run_meta_path))
# Import Graph
graph_def_path = os.path.join(model_dir, "graph.pbtxt")
graph = tf.Graph()
if gfile.Exists(graph_def_path):
with graph.as_default():
_register_function_ops(CUSTOM_OP_FUNCTIONS)
graph_def = tf.GraphDef()
with gfile.GFile(graph_def_path, "rb") as file:
text_format.Parse(file.read(), graph_def)
tf.import_graph_def(graph_def, name="")
print("Loaded Graph from {}".format(graph_def_path))
else:
print("Graph does not exist a {}. Skipping.".format(graph_def_path))
# Import OpLog
op_log_path = os.path.join(model_dir, "metadata/tfprof_log")
op_log = tfprof_log_pb2.OpLog()
if gfile.Exists(op_log_path):
with gfile.GFile(op_log_path, "rb") as file:
op_log.MergeFromString(file.read())
print("Loaded OpLog from {}".format(op_log_path))
else:
print("OpLog does not exist a {}. Skipping.".format(op_log_path))
return run_meta, graph, op_log
def __init__(self, config_path = None):
if config_path is not None:
self.load(config_path)
if self.time_trace:
self.run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
self.run_metadata = tf.RunMetadata()
# set workspace
self.workspace = os.path.join(self.workspace, self.dataset_name)
self.dataset_path = os.path.join(self.workspace, self.file_name)
self.map_path = os.path.join(self.workspace, "map/")
self.__set_save_path()
if self.eval_mode and self.save_ckpt:
print("Warning, in evaluation mode, automatically set config.save_ckpt to False")
self.save_ckpt = False
def run_op(op):
start_time = time.time()
print("%10.2f ms: starting op %s\n" % ((start_time-start_time0)*1000, op.name), flush=True, end='')
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
sess.run(op, options=options, run_metadata=run_metadata)
end_time = time.time()
print("%10.2f ms: ending op %s\n" % ((end_time-start_time0)*1000, op.name), flush=True, end='')
run_metadatas.append(run_metadata)
def run_op(op):
start_time = time.time()
print("%10.2f ms: starting op %s\n" % ((start_time-start_time0)*1000, op.name), flush=True, end='')
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
sess.run(op, options=options, run_metadata=run_metadata)
end_time = time.time()
print("%10.2f ms: ending op %s\n" % ((end_time-start_time0)*1000, op.name), flush=True, end='')
run_metadatas.append(run_metadata)
def sessrun(*args, **kwargs):
"""Helper to do sess.run and save run_metadata"""
global sess, run_metadata
run_metadata = tf.RunMetadata()
kwargs['options'] = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
kwargs['run_metadata'] = run_metadata
result = sess.run(*args, **kwargs)
first_entry = args[0]
# have to do this because sess.run(tensor) is same as sess.run([tensor])
if isinstance(first_entry, list):
if len(first_entry) == 0 and len(args) == 1:
return None
first_entry = first_entry[0]
def traced_run(fetches):
"""Runs fetches, dumps timeline files in current directory."""
global timeline_counter
run_metadata = tf.RunMetadata()
config = load_config()
log_fn = "%s-%s-%s"%(config.task_type, config.task_id, timeline_counter)
sess = tf.get_default_session()
root = os.getcwd()+"/data"
os.system('mkdir -p '+root)
from tensorflow.python.client import timeline
results = sess.run(fetches,
options=run_options,
run_metadata=run_metadata);
tl = timeline.Timeline(step_stats=run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format(show_memory=True,
show_dataflow=False)
open(root+"/timeline_%s.json"%(log_fn,), "w").write(ctf)
open(root+"/stepstats_%s.pbtxt"%(log_fn,), "w").write(str(
run_metadata.step_stats))
timeline_counter+=1
return results
def run_shit():
sess = tf.Session()
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
sess.run(tf.initialize_all_variables())
train_step_ = sess.run([train_step], options=run_options, run_metadata=run_metadata,
)#feed_dict={x: [[2,3],[5,1]]})
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open('o_100.json', 'w') as f:
f.write(ctf)
def load_metadata(model_dir):
"""Loads RunMetadata, Graph and OpLog from files
"""
# Import RunMetadata
run_meta_path = os.path.join(model_dir, "metadata/run_meta")
run_meta = tf.RunMetadata()
if gfile.Exists(run_meta_path):
with gfile.GFile(run_meta_path, "rb") as file:
run_meta.MergeFromString(file.read())
print("Loaded RunMetadata from {}".format(run_meta_path))
else:
print("RunMetadata does not exist a {}. Skipping.".format(run_meta_path))
# Import Graph
graph_def_path = os.path.join(model_dir, "graph.pbtxt")
graph = tf.Graph()
if gfile.Exists(graph_def_path):
with graph.as_default():
_register_function_ops(CUSTOM_OP_FUNCTIONS)
graph_def = tf.GraphDef()
with gfile.GFile(graph_def_path, "rb") as file:
text_format.Parse(file.read(), graph_def)
tf.import_graph_def(graph_def, name="")
print("Loaded Graph from {}".format(graph_def_path))
else:
print("Graph does not exist a {}. Skipping.".format(graph_def_path))
# Import OpLog
op_log_path = os.path.join(model_dir, "metadata/tfprof_log")
op_log = tfprof_log_pb2.OpLog()
if gfile.Exists(op_log_path):
with gfile.GFile(op_log_path, "rb") as file:
op_log.MergeFromString(file.read())
print("Loaded OpLog from {}".format(op_log_path))
else:
print("OpLog does not exist a {}. Skipping.".format(op_log_path))
return run_meta, graph, op_log
def train(self, images, labels):
self.steps += 1
feed_dict = {self.images: images, self.labels: labels}
if self.steps == 1:
metadata = tf.RunMetadata()
self.session.run(self.training, feed_dict, options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE), run_metadata = metadata)
self.summary_writer.add_run_metadata(metadata, 'step1')
elif self.steps % 100 == 0:
_, summary = self.session.run([self.training, self.summaries['training']], feed_dict)
self.summary_writer.add_summary(summary, self.steps)
else:
self.session.run(self.training, feed_dict)
def train(self, images, labels):
self.steps += 1
feed_dict = {self.images: images, self.labels: labels}
if self.steps == 1:
metadata = tf.RunMetadata()
self.session.run(self.training, feed_dict, options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE), run_metadata = metadata)
self.summary_writer.add_run_metadata(metadata, 'step1')
elif self.steps % 100 == 0:
_, summary = self.session.run([self.training, self.summaries['training']], feed_dict)
self.summary_writer.add_summary(summary, self.steps)
else:
self.session.run(self.training, feed_dict)
def testExtractGatedGrpcTensorsFoundGatedGrpcOps(self):
with tf.Session() as sess:
z, run_options = self._createTestGraphAndRunOptions(sess, gated_grpc=True)
sess.run(tf.global_variables_initializer())
run_metadata = tf.RunMetadata()
self.assertAllClose(
[10.0], sess.run(z, options=run_options, run_metadata=run_metadata))
graph_wrapper = debug_graphs_helper.DebugGraphWrapper(
run_metadata.partition_graphs[0])
gated_debug_ops = graph_wrapper.get_gated_grpc_tensors()
# Verify that the op types are available.
for item in gated_debug_ops:
self.assertTrue(item[1])
# Strip out the op types before further checks, because op type names can
# change in the future (e.g., 'VariableV2' --> 'VariableV3').
gated_debug_ops = [
(item[0], item[2], item[3]) for item in gated_debug_ops]
self.assertIn(('a', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('a/read', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('b', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('b/read', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('c', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('c/read', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('d', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('d/read', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('x', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('y', 0, 'DebugIdentity'), gated_debug_ops)
self.assertIn(('z', 0, 'DebugIdentity'), gated_debug_ops)
def testGraphDefProperty(self):
with tf.Session() as sess:
z, run_options = self._createTestGraphAndRunOptions(sess, gated_grpc=True)
sess.run(tf.global_variables_initializer())
run_metadata = tf.RunMetadata()
self.assertAllClose(
[10.0], sess.run(z, options=run_options, run_metadata=run_metadata))
graph_wrapper = debug_graphs_helper.DebugGraphWrapper(
run_metadata.partition_graphs[0])
self.assertProtoEquals(
run_metadata.partition_graphs[0], graph_wrapper.graph_def)
def testExtractGatedGrpcTensorsFoundNoGatedGrpcOps(self):
with tf.Session() as sess:
z, run_options = self._createTestGraphAndRunOptions(sess,
gated_grpc=False)
sess.run(tf.global_variables_initializer())
run_metadata = tf.RunMetadata()
self.assertAllClose(
[10.0], sess.run(z, options=run_options, run_metadata=run_metadata))
graph_wrapper = debug_graphs_helper.DebugGraphWrapper(
run_metadata.partition_graphs[0])
gated_debug_ops = graph_wrapper.get_gated_grpc_tensors()
self.assertEqual([], gated_debug_ops)
def generate_run(self, run_name, include_graph):
"""Create a run with a text summary, metadata, and optionally a graph."""
tf.reset_default_graph()
k1 = tf.constant(math.pi, name='k1')
k2 = tf.constant(math.e, name='k2')
result = (k1 ** k2) - k1
expected = tf.constant(20.0, name='expected')
error = tf.abs(result - expected, name='error')
message_prefix_value = 'error ' * 1000
true_length = len(message_prefix_value)
assert true_length > self._MESSAGE_PREFIX_LENGTH_LOWER_BOUND, true_length
message_prefix = tf.constant(message_prefix_value, name='message_prefix')
error_message = tf.string_join([message_prefix,
tf.as_string(error, name='error_string')],
name='error_message')
summary_message = tf.summary.text('summary_message', error_message)
sess = tf.Session()
writer = tf.summary.FileWriter(os.path.join(self.logdir, run_name))
if include_graph:
writer.add_graph(sess.graph)
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
s = sess.run(summary_message, options=options, run_metadata=run_metadata)
writer.add_summary(s)
writer.add_run_metadata(run_metadata, self._METADATA_TAG)
writer.close()
def test_run_metadata(self):
self.set_up_with_runs()
(metadata_pbtxt, mime_type) = self.plugin.run_metadata_impl(
self._RUN_WITH_GRAPH, self._METADATA_TAG)
self.assertEqual(mime_type, 'text/x-protobuf')
text_format.Parse(metadata_pbtxt, tf.RunMetadata())
# If it parses, we're happy.
def basic_train(loss_op, update_op,
profile=0, save_dir='asset/unamed',
**kwargs):
profile_state = _ShouldProfile(profile)
@stf.sg_train_func
def train_func(sess, arg):
profile_state.increment()
if profile_state.should_profile():
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
else:
options = None
run_metadata = None
loss = sess.run([loss_op] + update_op,
options=options,
run_metadata=run_metadata)[0]
if profile_state.should_profile():
tl = tf_timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open(path.join(save_dir, 'timeline.json'), 'w') as fd:
print(ctf, file=fd)
return loss
# run train function
train_func(save_dir=save_dir, **kwargs)
