def get_input_op(self, fq, *args, **kwargs):
"""
This is the main method that returns a tensorflow data reading operation.
This method will get called n_threads * n_attrs times in the method init_ops (see above).
Specifically, it is called once for each thread id and each attribute group.
The arguments are:
fq: filename queue object. When run in a tf session, this object will act
as a queue of filenames. When fq.dequeue() is called in a tf.Session, it
will produce the next filename to begin reading from. Note: it only makes
sense to dequeue from fq if the current file being read has been completed.
*args: any position arguments to the reader. these are specified on a
per-attribute-group basis (eg. across thread ids, calls for the same attribute
group will get the same args).
*kwargs: any keyward arguments to the reader. like for *args, these are specified
on a per-attribute-group basis.
As an example of this method, see the TFRecordParallelByFileProvider.get_input_ops.
"""
raise NotImplementedError()
python类Session()的实例源码
def s1_predict(config_file, model_dir, model_file, predict_file_list, out_dir):
"""
This function serves as a test/validation tool during the model development. It is not used as
a final product in part of the pipeline.
"""
with open(config_file) as config_buffer:
config = json.loads(config_buffer.read())
with tf.Graph().as_default() as graph:
converted_model = ConvertedModel(config, graph, 's1_keras', model_dir, model_file)
with tf.Session(graph=graph) as sess:
for img_file in predict_file_list:
image = cv2.imread(img_file)
boxes = converted_model.predict(sess, image)
image = draw_boxes(image, boxes)
_, filename = os.path.split(img_file)
cv2.imwrite(os.path.join(out_dir, filename), image)
def main(_):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
model = UNet(args.experiment_dir, batch_size=args.batch_size, experiment_id=args.experiment_id,
input_width=args.image_size, output_width=args.image_size, embedding_num=args.embedding_num,
embedding_dim=args.embedding_dim, L1_penalty=args.L1_penalty, Lconst_penalty=args.Lconst_penalty,
Ltv_penalty=args.Ltv_penalty, Lcategory_penalty=args.Lcategory_penalty)
model.register_session(sess)
if args.flip_labels:
model.build_model(is_training=True, inst_norm=args.inst_norm, no_target_source=True)
else:
model.build_model(is_training=True, inst_norm=args.inst_norm)
fine_tune_list = None
if args.fine_tune:
ids = args.fine_tune.split(",")
fine_tune_list = set([int(i) for i in ids])
model.train(lr=args.lr, epoch=args.epoch, resume=args.resume,
schedule=args.schedule, freeze_encoder=args.freeze_encoder, fine_tune=fine_tune_list,
sample_steps=args.sample_steps, checkpoint_steps=args.checkpoint_steps,
flip_labels=args.flip_labels, no_val=args.no_val)
def test_vgg():
vgg = Vgg16()
image_tensor = tf.placeholder(tf.float32)
with tf.Session() as sess:
vgg.build(image_tensor)
init = tf.initialize_all_variables()
sess.run(init)
load_feature_layer_params('/Users/dtong/code/data/tf-image-interpreter/pretrain/vgg16_weights.npz', sess)
for v in tf.get_collection(tf.GraphKeys.VARIABLES):
print_op = tf.Print(v, [v], message=v.name, first_n=10)
sess.run(print_op)
roidb = RoiDb('val.txt', 2007)
batch_gen = BatchGenerator(roidb)
for i in range(10):
image, scale, bboxes = batch_gen.next_batch()
print(sess.run(vgg.conv5_3, feed_dict={image_tensor: image}))
def main():
roidb = RoiDb('val.txt', 2007)
batch_gen = BatchGenerator(roidb)
image_tensor = tf.placeholder(dtype=tf.float32)
scale_tensor = tf.placeholder(dtype=tf.float32)
bboxes_tensor = tf.placeholder(dtype=tf.float32)
p_op = tf.Print(image_tensor, [tf.shape(image_tensor), scale_tensor, bboxes_tensor])
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
coord = tf.train.Coordinator()
queue_threads = queue_runner.start_queue_runners(sess, coord=coord)
for i in range(10):
if coord.should_stop():
break
image, scale, bboxes = batch_gen.next_batch()
sess.run([p_op], feed_dict={image_tensor: image, scale_tensor: scale, bboxes_tensor:bboxes})
coord.request_stop()
coord.join(queue_threads)
def test_rpn():
vgg = Vgg16()
rpn = RpnNet()
image_tensor = tf.placeholder(tf.float32)
with tf.Session() as sess:
vgg.build(image_tensor)
rpn.build(vgg.conv5_3, None)
init = tf.initialize_all_variables()
sess.run(init)
load_feature_layer_params('/Users/dtong/code/data/tf-image-interpreter/pretrain/vgg16_weights.npz', sess)
roidb = RoiDb('val.txt', 2007)
batch_gen = BatchGenerator(roidb)
for i in range(10):
image, scale, bboxes = batch_gen.next_batch()
feature_shape = tf.shape(rpn.rpn_cls_score_reshape)
print_feat_shape = tf.Print(feature_shape, [feature_shape], summarize=5)
sess.run(print_feat_shape, feed_dict={image_tensor: image})
# print(sess.run(vgg.conv5_3, feed_dict={image_tensor: image}))
def test(self):
def decode_greedily(beam_search: bool, merge_repeated: bool):
aa_ctc_blank_aa_logits = tf.constant(np.array([[[1.0, 0.0]], [[1.0, 0.0]], [[0.0, 1.0]],
[[1.0, 0.0]], [[1.0, 0.0]]], dtype=np.float32))
sequence_length = tf.constant(np.array([5], dtype=np.int32))
(decoded_list,), log_probabilities = \
tf.nn.ctc_beam_search_decoder(inputs=aa_ctc_blank_aa_logits,
sequence_length=sequence_length,
merge_repeated=merge_repeated,
beam_width=1) \
if beam_search else \
tf.nn.ctc_greedy_decoder(inputs=aa_ctc_blank_aa_logits,
sequence_length=sequence_length,
merge_repeated=merge_repeated)
return list(tf.Session().run(tf.sparse_tensor_to_dense(decoded_list)[0]))
self.assertEqual([0], decode_greedily(beam_search=True, merge_repeated=True))
self.assertEqual([0, 0], decode_greedily(beam_search=True, merge_repeated=False))
self.assertEqual([0, 0], decode_greedily(beam_search=False, merge_repeated=True))
self.assertEqual([0, 0, 0, 0], decode_greedily(beam_search=False, merge_repeated=False))
def retrain():
print 'Start retraining'
tf.reset_default_graph()
policy_network = PolicyNetwork(scope = 'supervised_policy')
f = open(relationPath)
training_pairs = f.readlines()
f.close()
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, 'models/policy_supervised_' + relation)
print "sl_policy restored"
episodes = len(training_pairs)
if episodes > 300:
episodes = 300
REINFORCE(training_pairs, policy_network, episodes)
saver.save(sess, 'models/policy_retrained' + relation)
print 'Retrained model saved'
def read_tensor_from_image_file(file_name='test.jpg', input_height=128, input_width=128,
input_mean=0, input_std=255):
input_name = "file_reader"
output_name = "normalized"
file_reader = tf.read_file(file_name, input_name)
image_reader = tf.image.decode_jpeg(file_reader, channels = 3, name='jpeg_reader')
float_caster = tf.cast(image_reader, tf.float32)
dims_expander = tf.expand_dims(float_caster, 0);
resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
sess = tf.Session()
result = sess.run(normalized)
return result
def begin_session(self):
"""
Begins the session
:return: None
"""
# start the tensorflow session
ops.reset_default_graph()
# initialize interactive session
self.sess = tf.Session()
def run():
if len(sys.argv) < 3:
print("** Usage: python3 " + sys.argv[0] + " <<Model Directory>> <<Test Set>>")
sys.exit(1)
np.random.seed(42)
model_dir = sys.argv[1]
config = Config.load(['./default.conf', os.path.join(model_dir, 'model.conf')])
model = create_model(config)
test_data = load_data(sys.argv[2], config.dictionary, config.grammar, config.max_length)
print("unknown", unknown_tokens)
with tf.Graph().as_default():
tf.set_random_seed(1234)
with tf.device('/cpu:0'):
model.build()
test_eval = Seq2SeqEvaluator(model, config.grammar, test_data, 'test', config.reverse_dictionary, beam_size=config.beam_size, batch_size=config.batch_size)
loader = tf.train.Saver()
with tf.Session() as sess:
loader.restore(sess, os.path.join(model_dir, 'best'))
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
#sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
test_eval.eval(sess, save_to_file=True)
eval_output_embeddings.py 文件源码
项目：almond-nnparser
作者: Stanford-Mobisocial-IoT-Lab
项目源码
文件源码
阅读 32
收藏 0
点赞 0
评论 0
def run():
if len(sys.argv) < 4:
print("** Usage: python3 " + sys.argv[0] + " <<Model Directory>> <<Everything Set>> <<Test Set>>")
sys.exit(1)
np.random.seed(42)
model_dir = sys.argv[1]
config = Config.load(['./default.conf', os.path.join(model_dir, 'model.conf')])
model = create_model(config)
everything_labels, everything_label_lengths = load_programs(config, sys.argv[2])
test_labels, test_label_lengths = load_programs(config, sys.argv[3])
#test_labels, test_label_lengths = sample(config.grammar, test_labels, test_label_lengths)
print("unknown", unknown_tokens)
with tf.Graph().as_default():
tf.set_random_seed(1234)
model.build()
loader = tf.train.Saver()
train_bag_of_tokens = bag_of_tokens(config, everything_labels, everything_label_lengths)
V, mean = pca_fit(train_bag_of_tokens, n_components=2)
eval_bag_of_tokens = bag_of_tokens(config, test_labels, test_label_lengths)
transformed = pca_transform(eval_bag_of_tokens, V, mean)
with tf.Session() as sess:
loader.restore(sess, os.path.join(model_dir, 'best'))
transformed = transformed.eval(session=sess)
programs = reconstruct_programs(test_labels, test_label_lengths, config.grammar.tokens)
show_pca(transformed, programs)
def __init__(self, export_dir, model_name, use_spell_check=False, use_visual_features=False):
'''
Args:
export_dir(type = str): Path to directory where trained model
has been exported (with trailing slash).
model_name(type = str): Name of the model exported.
'''
self.export_dir = export_dir
self.session = tf.Session()
self.name = model_name
self.use_spell_check = use_spell_check
self.use_visual_features = use_visual_features
def extract_feature(imgs):
x, fc6 = initModel()
# init = tf.initialize_all_variables()
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
return sess.run(fc6, feed_dict={x: imgs})
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_meta_graph(checkpoint_file + ".meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def single_threaded_session():
tf_config = tf.ConfigProto(
inter_op_parallelism_threads=1,
intra_op_parallelism_threads=1)
return tf.Session(config=tf_config)
