def make_session(num_cpu):
tf_config = tf.ConfigProto(
inter_op_parallelism_threads=num_cpu,
intra_op_parallelism_threads=num_cpu)
return tf.Session(config=tf_config)
python类ConfigProto()的实例源码
def get_genre(album_or_phrase):
try:
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
x_raw = [album_or_phrase]
all_predictions = []
x_test = np.array(list(vocab_processor.transform(x_raw)))
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
# Generate batches for one epoch
batches = data_loader.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False)
for x_test_batch in batches:
batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0})
all_predictions = np.concatenate([all_predictions, batch_predictions])
return data_loader.genre_ids[int(all_predictions[0])]
except Exception as e:
print e
def get_genre():
try:
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
albums = request.args.get('albums')
x_raw = albums.split(',')
all_predictions = []
x_test = np.array(list(vocab_processor.transform(x_raw)))
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
# Generate batches for one epoch
batches = data_loader.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False)
for x_test_batch in batches:
batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0})
all_predictions = np.concatenate([all_predictions, batch_predictions])
return jsonify({'results': map(lambda x: data_loader.genre_ids[int(x)], all_predictions)})
except Exception as e:
print e
def evaluate(num_votes):
is_training = False
with tf.device('/gpu:'+str(GPU_INDEX)):
pointclouds_pl, labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT)
is_training_pl = tf.placeholder(tf.bool, shape=())
# simple model
pred, end_points = MODEL.get_model(pointclouds_pl, is_training_pl)
loss = MODEL.get_loss(pred, labels_pl, end_points)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = True
sess = tf.Session(config=config)
# Restore variables from disk.
saver.restore(sess, MODEL_PATH)
log_string("Model restored.")
ops = {'pointclouds_pl': pointclouds_pl,
'labels_pl': labels_pl,
'is_training_pl': is_training_pl,
'pred': pred,
'loss': loss}
eval_one_epoch(sess, ops, num_votes)
def main():
if args.logdir is None:
raise ValueError('Please specify the logdir file')
ckpt = get_checkpoint(args.logdir)
if ckpt is None:
raise ValueError('No checkpoints in {}'.format(args.logdir))
with open(os.path.join(args.logdir, 'architecture.json')) as f:
arch = json.load(f)
reader = VCC2016TFRManager()
features = reader.read_whole(args.file_pattern, num_epochs=1)
x = features['frame']
y = features['label']
filename = features['filename']
y_conv = y * 0 + args.target_id
net = MLPcVAE(arch=arch, is_training=False)
z = net.encode(x)
xh = net.decode(z, y)
x_conv = net.decode(z, y_conv)
pre_train_saver = tf.train.Saver()
def load_pretrain(sess):
pre_train_saver.restore(sess, ckpt)
sv = tf.train.Supervisor(init_fn=load_pretrain)
gpu_options = tf.GPUOptions(allow_growth=True)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
gpu_options=gpu_options)
with sv.managed_session(config=sess_config) as sess:
for _ in range(reader.n_files):
if sv.should_stop():
break
fetch_dict = {'x': x, 'xh': xh, 'x_conv': x_conv, 'f': filename}
results = sess.run(fetch_dict)
plot_spectra(results)
def train(self, nIter, machine=None, summary_op=None):
# Xh = self._validate(machine=machine, n=10)
run_metadata = tf.RunMetadata()
sv = tf.train.Supervisor(
logdir=self.dirs['logdir'],
# summary_writer=summary_writer,
# summary_op=None,
# is_chief=True,
save_model_secs=300,
global_step=self.opt['global_step'])
# sess_config = configure_gpu_settings(args.gpu_cfg)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=True))
with sv.managed_session(config=sess_config) as sess:
sv.loop(60, self._refresh_status, (sess,))
for step in range(self.arch['training']['max_iter']):
if sv.should_stop():
break
# main loop
sess.run(self.opt['g'])
# # output img
# if step % 1000 == 0:
# xh = sess.run(Xh)
# with tf.gfile.GFile(
# os.path.join(
# self.dirs['logdir'],
# 'img-anime-{:03d}k.png'.format(step // 1000),
# ),
# mode='wb',
# ) as fp:
# fp.write(xh)
def _create_session(self, session_config: Optional[dict]) -> tf.Session:
"""
Create and return TF Session for this model.
By default the session is configured with ``tf.ConfigProto`` created with
the given ``session_config`` as ``**kwargs``. Nested dictionaries such as
``gpu_options`` or ``graph_options`` are handled automatically.
:param session_config: session configuration dict as specified in the config yaml
:return: TensorFlow session
"""
if session_config:
session_config = tf.ConfigProto(**session_config)
return tf.Session(graph=self._graph, config=session_config)
def __init__(self, energy_fn, prior, std=1.0,
inter_op_parallelism_threads=1, intra_op_parallelism_threads=1):
self.energy_fn = energy_fn
self.prior = prior
self.z = self.energy_fn.z
def fn(z, x):
z_ = z + tf.random_normal(tf.shape(self.z), 0.0, std)
accept = metropolis_hastings_accept(
energy_prev=energy_fn(z),
energy_next=energy_fn(z_)
)
return tf.where(accept, z_, z)
self.steps = tf.placeholder(tf.int32, [])
elems = tf.zeros([self.steps])
self.z_ = tf.scan(
fn, elems, self.z, back_prop=False
)
self.sess = tf.Session(
config=tf.ConfigProto(
inter_op_parallelism_threads=inter_op_parallelism_threads,
intra_op_parallelism_threads=intra_op_parallelism_threads
)
)
self.sess.run(tf.global_variables_initializer())
def train_test_validation():
M = read_dataset()
num_rating = np.count_nonzero(M)
idx = np.arange(num_rating)
np.random.seed(1)
np.random.shuffle(idx)
train_idx = idx[:int(0.85 * num_rating)]
valid_idx = idx[int(0.85 * num_rating):int(0.90 * num_rating)]
test_idx = idx[int(0.90 * num_rating):]
for hidden_encoder_dim, hidden_decoder_dim, latent_dim, learning_rate, batch_size, reg_param, vae in itertools.product(hedims, hddims, ldims, lrates, bsizes, regs, vaes):
result_path = "{0}_{1}_{2}_{3}_{4}_{5}_{6}".format(
hidden_encoder_dim, hidden_decoder_dim, latent_dim, learning_rate, batch_size, reg_param, vae)
if not os.path.exists(result_path + "/model.ckpt.index"):
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
with tf.Session(config=config) as sess:
model = VAEMF(sess, num_user, num_item,
hidden_encoder_dim=hidden_encoder_dim, hidden_decoder_dim=hidden_decoder_dim,
latent_dim=latent_dim, learning_rate=learning_rate, batch_size=batch_size, reg_param=reg_param, vae=vae)
print("Train size={0}, Validation size={1}, Test size={2}".format(
train_idx.size, valid_idx.size, test_idx.size))
print(result_path)
best_rmse = model.train_test_validation(M, train_idx=train_idx, test_idx=test_idx, valid_idx=valid_idx, n_steps=n_steps, result_path=result_path)
print("Best MSE = {0}".format(best_rmse))
with open('result.csv', 'a') as f:
f.write("{0},{1},{2},{3},{4},{5},{6},{7}\n".format(hidden_encoder_dim, hidden_decoder_dim,
latent_dim, learning_rate, batch_size, reg_param, vae, best_rmse))
tf.reset_default_graph()
def __init__(self, fnames, shuffle=True, num_epochs=None):
"""Init from a list of filenames to enqueue.
Args:
fnames: list of .tfrecords filenames to enqueue.
shuffle: if true, shuffle the list at each epoch
"""
self._fnames = fnames
self._fname_queue = tf.train.string_input_producer(
self._fnames,
capacity=1000,
shuffle=shuffle,
num_epochs=num_epochs,
shared_name='input_files')
self._reader = tf.TFRecordReader()
# Read first record to initialize the shape parameters
with tf.Graph().as_default():
fname_queue = tf.train.string_input_producer(self._fnames)
reader = tf.TFRecordReader()
_, serialized = reader.read(fname_queue)
shapes = self._parse_shape(serialized)
dtypes = self._parse_dtype(serialized)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
self.shapes = sess.run(shapes)
self.shapes = {k: self.shapes[k+'_sz'].tolist() for k in self.FEATURES}
self.dtypes = sess.run(dtypes)
self.dtypes = {k: REVERSE_TYPEMAP[self.dtypes[k+'_dtype'][0]] for k in self.FEATURES}
coord.request_stop()
coord.join(threads)
def main():
# Create the TensorFlow placeholders
placeholders = create_placeholders()
# Get the training feed dicts and define the length of the test set.
train_feed_dicts, vocab = load_data(placeholders)
num_test = int(len(train_feed_dicts) * (1 / 5))
print("Number of Feed Dicts: ", len(train_feed_dicts))
print("Number of Test Dicts: ", num_test)
# Slice the dictionary list into training and test sets
final_test_feed_dicts = train_feed_dicts[0:num_test]
test_feed_dicts = train_feed_dicts[0:50]
train_feed_dicts = train_feed_dicts[num_test:]
# Do not take up all the GPU memory, all the time.
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
logits, loss, preds, accuracy, saver = train(placeholders, train_feed_dicts, test_feed_dicts, vocab, sess=sess)
print('============')
# Test on train data - later, test on test data
avg_acc = 0
count = 0
for j, batch in enumerate(final_test_feed_dicts):
acc = sess.run(accuracy, feed_dict=batch)
print("Accuracy on test set is: ", acc)
avg_acc += acc
count += 1
print('-----')
print("Overall Average Accuracy on the Test Set Is: ", avg_acc / count)
def serialize_cifar_pool3(X,filename):
print 'About to generate file: %s' % filename
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1)
sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))
X_pool3 = batch_pool3_features(sess,X)
np.save(filename,X_pool3)
def get_session():
tf.reset_default_graph()
tf_config = tf.ConfigProto(
inter_op_parallelism_threads=1,
intra_op_parallelism_threads=1)
session = tf.Session(config=tf_config)
print("AVAILABLE GPUS: ", get_available_gpus())
return session
def get_session():
tf.reset_default_graph()
tf_config = tf.ConfigProto(
inter_op_parallelism_threads=1,
intra_op_parallelism_threads=1)
session = tf.Session(config=tf_config)
print("AVAILABLE GPUS: ", get_available_gpus())
return session
def compute_embeddings(images):
"""Runs inference on an image.
Args:
image: Image file names.
Returns:
Dict mapping image file name to embedding.
"""
# Creates graph from saved GraphDef.
create_graph()
filename_to_emb = {}
config = tf.ConfigProto(device_count = {'GPU': 0})
bar = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
with tf.Session(config=config) as sess:
i = 0
for image in bar(images):
if not tf.gfile.Exists(image):
tf.logging.fatal('File does not exist %s', image)
image_data = tf.gfile.FastGFile(image, 'rb').read()
# Some useful tensors:
# 'softmax:0': A tensor containing the normalized prediction across
# 1000 labels.
# 'pool_3:0': A tensor containing the next-to-last layer containing 2048
# float description of the image.
# 'DecodeJpeg/contents:0': A tensor containing a string providing JPEG
# encoding of the image.
# Runs the softmax tensor by feeding the image_data as input to the graph.
softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')
embedding_tensor = sess.graph.get_tensor_by_name('pool_3:0')
embedding = sess.run(embedding_tensor,
{'DecodeJpeg/contents:0': image_data})
filename_to_emb[image] = embedding.reshape(2048)
i += 1
# print(image, i, len(images))
return filename_to_emb
# temp_dir is a subdir of temp
def main(_):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
model = UNet(batch_size=args.batch_size)
model.register_session(sess)
model.build_model(is_training=False, inst_norm=args.inst_norm)
model.export_generator(save_dir=args.save_dir, model_dir=args.model_dir)
def main(_):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
with tf.Session(config=config) as sess:
model = UNet(batch_size=args.batch_size)
model.register_session(sess)
model.build_model(is_training=False, inst_norm=args.inst_norm)
embedding_ids = [int(i) for i in args.embedding_ids.split(",")]
if not args.interpolate:
if len(embedding_ids) == 1:
embedding_ids = embedding_ids[0]
if args.compare:
model.infer_compare(model_dir=args.model_dir, source_obj=args.source_obj, embedding_ids=embedding_ids,
save_dir=args.save_dir, show_ssim=args.show_ssim)
else:
model.infer(model_dir=args.model_dir, source_obj=args.source_obj, embedding_ids=embedding_ids,
save_dir=args.save_dir, progress_file=args.progress_file)
else:
if len(embedding_ids) < 2:
raise Exception("no need to interpolate yourself unless you are a narcissist")
chains = embedding_ids[:]
if args.uroboros:
chains.append(chains[0])
pairs = list()
for i in range(len(chains) - 1):
pairs.append((chains[i], chains[i + 1]))
for s, e in pairs:
model.interpolate(model_dir=args.model_dir, source_obj=args.source_obj, between=[s, e],
save_dir=args.save_dir, steps=args.steps)
if args.output_gif:
gif_path = os.path.join(args.save_dir, args.output_gif)
compile_frames_to_gif(args.save_dir, gif_path)
print("gif saved at %s" % gif_path)
def restrict_gpu_memory(per_process_gpu_memory_fraction: float = 0.9):
import os
import tensorflow as tf
import keras
thread_count = os.environ.get('OMP_NUM_THREADS')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=per_process_gpu_memory_fraction)
config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True,
intra_op_parallelism_threads=thread_count) \
if thread_count else tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True)
keras.backend.tensorflow_backend.set_session(tf.Session(config=config))
def main(flags):
# ???????
ds = mnist.Data(flags.directory)
# tensorflow ??
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# ??session
with tf.Session(config=config) as session:
# ??????
model = Model('Model',
session,
flags.height,
flags.width,
flags.depth,
flags.feature_size,
flags.num_classes).build()
# ??slot
train = model.get_slot('train')
predict = model.get_slot('predict')
# ?????
session.run(tf.global_variables_initializer())
# ????
for i in range(1, flags.nloop + 1):
# ????batch???
images_batch, labels_batch = ds.next_batch(flags.bsize)
loss, train_accuracy = train(images_batch, labels_batch, 0.5)
# ?100?????????????batch??accuracy
if i % 100 == 0:
print('Loop {}:\tloss={}\ttrain accuracy={}'.format(i, loss, train_accuracy))
# ????????accuracy
accuracy = predict(ds.test_images, ds.test_labels, 1.0)
print('Accuracy on test set: {}'.format(accuracy))
return 0
def main(flags):
# ???????
ds = DataSource(flags.directory)
# tensorflow ??
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# ??session
with tf.Session(config=config) as session:
# ??????
model = Model('Model', session, flags.input_size, flags.num_classes).build()
# ??slot
train = model.get_slot('train')
predict = model.get_slot('predict')
# ?????
session.run(tf.global_variables_initializer())
# ????
for i in range(1, flags.nloop + 1):
# ????batch???
images_batch, labels_batch = ds.next_batch(flags.bsize)
# ?????????
loss = train(images_batch, labels_batch)
print('Loop {}:\tloss= {}'.format(i, loss))
# ????????accuracy
accuracy = predict(ds.test_images, ds.test_labels)
print('Accuracy on test set: {}'.format(accuracy))
return 0
