def _upload_metrics(current_model):
bq.upload_metrics_to_bq(test_name=current_model.test_name,
total_time=current_model.total_time,
epochs=current_model.epochs,
batch_size=current_model.batch_size,
backend_type=keras.backend.backend(),
backend_version=get_backend_version(),
cpu_num_cores=config['cpu_num_cores'],
cpu_memory=config['cpu_memory'],
cpu_memory_info=config['cpu_memory_info'],
gpu_count=config['gpus'],
gpu_platform=config['gpu_platform'],
platform_type=config['platform_type'],
platform_machine_type=config['platform_machine_type'],
keras_version=keras.__version__,
sample_type=current_model.sample_type)
# MNIST MLP
python类__version__()的实例源码
def save_run(config, environment=None, comment=None, extra_config=None,
base_path=DEFAULT_BASE_PATH, filename=DEFAULT_FILENAME):
if environment == 'cloud':
# We don't write runs inside Google Cloud, we run it before.
return
diff = get_diff()
lumi_version = get_luminoth_version()
tf_version = get_tensorflow_version()
experiment = {
'environment': environment,
'datetime': str(datetime.datetime.utcnow()) + 'Z',
'diff': diff,
'luminoth_version': lumi_version,
'tensorflow_version': tf_version,
'config': config,
'extra_config': extra_config,
}
file_path = os.path.join(base_path, filename)
tf.gfile.MakeDirs(base_path)
with tf.gfile.Open(file_path, 'a') as log:
log.write(json.dumps(experiment) + '\n')
controller.py 文件源码
项目:language-translation-english-to-french
作者: Satyaki0924
项目源码
文件源码
阅读 30
收藏 0
点赞 0
评论 0
def main(choice):
try:
assert LooseVersion(tf.__version__) in [LooseVersion('1.0.0'), LooseVersion(
'1.0.1')], 'This project requires TensorFlow version 1.0 You are using {}' \
.format(tf.__version__)
print('TensorFlow Version: {}'.format(tf.__version__))
print('*****Author: Satyaki Sanyal*****')
print('***This project must only be used for educational purpose***')
if choice == 1:
if not tf.test.gpu_device_name():
print('*** ERROR: No GPU found. Please use a GPU to train your neural network. ***')
else:
print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
Main().main()
elif choice == 2:
Translate().translate()
elif choice == 3:
Plot().plot()
else:
print('*** Error: Wrong choice ***')
except Exception as exc:
print('*** Error: ' + str(exc) + ' ***')
def main(_):
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(name)-7s %(levelname)-7s %(message)s'
)
logger.info('tf version: {}'.format(tf.__version__))
parser = argparse.ArgumentParser(description='Run Dobot WebAPI.')
parser.add_argument('--data_dir', type=str, default='data', help="Directory for training data.")
parser.add_argument('--train_dir', type=str, default='train', help="Directory for checkpoints.")
args = parser.parse_args()
reader = FeaturesDataReader(args.data_dir)
predictor = Predictor(reader, args.train_dir, args.train_dir+'/params.json')
print(predictor.predict_to_json())
def create_model(session, forward_only):
"""Create model and initialize or load parameters"""
model = seq2seq_model.Seq2SeqModel( gConfig['enc_vocab_size'], gConfig['dec_vocab_size'], _buckets, gConfig['layer_size'], gConfig['num_layers'], gConfig['max_gradient_norm'], gConfig['batch_size'], gConfig['learning_rate'], gConfig['learning_rate_decay_factor'], forward_only=forward_only)
if 'pretrained_model' in gConfig:
model.saver.restore(session,gConfig['pretrained_model'])
return model
ckpt = tf.train.get_checkpoint_state(gConfig['working_directory'])
# the checkpoint filename has changed in recent versions of tensorflow
checkpoint_suffix = ""
if tf.__version__ > "0.12":
checkpoint_suffix = ".index"
if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + checkpoint_suffix):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
model.saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
session.run(tf.initialize_all_variables())
return model
def create_model(session, forward_only):
"""Create model and initialize or load parameters"""
model = seq2seq_model.Seq2SeqModel( gConfig['enc_vocab_size'], gConfig['dec_vocab_size'], _buckets, gConfig['layer_size'], gConfig['num_layers'], gConfig['max_gradient_norm'], gConfig['batch_size'], gConfig['learning_rate'], gConfig['learning_rate_decay_factor'], forward_only=forward_only)
if 'pretrained_model' in gConfig:
model.saver.restore(session,gConfig['pretrained_model'])
return model
ckpt = tf.train.get_checkpoint_state(gConfig['working_directory'])
# the checkpoint filename has changed in recent versions of tensorflow
checkpoint_suffix = ""
if tf.__version__ > "0.12":
checkpoint_suffix = ".index"
if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + checkpoint_suffix):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
model.saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
session.run(tf.initialize_all_variables())
return model
def get_inception_layer( inputs, conv11_size, conv33_11_size, conv33_size,
conv55_11_size, conv55_size, pool11_size ):
with tf.variable_scope("conv_1x1"):
conv11 = layers.conv2d( inputs, conv11_size, [ 1, 1 ] )
with tf.variable_scope("conv_3x3"):
conv33_11 = layers.conv2d( inputs, conv33_11_size, [ 1, 1 ] )
conv33 = layers.conv2d( conv33_11, conv33_size, [ 3, 3 ] )
with tf.variable_scope("conv_5x5"):
conv55_11 = layers.conv2d( inputs, conv55_11_size, [ 1, 1 ] )
conv55 = layers.conv2d( conv55_11, conv55_size, [ 5, 5 ] )
with tf.variable_scope("pool_proj"):
pool_proj = layers.max_pool2d( inputs, [ 3, 3 ], stride = 1 )
pool11 = layers.conv2d( pool_proj, pool11_size, [ 1, 1 ] )
if tf.__version__ == '0.11.0rc0':
return tf.concat(3, [conv11, conv33, conv55, pool11])
return tf.concat([conv11, conv33, conv55, pool11], 3)
def main(_):
print('tf version', tf.__version__)
topics, answers, num_topics = read_assistments_data(DATA_LOC)
full_data = load_data(topics, answers, num_topics)
model = DKTModel(num_topics, HIDDEN_SIZE, MAX_LENGTH)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
#We need to explicitly initialize local variables to use
#TensorFlow's AUC function for some reason...
session.run(tf.local_variables_initializer())
train_model(model, session, full_data)
#model1, model2 = train_paired_models(session, full_data, num_topics)
#test_paired_models(session, full_data, model1, model2)
#embed()
def create_model(session, forward_only):
"""Create model and initialize or load parameters"""
model = seq2seq_model.Seq2SeqModel( gConfig['enc_vocab_size'], gConfig['dec_vocab_size'], _buckets, gConfig['layer_size'], gConfig['num_layers'], gConfig['max_gradient_norm'], gConfig['batch_size'], gConfig['learning_rate'], gConfig['learning_rate_decay_factor'], forward_only=forward_only)
if 'pretrained_model' in gConfig:
model.saver.restore(session,gConfig['pretrained_model'])
return model
ckpt = tf.train.get_checkpoint_state(gConfig['working_directory'])
# the checkpoint filename has changed in recent versions of tensorflow
checkpoint_suffix = ""
if tf.__version__ > "0.12":
checkpoint_suffix = ".index"
if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + checkpoint_suffix):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
model.saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
session.run(tf.initialize_all_variables())
return model
def _model_deploy(args, cell):
parts = args['name'].split('.')
if len(parts) == 2:
model_name, version_name = parts[0], parts[1]
model_exists = False
try:
# If describe() works, the model already exists.
datalab_ml.Models(project_id=args['project']).get_model_details(model_name)
model_exists = True
except:
pass
if not model_exists:
datalab_ml.Models(project_id=args['project']).create(model_name)
versions = datalab_ml.ModelVersions(model_name, project_id=args['project'])
runtime_version = args['runtime_version']
if not runtime_version:
runtime_version = tf.__version__
versions.deploy(version_name, args['path'], runtime_version=runtime_version)
else:
raise ValueError('Name must be like "model.version".')
def create_model(session, forward_only):
"""Create model and initialize or load parameters"""
model = seq2seq_model.Seq2SeqModel( gConfig['enc_vocab_size'], gConfig['dec_vocab_size'], _buckets, gConfig['layer_size'], gConfig['num_layers'], gConfig['max_gradient_norm'], gConfig['batch_size'], gConfig['learning_rate'], gConfig['learning_rate_decay_factor'], forward_only=forward_only)
if 'pretrained_model' in gConfig:
model.saver.restore(session,gConfig['pretrained_model'])
return model
ckpt = tf.train.get_checkpoint_state(gConfig['working_directory'])
# the checkpoint filename has changed in recent versions of tensorflow
checkpoint_suffix = ""
if tf.__version__ > "0.12":
checkpoint_suffix = ".index"
if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + checkpoint_suffix):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
model.saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
session.run(tf.initialize_all_variables())
return model
def setup(tf, order=None):
"""
Sets up global variables (currently only the tensorflow version) to adapt to peculiarities of
different tensorflow versions. This function should only be called before :py:class:`Model`
creation, not for evaluation. Therefore, the tensorflow module *tf* must be passed:
.. code-block:: python
import tensorflow as tf
import tfdeploy as td
td.setup(tf)
# ...
Also, when *order* is not *None*, it is forwarded to :py:func:`optimize` for convenience.
"""
global _tf_version_string, _tf_version
_tf_version_string = tf.__version__
_tf_version = _parse_tf_version(_tf_version_string)
if order is not None:
optimize(order)
def create_model(session, forward_only):
"""Create model and initialize or load parameters"""
model = seq2seq_model.Seq2SeqModel( gConfig['enc_vocab_size'], gConfig['dec_vocab_size'], _buckets, gConfig['layer_size'], gConfig['num_layers'], gConfig['max_gradient_norm'], gConfig['batch_size'], gConfig['learning_rate'], gConfig['learning_rate_decay_factor'], forward_only=forward_only)
if 'pretrained_model' in gConfig:
model.saver.restore(session,gConfig['pretrained_model'])
return model
ckpt = tf.train.get_checkpoint_state(gConfig['working_directory'])
# the checkpoint filename has changed in recent versions of tensorflow
checkpoint_suffix = ""
if tf.__version__ > "0.12":
checkpoint_suffix = ".index"
if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + checkpoint_suffix):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
model.saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
session.run(tf.initialize_all_variables())
return model
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
check_video_id()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
check_video_id()
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
inference()
def tensorflow_version_tuple():
v = tf.__version__
major, minor, patch = v.split('.')
return (int(major), int(minor), patch)