def _upload_metrics(current_model):
bq.upload_metrics_to_bq(test_name=current_model.test_name,
total_time=current_model.total_time,
epochs=current_model.epochs,
batch_size=current_model.batch_size,
backend_type=keras.backend.backend(),
backend_version=get_backend_version(),
cpu_num_cores=config['cpu_num_cores'],
cpu_memory=config['cpu_memory'],
cpu_memory_info=config['cpu_memory_info'],
gpu_count=config['gpus'],
gpu_platform=config['gpu_platform'],
platform_type=config['platform_type'],
platform_machine_type=config['platform_machine_type'],
keras_version=keras.__version__,
sample_type=current_model.sample_type)
# MNIST MLP
python类__version__()的实例源码
def get_backend_version():
if keras.backend.backend() == "tensorflow":
return tf.__version__
if keras.backend.backend() == "theano":
return theano.__version__
if keras.backend.backend() == "cntk":
return cntk.__version__
return "undefined"
def _check_modules():
"""Checks whether all dependencies are installed"""
try:
import numpy
if numpy.__version__ < "1.6.0":
logger.warning("WARNING: You are using a numpy %s < 1.6.0. This "
"might not work", numpy.__version__)
except:
raise ImportError("Numpy cannot be imported. Are you sure that it's installed?")
try:
import scipy
if scipy.__version__ < "0.12.0":
logger.warning("WARNING: You are using a scipy %s < 0.12.0. "
"This might not work", scipy.__version__)
except:
raise ImportError("Scipy cannot be imported. Are you sure that it's installed?")
try:
import theano
logger.debug("\tTheano: %s" % str(theano.__version__))
except ImportError:
logger.warning("Theano not found. You might need this to run some "
"more complex benchmarks!")
if 'cuda' not in os.environ['PATH']:
logger.warning("CUDA not in $PATH")
def initialize(args, data_shape, class_labels, class_num):
cudnn_info=(theano.config.dnn.conv.algo_fwd, theano.config.dnn.conv.algo_bwd_data, theano.config.dnn.conv.algo_bwd_filter)
logging.info("Using theano version:", theano.__version__, "(cudnn fwd=%s,bwd data=%s,bwd filter=%s)"%cudnn_info)
if args.model is None:
#construct convolutional model
logging.info("Building convolutional model (%i classes)..."%class_num)
model = ModelCNN()
model.batch_size = args.batch_size
model.class_labels = class_labels
model.class_num = class_num
#allow padding to be specified in border mode
try:
n = int(args.border_mode)
border_mode = (n,n)
except ValueError:
border_mode = args.border_mode
model.build(args.model_desc, data_shape, args.activation, border_mode, list(args.weight_init))
else:
model = load_from_file(args.model, args.batch_size)
model.class_labels = class_labels
model.class_num = class_num
assert data_shape == model.data_shape, "Mismatching data shapes in .mdl and data: " + str(data_shape) + "!=" + str(model.data_shape)
model.skip_layer_updates = args.skip_layer_updates
if len(model.skip_layer_updates) > 0:
logging.info("Skipping layer updates:", model.skip_layer_updates)
return model
#
def _show_system_info(self):
import theano
print("Theano version %s" % theano.__version__)
theano_dir = os.path.dirname(theano.__file__)
print("theano is installed in %s" % theano_dir)
super(TheanoNoseTester, self)._show_system_info()
def __init__(self, input):
self.input = input
if "0.9.0" in theano.__version__:
self.output = T.flatten(self.input, outdim=2) # support theano 0.9.0 api
elif "0.10.0" in theano.__version__:
self.output = T.flatten(self.input, ndim=2) # support theano 0.10.0 api
else:
raise NotImplementedError("this version of theano is not supported") # I can't support all versions; I'm only human.
def print_model_settings(locals_var, path=None, sys_arg=False):
"""
Prints all variables in upper case in locals_var,
except for T which usually stands for theano.tensor.
If locals() passed as input to this method, will print
all the variables in upper case defined so far, that is
model settings.
With `path` as an address to a directory it will _append_ it
as a file named `model_settings.txt` as well.
With `sys_arg` set to True, log information about Python, Numpy,
and Theano and passed arguments to the script will be added too.
args.pkl would be overwritten, specially in case of resuming a job.
But again that wouldn't be much of a problem as all the passed args
to the script except for '--resume' should be the same.
With both `path` and `sys_arg` passed, dumps the theano.config.
:usage:
>>> import theano.tensor as T
>>> import lib
>>> BATCH_SIZE, DIM = 128, 512
>>> DATA_PATH = '/Path/to/dataset'
>>> lib.print_model_settings(locals(), path='./')
"""
log = ""
if sys_arg:
try:
log += "Python:\n"
log += "\tsys.version_info\t{}\n".format(str(sys.version_info))
log += "Numpy:\n"
log += "\t.__version__\t{}\n".format(numpy.__version__)
log += "Theano:\n"
log += "\t.__version__\t{}\n".format(theano.__version__)
log += "\n\nAll passed args:\n"
log += str(sys.argv)
log += "\n"
except:
print "Something went wrong during sys_arg logging. Continue anyway!"
log += "\nModel settings:"
all_vars = [(k,v) for (k,v) in locals_var.items() if (k.isupper() and k != 'T')]
all_vars = sorted(all_vars, key=lambda x: x[0])
for var_name, var_value in all_vars:
log += ("\n\t%-20s %s" % (var_name, var_value))
print log
if path is not None:
ensure_dir(path)
# Don't override, just append if by mistake there is something in the file.
with open(os.path.join(path, __model_setting_file_name), 'a+') as f:
f.write(log)
if sys_arg:
with open(os.path.join(path, 'th_conf.txt'), 'a+') as f:
f.write(str(theano.config))
with open(os.path.join(path, 'args.pkl'), 'wb') as f:
pickle.dump(sys.argv, f)
# To load:
# >>> import cPickle as pickle
# >>> args = pickle.load(open(os.path.join(path, 'args.pkl'), 'rb'))
def print_model_settings(locals_var, path=None, sys_arg=False):
"""
Prints all variables in upper case in locals_var,
except for T which usually stands for theano.tensor.
If locals() passed as input to this method, will print
all the variables in upper case defined so far, that is
model settings.
With `path` as an address to a directory it will _append_ it
as a file named `model_settings.txt` as well.
With `sys_arg` set to True, log information about Python, Numpy,
and Theano and passed arguments to the script will be added too.
args.pkl would be overwritten, specially in case of resuming a job.
But again that wouldn't be much of a problem as all the passed args
to the script except for '--resume' should be the same.
With both `path` and `sys_arg` passed, dumps the theano.config.
:usage:
>>> import theano.tensor as T
>>> import lib
>>> BATCH_SIZE, DIM = 128, 512
>>> DATA_PATH = '/Path/to/dataset'
>>> lib.print_model_settings(locals(), path='./')
"""
log = ""
if sys_arg:
try:
log += "Python:\n"
log += "\tsys.version_info\t{}\n".format(str(sys.version_info))
log += "Numpy:\n"
log += "\t.__version__\t{}\n".format(numpy.__version__)
log += "Theano:\n"
log += "\t.__version__\t{}\n".format(theano.__version__)
log += "\n\nAll passed args:\n"
log += str(sys.argv)
log += "\n"
except:
print "Something went wrong during sys_arg logging. Continue anyway!"
log += "\nModel settings:"
all_vars = [(k,v) for (k,v) in locals_var.items() if (k.isupper() and k != 'T')]
all_vars = sorted(all_vars, key=lambda x: x[0])
for var_name, var_value in all_vars:
log += ("\n\t%-20s %s" % (var_name, var_value))
print log
if path is not None:
ensure_dir(path)
# Don't override, just append if by mistake there is something in the file.
with open(os.path.join(path, __model_setting_file_name), 'a+') as f:
f.write(log)
if sys_arg:
with open(os.path.join(path, 'th_conf.txt'), 'a+') as f:
f.write(str(theano.config))
with open(os.path.join(path, 'args.pkl'), 'wb') as f:
pickle.dump(sys.argv, f)
# To load:
# >>> import cPickle as pickle
# >>> args = pickle.load(open(os.path.join(path, 'args.pkl'), 'rb'))
