def model(self,
input_vectors, input_gene, input_variation, output_label, batch_size,
embedding_size=EMBEDDINGS_SIZE,
output_classes=9,
learning_rate_initial=D2V_DOC_LEARNING_RATE_INITIAL,
learning_rate_decay=D2V_DOC_LEARNING_RATE_DECAY,
learning_rate_decay_steps=D2V_DOC_LEARNING_RATE_DECAY_STEPS):
self.global_step = training_util.get_or_create_global_step()
logits, targets = doc2vec_prediction_model(input_vectors, input_gene, input_variation,
output_label, batch_size,
is_training=True, embedding_size=embedding_size,
output_classes=output_classes)
self.prediction = tf.nn.softmax(logits)
self.loss = tf.nn.softmax_cross_entropy_with_logits(labels=targets, logits=logits)
self.loss = tf.reduce_mean(self.loss)
tf.summary.scalar('loss', self.loss)
# learning rate & optimizer
self.learning_rate = tf.train.exponential_decay(learning_rate_initial, self.global_step,
learning_rate_decay_steps,
learning_rate_decay,
staircase=True, name='learning_rate')
tf.summary.scalar('learning_rate', self.learning_rate)
sgd = tf.train.GradientDescentOptimizer(self.learning_rate)
self.optimizer = sgd.minimize(self.loss, global_step=self.global_step)
# metrics
self.metrics = metrics.single_label(self.prediction, targets)
# saver to save the model
self.saver = tf.train.Saver()
# check a nan value in the loss
self.loss = tf.check_numerics(self.loss, 'loss is nan')
return None
python类check_numerics()的实例源码
doc2vec_train_doc_prediction.py 文件源码
项目:kaggle_redefining_cancer_treatment
作者: jorgemf
项目源码
文件源码
阅读 24
收藏 0
点赞 0
评论 0
text_classification_train.py 文件源码
项目:kaggle_redefining_cancer_treatment
作者: jorgemf
项目源码
文件源码
阅读 23
收藏 0
点赞 0
评论 0
def model(self, input_text_begin, input_text_end, gene, variation, expected_labels, batch_size,
vocabulary_size=VOCABULARY_SIZE, embeddings_size=EMBEDDINGS_SIZE, output_classes=9):
# embeddings
embeddings = _load_embeddings(vocabulary_size, embeddings_size)
# global step
self.global_step = training_util.get_or_create_global_step()
# model
with slim.arg_scope(self.text_classification_model.model_arg_scope()):
outputs = self.text_classification_model.model(input_text_begin, input_text_end,
gene, variation, output_classes,
embeddings=embeddings,
batch_size=batch_size)
# loss
targets = self.text_classification_model.targets(expected_labels, output_classes)
self.loss = self.text_classification_model.loss(targets, outputs)
tf.summary.scalar('loss', self.loss)
# learning rate
self.optimizer, self.learning_rate = \
self.text_classification_model.optimize(self.loss, self.global_step)
if self.learning_rate is not None:
tf.summary.scalar('learning_rate', self.learning_rate)
# metrics
self.metrics = metrics.single_label(outputs['prediction'], targets)
# saver to save the model
self.saver = tf.train.Saver()
# check a nan value in the loss
self.loss = tf.check_numerics(self.loss, 'loss is nan')
return None
def build_graph(self, *layers):
weights = [layer.weight.node for layer in layers]
self.ph_weights = graph.Placeholders(variables=graph.TfNode(weights))
self.assign = graph.TfNode([tf.assign(variable, value) for variable, value in
utils.Utils.izip(weights, self.ph_weights.checked)])
self.check = graph.TfNode(tf.group(*[tf.check_numerics(w, 'weight_%d' % i) for i, w in
enumerate(utils.Utils.flatten(weights))]))
self.global_norm = tf.global_norm(list(utils.Utils.flatten(weights)))
return weights
def build_graph(self, variables):
phs = utils.Utils.map(variables.node, lambda v: tf.placeholder(shape=v.get_shape(), dtype=v.dtype))
self.checked = utils.Utils.map(phs, lambda ph: tf.check_numerics(ph, ''))
return phs
def train(self, batch):
_, _, l = tf.get_default_session().run([self.check_numerics, self.train_op, self.loss],
feed_dict={self.input_state: batch.state_1,
self.input_action: batch.action,
self.reward: batch.reward,
self.terminal_mask: batch.terminal_mask,
self.input_state_2: batch.state_2,
base_network.IS_TRAINING: True})
return l
def _calc_scores(self, attention_scores):
"""
Combine exact match scores & attention scores
:param attention_scores: [batch_size, max_question_length, table_size]
:return:
"""
with tf.variable_scope("scoring"):
scores = attention_scores
tf.check_numerics(scores, message="Score Nan...")
softmax_scores = tf.nn.softmax(scores, dim=-1)
tf.check_numerics(softmax_scores, message="Softmax Score Nan...")
return softmax_scores
def train(self, batch):
flip_horizontally = np.random.random() < 0.5
if VERBOSE_DEBUG:
print "batch.action"
print batch.action.T
print "batch.reward", batch.reward.T
print "batch.terminal_mask", batch.terminal_mask.T
print "flip_horizontally", flip_horizontally
print "weights", batch.weight.T
values = tf.get_default_session().run([self._l_values, self.value_net.value,
self.advantage, self.target_value_net.value,
self.print_gradient_norms],
feed_dict={self.input_state: batch.state_1,
self.input_action: batch.action,
self.reward: batch.reward,
self.terminal_mask: batch.terminal_mask,
self.input_state_2: batch.state_2,
self.importance_weight: batch.weight,
base_network.IS_TRAINING: True,
base_network.FLIP_HORIZONTALLY: flip_horizontally})
values = [np.squeeze(v) for v in values]
print "_l_values", values[0].T
print "value_net.value ", values[1].T
print "advantage ", values[2].T
print "target_value_net.value ", values[3].T
_, _, l = tf.get_default_session().run([self.check_numerics, self.train_op,
self.loss],
feed_dict={self.input_state: batch.state_1,
self.input_action: batch.action,
self.reward: batch.reward,
self.terminal_mask: batch.terminal_mask,
self.input_state_2: batch.state_2,
self.importance_weight: batch.weight,
base_network.IS_TRAINING: True,
base_network.FLIP_HORIZONTALLY: flip_horizontally})
return l
def detect(sess, model, names, image, path):
preprocess = eval(args.preprocess)
_, height, width, _ = image.get_shape().as_list()
_image = read_image(path)
image_original = np.array(np.uint8(_image))
if len(image_original.shape) == 2:
image_original = np.repeat(np.expand_dims(image_original, -1), 3, 2)
image_height, image_width, _ = image_original.shape
image_std = preprocess(np.array(np.uint8(_image.resize((width, height)))).astype(np.float32))
feed_dict = {image: np.expand_dims(image_std, 0)}
tensors = [model.conf, model.xy_min, model.xy_max]
conf, xy_min, xy_max = sess.run([tf.check_numerics(t, t.op.name) for t in tensors], feed_dict=feed_dict)
boxes = utils.postprocess.non_max_suppress(conf[0], xy_min[0], xy_max[0], args.threshold, args.threshold_iou)
scale = [image_width / model.cell_width, image_height / model.cell_height]
fig = plt.figure()
ax = fig.gca()
ax.imshow(image_original)
colors = [prop['color'] for _, prop in zip(names, itertools.cycle(plt.rcParams['axes.prop_cycle']))]
cnt = 0
for _conf, _xy_min, _xy_max in boxes:
index = np.argmax(_conf)
if _conf[index] > args.threshold:
wh = _xy_max - _xy_min
_xy_min = _xy_min * scale
_wh = wh * scale
linewidth = min(_conf[index] * 10, 3)
ax.add_patch(patches.Rectangle(_xy_min, _wh[0], _wh[1], linewidth=linewidth, edgecolor=colors[index], facecolor='none'))
ax.annotate(names[index] + ' (%.1f%%)' % (_conf[index] * 100), _xy_min, color=colors[index])
cnt += 1
fig.canvas.set_window_title('%d objects detected' % cnt)
ax.set_xticks([])
ax.set_yticks([])
return fig
def main():
model = config.get('config', 'model')
yolo = importlib.import_module('model.' + model)
width = config.getint(model, 'width')
height = config.getint(model, 'height')
preprocess = getattr(importlib.import_module('detect'), args.preprocess)
with tf.Session() as sess:
ph_image = tf.placeholder(tf.float32, [1, height, width, 3], name='ph_image')
builder = yolo.Builder(args, config)
builder(ph_image)
global_step = tf.contrib.framework.get_or_create_global_step()
model_path = tf.train.latest_checkpoint(utils.get_logdir(config))
tf.logging.info('load ' + model_path)
slim.assign_from_checkpoint_fn(model_path, tf.global_variables())(sess)
tf.logging.info('global_step=%d' % sess.run(global_step))
tensors = [builder.model.conf, builder.model.xy_min, builder.model.xy_max]
tensors = [tf.check_numerics(t, t.op.name) for t in tensors]
cap = cv2.VideoCapture(0)
try:
while True:
ret, image_bgr = cap.read()
assert ret
image_height, image_width, _ = image_bgr.shape
scale = [image_width / builder.model.cell_width, image_height / builder.model.cell_height]
image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
image_std = np.expand_dims(preprocess(cv2.resize(image_rgb, (width, height))).astype(np.float32), 0)
feed_dict = {ph_image: image_std}
conf, xy_min, xy_max = sess.run(tensors, feed_dict)
boxes = utils.postprocess.non_max_suppress(conf[0], xy_min[0], xy_max[0], args.threshold, args.threshold_iou)
for _conf, _xy_min, _xy_max in boxes:
index = np.argmax(_conf)
if _conf[index] > args.threshold:
_xy_min = (_xy_min * scale).astype(np.int)
_xy_max = (_xy_max * scale).astype(np.int)
cv2.rectangle(image_bgr, tuple(_xy_min), tuple(_xy_max), (255, 0, 255), 3)
cv2.putText(image_bgr, builder.names[index] + ' (%.1f%%)' % (_conf[index] * 100), tuple(_xy_min), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)
cv2.imshow('detection', image_bgr)
cv2.waitKey(1)
finally:
cv2.destroyAllWindows()
cap.release()
def cross_entropy_indirect(logprops, name):
loss = tf.reduce_mean(-logprops, axis=0)
loss = tf.check_numerics(loss, f'check/cross_entropy/{name}')
return loss
def cross_entropy_direct(logits, target, name):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=target)
loss *= tf.cast(tf.not_equal(target, tf.zeros_like(target)), tf.float32)
batch_loss = tf.reduce_sum(loss, axis=1)
batch_loss = tf.reduce_mean(batch_loss, axis=0)
batch_loss = tf.check_numerics(batch_loss, f'check/cross_entropy/{name}')
return batch_loss
def _mapper(self, grad, var):
# this is very slow...
#op = tf.Assert(tf.reduce_all(tf.is_finite(var)), [var], summarize=100)
grad = tf.check_numerics(grad, 'CheckGradient')
return grad
def forward_and_jacobian(self, x, sum_log_det_jacobians, z):
with tf.variable_scope(self.name):
xs = int_shape(x)
b = self.get_mask(xs, self.mask_type)
# masked half of x
x1 = x * b
l,m = self.function_l_m(x1, b)
y = x1 + tf.mul(-b+1.0, x*tf.check_numerics(tf.exp(l), "exp has NaN") + m)
log_det_jacobian = tf.reduce_sum(l, [1,2,3])
sum_log_det_jacobians += log_det_jacobian
return y,sum_log_det_jacobians, z
def _create_reparam_variables(self, eps=1e-8):
# noise for generating z
u = tf.random_uniform([self.n_samples, self.dim], dtype=tf.float32)
log_alpha = self._log_alpha
# logistic reparameterization z = g(u, log_alpha)
z = log_alpha + safe_log_prob(u) - safe_log_prob(1 - u)
# b = H(z)
b = tf.to_float(tf.stop_gradient(z > 0))
# g(u', log_alpha) = 0
u_prime = tf.nn.sigmoid(-log_alpha)
v_1 = (u - u_prime) / tf.clip_by_value(1 - u_prime, eps, 1)
v_1 = tf.clip_by_value(v_1, 0, 1)
v_1 = tf.stop_gradient(v_1)
v_1 = v_1 * (1 - u_prime) + u_prime
v_0 = u / tf.clip_by_value(u_prime, eps, 1)
v_0 = tf.clip_by_value(v_0, 0, 1)
v_0 = tf.stop_gradient(v_0)
v_0 = v_0 * u_prime
v = tf.where(u > u_prime, v_1, v_0)
v = tf.check_numerics(v, 'v sampling is not numerically stable.')
v = v + tf.stop_gradient(-v + u) # v and u are the same up to numerical errors
tf.summary.histogram("u-v", u-v)
z_tilde = log_alpha + safe_log_prob(v) - safe_log_prob(1 - v)
self.b = b
self.z = z
self.z_tilde = z_tilde
def embed_tokens(self, is_training, config, embedding_initializer):
"""Embedds input tokens.
"""
vocab_size = config.vocab_size
size = config.word_embed_size
max_question_length = self.max_question_length
max_sentence_length = self.max_sentence_length
max_sentence_num = self.max_sentence_num
with tf.variable_scope("embed"):
with tf.device("/cpu:0"):
embedding = tf.get_variable(
"embedding_mat", [vocab_size, size],
initializer=embedding_initializer,
dtype=config.data_type,
trainable=False # Continue to train pretrained word2vec
# trainable=True # Continue to train pretrained word2vec
)
self._embedding = embedding
embed_question= []
for i in xrange(max_question_length):
embed_question.append(
tf.nn.embedding_lookup(embedding, self._input_question[i]))
if is_training and config.w_embed_keep_prob < 1:
embed_question[i] = tf.nn.dropout(embed_question[i],
config.w_embed_keep_prob)
if NUMERIC_CHECK:
embed_question[i] = \
tf.check_numerics(embed_question[i],
"embed_question[{}][{}] numeric error".format(i))
embed_sentences = []
for i in xrange(max_sentence_num):
embed_sentences.append([])
for j in xrange(max_sentence_length):
embed_sentences[i].append(
tf.nn.embedding_lookup(embedding, self._input_sentences[i][j]))
if is_training and config.w_embed_keep_prob < 1:
embed_sentences[i][j] = tf.nn.dropout(embed_sentences[i][j],
config.w_embed_keep_prob)
if NUMERIC_CHECK:
embed_sentences[i][j] = \
tf.check_numerics(embed_sentences[i][j],
"embed_sentences[{}][{}] numeric error".format(i, j))
return embed_question, embed_sentences
# RESULTS
def __init__(self,
mean=0.,
logstd=None,
std=None,
group_ndims=0,
is_reparameterized=True,
use_path_derivative=False,
check_numerics=False,
**kwargs):
self._mean = tf.convert_to_tensor(mean)
warnings.warn("Normal: The order of arguments logstd/std will change "
"to std/logstd in the coming version.", FutureWarning)
if (logstd is None) == (std is None):
raise ValueError("Either std or logstd should be passed but not "
"both of them.")
elif logstd is None:
self._std = tf.convert_to_tensor(std)
dtype = assert_same_float_dtype([(self._mean, 'Normal.mean'),
(self._std, 'Normal.std')])
logstd = tf.log(self._std)
if check_numerics:
logstd = tf.check_numerics(logstd, "log(std)")
self._logstd = logstd
else:
# std is None
self._logstd = tf.convert_to_tensor(logstd)
dtype = assert_same_float_dtype([(self._mean, 'Normal.mean'),
(self._logstd, 'Normal.logstd')])
std = tf.exp(self._logstd)
if check_numerics:
std = tf.check_numerics(std, "exp(logstd)")
self._std = std
try:
tf.broadcast_static_shape(self._mean.get_shape(),
self._std.get_shape())
except ValueError:
raise ValueError(
"mean and std/logstd should be broadcastable to match each "
"other. ({} vs. {})".format(
self._mean.get_shape(), self._std.get_shape()))
self._check_numerics = check_numerics
super(Normal, self).__init__(
dtype=dtype,
param_dtype=dtype,
is_continuous=True,
is_reparameterized=is_reparameterized,
use_path_derivative=use_path_derivative,
group_ndims=group_ndims,
**kwargs)
def __init__(self,
mean=0.,
logstd=None,
std=None,
group_ndims=0,
is_reparameterized=True,
use_path_derivative=False,
check_numerics=False,
**kwargs):
self._mean = tf.convert_to_tensor(mean)
warnings.warn("FoldNormal: The order of arguments logstd/std will change "
"to std/logstd in the coming version.", FutureWarning)
if (logstd is None) == (std is None):
raise ValueError("Either std or logstd should be passed but not "
"both of them.")
elif logstd is None:
self._std = tf.convert_to_tensor(std)
dtype = assert_same_float_dtype([(self._mean, 'FoldNormal.mean'),
(self._std, 'FoldNormal.std')])
logstd = tf.log(self._std)
if check_numerics:
logstd = tf.check_numerics(logstd, "log(std)")
self._logstd = logstd
else:
# std is None
self._logstd = tf.convert_to_tensor(logstd)
dtype = assert_same_float_dtype([(self._mean, 'FoldNormal.mean'),
(self._logstd, 'FoldNormal.logstd')])
std = tf.exp(self._logstd)
if check_numerics:
std = tf.check_numerics(std, "exp(logstd)")
self._std = std
try:
tf.broadcast_static_shape(self._mean.get_shape(),
self._std.get_shape())
except ValueError:
raise ValueError(
"mean and std/logstd should be broadcastable to match each "
"other. ({} vs. {})".format(
self._mean.get_shape(), self._std.get_shape()))
self._check_numerics = check_numerics
super(FoldNormal, self).__init__(
dtype=dtype,
param_dtype=dtype,
is_continuous=True,
is_reparameterized=is_reparameterized,
use_path_derivative=use_path_derivative,
group_ndims=group_ndims,
**kwargs)
def __init__(self,
logits,
n_experiments,
dtype=None,
group_ndims=0,
check_numerics=False,
**kwargs):
self._logits = tf.convert_to_tensor(logits)
param_dtype = assert_same_float_dtype(
[(self._logits, 'Binomial.logits')])
if dtype is None:
dtype = tf.int32
assert_same_float_and_int_dtype([], dtype)
sign_err_msg = "n_experiments must be positive"
if isinstance(n_experiments, int):
if n_experiments <= 0:
raise ValueError(sign_err_msg)
self._n_experiments = n_experiments
else:
try:
n_experiments = tf.convert_to_tensor(n_experiments, tf.int32)
except ValueError:
raise TypeError('n_experiments must be int32')
_assert_rank_op = tf.assert_rank(
n_experiments, 0,
message="n_experiments should be a scalar (0-D Tensor).")
_assert_positive_op = tf.assert_greater(
n_experiments, 0, message=sign_err_msg)
with tf.control_dependencies([_assert_rank_op,
_assert_positive_op]):
self._n_experiments = tf.identity(n_experiments)
self._check_numerics = check_numerics
super(Binomial, self).__init__(
dtype=dtype,
param_dtype=param_dtype,
is_continuous=False,
is_reparameterized=False,
group_ndims=group_ndims,
**kwargs)
def __init__(self,
mean,
cov_tril,
group_ndims=0,
is_reparameterized=True,
use_path_derivative=False,
check_numerics=False,
**kwargs):
self._check_numerics = check_numerics
self._mean = tf.convert_to_tensor(mean)
self._mean = assert_rank_at_least_one(
self._mean, 'MultivariateNormalCholesky.mean')
self._n_dim = get_shape_at(self._mean, -1)
self._cov_tril = tf.convert_to_tensor(cov_tril)
self._cov_tril = assert_rank_at_least(
self._cov_tril, 2, 'MultivariateNormalCholesky.cov_tril')
# Static shape check
expected_shape = self._mean.get_shape().concatenate(
[self._n_dim if isinstance(self._n_dim, int) else None])
self._cov_tril.get_shape().assert_is_compatible_with(expected_shape)
# Dynamic
expected_shape = tf.concat(
[tf.shape(self._mean), [self._n_dim]], axis=0)
actual_shape = tf.shape(self._cov_tril)
msg = ['MultivariateNormalCholesky.cov_tril should have compatible '
'shape with mean. Expected', expected_shape, ' got ',
actual_shape]
assert_ops = [tf.assert_equal(expected_shape, actual_shape, msg)]
with tf.control_dependencies(assert_ops):
self._cov_tril = tf.identity(self._cov_tril)
dtype = assert_same_float_dtype(
[(self._mean, 'MultivariateNormalCholesky.mean'),
(self._cov_tril, 'MultivariateNormalCholesky.cov_tril')])
super(MultivariateNormalCholesky, self).__init__(
dtype=dtype,
param_dtype=dtype,
is_continuous=True,
is_reparameterized=is_reparameterized,
use_path_derivative=use_path_derivative,
group_ndims=group_ndims,
**kwargs)
word2vec_train.py 文件源码
项目:kaggle_redefining_cancer_treatment
作者: jorgemf
项目源码
文件源码
阅读 18
收藏 0
点赞 0
评论 0
def model(self, input_label, output_word, batch_size, vocabulary_size=VOCABULARY_SIZE,
embedding_size=EMBEDDINGS_SIZE, num_negative_samples=W2V_NEGATIVE_NUM_SAMPLES,
learning_rate_initial=W2V_LEARNING_RATE_INITIAL,
learning_rate_decay=W2V_LEARNING_RATE_DECAY,
learning_rate_decay_steps=W2V_LEARNING_RATE_DECAY_STEPS):
self.global_step = training_util.get_or_create_global_step()
# inputs/outputs
input_label_reshaped = tf.reshape(input_label, [batch_size])
output_word_reshaped = tf.reshape(output_word, [batch_size, 1])
# embeddings
matrix_dimension = [vocabulary_size, embedding_size]
self.embeddings = tf.get_variable(shape=matrix_dimension,
initializer=layers.xavier_initializer(), dtype=tf.float32,
name='embeddings')
embed = tf.nn.embedding_lookup(self.embeddings, input_label_reshaped)
# NCE loss
stddev = 1.0 / math.sqrt(embedding_size)
nce_weights = tf.Variable(tf.truncated_normal(matrix_dimension, stddev=stddev))
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
nce_loss = tf.nn.nce_loss(weights=nce_weights, biases=nce_biases,
labels=output_word_reshaped, inputs=embed,
num_sampled=num_negative_samples, num_classes=vocabulary_size)
self.loss = tf.reduce_mean(nce_loss)
tf.summary.scalar('loss', self.loss)
# learning rate & optimizer
self.learning_rate = tf.train.exponential_decay(learning_rate_initial, self.global_step,
learning_rate_decay_steps,
learning_rate_decay, staircase=True,
name='learning_rate')
tf.summary.scalar('learning_rate', self.learning_rate)
sgd = tf.train.GradientDescentOptimizer(self.learning_rate)
self.optimizer = sgd.minimize(self.loss, global_step=self.global_step)
# saver to save the model
self.saver = tf.train.Saver()
# check a nan value in the loss
self.loss = tf.check_numerics(self.loss, 'loss is nan')
# embeddings
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = self.embeddings.name
filename_tsv = '{}_{}.tsv'.format('word2vec_dataset', vocabulary_size)
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
shutil.copy(os.path.join(DIR_DATA_WORD2VEC, filename_tsv), self.log_dir)
embedding.metadata_path = filename_tsv
summary_writer = tf.summary.FileWriter(self.log_dir)
projector.visualize_embeddings(summary_writer, config)
# normalize the embeddings to save them
norm = tf.sqrt(tf.reduce_sum(tf.square(self.embeddings), 1, keep_dims=True))
self.normalized_embeddings = self.embeddings / norm
return None