def _anneal_weight(init_val, final_val, anneal_type, global_step, anneal_steps, hold_for=0., steps_div=1.,
dtype=tf.float64):
val, final, step, hold_for, anneal_steps, steps_div = (tf.cast(i, dtype) for i in
(init_val, final_val, global_step, hold_for, anneal_steps, steps_div))
step = tf.maximum(step - hold_for, 0.)
if anneal_type == 'exp':
decay_rate = tf.pow(final / val, steps_div / anneal_steps)
val = tf.train.exponential_decay(val, step, steps_div, decay_rate)
elif anneal_type == 'linear':
val = final + (val - final) * (1. - step / anneal_steps)
else:
raise NotImplementedError
anneal_weight = tf.maximum(final, val)
return anneal_weight
python类float64()的实例源码
def tabular_kl(p, q, zero_prob_value=0., logarg_clip=None):
"""Computes KL-divergence KL(p||q) for two probability mass functions (pmf) given in a tabular form.
:param p: iterable
:param q: iterable
:param zero_prob_value: float; values below this threshold are treated as zero
:param logarg_clip: float or None, clips the argument to the log to lie in [-logarg_clip, logarg_clip] if not None
:return: iterable of brodcasted shape of (p * q), per-coordinate value of KL(p||q)
"""
p, q = (tf.cast(i, tf.float64) for i in (p, q))
non_zero = tf.greater(p, zero_prob_value)
logarg = p / q
if logarg_clip is not None:
logarg = clip_preserve(logarg, 1. / logarg_clip, logarg_clip)
log = masked_apply(logarg, tf.log, non_zero)
kl = p * log
return tf.cast(kl, tf.float32)
def softmax_loss(self, antecedent_scores, antecedent_labels):
"""
Computes the value of the loss function using antecedent_scores and antecedent_labels.
Practically standard softmax function.
Args:
antecedent_scores: tf.float64, [num_mentions, max_ant + 1], output of fully-connected network that compute
antecedent scores.
antecedent_labels: True labels for antecedent.
Returns: [num_mentions]
The value of loss function.
"""
gold_scores = antecedent_scores + tf.log(tf.cast(antecedent_labels, tf.float64)) # [num_mentions, max_ant + 1]
marginalized_gold_scores = tf.reduce_logsumexp(gold_scores, [1]) # [num_mentions]
log_norm = tf.reduce_logsumexp(antecedent_scores, [1]) # [num_mentions]
return log_norm - marginalized_gold_scores # [num_mentions]
def create_torch_variable(self, value, gpu=False):
"""Convenience method that produces a tensor given the value of the defined type.
Returns: a torch tensor of same type.
"""
if isinstance(value, torch.autograd.Variable):
if gpu:
value = value.cuda()
return value
if not torch.is_tensor(value):
if not isinstance(value, np.ndarray):
value = np.array(value, dtype=self.dtype.as_numpy_dtype)
else:
value = value.astype(self.dtype.as_numpy_dtype)
if value.size == 0:
return value
allowed = [tf.int16, tf.int32, tf.int64, tf.float16, tf.float32, tf.float64, tf.int8]
if self.dtype in allowed:
value = torch.autograd.Variable(torch.from_numpy(value))
else:
value = torch.autograd.Variable(value)
if gpu and isinstance(value, torch.autograd.Variable):
value = value.cuda()
return value
def _convert_string_dtype(dtype):
if dtype == 'float16':
return tf.float16
if dtype == 'float32':
return tf.float32
elif dtype == 'float64':
return tf.float64
elif dtype == 'int16':
return tf.int16
elif dtype == 'int32':
return tf.int32
elif dtype == 'int64':
return tf.int64
elif dtype == 'uint8':
return tf.int8
elif dtype == 'uint16':
return tf.uint16
else:
raise ValueError('Unsupported dtype:', dtype)
def conv1d(x, kernel, stride=1, border_mode='valid',
image_shape=None, filter_shape=None):
'''1D convolution.
# Arguments
kernel: kernel tensor.
strides: stride integer.
border_mode: string, "same" or "valid".
'''
# pre-process dtype
if _FLOATX == 'float64':
x = tf.cast(x, 'float32')
kernel = tf.cast(kernel, 'float32')
padding = _preprocess_border_mode(border_mode)
x = tf.nn.conv1d(x, kernel, stride, padding=padding)
# post-process dtype
if _FLOATX == 'float64':
x = tf.cast(x, 'float64')
return x
def __init__(self, epsilon=1e-2, shape=()):
self._sum = tf.get_variable(
dtype=tf.float64,
shape=shape,
initializer=tf.constant_initializer(0.0),
name="runningsum", trainable=False)
self._sumsq = tf.get_variable(
dtype=tf.float64,
shape=shape,
initializer=tf.constant_initializer(epsilon),
name="runningsumsq", trainable=False)
self._count = tf.get_variable(
dtype=tf.float64,
shape=(),
initializer=tf.constant_initializer(epsilon),
name="count", trainable=False)
self.shape = shape
self.mean = tf.to_float(self._sum / self._count)
self.std = tf.sqrt( tf.maximum( tf.to_float(self._sumsq / self._count) - tf.square(self.mean) , 1e-2 ))
newsum = tf.placeholder(shape=self.shape, dtype=tf.float64, name='sum')
newsumsq = tf.placeholder(shape=self.shape, dtype=tf.float64, name='var')
newcount = tf.placeholder(shape=[], dtype=tf.float64, name='count')
self.incfiltparams = U.function([newsum, newsumsq, newcount], [],
updates=[tf.assign_add(self._sum, newsum),
tf.assign_add(self._sumsq, newsumsq),
tf.assign_add(self._count, newcount)])
def assert_same_float_dtype(tensors_with_name, dtype=None):
"""
Whether all types of tensors in `tensors` are the same and floating type.
:param tensors_with_name: A list of (tensor, tensor_name).
:param dtype: Expected type. If `None`, depend on the type of tensors.
:return: The type of `tensors`.
"""
floating_types = [tf.float16, tf.float32, tf.float64]
if dtype is None:
return assert_same_specific_dtype(tensors_with_name, floating_types)
elif dtype in floating_types:
return assert_same_dtype(tensors_with_name, dtype)
else:
raise TypeError("The argument 'dtype' must be in %s" % floating_types)
def _convert_string_dtype(dtype):
if dtype == 'float16':
return tf.float16
if dtype == 'float32':
return tf.float32
elif dtype == 'float64':
return tf.float64
elif dtype == 'int16':
return tf.int16
elif dtype == 'int32':
return tf.int32
elif dtype == 'int64':
return tf.int64
elif dtype == 'uint8':
return tf.int8
elif dtype == 'uint16':
return tf.uint16
else:
raise ValueError('Unsupported dtype:', dtype)
def testInputTypeError(self, use_bias):
"""Errors are thrown for invalid input types."""
conv1 = snt.Conv2D(output_channels=1,
kernel_shape=3,
stride=1,
padding=snt.SAME,
name="conv1",
use_bias=use_bias,
initializers=create_constant_initializers(
1.0, 1.0, use_bias))
for dtype in (tf.float16, tf.float64):
x = tf.constant(np.ones([1, 5, 5, 1]), dtype=dtype)
err = "Input must have dtype tf.float32.*"
with self.assertRaisesRegexp(TypeError, err):
conv1(x)
def testInputTypeError(self, use_bias):
"""Errors are thrown for invalid input types."""
conv1 = snt.Conv1D(output_channels=1,
kernel_shape=3,
stride=1,
padding=snt.VALID,
use_bias=use_bias,
name="conv1",
initializers=create_constant_initializers(
1.0, 1.0, use_bias))
for dtype in (tf.float16, tf.float64):
x = tf.constant(np.ones([1, 5, 1]), dtype=dtype)
err = "Input must have dtype tf.float32.*"
with self.assertRaisesRegexp(TypeError, err):
conv1(x)
def testInputTypeError(self, batch_size, in_length, in_channels, out_channels,
kernel_shape, padding, use_bias, out_shape,
stride_shape):
"""Errors are thrown for invalid input types."""
conv1 = snt.Conv1DTranspose(
output_channels=out_channels,
output_shape=out_shape,
kernel_shape=kernel_shape,
padding=padding,
stride=stride_shape,
name="conv1",
use_bias=use_bias)
for dtype in (tf.float16, tf.float64):
x = tf.constant(np.ones([batch_size, in_length,
in_channels]), dtype=dtype)
err = "Input must have dtype tf.float32.*"
with self.assertRaisesRegexp(TypeError, err):
conv1(x)
def testInputTypeError(self):
"""Errors are thrown for invalid input types."""
conv1 = snt.Conv3D(output_channels=1,
kernel_shape=3,
stride=1,
padding=snt.SAME,
name="conv1",
initializers={
"w": tf.constant_initializer(1.0),
"b": tf.constant_initializer(1.0),
})
for dtype in (tf.float16, tf.float64):
x = tf.constant(np.ones([1, 5, 5, 5, 1]), dtype=dtype)
self.assertRaisesRegexp(TypeError, "Input must have dtype tf.float32.*",
conv1, x)
def testVariableInitialization(self):
# Check that a simple operation involving the TrainableVariable
# matches the result of the corresponding operation in numpy
np.random.seed(100)
types = (tf.float16, tf.float32, tf.float64)
tol = (1e-2, 1e-6, 1e-9)
tolerance_map = dict(zip(types, tol))
lhs_shape = [3, 4]
rhs_shape = [4, 6]
for dtype in types:
x = tf.placeholder(dtype, shape=lhs_shape)
var = snt.TrainableVariable(shape=rhs_shape,
dtype=dtype,
initializers={"w": _test_initializer()})
y = tf.matmul(x, var())
with self.test_session() as sess:
lhs_matrix = np.random.randn(*lhs_shape)
sess.run(tf.global_variables_initializer())
product, w = sess.run([y, var.w], {x: lhs_matrix})
self.assertAllClose(product,
np.dot(
lhs_matrix.astype(dtype.as_numpy_dtype),
w.astype(dtype.as_numpy_dtype)),
atol=tolerance_map[dtype],
rtol=tolerance_map[dtype])
def test_hessian_quadratic(self):
rnd = np.random.RandomState(0)
dtype = tf.float64
with tf.Graph().as_default():
r = tf.Variable(0.0, dtype=dtype)
x = tf.constant(rnd.uniform(-1.0, 1.0, [2, 27]), dtype=dtype, name="x")
w2 = tf.constant(rnd.uniform(-1.0, 1.0, [27, 1]), dtype=dtype, name="w2")
v2 = tf.constant(rnd.uniform(-1.0, 1.0, [27, 1]), dtype=dtype, name="v2")
w2v = tf.add(w2, tf.multiply(r, v2))
h2 = tf.matmul(x, w2v)
y2 = tf.reduce_sum(h2 * h2)
grad_w = tf.gradients(y2, w2)
hv_fw = hessian_vec_fw(y2, [w2v], [v2])
hv_bk = hessian_vec_bk(y2, [w2], [v2])
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
grad_w = sess.run(grad_w)
hv_fw_val = sess.run(hv_fw)
hv_bk_val = sess.run(hv_bk)
np.testing.assert_allclose(hv_fw_val, hv_bk_val, rtol=1e-5)
def average_precision_voc12(precision, recall, name=None):
"""Compute (interpolated) average precision from precision and recall Tensors.
The implementation follows Pascal 2012 and ILSVRC guidelines.
See also: https://sanchom.wordpress.com/tag/average-precision/
"""
with tf.name_scope(name, 'average_precision_voc12', [precision, recall]):
# Convert to float64 to decrease error on Riemann sums.
precision = tf.cast(precision, dtype=tf.float64)
recall = tf.cast(recall, dtype=tf.float64)
# Add bounds values to precision and recall.
precision = tf.concat([[0.], precision, [0.]], axis=0)
recall = tf.concat([[0.], recall, [1.]], axis=0)
# Ensures precision is increasing in reverse order.
precision = tfe_math.cummax(precision, reverse=True)
# Riemann sums for estimating the integral.
# mean_pre = (precision[1:] + precision[:-1]) / 2.
mean_pre = precision[1:]
diff_rec = recall[1:] - recall[:-1]
ap = tf.reduce_sum(mean_pre * diff_rec)
return ap
def average_precision_voc07(precision, recall, name=None):
"""Compute (interpolated) average precision from precision and recall Tensors.
The implementation follows Pascal 2007 guidelines.
See also: https://sanchom.wordpress.com/tag/average-precision/
"""
with tf.name_scope(name, 'average_precision_voc07', [precision, recall]):
# Convert to float64 to decrease error on cumulated sums.
precision = tf.cast(precision, dtype=tf.float64)
recall = tf.cast(recall, dtype=tf.float64)
# Add zero-limit value to avoid any boundary problem...
precision = tf.concat([precision, [0.]], axis=0)
recall = tf.concat([recall, [np.inf]], axis=0)
# Split the integral into 10 bins.
l_aps = []
for t in np.arange(0., 1.1, 0.1):
mask = tf.greater_equal(recall, t)
v = tf.reduce_max(tf.boolean_mask(precision, mask))
l_aps.append(v / 11.)
ap = tf.add_n(l_aps)
return ap
def _precision_recall(n_gbboxes, n_detections, scores, tp, fp, scope=None):
"""Compute precision and recall from scores, true positives and false
positives booleans arrays
"""
# Sort by score.
with tf.name_scope(scope, 'prec_rec', [n_gbboxes, scores, tp, fp]):
# Sort detections by score.
scores, idxes = tf.nn.top_k(scores, k=n_detections, sorted=True)
tp = tf.gather(tp, idxes)
fp = tf.gather(fp, idxes)
# Computer recall and precision.
dtype = tf.float64
tp = tf.cumsum(tf.cast(tp, dtype), axis=0)
fp = tf.cumsum(tf.cast(fp, dtype), axis=0)
recall = _safe_div(tp, tf.cast(n_gbboxes, dtype), 'recall')
precision = _safe_div(tp, tp + fp, 'precision')
return tf.tuple([precision, recall])
tensorflow_backend.py 文件源码
项目：deep-learning-keras-projects
作者: jasmeetsb
项目源码
文件源码
阅读 27
收藏 0
点赞 0
评论 0
def _convert_string_dtype(dtype):
if dtype == 'float16':
return tf.float16
if dtype == 'float32':
return tf.float32
elif dtype == 'float64':
return tf.float64
elif dtype == 'int16':
return tf.int16
elif dtype == 'int32':
return tf.int32
elif dtype == 'int64':
return tf.int64
elif dtype == 'uint8':
return tf.int8
elif dtype == 'uint16':
return tf.uint16
else:
raise ValueError('Unsupported dtype:', dtype)
tensorflow_backend.py 文件源码
项目：deep-learning-keras-projects
作者: jasmeetsb
项目源码
文件源码
阅读 31
收藏 0
点赞 0
评论 0
def conv1d(x, kernel, stride=1, border_mode='valid',
image_shape=None, filter_shape=None):
"""1D convolution.
# Arguments
kernel: kernel tensor.
strides: stride integer.
border_mode: string, `"same"` or `"valid"`.
# Returns
A tensor, result of 1D convolution.
"""
# pre-process dtype
x_dtype = dtype(x)
if x_dtype == 'float64':
x = tf.cast(x, 'float32')
kernel = tf.cast(kernel, 'float32')
padding = _preprocess_border_mode(border_mode)
x = tf.nn.conv1d(x, kernel, stride, padding=padding)
# post-process dtype
if x_dtype == 'float64':
x = tf.cast(x, 'float64')
return x
def average_precision_voc12(precision, recall, name=None):
"""Compute (interpolated) average precision from precision and recall Tensors.
The implementation follows Pascal 2012 and ILSVRC guidelines.
See also: https://sanchom.wordpress.com/tag/average-precision/
"""
with tf.name_scope(name, 'average_precision_voc12', [precision, recall]):
# Convert to float64 to decrease error on Riemann sums.
precision = tf.cast(precision, dtype=tf.float64)
recall = tf.cast(recall, dtype=tf.float64)
# Add bounds values to precision and recall.
precision = tf.concat([[0.], precision, [0.]], axis=0)
recall = tf.concat([[0.], recall, [1.]], axis=0)
# Ensures precision is increasing in reverse order.
precision = tfe_math.cummax(precision, reverse=True)
# Riemann sums for estimating the integral.
# mean_pre = (precision[1:] + precision[:-1]) / 2.
mean_pre = precision[1:]
diff_rec = recall[1:] - recall[:-1]
ap = tf.reduce_sum(mean_pre * diff_rec)
return ap
def average_precision_voc07(precision, recall, name=None):
"""Compute (interpolated) average precision from precision and recall Tensors.
The implementation follows Pascal 2007 guidelines.
See also: https://sanchom.wordpress.com/tag/average-precision/
"""
with tf.name_scope(name, 'average_precision_voc07', [precision, recall]):
# Convert to float64 to decrease error on cumulated sums.
precision = tf.cast(precision, dtype=tf.float64)
recall = tf.cast(recall, dtype=tf.float64)
# Add zero-limit value to avoid any boundary problem...
precision = tf.concat([precision, [0.]], axis=0)
recall = tf.concat([recall, [np.inf]], axis=0)
# Split the integral into 10 bins.
l_aps = []
for t in np.arange(0., 1.1, 0.1):
mask = tf.greater_equal(recall, t)
v = tf.reduce_max(tf.boolean_mask(precision, mask))
l_aps.append(v / 11.)
ap = tf.add_n(l_aps)
return ap
def _precision_recall(n_gbboxes, n_detections, scores, tp, fp, scope=None):
"""Compute precision and recall from scores, true positives and false
positives booleans arrays
"""
# Sort by score.
with tf.name_scope(scope, 'prec_rec', [n_gbboxes, scores, tp, fp]):
# Sort detections by score.
scores, idxes = tf.nn.top_k(scores, k=n_detections, sorted=True)
tp = tf.gather(tp, idxes)
fp = tf.gather(fp, idxes)
# Computer recall and precision.
dtype = tf.float64
tp = tf.cumsum(tf.cast(tp, dtype), axis=0)
fp = tf.cumsum(tf.cast(fp, dtype), axis=0)
recall = _safe_div(tp, tf.cast(n_gbboxes, dtype), 'recall')
precision = _safe_div(tp, tp + fp, 'precision')
return tf.tuple([precision, recall])
def testTensorSignatureExampleParserDict(self):
examples = tf.placeholder(name='example', shape=[None], dtype=tf.string)
placeholder_a = tf.placeholder(name='test',
shape=[None, 100],
dtype=tf.int32)
placeholder_b = tf.placeholder(name='bb',
shape=[None, 100],
dtype=tf.float64)
inputs = {'a': placeholder_a, 'b': placeholder_b}
signatures = tensor_signature.create_signatures(inputs)
result = tensor_signature.create_example_parser_from_signatures(
signatures, examples)
self.assertTrue(tensor_signature.tensors_compatible(result, signatures))
new_signatures = tensor_signature.create_signatures(result)
self.assertTrue(new_signatures['a'].is_compatible_with(signatures['a']))
self.assertTrue(new_signatures['b'].is_compatible_with(signatures['b']))
def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights(
self):
with self.test_session():
embedding_weights = self._random_weights(num_shards=3)
sparse_ids, sparse_weights = self._ids_and_weights_3d()
embedding_weights[1] = embedding_weights[1].astype(np.float64)
self.assertRaises(ValueError,
tf.contrib.layers.safe_embedding_lookup_sparse,
embedding_weights, sparse_ids)
embedding_weights = [
tf.constant(w, dtype=tf.float64) for w in embedding_weights
]
self.assertRaises(ValueError,
tf.contrib.layers.safe_embedding_lookup_sparse,
embedding_weights, sparse_ids, sparse_weights)
def testTensorSignatureExampleParserDict(self):
examples = tf.placeholder(name='example', shape=[None], dtype=tf.string)
placeholder_a = tf.placeholder(name='test',
shape=[None, 100],
dtype=tf.int32)
placeholder_b = tf.placeholder(name='bb',
shape=[None, 100],
dtype=tf.float64)
inputs = {'a': placeholder_a, 'b': placeholder_b}
signatures = tensor_signature.create_signatures(inputs)
result = tensor_signature.create_example_parser_from_signatures(
signatures, examples)
self.assertTrue(tensor_signature.tensors_compatible(result, signatures))
new_signatures = tensor_signature.create_signatures(result)
self.assertTrue(new_signatures['a'].is_compatible_with(signatures['a']))
self.assertTrue(new_signatures['b'].is_compatible_with(signatures['b']))
def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights(
self):
with self.test_session():
embedding_weights = self._random_weights(num_shards=3)
sparse_ids, sparse_weights = self._ids_and_weights_3d()
embedding_weights[1] = embedding_weights[1].astype(np.float64)
self.assertRaises(ValueError,
tf.contrib.layers.safe_embedding_lookup_sparse,
embedding_weights, sparse_ids)
embedding_weights = [
tf.constant(w, dtype=tf.float64) for w in embedding_weights
]
self.assertRaises(ValueError,
tf.contrib.layers.safe_embedding_lookup_sparse,
embedding_weights, sparse_ids, sparse_weights)
def test_soft_lif(Simulator, sigma, seed):
with nengo.Network(seed=seed) as net:
inp = nengo.Node([0])
ens = nengo.Ensemble(10, 1, neuron_type=SoftLIFRate(sigma=sigma))
nengo.Connection(inp, ens)
p = nengo.Probe(ens.neurons)
x = str(ens.neuron_type)
if sigma == 1:
assert "sigma" not in x
else:
assert "sigma=%s" % sigma in x
with nengo.Simulator(net) as sim:
_, nengo_curves = nengo.utils.ensemble.tuning_curves(ens, sim)
sim.run_steps(30)
with Simulator(net, dtype=tf.float64) as sim2:
_, nengo_dl_curves = nengo.utils.ensemble.tuning_curves(ens, sim2)
sim2.run_steps(30)
assert np.allclose(nengo_curves, nengo_dl_curves)
assert np.allclose(sim.data[p], sim2.data[p])
def __init__(self, *args, **kwargs):
logging.basicConfig(level=logging.WARNING)
if "NENGO_DL_TEST_PRECISION" in os.environ:
if os.environ["NENGO_DL_TEST_PRECISION"] == "32":
kwargs.setdefault("dtype", tf.float32)
else:
kwargs.setdefault("dtype", tf.float64)
if "NENGO_DL_TEST_UNROLL" in os.environ:
kwargs.setdefault("unroll_simulation",
int(os.environ["NENGO_DL_TEST_UNROLL"]))
if "NENGO_DL_TEST_DEVICE" in os.environ:
device = os.environ["NENGO_DL_TEST_DEVICE"]
if device == "None":
kwargs.setdefault("device", None)
else:
kwargs.setdefault("device", os.environ["NENGO_DL_TEST_DEVICE"])
super(Simulator, self).__init__(*args, **kwargs)
def create_training(self, image_size = [151,151]):
"""Create the cost function and trainer"""
self.phi_input = tf.stop_gradient(tf.placeholder("float32", [None, image_size[0], image_size[1], 1]));
def cost(output, phi_in):
#return np.array([self.cost(o, phi_in) for o in output]);
return np.sum(self.cost_func(output, phi_in));
def cost_grad(op, grad):
#print op
output = op.inputs[0];
phi = op.inputs[1];
grad = tf.py_func(self.cost_func_grad, [output, phi], [tf.float32])[0];
#return [self.cost_func_grad(output, phi_in, epsilon = 0.01), np.zeros((phi_in.shape))];
return [grad, None];
self.cost_tf = py_func(cost, [self.output, self.phi_input], [tf.float32], grad = cost_grad)[0];
#self.cost_tf = tf.py_func(cost, [self.output, self.phi_input], [tf.float64])[0];
#self.phi = tf.py_func(phi_func, [self.output], [tf.float64]);
#self.cost = tf.reduce_mean(tf.squared_difference(self.phi_input, self.phi));
self.train_tf = tf.train.RMSPropOptimizer(0.00025,0.99,0.0,1e-6).minimize(self.cost_tf)
