def testInvalidInitializationParameters(self):
variable_name = "trainable_variable"
with self.assertRaisesRegexp(KeyError, "Invalid initializer keys.*"):
snt.TrainableVariable(
name=variable_name,
shape=[1],
initializers={"w": tf.truncated_normal_initializer(stddev=1.0),
"extra": tf.truncated_normal_initializer(stddev=1.0)})
with self.assertRaisesRegexp(KeyError, "Invalid initializer keys.*"):
snt.TrainableVariable(
name=variable_name,
shape=[1],
initializers={"not_w": tf.truncated_normal_initializer(stddev=1.0)})
err = "Initializer for 'w' is not a callable function"
with self.assertRaisesRegexp(TypeError, err):
snt.TrainableVariable(name=variable_name,
shape=[1],
initializers={"w": tf.zeros([1, 2, 3])})
python类truncated_normal_initializer()的实例源码
def _conv(inpOp, nIn, nOut, kH, kW, dH, dW, padType):
global conv_counter
name = 'conv' + str(conv_counter)
conv_counter += 1
with tf.variable_scope(name):
kernel_initializer = tf.truncated_normal_initializer(stddev=1e-2)
conv = tf.layers.conv2d(inpOp,
nOut,
[kH, kW],
strides=[dH, dW],
padding=padType,
data_format=data_format_c,
kernel_initializer=kernel_initializer,
use_bias=False)
biases = tf.get_variable(
'biases', [nOut], tf.float32,
tf.constant_initializer(0.0))
bias = tf.reshape(tf.nn.bias_add(conv, biases, data_format=data_format),
conv.get_shape())
return conv
def _conv(inpOp, nIn, nOut, kH, kW, dH, dW, padType):
global conv_counter
name = 'conv' + str(conv_counter)
conv_counter += 1
with tf.variable_scope(name):
kernel_initializer = tf.truncated_normal_initializer(stddev=1e-2)
conv = tf.layers.conv2d(inpOp,
nOut,
[kH, kW],
strides=[dH, dW],
padding=padType,
data_format=data_format_c,
kernel_initializer=kernel_initializer,
use_bias=False)
biases = tf.get_variable(
'biases', [nOut], tf.float32,
tf.constant_initializer(0.0))
bias = tf.reshape(tf.nn.bias_add(conv, biases, data_format=data_format),
conv.get_shape())
return bias
def _conv(inpOp, nIn, nOut, kH, kW, dH, dW, padType):
global conv_counter
global parameters
name = 'conv' + str(conv_counter)
conv_counter += 1
with tf.variable_scope(name) as scope:
#kernel = tf.get_variable(name='weights', initializer=tf.random_normal([kH, kW, nIn, nOut], dtype=tf.float32, stddev=1e-2))
kernel = tf.get_variable(name='weights', shape=[kH, kW, nIn, nOut], initializer=tf.truncated_normal_initializer(dtype=tf.float32, stddev=1e-2))
strides = [1, dH, dW, 1]
conv = tf.nn.conv2d(inpOp, kernel, strides, padding=padType)
#biases = tf.Variable(tf.constant(0.0, shape=[nOut], dtype=tf.float32),
# trainable=True, name='biases')
biases = tf.get_variable(name='biases', initializer=tf.constant(0.0, shape=[nOut], dtype=tf.float32), dtype=tf.float32)
bias = tf.reshape(tf.nn.bias_add(conv, biases),
conv.get_shape())
parameters += [kernel, biases]
return bias
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(name,
shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def _variable_with_weight_decay(self, name, shape, stddev, wd=None):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = self._variable_on_cpu(
name,
shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd is not None:
# weight_decay = tf.mul(tf.constant(0.1), wd, name='weight_loss')
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
# tf.add_to_collection('losses', wd)
return var
def cifarnet_arg_scope(weight_decay=0.004):
"""Defines the default cifarnet argument scope.
Args:
weight_decay: The weight decay to use for regularizing the model.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
with slim.arg_scope(
[slim.conv2d],
weights_initializer=tf.truncated_normal_initializer(stddev=5e-2),
activation_fn=tf.nn.relu):
with slim.arg_scope(
[slim.fully_connected],
biases_initializer=tf.constant_initializer(0.1),
weights_initializer=trunc_normal(0.04),
weights_regularizer=slim.l2_regularizer(weight_decay),
activation_fn=tf.nn.relu) as sc:
return sc
def spoofnet_y_arg_scope(weight_decay=0.0004):
"""Defines the default cifarnet argument scope.
Args:
weight_decay: The weight decay to use for regularizing the model.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
with slim.arg_scope(
[slim.conv2d],
weights_initializer=tf.truncated_normal_initializer(stddev=5e-2), #TODO: or: weights_initializer=slim.variance_scaling_initializer(), as inception/vgg/resnet
# weights_regularizer=slim.l2_regularizer(weight_decay),
activation_fn=tf.nn.relu
):
# with slim.arg_scope(
# [slim.fully_connected],
# biases_initializer=tf.constant_initializer(0.1),
# weights_initializer=trunc_normal(0.04),
# weights_regularizer=slim.l2_regularizer(weight_decay),
# activation_fn=tf.nn.relu):
with slim.arg_scope([slim.max_pool2d], padding='SAME') as sc:
return sc
def variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def conv(x, c):
ksize = c['ksize']
stride = c['stride']
filters_out = c['conv_filters_out']
filters_in = x.get_shape()[-1]
shape = [ksize, ksize, filters_in, filters_out]
# initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV)
initializer = tf.contrib.layers.xavier_initializer()
weights = _get_variable('weights',
shape=shape,
#dtype='float',
initializer=initializer,
weight_decay=CONV_WEIGHT_DECAY)
bias = tf.get_variable('bias', [filters_out], 'float', tf.constant_initializer(0.05, dtype='float'))
x = tf.nn.conv2d(x, weights, [1, stride, stride, 1], padding='SAME')
return tf.nn.bias_add(x, bias)
def conv_3d(x, c):
ksize = c['ksize']
stride = c['stride']
filters_out = c['conv_filters_out']
filters_in = x.get_shape()[-1]
shape = [ksize, ksize, ksize, filters_in, filters_out]
# initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV)
initializer = tf.contrib.layers.xavier_initializer()
weights = _get_variable('weights',
shape=shape,
#dtype='float',
initializer=initializer,
weight_decay=CONV_WEIGHT_DECAY)
bias = tf.get_variable('bias', [filters_out], 'float', tf.constant_initializer(0.05, dtype='float'))
x = tf.nn.conv3d(x, weights, [1, stride, stride, stride, 1], padding='SAME')
return tf.nn.bias_add(x, bias)
def deconv_3d(x, c):
ksize = c['ksize']
stride = c['stride']
filters_out = c['conv_filters_out']
filters_in = x.get_shape()[-1]
# must have as_list to get a python list!!!!!!!!!!!!!!
x_shape = x.get_shape().as_list()
d = x_shape[1] * stride
height = x_shape[2] * stride
width = x_shape[3] * stride
output_shape = [1, d, height, width, filters_out]
strides = [1, stride, stride, stride, 1]
shape = [ksize, ksize, ksize, filters_out, filters_in]
# initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV)
initializer = tf.contrib.layers.xavier_initializer()
weights = _get_variable('weights',
shape=shape,
dtype='float32',
initializer=initializer,
weight_decay=CONV_WEIGHT_DECAY)
bias = tf.get_variable('bias', [filters_out], 'float32', tf.constant_initializer(0.05, dtype='float32'))
x = tf.nn.conv3d_transpose(x, weights, output_shape=output_shape, strides=strides, padding='SAME')
return tf.nn.bias_add(x, bias)
# wrapper for batch-norm op
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(name,
shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(name,
shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def conv2d(self, x, w_shape, strides, padding, name, reuse=False,
initializer_w=tf.truncated_normal_initializer(mean=0.0, stddev=1e-2),
initializer_b=tf.truncated_normal_initializer(mean=0.0, stddev=1e-2)
):
'''
convolution layer:
Input
- x:input tensor
- w_shape:weight shape for convolution kernel
- strides
- padding:'SAME' or 'VALID'
- name:variable name scope
- initializer_w/b:initializer of weight and bias
'''
_, _, _, num_out = w_shape
with tf.variable_scope(name, reuse=reuse) as scope:
weights = tf.get_variable('weights', w_shape, initializer=initializer_w)
biases = tf.get_variable('biases', [num_out], initializer=initializer_b)
#conv
conv = tf.nn.conv2d(x, weights, strides, padding)
#relu
relu = tf.nn.relu(conv + biases, name=scope.name)
return relu
def cifarnet_arg_scope(weight_decay=0.004):
"""Defines the default cifarnet argument scope.
Args:
weight_decay: The weight decay to use for regularizing the model.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
with slim.arg_scope(
[slim.conv2d],
weights_initializer=tf.truncated_normal_initializer(stddev=5e-2),
activation_fn=tf.nn.relu):
with slim.arg_scope(
[slim.fully_connected],
biases_initializer=tf.constant_initializer(0.1),
weights_initializer=trunc_normal(0.04),
weights_regularizer=slim.l2_regularizer(weight_decay),
activation_fn=tf.nn.relu) as sc:
return sc
def variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
var = _variable_on_cpu(
name,
shape,
tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
if wd is not None:
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def weightVariable(shape,std=1.0,name=None):
# Create a set of weights initialized with truncated normal random values
name = 'weights' if name is None else name
return tf.get_variable(name,shape,initializer=tf.truncated_normal_initializer(stddev=std/math.sqrt(shape[0])))
def weightVariable(shape,std=1.0,name=None):
# Create a set of weights initialized with truncated normal random values
name = 'weights' if name is None else name
return tf.get_variable(name,shape,initializer=tf.truncated_normal_initializer(stddev=std/math.sqrt(shape[0])))
