def dense(inputs, units, bias_shape, w_i, b_i=None, activation=tf.nn.relu):
# ??tf.layers?????flatten
# dense1 = tf.layers.dense(tf.contrib.layers.flatten(relu5), activation=tf.nn.relu, units=50)
if not isinstance(inputs, ops.Tensor):
inputs = ops.convert_to_tensor(inputs, dtype='float')
# dim_list = inputs.get_shape().as_list()
# flatten_shape = dim_list[1] if len(dim_list) <= 2 else reduce(lambda x, y: x * y, dim_list[1:])
# reshaped = tf.reshape(inputs, [dim_list[0], flatten_shape])
if len(inputs.shape) > 2:
inputs = tf.contrib.layers.flatten(inputs)
flatten_shape = inputs.shape[1]
weights = tf.get_variable('weights', shape=[flatten_shape, units], initializer=w_i)
dense = tf.matmul(inputs, weights)
if bias_shape is not None:
assert bias_shape[0] == units
biases = tf.get_variable('biases', shape=bias_shape, initializer=b_i)
return activation(dense + biases) if activation is not None else dense + biases
return activation(dense) if activation is not None else dense
python类layers()的实例源码
def _conv2d_impl(self, input_layer, num_channels_in, filters, kernel_size,
strides, padding, kernel_initializer):
if self.use_tf_layers:
return conv_layers.conv2d(input_layer, filters, kernel_size, strides,
padding, self.channel_pos,
kernel_initializer=kernel_initializer,
use_bias=False)
else:
weights_shape = [kernel_size[0], kernel_size[1], num_channels_in, filters]
# We use the name 'conv2d/kernel' so the variable has the same name as its
# tf.layers equivalent. This way, if a checkpoint is written when
# self.use_tf_layers == True, it can be loaded when
# self.use_tf_layers == False, and vice versa.
weights = self.get_variable('conv2d/kernel', weights_shape,
self.variable_dtype, self.dtype,
initializer=kernel_initializer)
if self.data_format == 'NHWC':
strides = [1] + strides + [1]
else:
strides = [1, 1] + strides
return tf.nn.conv2d(input_layer, weights, strides, padding,
data_format=self.data_format)
def _conv2d_impl(self, input_layer, num_channels_in, filters, kernel_size,
strides, padding, kernel_initializer):
if self.use_tf_layers:
return conv_layers.conv2d(input_layer, filters, kernel_size, strides,
padding, self.channel_pos,
kernel_initializer=kernel_initializer,
use_bias=False)
else:
weights_shape = [kernel_size[0], kernel_size[1], num_channels_in, filters]
# We use the name 'conv2d/kernel' so the variable has the same name as its
# tf.layers equivalent. This way, if a checkpoint is written when
# self.use_tf_layers == True, it can be loaded when
# self.use_tf_layers == False, and vice versa.
weights = self.get_variable('conv2d/kernel', weights_shape,
self.variable_dtype, self.dtype,
initializer=kernel_initializer)
if self.data_format == 'NHWC':
strides = [1] + strides + [1]
else:
strides = [1, 1] + strides
return tf.nn.conv2d(input_layer, weights, strides, padding,
data_format=self.data_format)
def conv(inputs, kernel_shape, bias_shape, strides, w_i, b_i=None, activation=tf.nn.relu):
# ??tf.layers
# relu1 = tf.layers.conv2d(input_imgs, filters=24, kernel_size=[5, 5], strides=[2, 2],
# padding='SAME', activation=tf.nn.relu,
# kernel_initializer=w_i, bias_initializer=b_i)
weights = tf.get_variable('weights', shape=kernel_shape, initializer=w_i)
conv = tf.nn.conv2d(inputs, weights, strides=strides, padding='SAME')
if bias_shape is not None:
biases = tf.get_variable('biases', shape=bias_shape, initializer=b_i)
return activation(conv + biases) if activation is not None else conv + biases
return activation(conv) if activation is not None else conv
# ???bias??????relu
def noisy_dense(inputs, units, bias_shape, c_names, w_i, b_i=None, activation=tf.nn.relu, noisy_distribution='factorised'):
def f(e_list):
return tf.multiply(tf.sign(e_list), tf.pow(tf.abs(e_list), 0.5))
# ??tf.layers?????flatten
# dense1 = tf.layers.dense(tf.contrib.layers.flatten(relu5), activation=tf.nn.relu, units=50)
if not isinstance(inputs, ops.Tensor):
inputs = ops.convert_to_tensor(inputs, dtype='float')
# dim_list = inputs.get_shape().as_list()
# flatten_shape = dim_list[1] if len(dim_list) <= 2 else reduce(lambda x, y: x * y, dim_list[1:])
# reshaped = tf.reshape(inputs, [dim_list[0], flatten_shape])
if len(inputs.shape) > 2:
inputs = tf.contrib.layers.flatten(inputs)
flatten_shape = inputs.shape[1]
weights = tf.get_variable('weights', shape=[flatten_shape, units], initializer=w_i)
w_noise = tf.get_variable('w_noise', [flatten_shape, units], initializer=w_i, collections=c_names)
if noisy_distribution == 'independent':
weights += tf.multiply(tf.random_normal(shape=w_noise.shape), w_noise)
elif noisy_distribution == 'factorised':
noise_1 = f(tf.random_normal(tf.TensorShape([flatten_shape, 1]), dtype=tf.float32)) # ???????????????
noise_2 = f(tf.random_normal(tf.TensorShape([1, units]), dtype=tf.float32))
weights += tf.multiply(noise_1 * noise_2, w_noise)
dense = tf.matmul(inputs, weights)
if bias_shape is not None:
assert bias_shape[0] == units
biases = tf.get_variable('biases', shape=bias_shape, initializer=b_i)
b_noise = tf.get_variable('b_noise', [1, units], initializer=b_i, collections=c_names)
if noisy_distribution == 'independent':
biases += tf.multiply(tf.random_normal(shape=b_noise.shape), b_noise)
elif noisy_distribution == 'factorised':
biases += tf.multiply(noise_2, b_noise)
return activation(dense + biases) if activation is not None else dense + biases
return activation(dense) if activation is not None else dense
# ???bias??????relu
def flatten(inputs):
# ??tf.layers
# return tf.contrib.layers.flatten(inputs)
return tf.reshape(inputs, [-1, np.prod(inputs.get_shape().as_list()[1:])])
# flatten = tf.reshape(relu5, [-1, np.prod(relu5.shape.as_list()[1:])])
def inception_module(self, name, cols, input_layer=None, in_size=None):
if input_layer is None:
input_layer = self.top_layer
if in_size is None:
in_size = self.top_size
name += str(self.counts[name])
self.counts[name] += 1
with tf.variable_scope(name):
col_layers = []
col_layer_sizes = []
for c, col in enumerate(cols):
col_layers.append([])
col_layer_sizes.append([])
for l, layer in enumerate(col):
ltype, args = layer[0], layer[1:]
kwargs = {
'input_layer': input_layer,
'num_channels_in': in_size
} if l == 0 else {}
if ltype == 'conv':
self.conv(*args, **kwargs)
elif ltype == 'mpool':
self.mpool(*args, **kwargs)
elif ltype == 'apool':
self.apool(*args, **kwargs)
elif ltype == 'share': # Share matching layer from previous column
self.top_layer = col_layers[c - 1][l]
self.top_size = col_layer_sizes[c - 1][l]
else:
raise KeyError(
'Invalid layer type for inception module: \'%s\'' % ltype)
col_layers[c].append(self.top_layer)
col_layer_sizes[c].append(self.top_size)
catdim = 3 if self.data_format == 'NHWC' else 1
self.top_layer = tf.concat([layers[-1] for layers in col_layers], catdim)
self.top_size = sum([sizes[-1] for sizes in col_layer_sizes])
return self.top_layer
def _batch_norm_without_layers(self, input_layer, decay, use_scale, epsilon):
"""Batch normalization on `input_layer` without tf.layers."""
# We make this function as similar as possible to the
# tf.contrib.layers.batch_norm, to minimize the differences between using
# layers and not using layers.
shape = input_layer.shape
num_channels = shape[3] if self.data_format == 'NHWC' else shape[1]
beta = self.get_variable('beta', [num_channels], tf.float32, tf.float32,
initializer=tf.zeros_initializer())
if use_scale:
gamma = self.get_variable('gamma', [num_channels], tf.float32,
tf.float32, initializer=tf.ones_initializer())
else:
gamma = tf.constant(1.0, tf.float32, [num_channels])
# For moving variables, we use tf.get_variable instead of self.get_variable,
# since self.get_variable returns the result of tf.cast which we cannot
# assign to.
moving_mean = tf.get_variable('moving_mean', [num_channels],
tf.float32,
initializer=tf.zeros_initializer(),
trainable=False)
moving_variance = tf.get_variable('moving_variance', [num_channels],
tf.float32,
initializer=tf.ones_initializer(),
trainable=False)
if self.phase_train:
bn, batch_mean, batch_variance = tf.nn.fused_batch_norm(
input_layer, gamma, beta, epsilon=epsilon,
data_format=self.data_format, is_training=True)
mean_update = moving_averages.assign_moving_average(
moving_mean, batch_mean, decay=decay, zero_debias=False)
variance_update = moving_averages.assign_moving_average(
moving_variance, batch_variance, decay=decay, zero_debias=False)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, mean_update)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, variance_update)
else:
bn, _, _ = tf.nn.fused_batch_norm(
input_layer, gamma, beta, mean=moving_mean,
variance=moving_variance, epsilon=epsilon,
data_format=self.data_format, is_training=False)
return bn
def batch_norm(self, input_layer=None, decay=0.999, scale=False,
epsilon=0.001):
"""Adds a Batch Normalization layer."""
if input_layer is None:
input_layer = self.top_layer
else:
self.top_size = None
name = 'batchnorm' + str(self.counts['batchnorm'])
self.counts['batchnorm'] += 1
with tf.variable_scope(name) as scope:
if self.use_tf_layers:
bn = tf.contrib.layers.batch_norm(
input_layer,
decay=decay,
scale=scale,
epsilon=epsilon,
is_training=self.phase_train,
fused=True,
data_format=self.data_format,
scope=scope)
else:
bn = self._batch_norm_without_layers(input_layer, decay, scale, epsilon)
self.top_layer = bn
self.top_size = bn.shape[3] if self.data_format == 'NHWC' else bn.shape[1]
self.top_size = int(self.top_size)
return bn
def inception_module(self, name, cols, input_layer=None, in_size=None):
if input_layer is None:
input_layer = self.top_layer
if in_size is None:
in_size = self.top_size
name += str(self.counts[name])
self.counts[name] += 1
with tf.variable_scope(name):
col_layers = []
col_layer_sizes = []
for c, col in enumerate(cols):
col_layers.append([])
col_layer_sizes.append([])
for l, layer in enumerate(col):
ltype, args = layer[0], layer[1:]
kwargs = {
'input_layer': input_layer,
'num_channels_in': in_size
} if l == 0 else {}
if ltype == 'conv':
self.conv(*args, **kwargs)
elif ltype == 'mpool':
self.mpool(*args, **kwargs)
elif ltype == 'apool':
self.apool(*args, **kwargs)
elif ltype == 'share': # Share matching layer from previous column
self.top_layer = col_layers[c - 1][l]
self.top_size = col_layer_sizes[c - 1][l]
else:
raise KeyError(
'Invalid layer type for inception module: \'%s\'' % ltype)
col_layers[c].append(self.top_layer)
col_layer_sizes[c].append(self.top_size)
catdim = 3 if self.data_format == 'NHWC' else 1
self.top_layer = tf.concat([layers[-1] for layers in col_layers], catdim)
self.top_size = sum([sizes[-1] for sizes in col_layer_sizes])
return self.top_layer
def _batch_norm_without_layers(self, input_layer, decay, use_scale, epsilon):
"""Batch normalization on `input_layer` without tf.layers."""
# We make this function as similar as possible to the
# tf.contrib.layers.batch_norm, to minimize the differences between using
# layers and not using layers.
shape = input_layer.shape
num_channels = shape[3] if self.data_format == 'NHWC' else shape[1]
beta = self.get_variable('beta', [num_channels], tf.float32, tf.float32,
initializer=tf.zeros_initializer())
if use_scale:
gamma = self.get_variable('gamma', [num_channels], tf.float32,
tf.float32, initializer=tf.ones_initializer())
else:
gamma = tf.constant(1.0, tf.float32, [num_channels])
# For moving variables, we use tf.get_variable instead of self.get_variable,
# since self.get_variable returns the result of tf.cast which we cannot
# assign to.
moving_mean = tf.get_variable('moving_mean', [num_channels],
tf.float32,
initializer=tf.zeros_initializer(),
trainable=False)
moving_variance = tf.get_variable('moving_variance', [num_channels],
tf.float32,
initializer=tf.ones_initializer(),
trainable=False)
if self.phase_train:
bn, batch_mean, batch_variance = tf.nn.fused_batch_norm(
input_layer, gamma, beta, epsilon=epsilon,
data_format=self.data_format, is_training=True)
mean_update = moving_averages.assign_moving_average(
moving_mean, batch_mean, decay=decay, zero_debias=False)
variance_update = moving_averages.assign_moving_average(
moving_variance, batch_variance, decay=decay, zero_debias=False)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, mean_update)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, variance_update)
else:
bn, _, _ = tf.nn.fused_batch_norm(
input_layer, gamma, beta, mean=moving_mean,
variance=moving_variance, epsilon=epsilon,
data_format=self.data_format, is_training=False)
return bn
def batch_norm(self, input_layer=None, decay=0.999, scale=False,
epsilon=0.001):
"""Adds a Batch Normalization layer."""
if input_layer is None:
input_layer = self.top_layer
else:
self.top_size = None
name = 'batchnorm' + str(self.counts['batchnorm'])
self.counts['batchnorm'] += 1
with tf.variable_scope(name) as scope:
if self.use_tf_layers:
bn = tf.contrib.layers.batch_norm(
input_layer,
decay=decay,
scale=scale,
epsilon=epsilon,
is_training=self.phase_train,
fused=True,
data_format=self.data_format,
scope=scope)
else:
bn = self._batch_norm_without_layers(input_layer, decay, scale, epsilon)
self.top_layer = bn
self.top_size = bn.shape[3] if self.data_format == 'NHWC' else bn.shape[1]
self.top_size = int(self.top_size)
return bn
def tensor_layer(input, layer_func, shape_in=None, synapse=None,
transform=1, return_conn=False, **layer_args):
"""A utility function to construct TensorNodes that apply some function
to their input (analogous to the ``tf.layers`` syntax).
Parameters
----------
input : :class:`~nengo:nengo.base.NengoObject`
Object providing input to the layer
layer_func : callable or :class:`~nengo:nengo.neurons.NeuronType`
A function that takes the value from ``input`` (represented as a
``tf.Tensor``) and maps it to some output value, or a Nengo neuron
type, defining a nonlinearity that will be applied to ``input``.
shape_in : tuple of int, optional
If not None, reshape the input to the given shape
synapse : float or :class:`~nengo:nengo.synapses.Synapse`, optional
Synapse to apply on connection from ``input`` to this layer
transform : :class:`~numpy:numpy.ndarray`, optional
Transform matrix to apply on connection from ``input`` to this layer
return_conn : bool, optional
If True, also return the connection linking this layer to ``input``
layer_args : dict, optional
These arguments will be passed to ``layer_func`` if it is callable, or
:class:`~nengo:nengo.Ensemble` if ``layer_func`` is a
:class:`~nengo:nengo.neurons.NeuronType`
Returns
-------
:class:`.TensorNode` or :class:`~nengo:nengo.ensemble.Neurons`
A TensorNode that implements the given layer function (if
``layer_func`` was a callable), or a Neuron object with the given
neuron type, connected to ``input``
"""
if isinstance(transform, np.ndarray) and transform.ndim == 2:
size_in = transform.shape[0]
elif shape_in is not None:
size_in = np.prod(shape_in)
else:
size_in = input.size_out
if isinstance(layer_func, NeuronType):
node = Ensemble(size_in, 1, neuron_type=layer_func,
**layer_args).neurons
else:
# add (ignored) time input and pass kwargs
def node_func(_, x):
return layer_func(x, **layer_args)
# reshape input if necessary
if shape_in is not None:
node_func = reshaped(shape_in)(node_func)
node = TensorNode(node_func, size_in=size_in)
conn = Connection(input, node, synapse=synapse, transform=transform)
return (node, conn) if return_conn else node
def get_estimator(args, output_dir, features, stats, target_vocab_size):
# Check layers used for dnn models.
if is_dnn_model(args.model) and not args.hidden_layer_sizes:
raise ValueError('--hidden-layer-size* must be used with DNN models')
if is_linear_model(args.model) and args.hidden_layer_sizes:
raise ValueError('--hidden-layer-size* cannot be used with linear models')
# Build tf.learn features
feature_columns = build_feature_columns(features, stats, args.model)
# Set how often to run checkpointing in terms of steps.
config = tf.contrib.learn.RunConfig(
save_checkpoints_steps=args.min_eval_frequency)
train_dir = os.path.join(output_dir, 'train')
if args.model == 'dnn_regression':
estimator = tf.contrib.learn.DNNRegressor(
feature_columns=feature_columns,
hidden_units=args.hidden_layer_sizes,
config=config,
model_dir=train_dir,
optimizer=tf.train.AdamOptimizer(
args.learning_rate, epsilon=args.epsilon))
elif args.model == 'linear_regression':
estimator = tf.contrib.learn.LinearRegressor(
feature_columns=feature_columns,
config=config,
model_dir=train_dir,
optimizer=tf.train.FtrlOptimizer(
args.learning_rate,
l1_regularization_strength=args.l1_regularization,
l2_regularization_strength=args.l2_regularization))
elif args.model == 'dnn_classification':
estimator = tf.contrib.learn.DNNClassifier(
feature_columns=feature_columns,
hidden_units=args.hidden_layer_sizes,
n_classes=target_vocab_size,
config=config,
model_dir=train_dir,
optimizer=tf.train.AdamOptimizer(
args.learning_rate, epsilon=args.epsilon))
elif args.model == 'linear_classification':
estimator = tf.contrib.learn.LinearClassifier(
feature_columns=feature_columns,
n_classes=target_vocab_size,
config=config,
model_dir=train_dir,
optimizer=tf.train.FtrlOptimizer(
args.learning_rate,
l1_regularization_strength=args.l1_regularization,
l2_regularization_strength=args.l2_regularization))
else:
raise ValueError('bad --model-type value')
return estimator
def get_estimator(args, output_dir, features, stats, target_vocab_size):
# Check layers used for dnn models.
if is_dnn_model(args.model) and not args.hidden_layer_sizes:
raise ValueError('--hidden-layer-size* must be used with DNN models')
if is_linear_model(args.model) and args.hidden_layer_sizes:
raise ValueError('--hidden-layer-size* cannot be used with linear models')
# Build tf.learn features
feature_columns = build_feature_columns(features, stats, args.model)
# Set how often to run checkpointing in terms of steps.
config = tf.contrib.learn.RunConfig(
save_checkpoints_steps=args.min_eval_frequency)
train_dir = os.path.join(output_dir, 'train')
if args.model == 'dnn_regression':
estimator = tf.contrib.learn.DNNRegressor(
feature_columns=feature_columns,
hidden_units=args.hidden_layer_sizes,
config=config,
model_dir=train_dir,
optimizer=tf.train.AdamOptimizer(
args.learning_rate, epsilon=args.epsilon))
elif args.model == 'linear_regression':
estimator = tf.contrib.learn.LinearRegressor(
feature_columns=feature_columns,
config=config,
model_dir=train_dir,
optimizer=tf.train.FtrlOptimizer(
args.learning_rate,
l1_regularization_strength=args.l1_regularization,
l2_regularization_strength=args.l2_regularization))
elif args.model == 'dnn_classification':
estimator = tf.contrib.learn.DNNClassifier(
feature_columns=feature_columns,
hidden_units=args.hidden_layer_sizes,
n_classes=target_vocab_size,
config=config,
model_dir=train_dir,
optimizer=tf.train.AdamOptimizer(
args.learning_rate, epsilon=args.epsilon))
elif args.model == 'linear_classification':
estimator = tf.contrib.learn.LinearClassifier(
feature_columns=feature_columns,
n_classes=target_vocab_size,
config=config,
model_dir=train_dir,
optimizer=tf.train.FtrlOptimizer(
args.learning_rate,
l1_regularization_strength=args.l1_regularization,
l2_regularization_strength=args.l2_regularization))
else:
raise ValueError('bad --model-type value')
return estimator