def ResidualBlock1D_helper(layers, kernel_size, filters, final_stride=1):
def f(_input):
basic = _input
for ln in range(layers):
#basic = BatchNormalization()( basic ) # triggers known keras bug w/ TimeDistributed: https://github.com/fchollet/keras/issues/5221
basic = ELU()(basic)
basic = Conv1D(filters, kernel_size, kernel_initializer='he_normal',
kernel_regularizer=l2(1.e-4), padding='same')(basic)
# note that this strides without averaging
return AveragePooling1D(pool_size=1, strides=final_stride)(Add()([_input, basic]))
return f
python类ELU的实例源码
def convresblock(x, nfeats=8, ksize=3, nskipped=2, elu=True):
"""The proposed residual block from [4].
Running with elu=True will use ELU nonlinearity and running with
elu=False will use BatchNorm + RELU nonlinearity. While ELU's are fast
due to the fact they do not suffer from BatchNorm overhead, they may
overfit because they do not offer the stochastic element of the batch
formation process of BatchNorm, which acts as a good regularizer.
# Arguments
x: 4D tensor, the tensor to feed through the block
nfeats: Integer, number of feature maps for conv layers.
ksize: Integer, width and height of conv kernels in first convolution.
nskipped: Integer, number of conv layers for the residual function.
elu: Boolean, whether to use ELU or BN+RELU.
# Input shape
4D tensor with shape:
`(batch, channels, rows, cols)`
# Output shape
4D tensor with shape:
`(batch, filters, rows, cols)`
"""
y0 = Conv2D(nfeats, ksize, padding='same')(x)
y = y0
for i in range(nskipped):
if elu:
y = ELU()(y)
else:
y = BatchNormalization(axis=1)(y)
y = Activation('relu')(y)
y = Conv2D(nfeats, 1, padding='same')(y)
return layers.add([y0, y])
def generate_dense_model(input_shape, layers, nb_actions):
model = Sequential()
model.add(Flatten(input_shape=input_shape))
model.add(Dropout(0.1)) # drop out the input to make model less sensitive to any 1 feature
for layer in layers:
model.add(Dense(layer))
model.add(BatchNormalization())
model.add(ELU(alpha=1.0))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
return model
def Encoder(hidden_size, activation=None, return_sequences=True, bidirectional=False, use_gru=True):
if activation is None:
activation = ELU()
if use_gru:
def _encoder(x):
if bidirectional:
branch_1 = GRU(int(hidden_size/2), activation='linear',
return_sequences=return_sequences, go_backwards=False)(x)
branch_2 = GRU(int(hidden_size/2), activation='linear',
return_sequences=return_sequences, go_backwards=True)(x)
x = concatenate([branch_1, branch_2])
x = activation(x)
return x
else:
x = GRU(hidden_size, activation='linear',
return_sequences=return_sequences)(x)
x = activation(x)
return x
else:
def _encoder(x):
if bidirectional:
branch_1 = LSTM(int(hidden_size/2), activation='linear',
return_sequences=return_sequences, go_backwards=False)(x)
branch_2 = LSTM(int(hidden_size/2), activation='linear',
return_sequences=return_sequences, go_backwards=True)(x)
x = concatenate([branch_1, branch_2])
x = activation(x)
return x
else:
x = LSTM(hidden_size, activation='linear',
return_sequences=return_sequences)(x)
x = activation(x)
return x
return _encoder
def AttentionDecoder(hidden_size, activation=None, return_sequences=True, bidirectional=False, use_gru=True):
if activation is None:
activation = ELU()
if use_gru:
def _decoder(x, attention):
if bidirectional:
branch_1 = AttentionWrapper(GRU(int(hidden_size/2), activation='linear', return_sequences=return_sequences,
go_backwards=False), attention, single_attention_param=True)(x)
branch_2 = AttentionWrapper(GRU(int(hidden_size/2), activation='linear', return_sequences=return_sequences,
go_backwards=True), attention, single_attention_param=True)(x)
x = concatenate([branch_1, branch_2])
return activation(x)
else:
x = AttentionWrapper(GRU(hidden_size, activation='linear',
return_sequences=return_sequences), attention, single_attention_param=True)(x)
x = activation(x)
return x
else:
def _decoder(x, attention):
if bidirectional:
branch_1 = AttentionWrapper(LSTM(int(hidden_size/2), activation='linear', return_sequences=return_sequences,
go_backwards=False), attention, single_attention_param=True)(x)
branch_2 = AttentionWrapper(LSTM(hidden_size, activation='linear', return_sequences=return_sequences,
go_backwards=True), attention, single_attention_param=True)(x)
x = concatenate([branch_1, branch_2])
x = activation(x)
return x
else:
x = AttentionWrapper(LSTM(hidden_size, activation='linear', return_sequences=return_sequences),
attention, single_attention_param=True)(x)
x = activation(x)
return x
return _decoder
def Decoder(hidden_size, activation=None, return_sequences=True, bidirectional=False, use_gru=True):
if activation is None:
activation = ELU()
if use_gru:
def _decoder(x):
if bidirectional:
x = Bidirectional(
GRU(int(hidden_size/2), activation='linear', return_sequences=return_sequences))(x)
x = activation(x)
return x
else:
x = GRU(hidden_size, activation='linear',
return_sequences=return_sequences)(x)
x = activation(x)
return x
else:
def _decoder(x):
if bidirectional:
x = Bidirectional(
LSTM(int(hidden_size/2), activation='linear', return_sequences=return_sequences))(x)
x = activation(x)
return x
else:
x = LSTM(hidden_size, activation='linear',
return_sequences=return_sequences)(x)
x = activation(x)
return x
return _decoder
def create_model(input_shape, hidden_layers=[1024, 512, 256], input_dropout=0.1, hidden_dropout=0.5):
'''Define a simple multilayer perceptron.
Args:
input_shape (tuple): input shape to the model. For this model, should be of shape (dim,)
input_dropout (float): fraction of input features to drop out during training
hidden_layers (tuple): a tuple/list with number of hidden units in each hidden layer
Returns:
keras.models.Sequential : a model to train
'''
model = Sequential()
# dropout the input to prevent overfitting to any one feature
# (a similar concept to randomization in random forests,
# but we choose less severe feature sampling )
model.add(Dropout(input_dropout, input_shape=input_shape))
# set up hidden layers
for n_hidden_units in hidden_layers:
# the layer...activation will come later
model.add(Dense(n_hidden_units))
# dropout to prevent overfitting
model.add(Dropout(hidden_dropout))
# batchnormalization helps training
model.add(BatchNormalization())
# ...the activation!
model.add(ELU())
# the output layer
model.add(Dense(1, activation='sigmoid'))
# we'll optimize with plain old sgd
model.compile(loss='binary_crossentropy',
optimizer='sgd', metrics=['accuracy'])
return model
def test_delete_channels_advanced_activations(channel_index, data_format):
layer_test_helper_flatten_2d(LeakyReLU(), channel_index, data_format)
layer_test_helper_flatten_2d(ELU(), channel_index, data_format)
layer_test_helper_flatten_2d(ThresholdedReLU(), channel_index, data_format)
def comma_model():
row, col, depth = 66, 200, 3
shape = (row, col, depth)
model = Sequential()
model.add(Lambda(lambda x: x/127.5 -1., input_shape=shape, output_shape=shape))
model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode='same'))
model.add(ELU())
model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode='same'))
model.add(ELU())
model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode='same'))
model.add(Flatten())
model.add(Dropout(.2))
model.add(ELU())
model.add(Dense(512))
model.add(Dropout(.5))
model.add(ELU())
#the fully connected layer accounts for huge % of parameters (50+)
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
model.summary()
return model
def convresblock(x, nfeats=8, ksize=3, nskipped=2, elu=True):
y0 = Conv2D(nfeats, ksize, padding='same')(x)
y = y0
for i in range(nskipped):
if elu:
y = ELU()(y)
else:
y = BatchNormalization(axis=1)(y)
y = Activation('relu')(y)
y = Conv2D(nfeats, 1, padding='same')(y)
return layers.add([y0, y])
# This example assume 'channels_first' data format.
def __init__(self):
super(ELUNet, self).__init__()
self.elu = nn.ELU()
def test_elu(self):
keras_model = Sequential()
keras_model.add(ELU(input_shape=(3, 32, 32), name='elu'))
keras_model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.SGD())
pytorch_model = ELUNet()
self.transfer(keras_model, pytorch_model)
self.assertEqualPrediction(keras_model, pytorch_model, self.test_data)
# Tests activation function with learned parameters
def buildModel(cameraFormat=(3, 480, 640)):
"""
Build and return a CNN; details in the comments.
The intent is a scaled down version of the model from "End to End Learning
for Self-Driving Cars": https://arxiv.org/abs/1604.07316.
Args:
cameraFormat: (3-tuple) Ints to specify the input dimensions (color
channels, rows, columns).
Returns:
A compiled Keras model.
"""
print "Building model..."
ch, row, col = cameraFormat
model = Sequential()
# Use a lambda layer to normalize the input data
model.add(Lambda(
lambda x: x/127.5 - 1.,
input_shape=(ch, row, col),
output_shape=(ch, row, col))
)
# Several convolutional layers, each followed by ELU activation
# 8x8 convolution (kernel) with 4x4 stride over 16 output filters
model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same"))
model.add(ELU())
# 5x5 convolution (kernel) with 2x2 stride over 32 output filters
model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same"))
model.add(ELU())
# 5x5 convolution (kernel) with 2x2 stride over 64 output filters
model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same"))
# Flatten the input to the next layer
model.add(Flatten())
# Apply dropout to reduce overfitting
model.add(Dropout(.2))
model.add(ELU())
# Fully connected layer
model.add(Dense(512))
# More dropout
model.add(Dropout(.5))
model.add(ELU())
# Fully connected layer with one output dimension (representing the speed).
model.add(Dense(1))
# Adam optimizer is a standard, efficient SGD optimization method
# Loss function is mean squared error, standard for regression problems
model.compile(optimizer="adam", loss="mse")
return model
def keras_nn(X_train, y_train):
"""
Constructs a neural network using keras and trains it.
The best final networks is saved in model.json and model.h5
Parameters
----------
X_train : numpy array
The images
y_train : numpy array
The angles
"""
model = Sequential()
# Further reduces the dimension of the image to 8x16
model.add(AveragePooling2D((2, 2), border_mode='valid', input_shape=(16, 32, 1)))
# Applies 2x2 convolution
model.add(Convolution2D(1, 2, 2, subsample=(1, 1)))
model.add(ELU())
# Max Pooling to reduce the dimensions. 2X4 used because it matches the aspect ratio of the input
model.add(MaxPooling2D((2, 4), border_mode='valid'))
# Droput - We only have 10 connections at this point, but it still improves performance. However it should be kept low, e.g. 0.5 doesn't work
model.add(Dropout(0.25))
model.add(Flatten())
# The final layer - outputs a float number (the steering angle)
model.add(Dense(1)) #
# Show a summary of the neural network
model.summary()
# Save the best model by validation mean squared error
checkpoint = ModelCheckpoint("model.h5", monitor='val_mean_squared_error', verbose=1, save_best_only=True, mode='min')
# Stop training when there is no improvment.
# This is to speed up training, the accuracy is not affected, because the checkpoint will pick-up the best model anyway
early_stop = EarlyStopping(monitor='val_mean_squared_error', min_delta=0.0001, patience=4, verbose=1, mode='min')
# Compile the model with Adam optimizer and monitor mean squared error
model.compile('adam', 'mean_squared_error', ['mean_squared_error'])
# Save the model to JSON
model_json = model.to_json()
with open("model.json", "w") as model_file:
model_file.write(model_json)
# Start training.
# nb_epoch should be a big number, there is early stopping callback anyway
# Data is split by keras to training and validation
history = model.fit(X_train, y_train, batch_size=32, nb_epoch=150, verbose=1, callbacks=[checkpoint, early_stop], validation_split=0.2, shuffle=True)
