def create_updates(loss, network, opt, learning_rate, momentum, beta1, beta2):
params = lasagne.layers.get_all_params(network, trainable=True)
grads = theano.grad(loss, params)
# if max_norm:
# names = ['crf.U', 'crf.W_h', 'crf.W_c', 'crf.b']
# constraints = [grad for param, grad in zip(params, grads) if param.name in names]
# assert len(constraints) == 4
# scaled_grads = total_norm_constraint(constraints, max_norm=max_norm)
# counter = 0
# for i in xrange(len(params)):
# param = params[i]
# if param.name in names:
# grads[i] = scaled_grads[counter]
# counter += 1
# assert counter == 4
if opt == 'adam':
updates = adam(grads, params=params, learning_rate=learning_rate, beta1=beta1, beta2=beta2)
elif opt == 'momentum':
updates = nesterov_momentum(grads, params=params, learning_rate=learning_rate, momentum=momentum)
else:
raise ValueError('unkown optimization algorithm: %s' % opt)
return updates
python类nesterov_momentum()的实例源码
def build_network(layers):
"""
Builds and returns a NeuralNet object
"""
return NeuralNet(
layers = layers,
batch_iterator_train = CropBatchIterator(batch_size=256),
update = nesterov_momentum,
update_learning_rate = theano.shared(float32(0.03)),
update_momentum = theano.shared(float32(0.9)),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.03, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=10),
],
regression = False,
max_epochs = 100,
verbose = 2
)
# --------------- Training the network ---------------
cnn_cascade_lasagne.py 文件源码
项目：Cascade-CNN-Face-Detection
作者: gogolgrind
项目源码
文件源码
阅读 48
收藏 0
点赞 0
评论 0
def __build_loss_train__fn__(self):
# create loss function
prediction = layers.get_output(self.net)
loss = objectives.categorical_crossentropy(prediction, self.__target_var__)
loss = loss.mean() + 1e-4 * regularization.regularize_network_params(self.net, regularization.l2)
val_acc = T.mean(T.eq(T.argmax(prediction, axis=1), self.__target_var__),dtype=theano.config.floatX)
# create parameter update expressions
params = layers.get_all_params(self.net, trainable=True)
self.eta = theano.shared(sp.array(sp.float32(0.05), dtype=sp.float32))
update_rule = updates.nesterov_momentum(loss, params, learning_rate=self.eta,
momentum=0.9)
# compile training function that updates parameters and returns training loss
self.__train_fn__ = theano.function([self.__input_var__,self.__target_var__], loss, updates=update_rule)
self.__predict_fn__ = theano.function([self.__input_var__], layers.get_output(self.net,deterministic=True))
self.__val_fn__ = theano.function([self.__input_var__,self.__target_var__], [loss,val_acc])
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def create_net(config, **kwargs):
args = {
'layers': config.layers,
'batch_iterator_train': iterator.ResampleIterator(
config, batch_size=config.get('batch_size_train')),
'batch_iterator_test': iterator.SharedIterator(
config, deterministic=True,
batch_size=config.get('batch_size_test')),
'on_epoch_finished': [
Schedule('update_learning_rate', config.get('schedule'),
weights_file=config.final_weights_file),
SaveBestWeights(weights_file=config.weights_file,
loss='kappa', greater_is_better=True,),
SaveWeights(config.weights_epoch, every_n_epochs=5),
SaveWeights(config.weights_best, every_n_epochs=1, only_best=True),
],
'objective': get_objective(),
'use_label_encoder': False,
'eval_size': 0.1,
'regression': False,
'max_epochs': 1000,
'verbose': 1,
'update_learning_rate': theano.shared(
util.float32(config.get('schedule')[0])),
'update': nesterov_momentum,
'update_momentum': 0.1,
'custom_scores': [('kappa', util.kappa)],
}
args.update(kwargs)
net = Net(**args)
return net
def get_updates(nnet,
train_obj,
trainable_params):
implemented_solvers = ("nesterov", "adagrad", "adadelta", "adam")
if not hasattr(nnet, "solver") or nnet.solver not in implemented_solvers:
nnet.sgd_solver = "nesterov"
else:
nnet.sgd_solver = nnet.solver
if nnet.sgd_solver == "nesterov":
updates = l_updates.nesterov_momentum(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate,
momentum=0.9)
elif nnet.sgd_solver == "adagrad":
updates = l_updates.adagrad(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate)
elif nnet.sgd_solver == "adadelta":
updates = l_updates.adadelta(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate)
elif nnet.sgd_solver == "adam":
updates = l_updates.adam(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate)
return updates
def build_network(layers):
"""
Builds and returns a NeuralNet object
"""
return NeuralNet(
layers = layers,
update = nesterov_momentum,
update_learning_rate = theano.shared(float32(0.01)),
update_momentum = theano.shared(float32(0.9)),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=200),
],
# Automatically split the data (90% training 10% validation).
train_split = TrainSplit(eval_size=0.10),
regression = False,
max_epochs = 300,
verbose = 2,
)
# --------------- Training the network ---------------
def define_updates(output_layer, X, Y):
output_train = lasagne.layers.get_output(output_layer)
output_test = lasagne.layers.get_output(output_layer, deterministic=True)
# set up the loss that we aim to minimize when using cat cross entropy our Y should be ints not one-hot
loss = lasagne.objectives.categorical_crossentropy(T.clip(output_train,0.000001,0.999999), Y)
loss = loss.mean()
acc = T.mean(T.eq(T.argmax(output_train, axis=1), Y), dtype=theano.config.floatX)
# if using ResNet use L2 regularization
all_layers = lasagne.layers.get_all_layers(output_layer)
l2_penalty = lasagne.regularization.regularize_layer_params(all_layers, lasagne.regularization.l2) * P.L2_LAMBDA
loss = loss + l2_penalty
# set up loss functions for validation dataset
test_loss = lasagne.objectives.categorical_crossentropy(T.clip(output_test,0.000001,0.999999), Y)
test_loss = test_loss.mean()
test_loss = test_loss + l2_penalty
test_acc = T.mean(T.eq(T.argmax(output_test, axis=1), Y), dtype=theano.config.floatX)
# get parameters from network and set up sgd with nesterov momentum to update parameters, l_r is shared var so it can be changed
l_r = theano.shared(np.array(LR_SCHEDULE[0], dtype=theano.config.floatX))
params = lasagne.layers.get_all_params(output_layer, trainable=True)
updates = nesterov_momentum(loss, params, learning_rate=l_r, momentum=P.MOMENTUM)
#updates = adam(loss, params, learning_rate=l_r)
prediction_binary = T.argmax(output_train, axis=1)
test_prediction_binary = T.argmax(output_test, axis=1)
# set up training and prediction functions
train_fn = theano.function(inputs=[X,Y], outputs=[loss, l2_penalty, acc, prediction_binary, output_train[:,1]], updates=updates)
valid_fn = theano.function(inputs=[X,Y], outputs=[test_loss, l2_penalty, test_acc, test_prediction_binary, output_test[:,1]])
return train_fn, valid_fn, l_r
def __init__(self, isTrain, isOutlierRemoval, isNN=1):
super(ClassificationNN, self).__init__(isTrain, isOutlierRemoval, isNN=1)
# data preprocessing
self.dataPreprocessing()
self.net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
#('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 12), # inut dimension is 12
hidden_num_units=6, # number of units in hidden layer
#hidden2_num_units=3, # number of units in hidden layer
output_nonlinearity=lasagne.nonlinearities.sigmoid, # output layer uses sigmoid function
output_num_units=1, # output dimension is 1
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.002,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=25, # we want to train this many epochs
verbose=0,
)
def define_updates(output_layer, X, Y):
output_train = lasagne.layers.get_output(output_layer)
output_test = lasagne.layers.get_output(output_layer, deterministic=True)
# set up the loss that we aim to minimize when using cat cross entropy our Y should be ints not one-hot
loss = lasagne.objectives.categorical_crossentropy(T.clip(output_train,0.000001,0.999999), Y)
loss = loss.mean()
acc = T.mean(T.eq(T.argmax(output_train, axis=1), Y), dtype=theano.config.floatX)
# if using ResNet use L2 regularization
all_layers = lasagne.layers.get_all_layers(output_layer)
l2_penalty = lasagne.regularization.regularize_layer_params(all_layers, lasagne.regularization.l2) * P.L2_LAMBDA
loss = loss + l2_penalty
# set up loss functions for validation dataset
test_loss = lasagne.objectives.categorical_crossentropy(T.clip(output_test,0.000001,0.999999), Y)
test_loss = test_loss.mean()
test_loss = test_loss + l2_penalty
test_acc = T.mean(T.eq(T.argmax(output_test, axis=1), Y), dtype=theano.config.floatX)
# get parameters from network and set up sgd with nesterov momentum to update parameters, l_r is shared var so it can be changed
l_r = theano.shared(np.array(LR_SCHEDULE[0], dtype=theano.config.floatX))
params = lasagne.layers.get_all_params(output_layer, trainable=True)
updates = nesterov_momentum(loss, params, learning_rate=l_r, momentum=P.MOMENTUM)
#updates = adam(loss, params, learning_rate=l_r)
prediction_binary = T.argmax(output_train, axis=1)
test_prediction_binary = T.argmax(output_test, axis=1)
# set up training and prediction functions
train_fn = theano.function(inputs=[X,Y], outputs=[loss, l2_penalty, acc, prediction_binary, output_train[:,1]], updates=updates)
valid_fn = theano.function(inputs=[X,Y], outputs=[test_loss, l2_penalty, test_acc, test_prediction_binary, output_test[:,1]])
return train_fn, valid_fn, l_r
def create_train_func(layers):
Xa, Xb = T.tensor4('Xa'), T.tensor4('Xb')
Xa_batch, Xb_batch = T.tensor4('Xa_batch'), T.tensor4('Xb_batch')
Tp = get_output(
layers['trans'],
inputs={
layers['inputa']: Xa, layers['inputb']: Xb,
}, deterministic=False,
)
# transforms: ground-truth, predicted
Tg = T.fmatrix('Tg')
Tg_batch = T.fmatrix('Tg_batch')
theta_gt = Tg.reshape((-1, 2, 3))
theta_pr = Tp.reshape((-1, 2, 3))
# grids: ground-truth, predicted
Gg = T.dot(theta_gt, _meshgrid(20, 20))
Gp = T.dot(theta_pr, _meshgrid(20, 20))
train_loss = T.mean(T.sqr(Gg - Gp))
params = get_all_params(layers['trans'], trainable=True)
updates = nesterov_momentum(train_loss, params, 1e-3, 0.9)
corr_func = theano.function(
inputs=[theano.In(Xa_batch), theano.In(Xb_batch), theano.In(Tg_batch)],
outputs=[Tp, train_loss],
updates=updates,
givens={
Xa: Xa_batch, Xb: Xb_batch, # Ia, Ib
Tg: Tg_batch, # transform Ia --> Ib
}
)
return corr_func
def load_finetuned_dbn(path):
"""
Load a fine tuned Deep Belief Net from file
:param path: path to deep belief net parameters
:return: deep belief net
"""
dbn = NeuralNet(
layers=[
('input', las.layers.InputLayer),
('l1', las.layers.DenseLayer),
('l2', las.layers.DenseLayer),
('l3', las.layers.DenseLayer),
('l4', las.layers.DenseLayer),
('l5', las.layers.DenseLayer),
('l6', las.layers.DenseLayer),
('l7', las.layers.DenseLayer),
('output', las.layers.DenseLayer)
],
input_shape=(None, 1200),
l1_num_units=2000, l1_nonlinearity=sigmoid,
l2_num_units=1000, l2_nonlinearity=sigmoid,
l3_num_units=500, l3_nonlinearity=sigmoid,
l4_num_units=50, l4_nonlinearity=linear,
l5_num_units=500, l5_nonlinearity=sigmoid,
l6_num_units=1000, l6_nonlinearity=sigmoid,
l7_num_units=2000, l7_nonlinearity=sigmoid,
output_num_units=1200, output_nonlinearity=linear,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
with open(path, 'rb') as f:
pretrained_nn = pickle.load(f)
if pretrained_nn is not None:
dbn.load_params_from(path)
return dbn
def load_encoder(path):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained unrolled encoder
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': (None, 1200)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid,
'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid,
'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid,
'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear,
'W': w4, 'b': b4}),
],
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def load_finetuned_dbn(path):
"""
Load a fine tuned Deep Belief Net from file
:param path: path to deep belief net parameters
:return: deep belief net
"""
dbn = NeuralNet(
layers=[
('input', las.layers.InputLayer),
('l1', las.layers.DenseLayer),
('l2', las.layers.DenseLayer),
('l3', las.layers.DenseLayer),
('l4', las.layers.DenseLayer),
('l5', las.layers.DenseLayer),
('l6', las.layers.DenseLayer),
('l7', las.layers.DenseLayer),
('output', las.layers.DenseLayer)
],
input_shape=(None, 1200),
l1_num_units=2000, l1_nonlinearity=sigmoid,
l2_num_units=1000, l2_nonlinearity=sigmoid,
l3_num_units=500, l3_nonlinearity=sigmoid,
l4_num_units=50, l4_nonlinearity=linear,
l5_num_units=500, l5_nonlinearity=sigmoid,
l6_num_units=1000, l6_nonlinearity=sigmoid,
l7_num_units=2000, l7_nonlinearity=sigmoid,
output_num_units=1200, output_nonlinearity=linear,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
with open(path, 'rb') as f:
pretrained_nn = pickle.load(f)
if pretrained_nn is not None:
dbn.load_params_from(path)
return dbn
def load_finetuned_dbn(path):
"""
Load a fine tuned Deep Belief Net from file
:param path: path to deep belief net parameters
:return: deep belief net
"""
dbn = NeuralNet(
layers=[
('input', las.layers.InputLayer),
('l1', las.layers.DenseLayer),
('l2', las.layers.DenseLayer),
('l3', las.layers.DenseLayer),
('l4', las.layers.DenseLayer),
('l5', las.layers.DenseLayer),
('l6', las.layers.DenseLayer),
('l7', las.layers.DenseLayer),
('output', las.layers.DenseLayer)
],
input_shape=(None, 1200),
l1_num_units=2000, l1_nonlinearity=sigmoid,
l2_num_units=1000, l2_nonlinearity=sigmoid,
l3_num_units=500, l3_nonlinearity=sigmoid,
l4_num_units=50, l4_nonlinearity=linear,
l5_num_units=500, l5_nonlinearity=sigmoid,
l6_num_units=1000, l6_nonlinearity=sigmoid,
l7_num_units=2000, l7_nonlinearity=sigmoid,
output_num_units=1200, output_nonlinearity=linear,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
with open(path, 'rb') as f:
pretrained_nn = pickle.load(f)
if pretrained_nn is not None:
dbn.load_params_from(path)
return dbn
def load_finetuned_dbn(path):
"""
Load a fine tuned Deep Belief Net from file
:param path: path to deep belief net parameters
:return: deep belief net
"""
dbn = NeuralNet(
layers=[
('input', las.layers.InputLayer),
('l1', las.layers.DenseLayer),
('l2', las.layers.DenseLayer),
('l3', las.layers.DenseLayer),
('l4', las.layers.DenseLayer),
('l5', las.layers.DenseLayer),
('l6', las.layers.DenseLayer),
('l7', las.layers.DenseLayer),
('output', las.layers.DenseLayer)
],
input_shape=(None, 1200),
l1_num_units=2000, l1_nonlinearity=sigmoid,
l2_num_units=1000, l2_nonlinearity=sigmoid,
l3_num_units=500, l3_nonlinearity=sigmoid,
l4_num_units=50, l4_nonlinearity=linear,
l5_num_units=500, l5_nonlinearity=sigmoid,
l6_num_units=1000, l6_nonlinearity=sigmoid,
l7_num_units=2000, l7_nonlinearity=sigmoid,
output_num_units=1200, output_nonlinearity=linear,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
with open(path, 'rb') as f:
pretrained_nn = pickle.load(f)
if pretrained_nn is not None:
dbn.load_params_from(path)
return dbn
def load_finetuned_dbn(path):
"""
Load a fine tuned Deep Belief Net from file
:param path: path to deep belief net parameters
:return: deep belief net
"""
dbn = NeuralNet(
layers=[
('input', las.layers.InputLayer),
('l1', las.layers.DenseLayer),
('l2', las.layers.DenseLayer),
('l3', las.layers.DenseLayer),
('l4', las.layers.DenseLayer),
('l5', las.layers.DenseLayer),
('l6', las.layers.DenseLayer),
('l7', las.layers.DenseLayer),
('output', las.layers.DenseLayer)
],
input_shape=(None, 1200),
l1_num_units=2000, l1_nonlinearity=sigmoid,
l2_num_units=1000, l2_nonlinearity=sigmoid,
l3_num_units=500, l3_nonlinearity=sigmoid,
l4_num_units=50, l4_nonlinearity=linear,
l5_num_units=500, l5_nonlinearity=sigmoid,
l6_num_units=1000, l6_nonlinearity=sigmoid,
l7_num_units=2000, l7_nonlinearity=sigmoid,
output_num_units=1200, output_nonlinearity=linear,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
with open(path, 'rb') as f:
pretrained_nn = pickle.load(f)
if pretrained_nn is not None:
dbn.load_params_from(path)
return dbn
def load_finetuned_dbn(path):
"""
Load a fine tuned Deep Belief Net from file
:param path: path to deep belief net parameters
:return: deep belief net
"""
dbn = NeuralNet(
layers=[
('input', las.layers.InputLayer),
('l1', las.layers.DenseLayer),
('l2', las.layers.DenseLayer),
('l3', las.layers.DenseLayer),
('l4', las.layers.DenseLayer),
('l5', las.layers.DenseLayer),
('l6', las.layers.DenseLayer),
('l7', las.layers.DenseLayer),
('output', las.layers.DenseLayer)
],
input_shape=(None, 1200),
l1_num_units=2000, l1_nonlinearity=sigmoid,
l2_num_units=1000, l2_nonlinearity=sigmoid,
l3_num_units=500, l3_nonlinearity=sigmoid,
l4_num_units=50, l4_nonlinearity=linear,
l5_num_units=500, l5_nonlinearity=sigmoid,
l6_num_units=1000, l6_nonlinearity=sigmoid,
l7_num_units=2000, l7_nonlinearity=sigmoid,
output_num_units=1200, output_nonlinearity=linear,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.5,
objective_l2=0.005,
verbose=1,
regression=True
)
with open(path, 'rb') as f:
pretrained_nn = pickle.load(f)
if pretrained_nn is not None:
dbn.load_params_from(path)
return dbn
ClassificationLinearBlending.py 文件源码
项目：AirTicketPredicting
作者: junlulocky
项目源码
文件源码
阅读 20
收藏 0
点赞 0
评论 0
def __init__(self, isTrain, isOutlierRemoval=0):
super(ClassificationLinearBlending, self).__init__(isTrain, isOutlierRemoval)
# data preprocessing
self.dataPreprocessing()
# create logistic regression object
self.logreg = linear_model.LogisticRegression(tol=1e-6, penalty='l1', C=0.0010985411419875584)
# create adaboost object
self.dt_stump = DecisionTreeClassifier(max_depth=10)
self.ada = AdaBoostClassifier(
base_estimator=self.dt_stump,
learning_rate=1,
n_estimators=5,
algorithm="SAMME.R")
# create knn object
self.knn = neighbors.KNeighborsClassifier(6, weights='uniform')
# create decision tree object
self.decisiontree = DecisionTreeClassifier(max_depth=50)
# create neural network object
self.net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
#('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 12), # inut dimension is 12
hidden_num_units=6, # number of units in hidden layer
#hidden2_num_units=3, # number of units in hidden layer
output_nonlinearity=lasagne.nonlinearities.sigmoid, # output layer uses sigmoid function
output_num_units=1, # output dimension is 1
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.002,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=25, # we want to train this many epochs
verbose=0,
)
ClassificationUniformBlending.py 文件源码
项目：AirTicketPredicting
作者: junlulocky
项目源码
文件源码
阅读 23
收藏 0
点赞 0
评论 0
def __init__(self, isTrain, isOutlierRemoval=0):
super(ClassificationUniformBlending, self).__init__(isTrain, isOutlierRemoval)
# data preprocessing
self.dataPreprocessing()
# create logistic regression object
self.logreg = linear_model.LogisticRegression(tol=1e-6, penalty='l1', C=0.0010985411419875584)
# create adaboost object
self.dt_stump = DecisionTreeClassifier(max_depth=10)
self.ada = AdaBoostClassifier(
base_estimator=self.dt_stump,
learning_rate=1,
n_estimators=5,
algorithm="SAMME.R")
# create knn object
self.knn = neighbors.KNeighborsClassifier(2, weights='uniform')
# create decision tree object
self.decisiontree = DecisionTreeClassifier(max_depth=45, max_features='log2')
# create neural network object
self.net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
#('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 12), # inut dimension is 12
hidden_num_units=6, # number of units in hidden layer
#hidden2_num_units=3, # number of units in hidden layer
output_nonlinearity=lasagne.nonlinearities.sigmoid, # output layer uses sigmoid function
output_num_units=1, # output dimension is 1
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.002,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=25, # we want to train this many epochs
verbose=0,
)
# create PLA object
self.pla = Perceptron()
# create random forest object
self.rf = RandomForestClassifier(max_features='log2', n_estimators=20, max_depth=30)
def __init__(self, conf):
self.conf = conf
if self.conf.act == "linear":
self.conf.act = linear
elif self.conf.act == "sigmoid":
self.conf.act = sigmoid
elif self.conf.act == "relu":
self.conf.act = rectify
elif self.conf.act == "tanh":
self.conf.act = tanh
else:
raise ValueError("Unknown activation function", self.conf.act)
input_var_first = T.matrix('inputs1')
input_var_second = T.matrix('inputs2')
target_var = T.matrix('targets')
# create network
self.autoencoder, encoder_first, encoder_second = self.__create_toplogy__(input_var_first, input_var_second)
self.out = get_output(self.autoencoder)
loss = squared_error(self.out, target_var)
loss = loss.mean()
params = get_all_params(self.autoencoder, trainable=True)
updates = nesterov_momentum(loss, params, learning_rate=self.conf.lr, momentum=self.conf.momentum)
# training function
self.train_fn = theano.function([input_var_first, input_var_second, target_var], loss, updates=updates)
# fuction to reconstruct
test_reconstruction = get_output(self.autoencoder, deterministic=True)
self.reconstruction_fn = theano.function([input_var_first, input_var_second], test_reconstruction)
# encoding function
test_encode = get_output([encoder_first, encoder_second], deterministic=True)
self.encoding_fn = theano.function([input_var_first, input_var_second], test_encode)
# utils
blas = lambda name, ndarray: scipy.linalg.get_blas_funcs((name,), (ndarray,))[0]
self.blas_nrm2 = blas('nrm2', np.array([], dtype=float))
self.blas_scal = blas('scal', np.array([], dtype=float))
# load weights if necessary
if self.conf.load_model is not None:
self.load_model()
def load_dbn(path='models/oulu_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1144)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1144, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=30,
objective_loss_function=squared_error,
update=nesterov_momentum,
regression=True,
verbose=1,
update_learning_rate=0.001,
update_momentum=0.05,
objective_l2=0.005,
)
return dbn
def load_dbn(path='models/oulu_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1144)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1144, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=30,
objective_loss_function=squared_error,
update=nesterov_momentum,
regression=True,
verbose=1,
update_learning_rate=0.001,
update_momentum=0.05,
objective_l2=0.005,
)
return dbn
def load_dbn(path='models/oulu_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1144)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1144, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=30,
objective_loss_function=squared_error,
update=nesterov_momentum,
regression=True,
verbose=1,
update_learning_rate=0.001,
update_momentum=0.05,
objective_l2=0.005,
)
return dbn
def load_dbn(path='models/oulu_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1144)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1144, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=30,
objective_loss_function=squared_error,
update=nesterov_momentum,
regression=True,
verbose=1,
update_learning_rate=0.001,
update_momentum=0.05,
objective_l2=0.005,
)
return dbn
