def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=adadelta,
update_learning_rate=0.01,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
python类adadelta()的实例源码
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=adadelta,
update_learning_rate=0.01,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def extract_encoder(dbn):
dbn_layers = dbn.get_all_layers()
encoder = NeuralNet(
layers=[
(InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}),
(DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[1].W, 'b': dbn_layers[1].b}),
(DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[2].W, 'b': dbn_layers[2].b}),
(DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid,
'W': dbn_layers[3].W, 'b': dbn_layers[3].b}),
(DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear,
'W': dbn_layers[4].W, 'b': dbn_layers[4].b}),
],
update=adadelta,
update_learning_rate=0.01,
objective_l2=0.005,
verbose=1,
regression=True
)
encoder.initialize()
return encoder
def get_updates(nnet,
train_obj,
trainable_params):
implemented_solvers = ("nesterov", "adagrad", "adadelta", "adam")
if not hasattr(nnet, "solver") or nnet.solver not in implemented_solvers:
nnet.sgd_solver = "nesterov"
else:
nnet.sgd_solver = nnet.solver
if nnet.sgd_solver == "nesterov":
updates = l_updates.nesterov_momentum(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate,
momentum=0.9)
elif nnet.sgd_solver == "adagrad":
updates = l_updates.adagrad(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate)
elif nnet.sgd_solver == "adadelta":
updates = l_updates.adadelta(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate)
elif nnet.sgd_solver == "adam":
updates = l_updates.adam(train_obj,
trainable_params,
learning_rate=Cfg.learning_rate)
return updates
def create_decoder(input_size, decode_size, weights):
decoder_layers = [
(InputLayer, {'shape': (None, input_size)}),
(DenseLayer, {'name': 'decoder', 'num_units': decode_size, 'nonlinearity': linear, 'W': weights})
]
decoder = NeuralNet(
layers=decoder_layers,
max_epochs=50,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1
)
return decoder
def load_dbn(path='models/oulu_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1144)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1144, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=10,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1,
update_learning_rate=0.01,
objective_l2=0.005,
)
return dbn
def load_dbn(path='models/oulu_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1144)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1144, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=10,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1,
update_learning_rate=0.01,
objective_l2=0.005,
)
return dbn
def load_dbn(path='models/avletters_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1'].astype('float32')
w2 = nn['w2'].astype('float32')
w3 = nn['w3'].astype('float32')
w4 = nn['w4'].astype('float32')
w5 = nn['w5'].astype('float32')
w6 = nn['w6'].astype('float32')
w7 = nn['w7'].astype('float32')
w8 = nn['w8'].astype('float32')
b1 = nn['b1'][0].astype('float32')
b2 = nn['b2'][0].astype('float32')
b3 = nn['b3'][0].astype('float32')
b4 = nn['b4'][0].astype('float32')
b5 = nn['b5'][0].astype('float32')
b6 = nn['b6'][0].astype('float32')
b7 = nn['b7'][0].astype('float32')
b8 = nn['b8'][0].astype('float32')
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1200)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1200, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=10,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1,
update_learning_rate=0.01,
# update_learning_rate=0.001,
# update_momentum=0.05,
objective_l2=0.005,
)
return dbn
def parse_options():
options = dict()
options['config'] = 'config/bimodal.ini'
options['no_plot'] = False
parser = argparse.ArgumentParser()
parser.add_argument('--config', help='config file to use, default=config/bimodal.ini')
parser.add_argument('--write_results', help='write results to file')
parser.add_argument('--update_rule', help='adadelta, sgdm, sgdnm, adam')
parser.add_argument('--learning_rate', help='learning rate')
parser.add_argument('--decay_rate', help='learning rate decay')
parser.add_argument('--momentum', help='momentum')
parser.add_argument('--momentum_schedule', help='eg: 0.9,0.9,0.95,0.99')
parser.add_argument('--validation_window', help='validation window length, eg: 6')
parser.add_argument('--t1', help='epoch to start learning rate decay, eg: 10')
parser.add_argument('--weight_init', help='norm,glorot,ortho,uniform')
parser.add_argument('--num_epoch', help='number of epochs to run')
parser.add_argument('--use_peepholes', help='use peephole connections in LSTM')
parser.add_argument('--no_plot', dest='no_plot', action='store_true', help='disable plots')
parser.set_defaults(no_plot=False)
parser.set_defaults(use_peepholes=False)
args = parser.parse_args()
if args.config:
options['config'] = args.config
if args.write_results:
options['write_results'] = args.write_results
if args.update_rule:
options['update_rule'] = args.update_rule
if args.learning_rate:
options['learning_rate'] = args.learning_rate
if args.decay_rate:
options['decay_rate'] = args.decay_rate
if args.momentum:
options['momentum'] = args.momentum
if args.momentum_schedule:
options['momentum_schedule'] = args.momentum_schedule
if args.validation_window:
options['validation_window'] = args.validation_window
if args.t1:
options['t1'] = args.t1
if args.weight_init:
options['weight_init'] = args.weight_init
if args.num_epoch:
options['num_epoch'] = args.num_epoch
if args.no_plot:
options['no_plot'] = True
if args.use_peepholes:
options['use_peepholes'] = True
return options
def load_dbn(path='models/cuave_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1500)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4}),
(DenseLayer, {'name': 'l5', 'num_units': 500, 'nonlinearity': sigmoid, 'W': w5, 'b': b5}),
(DenseLayer, {'name': 'l6', 'num_units': 1000, 'nonlinearity': sigmoid, 'W': w6, 'b': b6}),
(DenseLayer, {'name': 'l7', 'num_units': 2000, 'nonlinearity': sigmoid, 'W': w7, 'b': b7}),
(DenseLayer, {'name': 'output', 'num_units': 1144, 'nonlinearity': linear, 'W': w8, 'b': b8}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=10,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1,
update_learning_rate=0.01,
objective_l2=0.005,
)
return dbn
def load_ae_encoder(path, nonlinearity=sigmoid):
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
layers = [
(InputLayer, {'name': 'input', 'shape': (None, 1500)}),
(DenseLayer, {'name': 'l1', 'num_units': 2000, 'nonlinearity': nonlinearity, 'W': w1, 'b': b1}),
(DenseLayer, {'name': 'l2', 'num_units': 1000, 'nonlinearity': nonlinearity, 'W': w2, 'b': b2}),
(DenseLayer, {'name': 'l3', 'num_units': 500, 'nonlinearity': nonlinearity, 'W': w3, 'b': b3}),
(DenseLayer, {'name': 'l4', 'num_units': 50, 'nonlinearity': linear, 'W': w4, 'b': b4})
]
'''
dbn = NeuralNet(
layers=layers,
max_epochs=30,
objective_loss_function=squared_error,
update=nesterov_momentum,
regression=True,
verbose=1,
update_learning_rate=0.001,
update_momentum=0.05,
objective_l2=0.005,
)
'''
dbn = NeuralNet(
layers=layers,
max_epochs=10,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1,
update_learning_rate=0.01,
# update_learning_rate=0.001,
# update_momentum=0.05,
objective_l2=0.005,
)
return dbn
def build_instrument_model(self, n_vars, **kwargs):
targets = TT.vector()
instrument_vars = TT.matrix()
instruments = layers.InputLayer((None, n_vars), instrument_vars)
instruments = layers.DropoutLayer(instruments, p=0.2)
dense_layer = layers.DenseLayer(instruments, kwargs['dense_size'], nonlinearity=nonlinearities.tanh)
dense_layer = layers.DropoutLayer(dense_layer, p=0.2)
for _ in xrange(kwargs['n_dense_layers'] - 1):
dense_layer = layers.DenseLayer(dense_layer, kwargs['dense_size'], nonlinearity=nonlinearities.tanh)
dense_layer = layers.DropoutLayer(dense_layer, p=0.5)
self.instrument_output = layers.DenseLayer(dense_layer, 1, nonlinearity=nonlinearities.linear)
init_params = layers.get_all_param_values(self.instrument_output)
prediction = layers.get_output(self.instrument_output, deterministic=False)
test_prediction = layers.get_output(self.instrument_output, deterministic=True)
# flexible here, endog variable can be categorical, continuous, etc.
l2_cost = regularization.regularize_network_params(self.instrument_output, regularization.l2)
loss = objectives.squared_error(prediction.flatten(), targets.flatten()).mean() + 1e-4 * l2_cost
loss_total = objectives.squared_error(prediction.flatten(), targets.flatten()).mean()
params = layers.get_all_params(self.instrument_output, trainable=True)
param_updates = updates.adadelta(loss, params)
self._instrument_train_fn = theano.function(
[
targets,
instrument_vars,
],
loss,
updates=param_updates
)
self._instrument_loss_fn = theano.function(
[
targets,
instrument_vars,
],
loss_total
)
self._instrument_output_fn = theano.function([instrument_vars], test_prediction)
return init_params
def build_treatment_model(self, n_vars, **kwargs):
input_vars = TT.matrix()
instrument_vars = TT.matrix()
targets = TT.vector()
inputs = layers.InputLayer((None, n_vars), input_vars)
inputs = layers.DropoutLayer(inputs, p=0.2)
dense_layer = layers.DenseLayer(inputs, 2 * kwargs['dense_size'], nonlinearity=nonlinearities.rectify)
dense_layer = layers.batch_norm(dense_layer)
dense_layer= layers.DropoutLayer(dense_layer, p=0.2)
for _ in xrange(kwargs['n_dense_layers'] - 1):
dense_layer = layers.DenseLayer(dense_layer, kwargs['dense_size'], nonlinearity=nonlinearities.rectify)
dense_layer = layers.batch_norm(dense_layer)
self.treatment_output = layers.DenseLayer(dense_layer, 1, nonlinearity=nonlinearities.linear)
init_params = layers.get_all_param_values(self.treatment_output)
prediction = layers.get_output(self.treatment_output, deterministic=False)
test_prediction = layers.get_output(self.treatment_output, deterministic=True)
l2_cost = regularization.regularize_network_params(self.treatment_output, regularization.l2)
loss = gmm_loss(prediction, targets, instrument_vars) + 1e-4 * l2_cost
params = layers.get_all_params(self.treatment_output, trainable=True)
param_updates = updates.adadelta(loss, params)
self._train_fn = theano.function(
[
input_vars,
targets,
instrument_vars,
],
loss,
updates=param_updates
)
self._loss_fn = theano.function(
[
input_vars,
targets,
instrument_vars,
],
loss,
)
self._output_fn = theano.function(
[
input_vars,
],
test_prediction,
)
return init_params
