def build_mlp(input_var=None):
l_in = InputLayer(shape=(None, 1, 28, 28), input_var=input_var)
l_hid1 = DenseLayer(
l_in, num_units=500,
nonlinearity=rectify,
W=lasagne.init.GlorotUniform())
l_hid1_drop = DropoutLayer(l_hid1, p=0.4)
l_hid2 = DenseLayer(
l_hid1_drop, num_units=300,
nonlinearity=rectify)
l_hid2_drop = DropoutLayer(l_hid2, p=0.4)
l_out = DenseLayer(
l_hid2_drop, num_units=10,
nonlinearity=softmax)
return l_out
# generator giving the batches
python类DropoutLayer()的实例源码
def build_model(input_var=None):
layers = [1, 5, 10, 1]
l_in = InputLayer((None, None, layers[0]),
input_var=input_var)
l_lstm1 = LSTMLayer(l_in, layers[1])
l_lstm1_dropout = DropoutLayer(l_lstm1, p=0.2)
l_lstm2 = LSTMLayer(l_lstm1_dropout, layers[2])
l_lstm2_dropout = DropoutLayer(l_lstm2, p=0.2)
# The objective of this task depends only on the final value produced by
# the network. So, we'll use SliceLayers to extract the LSTM layer's
# output after processing the entire input sequence. For the forward
# layer, this corresponds to the last value of the second (sequence length)
# dimension.
l_slice = SliceLayer(l_lstm2_dropout, -1, 1)
l_out = DenseLayer(l_slice, 1, nonlinearity=lasagne.nonlinearities.linear)
return l_out
def build_model(input_var=None):
layers = [1, 5, 10, 1]
l_in = InputLayer((None, None, layers[0]),
input_var=input_var)
l_lstm1 = LSTMLayer(l_in, layers[1])
l_lstm1_dropout = DropoutLayer(l_lstm1, p=0.2)
l_lstm2 = LSTMLayer(l_lstm1_dropout, layers[2])
l_lstm2_dropout = DropoutLayer(l_lstm2, p=0.2)
# The objective of this task depends only on the final value produced by
# the network. So, we'll use SliceLayers to extract the LSTM layer's
# output after processing the entire input sequence. For the forward
# layer, this corresponds to the last value of the second (sequence length)
# dimension.
l_slice = SliceLayer(l_lstm2_dropout, -1, 1)
l_out = DenseLayer(l_slice, 1, nonlinearity=lasagne.nonlinearities.linear)
return l_out
def discriminator(input_var):
network = lasagne.layers.InputLayer(shape=(None, 1, 28, 28),
input_var=input_var)
network = ll.DropoutLayer(network, p=0.5)
network = weight_norm(conv_layer(network, 3, 32, 1, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 3, 32, 2, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 3, 64, 2, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 3, 128, 2, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 4, 128, 1, 'valid', nonlinearity=lrelu), train_g=False)
network =weight_norm(conv_layer(network, 1, 1, 1, 'valid', nonlinearity=None), train_g=True)
return network
def discriminator(input_var):
network = lasagne.layers.InputLayer(shape=(None, 1, 28, 28),
input_var=input_var)
network = ll.DropoutLayer(network, p=0.5)
network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 64, (4,4), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu))
network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 32, (5,5), stride=2, pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu))
network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 32, (5,5), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu))
network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 32, (5,5), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu))
network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 16, (3,3), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu))
network =nn.weight_norm(ll.DenseLayer(network, num_units=1, W=Normal(0.05), nonlinearity=None), train_g=True, init_stdv=0.1)
return network
def discriminator(input_var):
network = lasagne.layers.InputLayer(shape=(None, 1, 28, 28),
input_var=input_var)
network = ll.DropoutLayer(network, p=0.5)
network = weight_norm(conv_layer(network, 3, 32, 1, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 3, 32, 2, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 3, 64, 2, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 3, 128, 2, 'same', nonlinearity=lrelu), train_g=False)
network = weight_norm(conv_layer(network, 4, 128, 1, 'valid', nonlinearity=lrelu), train_g=False)
network =weight_norm(conv_layer(network, 1, 1, 1, 'valid', nonlinearity=None), train_g=True)
return network
def _invert_DropoutLayer(self, layer, feeder):
assert isinstance(layer, L.DropoutLayer)
return feeder
def _invert_layer(self, layer, feeder):
layer_type = type(layer)
if L.get_output_shape(feeder) != L.get_output_shape(layer):
feeder = L.ReshapeLayer(feeder, (-1,)+L.get_output_shape(layer)[1:])
if layer_type is L.InputLayer:
return self._invert_InputLayer(layer, feeder)
elif layer_type is L.FlattenLayer:
return self._invert_FlattenLayer(layer, feeder)
elif layer_type is L.DenseLayer:
return self._invert_DenseLayer(layer, feeder)
elif layer_type is L.Conv2DLayer:
return self._invert_Conv2DLayer(layer, feeder)
elif layer_type is L.DropoutLayer:
return self._invert_DropoutLayer(layer, feeder)
elif layer_type in [L.MaxPool2DLayer, L.MaxPool1DLayer]:
return self._invert_MaxPoolingLayer(layer, feeder)
elif layer_type is L.PadLayer:
return self._invert_PadLayer(layer, feeder)
elif layer_type is L.SliceLayer:
return self._invert_SliceLayer(layer, feeder)
elif layer_type is L.LocalResponseNormalization2DLayer:
return self._invert_LocalResponseNormalisation2DLayer(layer, feeder)
elif layer_type is L.GlobalPoolLayer:
return self._invert_GlobalPoolLayer(layer, feeder)
else:
return self._invert_UnknownLayer(layer, feeder)
def create_network():
l = 1000
pool_size = 5
test_size1 = 13
test_size2 = 7
test_size3 = 5
kernel1 = 128
kernel2 = 128
kernel3 = 128
layer1 = InputLayer(shape=(None, 1, 4, l+1024))
layer2_1 = SliceLayer(layer1, indices=slice(0, l), axis = -1)
layer2_2 = SliceLayer(layer1, indices=slice(l, None), axis = -1)
layer2_3 = SliceLayer(layer2_2, indices = slice(0,4), axis = -2)
layer2_f = FlattenLayer(layer2_3)
layer3 = Conv2DLayer(layer2_1,num_filters = kernel1, filter_size = (4,test_size1))
layer4 = Conv2DLayer(layer3,num_filters = kernel1, filter_size = (1,test_size1))
layer5 = Conv2DLayer(layer4,num_filters = kernel1, filter_size = (1,test_size1))
layer6 = MaxPool2DLayer(layer5, pool_size = (1,pool_size))
layer7 = Conv2DLayer(layer6,num_filters = kernel2, filter_size = (1,test_size2))
layer8 = Conv2DLayer(layer7,num_filters = kernel2, filter_size = (1,test_size2))
layer9 = Conv2DLayer(layer8,num_filters = kernel2, filter_size = (1,test_size2))
layer10 = MaxPool2DLayer(layer9, pool_size = (1,pool_size))
layer11 = Conv2DLayer(layer10,num_filters = kernel3, filter_size = (1,test_size3))
layer12 = Conv2DLayer(layer11,num_filters = kernel3, filter_size = (1,test_size3))
layer13 = Conv2DLayer(layer12,num_filters = kernel3, filter_size = (1,test_size3))
layer14 = MaxPool2DLayer(layer13, pool_size = (1,pool_size))
layer14_d = DenseLayer(layer14, num_units= 256)
layer3_2 = DenseLayer(layer2_f, num_units = 128)
layer15 = ConcatLayer([layer14_d,layer3_2])
layer16 = DropoutLayer(layer15,p=0.5)
layer17 = DenseLayer(layer16, num_units=256)
network = DenseLayer(layer17, num_units= 2, nonlinearity=softmax)
return network
#random search to initialize the weights
def create_network():
l = 1000
pool_size = 5
test_size1 = 13
test_size2 = 7
test_size3 = 5
kernel1 = 128
kernel2 = 128
kernel3 = 128
layer1 = InputLayer(shape=(None, 1, 4, l+1024))
layer2_1 = SliceLayer(layer1, indices=slice(0, l), axis = -1)
layer2_2 = SliceLayer(layer1, indices=slice(l, None), axis = -1)
layer2_3 = SliceLayer(layer2_2, indices = slice(0,4), axis = -2)
layer2_f = FlattenLayer(layer2_3)
layer3 = Conv2DLayer(layer2_1,num_filters = kernel1, filter_size = (4,test_size1))
layer4 = Conv2DLayer(layer3,num_filters = kernel1, filter_size = (1,test_size1))
layer5 = Conv2DLayer(layer4,num_filters = kernel1, filter_size = (1,test_size1))
layer6 = MaxPool2DLayer(layer5, pool_size = (1,pool_size))
layer7 = Conv2DLayer(layer6,num_filters = kernel2, filter_size = (1,test_size2))
layer8 = Conv2DLayer(layer7,num_filters = kernel2, filter_size = (1,test_size2))
layer9 = Conv2DLayer(layer8,num_filters = kernel2, filter_size = (1,test_size2))
layer10 = MaxPool2DLayer(layer9, pool_size = (1,pool_size))
layer11 = Conv2DLayer(layer10,num_filters = kernel3, filter_size = (1,test_size3))
layer12 = Conv2DLayer(layer11,num_filters = kernel3, filter_size = (1,test_size3))
layer13 = Conv2DLayer(layer12,num_filters = kernel3, filter_size = (1,test_size3))
layer14 = MaxPool2DLayer(layer13, pool_size = (1,pool_size))
layer14_d = DenseLayer(layer14, num_units= 256)
layer3_2 = DenseLayer(layer2_f, num_units = 128)
layer15 = ConcatLayer([layer14_d,layer3_2])
#layer16 = DropoutLayer(layer15,p=0.5)
layer17 = DenseLayer(layer15, num_units=256)
network = DenseLayer(layer17, num_units= 1, nonlinearity=None)
return network
#random search to initialize the weights
def build_model():
net = {}
net['input'] = InputLayer((None, 512*20, 3, 3))
au_fc_layers=[]
for i in range(20):
net['roi_AU_N_'+str(i)]=SliceLayer(net['input'],indices=slice(i*512,(i+1)*512),axis=1)
#try to adding upsampling here for more conv
net['Roi_upsample_'+str(i)]=Upscale2DLayer(net['roi_AU_N_'+str(i)],scale_factor=2)
net['conv_roi_'+str(i)]=ConvLayer(net['Roi_upsample_'+str(i)],512,3)
net['au_fc_'+str(i)]=DenseLayer(net['conv_roi_'+str(i)],num_units=150)
au_fc_layers+=[net['au_fc_'+str(i)]]
#
net['local_fc']=concat(au_fc_layers)
net['local_fc2']=DenseLayer(net['local_fc'],num_units=2048)
net['local_fc_dp']=DropoutLayer(net['local_fc2'],p=0.5)
# net['fc_comb']=concat([net['au_fc_layer'],net['local_fc_dp']])
# net['fc_dense']=DenseLayer(net['fc_comb'],num_units=1024)
# net['fc_dense_dp']=DropoutLayer(net['fc_dense'],p=0.3)
net['real_out']=DenseLayer(net['local_fc_dp'],num_units=12,nonlinearity=sigmoid)
# net['final']=concat([net['pred_pos_layer'],net['output_layer']])
return net
def __create_toplogy__(self, input_var_first=None, input_var_second=None):
# define network topology
if (self.conf.rep % 2 != 0):
raise ValueError("Representation size should be divisible by two as it's formed by combining two crossmodal translations", self.conf.rep)
# input layers
l_in_first = InputLayer(shape=(self.conf.batch_size, self.conf.mod1size), input_var=input_var_first)
l_in_second = InputLayer(shape=(self.conf.batch_size, self.conf.mod2size), input_var=input_var_second)
# first -> second
l_hidden1_first = DenseLayer(l_in_first, num_units=self.conf.hdn, nonlinearity=self.conf.act, W=GlorotUniform()) # enc1
l_hidden2_first = DenseLayer(l_hidden1_first, num_units=self.conf.rep//2, nonlinearity=self.conf.act, W=GlorotUniform()) # enc2
l_hidden2_first_d = DropoutLayer(l_hidden2_first, p=self.conf.dropout)
l_hidden3_first = DenseLayer(l_hidden2_first_d, num_units=self.conf.hdn, nonlinearity=self.conf.act, W=GlorotUniform()) # dec1
l_out_first = DenseLayer(l_hidden3_first, num_units=self.conf.mod2size, nonlinearity=self.conf.act, W=GlorotUniform()) # dec2
if self.conf.untied:
# FREE
l_hidden1_second = DenseLayer(l_in_second, num_units=self.conf.hdn, nonlinearity=self.conf.act, W=GlorotUniform()) # enc1
l_hidden2_second = DenseLayer(l_hidden1_second, num_units=self.conf.rep//2, nonlinearity=self.conf.act, W=GlorotUniform()) # enc2
l_hidden2_second_d = DropoutLayer(l_hidden2_second, p=self.conf.dropout)
l_hidden3_second = DenseLayer(l_hidden2_second_d, num_units=self.conf.hdn, nonlinearity=self.conf.act, W=GlorotUniform()) # dec1
l_out_second = DenseLayer(l_hidden3_second, num_units=self.conf.mod1size, nonlinearity=self.conf.act, W=GlorotUniform()) # dec2
else:
# TIED middle
l_hidden1_second = DenseLayer(l_in_second, num_units=self.conf.hdn, nonlinearity=self.conf.act, W=GlorotUniform()) # enc1
l_hidden2_second = DenseLayer(l_hidden1_second, num_units=self.conf.rep//2, nonlinearity=self.conf.act, W=l_hidden3_first.W.T) # enc2
l_hidden2_second_d = DropoutLayer(l_hidden2_second, p=self.conf.dropout)
l_hidden3_second = DenseLayer(l_hidden2_second_d, num_units=self.conf.hdn, nonlinearity=self.conf.act, W=l_hidden2_first.W.T) # dec1
l_out_second = DenseLayer(l_hidden3_second, num_units=self.conf.mod1size, nonlinearity=self.conf.act, W=GlorotUniform()) # dec2
l_out = concat([l_out_first, l_out_second])
return l_out, l_hidden2_first, l_hidden2_second
cnn_cascade_lasagne.py 文件源码
项目：Cascade-CNN-Face-Detection
作者: gogolgrind
项目源码
文件源码
阅读 22
收藏 0
点赞 0
评论 0
def __build_12_net__(self):
network = layers.InputLayer((None, 3, 12, 12), input_var=self.__input_var__)
network = layers.dropout(network, p=0.1)
network = layers.Conv2DLayer(network,num_filters=16,filter_size=(3,3),stride=1,nonlinearity=relu)
network = layers.batch_norm(network)
network = layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2)
network = layers.DropoutLayer(network,p=0.3)
network = layers.DenseLayer(network,num_units = 16,nonlinearity = relu)
network = layers.batch_norm(network)
network = layers.DropoutLayer(network,p=0.3)
network = layers.DenseLayer(network,num_units = 2, nonlinearity = softmax)
return network
cnn_cascade_lasagne.py 文件源码
项目：Cascade-CNN-Face-Detection
作者: gogolgrind
项目源码
文件源码
阅读 21
收藏 0
点赞 0
评论 0
def __build_24_net__(self):
network = layers.InputLayer((None, 3, 24, 24), input_var=self.__input_var__)
network = layers.dropout(network, p=0.1)
network = layers.Conv2DLayer(network,num_filters=64,filter_size=(5,5),stride=1,nonlinearity=relu)
network = layers.batch_norm(network)
network = layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2)
network = layers.DropoutLayer(network,p=0.5)
network = layers.batch_norm(network)
network = layers.DenseLayer(network,num_units = 64,nonlinearity = relu)
network = layers.DropoutLayer(network,p=0.5)
network = layers.DenseLayer(network,num_units = 2, nonlinearity = softmax)
return network
def build_model(input_var):
net = {}
net['input'] = InputLayer((None, 3, 224, 224), input_var=input_var)
net['conv1_1'] = ConvLayer(net['input'], 64, 3, pad=1, flip_filters=False)
net['conv1_2'] = ConvLayer(net['conv1_1'], 64, 3, pad=1, flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
net['conv2_1'] = ConvLayer(net['pool1'], 128, 3, pad=1, flip_filters=False)
net['conv2_2'] = ConvLayer(net['conv2_1'], 128, 3, pad=1, flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
net['conv3_1'] = ConvLayer(net['pool2'], 256, 3, pad=1, flip_filters=False)
net['conv3_2'] = ConvLayer(net['conv3_1'], 256, 3, pad=1, flip_filters=False)
net['conv3_3'] = ConvLayer(net['conv3_2'], 256, 3, pad=1, flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
net['conv4_1'] = ConvLayer(net['pool3'], 512, 3, pad=1, flip_filters=False)
net['conv4_2'] = ConvLayer(net['conv4_1'], 512, 3, pad=1, flip_filters=False)
net['conv4_3'] = ConvLayer(net['conv4_2'], 512, 3, pad=1, flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
net['conv5_1'] = ConvLayer(net['pool4'], 512, 3, pad=1, flip_filters=False)
net['conv5_2'] = ConvLayer(net['conv5_1'], 512, 3, pad=1, flip_filters=False)
net['conv5_3'] = ConvLayer(net['conv5_2'], 512, 3, pad=1, flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
net['fc6'] = DenseLayer(net['pool5'], num_units=4096)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=0.5)
net['fc7'] = DenseLayer(net['fc6_dropout'], num_units=4096)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['fc8'] = DenseLayer(net['fc7_dropout'], num_units=1000, nonlinearity=None)
net['prob'] = NonlinearityLayer(net['fc8'], softmax)
return net
def discriminator(input_var,Y):
yb = Y.dimshuffle(0, 1, 'x', 'x')
D_1 = lasagne.layers.InputLayer(shape=(None, 1, 28, 28),
input_var=input_var)
D_2 = lasagne.layers.InputLayer(shape=(None, 10),input_var=Y)
network=D_1
network_yb=D_2
network = CondConvConcatLayer([network,network_yb])
network = ll.DropoutLayer(network, p=0.4)
network = conv_layer(network, 3, 32, 1, 'same', nonlinearity=lrelu)
network = CondConvConcatLayer([network,network_yb])
network = conv_layer(network, 3, 64, 2, 'same', nonlinearity=lrelu)
network = CondConvConcatLayer([network,network_yb])
network = conv_layer(network, 3, 64, 2, 'same', nonlinearity=lrelu)
#network = batch_norm(conv_layer(network, 3, 128, 1, 'same', nonlinearity=lrelu))
#network = ll.DropoutLayer(network, p=0.2)
network = conv_layer(network, 3, 128, 2, 'same', nonlinearity=lrelu)
network = CondConvConcatLayer([network,network_yb])
network = batch_norm(conv_layer(network, 4, 128, 1, 'valid', nonlinearity=lrelu))
network = CondConvConcatLayer([network,network_yb])
#network= DropoutLayer(network, p=0.5)
network =conv_layer(network, 1, 1, 1, 'valid', nonlinearity=None)
return network, D_1,D_2
def gen_layers(self):
"""Construct the layers"""
# Init <TODO>
pad_in = 'valid'
self.layers = []
# input layer
l_input = (InputLayer,
{'shape': (None, self.X_in.shape[1],
self.X_in.shape[2],
self.X_in.shape[3])})
self.layers.append(l_input)
# Conv and pool layers
rows, cols = self.X_in.shape[2:]
for i in range(len(self.kernel_size)):
# conv
l_conv = (Conv2DLayer,
{'num_filters': self.kernel_num[i],
'filter_size': self.kernel_size[i],
'nonlinearity': lasagne.nonlinearities.rectify,
'W': lasagne.init.GlorotUniform(),
'pad': pad_in})
self.layers.append(l_conv)
rows = rows - self.kernel_size[i] + 1
cols = cols - self.kernel_size[i] + 1
# pool
if self.pool_flag[i]:
l_pool = (MaxPool2DLayer,
{'pool_size': self.pool_size})
self.layers.append(l_pool)
rows = rows // 2
cols = cols // 2
# dropout
if not self.dropflag:
self.droprate = 0
l_drop = (DropoutLayer, {'p': self.droprate})
# self.layers.append(l_drop)
# full connected layer
num_fc = rows * cols * self.kernel_num[-1]
l_fc = (DenseLayer,
{'num_units': num_fc,
'nonlinearity': lasagne.nonlinearities.rectify,
'W': lasagne.init.GlorotUniform(),
'b': lasagne.init.Constant(0.)}
)
self.layers.append(l_fc)
# dense
if not self.fc_nodes is None:
for i in range(len(self.fc_nodes)):
self.layers.append(l_drop)
l_dense = (DenseLayer, {'num_units': self.fc_nodes[i]})
self.layers.append(l_dense)
# output layer
self.layers.append(l_drop)
l_out = (DenseLayer,
{'name': 'output',
'num_units': self.numclass,
'nonlinearity': lasagne.nonlinearities.softmax})
self.layers.append(l_out)
def buildModel(mtype=1):
print "BUILDING MODEL TYPE", mtype, "..."
#default settings (Model 1)
filters = 64
first_stride = 2
last_filter_multiplier = 16
#specific model type settings (see working notes for details)
if mtype == 2:
first_stride = 1
elif mtype == 3:
filters = 32
last_filter_multiplier = 8
#input layer
net = l.InputLayer((None, IM_DIM, IM_SIZE[1], IM_SIZE[0]))
#conv layers
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters, filter_size=7, pad='same', stride=first_stride, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
if mtype == 2:
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters, filter_size=5, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 2, filter_size=5, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 4, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 8, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * last_filter_multiplier, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
print "\tFINAL POOL OUT SHAPE:", l.get_output_shape(net)
#dense layers
net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.DropoutLayer(net, DROPOUT)
net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.DropoutLayer(net, DROPOUT)
#Classification Layer
if MULTI_LABEL:
net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.sigmoid, W=init.HeNormal(gain=1))
else:
net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.softmax, W=init.HeNormal(gain=1))
print "...DONE!"
#model stats
print "MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"
print "MODEL HAS", l.count_params(net), "PARAMS"
return net
def main():
################
# LOAD DATASET #
################
dataset = './data/ubiquitous_aug.hkl'
kfd = './data/ubiquitous_kfold.hkl'
print('Loading dataset {}...'.format(dataset))
X, y = hkl.load(open(dataset, 'r'))
X = X.reshape(-1, 4, 1, 400).astype(floatX)
y = y.astype('int32')
print('X shape: {}, y shape: {}'.format(X.shape, y.shape))
kf = hkl.load(open(kfd, 'r'))
kfold = [(train, test) for train, test in kf]
(train, test) = kfold[0]
print('train_set size: {}, test_set size: {}'.format(len(train), len(test)))
# shuffle +/- labels in minibatch
print('shuffling train_set and test_set')
shuffle(train)
shuffle(test)
X_train = X[train]
X_test = X[test]
y_train = y[train]
y_test = y[test]
print('data prepared!')
layers = [
(InputLayer, {'shape': (None, 4, 1, 400)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 4)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 3)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 3)}),
(MaxPool2DLayer, {'pool_size': (1, 2)}),
(Conv2DLayer, {'num_filters': 32, 'filter_size': (1, 2)}),
(Conv2DLayer, {'num_filters': 32, 'filter_size': (1, 2)}),
(Conv2DLayer, {'num_filters': 32, 'filter_size': (1, 2)}),
(MaxPool2DLayer, {'pool_size': (1, 2)}),
(DenseLayer, {'num_units': 64}),
(DropoutLayer, {}),
(DenseLayer, {'num_units': 64}),
(DenseLayer, {'num_units': 2, 'nonlinearity': softmax})]
net = NeuralNet(
layers=layers,
max_epochs=100,
update=adam,
update_learning_rate=1e-4,
train_split=TrainSplit(eval_size=0.1),
on_epoch_finished=[
AdjustVariable(1e-4, target=0, half_life=20)],
verbose=2)
net.fit(X_train, y_train)
plot_loss(net)
def main(resume=None):
l = 300
dataset = './data/ubiquitous_train.hkl'
print('Loading dataset {}...'.format(dataset))
X_train, y_train = hkl.load(dataset)
X_train = X_train.reshape(-1, 4, 1, l).astype(floatX)
y_train = np.array(y_train, dtype='int32')
indice = np.arange(X_train.shape[0])
np.random.shuffle(indice)
X_train = X_train[indice]
y_train = y_train[indice]
print('X_train shape: {}, y_train shape: {}'.format(X_train.shape, y_train.shape))
layers = [
(InputLayer, {'shape': (None, 4, 1, l)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 4)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 3)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 3)}),
(MaxPool2DLayer, {'pool_size': (1, 2)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 2)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 2)}),
(Conv2DLayer, {'num_filters': 64, 'filter_size': (1, 2)}),
(MaxPool2DLayer, {'pool_size': (1, 2)}),
(DenseLayer, {'num_units': 64}),
(DropoutLayer, {}),
(DenseLayer, {'num_units': 64}),
(DenseLayer, {'num_units': 2, 'nonlinearity': softmax})]
lr = theano.shared(np.float32(1e-4))
net = NeuralNet(
layers=layers,
max_epochs=100,
update=adam,
update_learning_rate=lr,
train_split=TrainSplit(eval_size=0.1),
on_epoch_finished=[
AdjustVariable(lr, target=1e-8, half_life=20)],
verbose=4)
if resume != None:
net.load_params_from(resume)
net.fit(X_train, y_train)
net.save_params_to('./models/net_params.pkl')
def get_discriminator(self):
''' specify discriminator D0 '''
"""
disc0_layers = [LL.InputLayer(shape=(self.args.batch_size, 3, 32, 32))]
disc0_layers.append(LL.GaussianNoiseLayer(disc0_layers[-1], sigma=0.05))
disc0_layers.append(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 16x16
disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu)))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 8x8
disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=0, W=Normal(0.02), nonlinearity=nn.lrelu))) # 6x6
disc0_layer_shared = LL.NINLayer(disc0_layers[-1], num_units=192, W=Normal(0.02), nonlinearity=nn.lrelu) # 6x6
disc0_layers.append(disc0_layer_shared)
disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared, num_units=50, W=Normal(0.02), nonlinearity=None)
disc0_layers.append(disc0_layer_z_recon) # also need to recover z from x
disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared))
disc0_layer_adv = LL.DenseLayer(disc0_layers[-1], num_units=10, W=Normal(0.02), nonlinearity=None)
disc0_layers.append(disc0_layer_adv)
return disc0_layers, disc0_layer_adv, disc0_layer_z_recon
"""
disc_x_layers = [LL.InputLayer(shape=(None, 3, 32, 32))]
disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2))
disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu)
disc_x_layers.append(disc_x_layers_shared)
disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=self.args.z0dim, nonlinearity=None)
disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x
# disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100))
disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared))
disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None)
disc_x_layers.append(disc_x_layer_adv)
#output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False)
#output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False)
# temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True)
# temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True)
# init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])]
return disc_x_layers, disc_x_layer_adv, disc_x_layer_z_recon
def build_network():
conv_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'filter_size': (3, 3),
'stride': (1, 1),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
nin_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
dense_defs = {
'W': lasagne.init.HeNormal(1.0),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.softmax
}
wn_defs = {
'momentum': .999
}
net = InputLayer ( name='input', shape=(None, 3, 32, 32))
net = GaussianNoiseLayer(net, name='noise', sigma=.15)
net = WN(Conv2DLayer (net, name='conv1a', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv1b', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv1c', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool1', pool_size=(2, 2))
net = DropoutLayer (net, name='drop1', p=.5)
net = WN(Conv2DLayer (net, name='conv2a', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv2b', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv2c', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool2', pool_size=(2, 2))
net = DropoutLayer (net, name='drop2', p=.5)
net = WN(Conv2DLayer (net, name='conv3a', num_filters=512, pad=0, **conv_defs), **wn_defs)
net = WN(NINLayer (net, name='conv3b', num_units=256, **nin_defs), **wn_defs)
net = WN(NINLayer (net, name='conv3c', num_units=128, **nin_defs), **wn_defs)
net = GlobalPoolLayer (net, name='pool3')
net = WN(DenseLayer (net, name='dense', num_units=10, **dense_defs), **wn_defs)
return net
def create_model(incoming, options):
input_p = 0.2
hidden_p = 0.5
conv_num_filters1 = int(100 / (1.0 - input_p))
conv_num_filters2 = int(150 / (1.0 - hidden_p))
conv_num_filters3 = int(200 / (1.0 - hidden_p))
filter_size1 = 5
filter_size2 = 5
filter_size3 = 3
pool_size = 2
encode_size = int(options['BOTTLENECK'] / 0.5)
dense_mid_size = int(options['DENSE'] / 0.5)
pad_in = 'valid'
pad_out = 'full'
scaled_tanh = create_scaled_tanh()
dropout0 = DropoutLayer(incoming, p=0.2, name='dropout0')
conv2d1 = Conv2DLayer(dropout0, num_filters=conv_num_filters1, filter_size=filter_size1, pad=pad_in, name='conv2d1', nonlinearity=scaled_tanh)
maxpool2d2 = MaxPool2DLayer(conv2d1, pool_size=pool_size, name='maxpool2d2')
dropout1 = DropoutLayer(maxpool2d2, name='dropout1')
conv2d3 = Conv2DLayer(dropout1, num_filters=conv_num_filters2, filter_size=filter_size2, pad=pad_in, name='conv2d3', nonlinearity=scaled_tanh)
maxpool2d4 = MaxPool2DLayer(conv2d3, pool_size=pool_size, name='maxpool2d4', pad=(1,0))
dropout2 = DropoutLayer(maxpool2d4, name='dropout2')
conv2d5 = Conv2DLayer(dropout2, num_filters=conv_num_filters3, filter_size=filter_size3, pad=pad_in, name='conv2d5', nonlinearity=scaled_tanh)
reshape6 = ReshapeLayer(conv2d5, shape=([0], -1), name='reshape6') # 3000
reshape6_output = reshape6.output_shape[1]
dropout3 = DropoutLayer(reshape6, name='dropout3')
dense7 = DenseLayer(dropout3, num_units=dense_mid_size, name='dense7', nonlinearity=scaled_tanh)
dropout4 = DropoutLayer(dense7, name='dropout4')
bottleneck = DenseLayer(dropout4, num_units=encode_size, name='bottleneck', nonlinearity=linear)
# print_network(bottleneck)
dense8 = DenseLayer(bottleneck, num_units=dense_mid_size, W=bottleneck.W.T, name='dense8', nonlinearity=linear)
dense9 = DenseLayer(dense8, num_units=reshape6_output, W=dense7.W.T, nonlinearity=scaled_tanh, name='dense9')
reshape10 = ReshapeLayer(dense9, shape=([0], conv_num_filters3, 3, 5), name='reshape10') # 32 x 4 x 7
deconv2d11 = Deconv2DLayer(reshape10, conv2d5.input_shape[1], conv2d5.filter_size, stride=conv2d5.stride,
W=conv2d5.W, flip_filters=not conv2d5.flip_filters, name='deconv2d11', nonlinearity=scaled_tanh)
upscale2d12 = Upscale2DLayer(deconv2d11, scale_factor=pool_size, name='upscale2d12')
deconv2d13 = Deconv2DLayer(upscale2d12, conv2d3.input_shape[1], conv2d3.filter_size, stride=conv2d3.stride,
W=conv2d3.W, flip_filters=not conv2d3.flip_filters, name='deconv2d13', nonlinearity=scaled_tanh)
upscale2d14 = Upscale2DLayer(deconv2d13, scale_factor=pool_size, name='upscale2d14')
deconv2d15 = Deconv2DLayer(upscale2d14, conv2d1.input_shape[1], conv2d1.filter_size, stride=conv2d1.stride,
crop=(1, 0), W=conv2d1.W, flip_filters=not conv2d1.flip_filters, name='deconv2d14', nonlinearity=scaled_tanh)
reshape16 = ReshapeLayer(deconv2d15, ([0], -1), name='reshape16')
return reshape16, bottleneck
def create_model(incoming, options):
conv_num_filters1 = 100
conv_num_filters2 = 150
conv_num_filters3 = 200
filter_size1 = 5
filter_size2 = 5
filter_size3 = 3
pool_size = 2
encode_size = options['BOTTLENECK']
dense_mid_size = options['DENSE']
pad_in = 'valid'
pad_out = 'full'
scaled_tanh = create_scaled_tanh()
dropout0 = DropoutLayer(incoming, p=0.2, name='dropout0')
conv2d1 = Conv2DLayer(dropout0, num_filters=conv_num_filters1, filter_size=filter_size1, pad=pad_in, name='conv2d1', nonlinearity=scaled_tanh)
bn1 = BatchNormLayer(conv2d1, name='batchnorm1')
maxpool2d2 = MaxPool2DLayer(bn1, pool_size=pool_size, name='maxpool2d2')
dropout1 = DropoutLayer(maxpool2d2, name='dropout1')
conv2d3 = Conv2DLayer(dropout1, num_filters=conv_num_filters2, filter_size=filter_size2, pad=pad_in, name='conv2d3', nonlinearity=scaled_tanh)
bn2 = BatchNormLayer(conv2d3, name='batchnorm2')
maxpool2d4 = MaxPool2DLayer(bn2, pool_size=pool_size, name='maxpool2d4', pad=(1,0))
dropout2 = DropoutLayer(maxpool2d4, name='dropout2')
conv2d5 = Conv2DLayer(dropout2, num_filters=conv_num_filters3, filter_size=filter_size3, pad=pad_in, name='conv2d5', nonlinearity=scaled_tanh)
bn3 = BatchNormLayer(conv2d5, name='batchnorm3')
reshape6 = ReshapeLayer(bn3, shape=([0], -1), name='reshape6') # 3000
reshape6_output = reshape6.output_shape[1]
dropout3 = DropoutLayer(reshape6, name='dropout3')
dense7 = DenseLayer(dropout3, num_units=dense_mid_size, name='dense7', nonlinearity=scaled_tanh)
bn4 = BatchNormLayer(dense7, name='batchnorm4')
dropout4 = DropoutLayer(bn4, name='dropout4')
bottleneck = DenseLayer(dropout4, num_units=encode_size, name='bottleneck', nonlinearity=linear)
# print_network(bottleneck)
dense8 = DenseLayer(bottleneck, num_units=dense_mid_size, W=bottleneck.W.T, name='dense8', nonlinearity=linear)
dense9 = DenseLayer(dense8, num_units=reshape6_output, W=dense7.W.T, nonlinearity=scaled_tanh, name='dense9')
reshape10 = ReshapeLayer(dense9, shape=([0], conv_num_filters3, 3, 5), name='reshape10') # 32 x 4 x 7
deconv2d11 = Deconv2DLayer(reshape10, conv2d5.input_shape[1], conv2d5.filter_size, stride=conv2d5.stride,
W=conv2d5.W, flip_filters=not conv2d5.flip_filters, name='deconv2d11', nonlinearity=scaled_tanh)
upscale2d12 = Upscale2DLayer(deconv2d11, scale_factor=pool_size, name='upscale2d12')
deconv2d13 = Deconv2DLayer(upscale2d12, conv2d3.input_shape[1], conv2d3.filter_size, stride=conv2d3.stride,
W=conv2d3.W, flip_filters=not conv2d3.flip_filters, name='deconv2d13', nonlinearity=scaled_tanh)
upscale2d14 = Upscale2DLayer(deconv2d13, scale_factor=pool_size, name='upscale2d14')
deconv2d15 = Deconv2DLayer(upscale2d14, conv2d1.input_shape[1], conv2d1.filter_size, stride=conv2d1.stride,
crop=(1, 0), W=conv2d1.W, flip_filters=not conv2d1.flip_filters, name='deconv2d14', nonlinearity=scaled_tanh)
reshape16 = ReshapeLayer(deconv2d15, ([0], -1), name='reshape16')
return reshape16, bottleneck
def get_net():
return NeuralNet(
layers=[
('input', layers.InputLayer),
('conv1', Conv2DLayer),
('pool1', MaxPool2DLayer),
('dropout1', layers.DropoutLayer),
('conv2', Conv2DLayer),
('pool2', MaxPool2DLayer),
('dropout2', layers.DropoutLayer),
('conv3', Conv2DLayer),
('pool3', MaxPool2DLayer),
('dropout3', layers.DropoutLayer),
('hidden4', layers.DenseLayer),
('dropout4', layers.DropoutLayer),
('hidden5', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape=(None, 1, 96, 96),
conv1_num_filters=32, conv1_filter_size=(3, 3), pool1_pool_size=(2, 2),
dropout1_p=0.1,
conv2_num_filters=64, conv2_filter_size=(2, 2), pool2_pool_size=(2, 2),
dropout2_p=0.2,
conv3_num_filters=128, conv3_filter_size=(2, 2), pool3_pool_size=(2, 2),
dropout3_p=0.3,
hidden4_num_units=1000,
dropout4_p=0.5,
hidden5_num_units=1000,
output_num_units=30, output_nonlinearity=None,
update_learning_rate=theano.shared(float32(0.03)),
update_momentum=theano.shared(float32(0.9)),
regression=True,
batch_iterator_train=FlipBatchIterator(batch_size=128),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.03, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=200),
],
max_epochs=3000,
verbose=1,
)
def build_instrument_model(self, n_vars, **kwargs):
targets = TT.vector()
instrument_vars = TT.matrix()
instruments = layers.InputLayer((None, n_vars), instrument_vars)
instruments = layers.DropoutLayer(instruments, p=0.2)
dense_layer = layers.DenseLayer(instruments, kwargs['dense_size'], nonlinearity=nonlinearities.tanh)
dense_layer = layers.DropoutLayer(dense_layer, p=0.2)
for _ in xrange(kwargs['n_dense_layers'] - 1):
dense_layer = layers.DenseLayer(dense_layer, kwargs['dense_size'], nonlinearity=nonlinearities.tanh)
dense_layer = layers.DropoutLayer(dense_layer, p=0.5)
self.instrument_output = layers.DenseLayer(dense_layer, 1, nonlinearity=nonlinearities.linear)
init_params = layers.get_all_param_values(self.instrument_output)
prediction = layers.get_output(self.instrument_output, deterministic=False)
test_prediction = layers.get_output(self.instrument_output, deterministic=True)
# flexible here, endog variable can be categorical, continuous, etc.
l2_cost = regularization.regularize_network_params(self.instrument_output, regularization.l2)
loss = objectives.squared_error(prediction.flatten(), targets.flatten()).mean() + 1e-4 * l2_cost
loss_total = objectives.squared_error(prediction.flatten(), targets.flatten()).mean()
params = layers.get_all_params(self.instrument_output, trainable=True)
param_updates = updates.adadelta(loss, params)
self._instrument_train_fn = theano.function(
[
targets,
instrument_vars,
],
loss,
updates=param_updates
)
self._instrument_loss_fn = theano.function(
[
targets,
instrument_vars,
],
loss_total
)
self._instrument_output_fn = theano.function([instrument_vars], test_prediction)
return init_params
def build_treatment_model(self, n_vars, **kwargs):
input_vars = TT.matrix()
instrument_vars = TT.matrix()
targets = TT.vector()
inputs = layers.InputLayer((None, n_vars), input_vars)
inputs = layers.DropoutLayer(inputs, p=0.2)
dense_layer = layers.DenseLayer(inputs, 2 * kwargs['dense_size'], nonlinearity=nonlinearities.rectify)
dense_layer = layers.batch_norm(dense_layer)
dense_layer= layers.DropoutLayer(dense_layer, p=0.2)
for _ in xrange(kwargs['n_dense_layers'] - 1):
dense_layer = layers.DenseLayer(dense_layer, kwargs['dense_size'], nonlinearity=nonlinearities.rectify)
dense_layer = layers.batch_norm(dense_layer)
self.treatment_output = layers.DenseLayer(dense_layer, 1, nonlinearity=nonlinearities.linear)
init_params = layers.get_all_param_values(self.treatment_output)
prediction = layers.get_output(self.treatment_output, deterministic=False)
test_prediction = layers.get_output(self.treatment_output, deterministic=True)
l2_cost = regularization.regularize_network_params(self.treatment_output, regularization.l2)
loss = gmm_loss(prediction, targets, instrument_vars) + 1e-4 * l2_cost
params = layers.get_all_params(self.treatment_output, trainable=True)
param_updates = updates.adadelta(loss, params)
self._train_fn = theano.function(
[
input_vars,
targets,
instrument_vars,
],
loss,
updates=param_updates
)
self._loss_fn = theano.function(
[
input_vars,
targets,
instrument_vars,
],
loss,
)
self._output_fn = theano.function(
[
input_vars,
],
test_prediction,
)
return init_params
def build_model(self, input_var, forward, dropout):
net = dict()
net['input'] = InputLayer((None, 3, None, None), input_var=input_var)
net['conv1/7x7_s2'] = ConvLayer(
net['input'], 64, 7, stride=2, pad=3, flip_filters=False)
net['pool1/3x3_s2'] = PoolLayer(
net['conv1/7x7_s2'], pool_size=3, stride=2, ignore_border=False)
net['pool1/norm1'] = LRNLayer(net['pool1/3x3_s2'], alpha=0.00002, k=1)
net['conv2/3x3_reduce'] = ConvLayer(
net['pool1/norm1'], 64, 1, flip_filters=False)
net['conv2/3x3'] = ConvLayer(
net['conv2/3x3_reduce'], 192, 3, pad=1, flip_filters=False)
net['conv2/norm2'] = LRNLayer(net['conv2/3x3'], alpha=0.00002, k=1)
net['pool2/3x3_s2'] = PoolLayerDNN(net['conv2/norm2'], pool_size=3, stride=2)
net.update(self.build_inception_module('inception_3a',
net['pool2/3x3_s2'],
[32, 64, 96, 128, 16, 32]))
net.update(self.build_inception_module('inception_3b',
net['inception_3a/output'],
[64, 128, 128, 192, 32, 96]))
net['pool3/3x3_s2'] = PoolLayerDNN(net['inception_3b/output'],
pool_size=3, stride=2)
net.update(self.build_inception_module('inception_4a',
net['pool3/3x3_s2'],
[64, 192, 96, 208, 16, 48]))
net.update(self.build_inception_module('inception_4b',
net['inception_4a/output'],
[64, 160, 112, 224, 24, 64]))
net.update(self.build_inception_module('inception_4c',
net['inception_4b/output'],
[64, 128, 128, 256, 24, 64]))
net.update(self.build_inception_module('inception_4d',
net['inception_4c/output'],
[64, 112, 144, 288, 32, 64]))
net.update(self.build_inception_module('inception_4e',
net['inception_4d/output'],
[128, 256, 160, 320, 32, 128]))
net['pool4/3x3_s2'] = PoolLayerDNN(net['inception_4e/output'],
pool_size=3, stride=2)
net.update(self.build_inception_module('inception_5a',
net['pool4/3x3_s2'],
[128, 256, 160, 320, 32, 128]))
net.update(self.build_inception_module('inception_5b',
net['inception_5a/output'],
[128, 384, 192, 384, 48, 128]))
net['pool5/7x7_s1'] = GlobalPoolLayer(net['inception_5b/output'])
if forward:
#net['fc6'] = DenseLayer(net['pool5/7x7_s1'], num_units=1000)
net['prob'] = DenseLayer(net['pool5/7x7_s1'], num_units=4, nonlinearity=softmax)
else:
net['dropout1'] = DropoutLayer(net['pool5/7x7_s1'], p=dropout)
#net['fc6'] = DenseLayer(net['dropout1'], num_units=1000)
#net['dropout2'] = DropoutLayer(net['fc6'], p=dropout)
net['prob'] = DenseLayer(net['dropout1'], num_units=4, nonlinearity=softmax)
return net
def buildModel():
print "BUILDING MODEL TYPE..."
#default settings
filters = 64
first_stride = 2
last_filter_multiplier = 16
#input layer
net = l.InputLayer((None, IM_DIM, IM_SIZE[1], IM_SIZE[0]))
#conv layers
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters, filter_size=7, pad='same', stride=first_stride, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 2, filter_size=5, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 4, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 8, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * last_filter_multiplier, filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.MaxPool2DLayer(net, pool_size=2)
print "\tFINAL POOL OUT SHAPE:", l.get_output_shape(net)
#dense layers
net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.DropoutLayer(net, DROPOUT)
net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY))
net = l.DropoutLayer(net, DROPOUT)
#Classification Layer
if MULTI_LABEL:
net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.sigmoid, W=init.HeNormal(gain=1))
else:
net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.softmax, W=init.HeNormal(gain=1))
print "...DONE!"
#model stats
print "MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"
print "MODEL HAS", l.count_params(net), "PARAMS"
return net
