def conv1d(name,input,kernel,stride,n_filters,depth,bias=False,batchnorm=False,pad='valid',filter_dilation=(1,1),run_mode=0):
W = lib.param(
name+'.W',
lasagne.init.HeNormal().sample((n_filters,depth,kernel,1)).astype('float32')
)
out = T.nnet.conv2d(input,W,subsample=(stride,1),border_mode=pad,filter_dilation=filter_dilation)
if bias:
b = lib.param(
name + '.b',
np.zeros(n_filters).astype('float32')
)
out += b[None,:,None,None]
if batchnorm:
out = BatchNorm(name,out,n_filters,mode=1,run_mode=run_mode)
return out
python类param()的实例源码
def create_wavenet_block(inp, num_dilation_layer, input_dim, output_dim, name =None):
assert name is not None
layer_out = inp
skip_contrib = []
skip_weights = lib.param(name+".parametrized_weights", lib.floatX(numpy.ones((num_dilation_layer,))))
for i in range(num_dilation_layer):
layer_out, skip_c = lib.ops.dil_conv_1D(
layer_out,
output_dim,
input_dim if i == 0 else output_dim,
2,
dilation = 2**i,
non_linearity = 'gated',
name = name+".dilation_{}".format(i+1)
)
skip_c = skip_c*skip_weights[i]
skip_contrib.append(skip_c)
skip_out = skip_contrib[-1]
j = 0
for i in range(num_dilation_layer-1):
j += 2**(num_dilation_layer-i-1)
skip_out = skip_out + skip_contrib[num_dilation_layer-2 - i][:,j:]
return layer_out, skip_out
def Embedding(name, n_symbols, output_dim, indices):
vectors = lib.param(
name,
numpy.random.randn(
n_symbols,
output_dim
).astype(theano.config.floatX)
)
output_shape = [
indices.shape[i]
for i in xrange(indices.ndim)
] + [output_dim]
return vectors[indices.flatten()].reshape(output_shape)
def __Recurrent(name, hidden_dims, step_fn, inputs, non_sequences=[], h0s=None):
if not isinstance(inputs, list):
inputs = [inputs]
if not isinstance(hidden_dims, list):
hidden_dims = [hidden_dims]
if h0s is None:
h0s = [None]*len(hidden_dims)
for i in xrange(len(hidden_dims)):
if h0s[i] is None:
h0_unbatched = lib.param(
name + '.h0_' + str(i),
numpy.zeros((hidden_dims[i],), dtype=theano.config.floatX)
)
num_batches = inputs[0].shape[1]
h0s[i] = T.alloc(h0_unbatched, num_batches, hidden_dims[i])
h0s[i] = T.patternbroadcast(h0s[i], [False] * h0s[i].ndim)
outputs, _ = theano.scan(
step_fn,
sequences=inputs,
outputs_info=h0s,
non_sequences=non_sequences
)
return outputs
def Recurrence(processed_frames, h0, reset):
"""
processed_frames.shape: (batch size, n frames, DIM)
h0.shape: (batch size, N_GRUS, DIM)
reset.shape: ()
output.shape: (batch size, n frames, DIM)
"""
# print "warning no recurrence"
# return T.zeros_like(processed_frames), h0
learned_h0 = lib.param(
'Recurrence.h0',
numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM)
learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
gru0 = lib.ops.LowMemGRU('Recurrence.GRU0', DIM, DIM, processed_frames, h0=h0[:, 0])
grus = [gru0]
for i in xrange(1, N_GRUS):
gru = lib.ops.LowMemGRU('Recurrence.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i])
grus.append(gru)
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
return (grus[-1], last_hidden)
def Embedding(name, n_symbols, output_dim, indices):
vectors = lib.param(
name,
numpy.random.randn(
n_symbols,
output_dim
).astype(theano.config.floatX)
)
output_shape = [
indices.shape[i]
for i in xrange(indices.ndim)
] + [output_dim]
return vectors[indices.flatten()].reshape(output_shape)
def Recurrent(name, hidden_dims, step_fn, inputs, non_sequences=[], h0s=None):
if not isinstance(inputs, list):
inputs = [inputs]
if not isinstance(hidden_dims, list):
hidden_dims = [hidden_dims]
if h0s is None:
h0s = [None]*len(hidden_dims)
for i in xrange(len(hidden_dims)):
if h0s[i] is None:
h0_unbatched = lib.param(
name + '.h0_' + str(i),
numpy.zeros((hidden_dims[i],), dtype=theano.config.floatX)
)
num_batches = inputs[0].shape[1]
h0s[i] = T.alloc(h0_unbatched, num_batches, hidden_dims[i])
h0s[i] = T.patternbroadcast(h0s[i], [False] * h0s[i].ndim)
outputs, _ = theano.scan(
step_fn,
sequences=inputs,
outputs_info=h0s,
non_sequences=non_sequences
)
return outputs
def Recurrence(processed_frames, h0, reset):
"""
processed_frames.shape: (batch size, n frames, DIM)
h0.shape: (batch size, N_GRUS, DIM)
reset.shape: ()
output.shape: (batch size, n frames, DIM)
"""
# print "warning no recurrence"
# return T.zeros_like(processed_frames), h0
learned_h0 = lib.param(
'Recurrence.h0',
numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM)
learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
gru0 = lib.ops.LowMemGRU('Recurrence.GRU0', DIM, DIM, processed_frames, h0=h0[:, 0])
grus = [gru0]
for i in xrange(1, N_GRUS):
gru = lib.ops.LowMemGRU('Recurrence.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i])
grus.append(gru)
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
return (grus[-1], last_hidden)
def Embedding(name, n_symbols, output_dim, indices):
vectors = lib.param(
name,
numpy.random.randn(
n_symbols,
output_dim
).astype(theano.config.floatX)
)
output_shape = [
indices.shape[i]
for i in xrange(indices.ndim)
] + [output_dim]
return vectors[indices.flatten()].reshape(output_shape)
def __Recurrent(name, hidden_dims, step_fn, inputs, non_sequences=[], h0s=None):
if not isinstance(inputs, list):
inputs = [inputs]
if not isinstance(hidden_dims, list):
hidden_dims = [hidden_dims]
if h0s is None:
h0s = [None]*len(hidden_dims)
for i in xrange(len(hidden_dims)):
if h0s[i] is None:
h0_unbatched = lib.param(
name + '.h0_' + str(i),
numpy.zeros((hidden_dims[i],), dtype=theano.config.floatX)
)
num_batches = inputs[0].shape[1]
h0s[i] = T.alloc(h0_unbatched, num_batches, hidden_dims[i])
h0s[i] = T.patternbroadcast(h0s[i], [False] * h0s[i].ndim)
outputs, _ = theano.scan(
step_fn,
sequences=inputs,
outputs_info=h0s,
non_sequences=non_sequences
)
return outputs
def Dense(name, input_dim, output_dim, inputs, bias=True, init=None, weightnorm=True,hidden_dim=None):
weight_values = init_weights(input_dim,output_dim,init)
weight = lib.param(
name + '.W',
weight_values
)
batch_size = None
if inputs.ndim==3:
batch_size = inputs.shape[0]
inputs = inputs.reshape((-1,input_dim))
if weightnorm:
norm_values = numpy.linalg.norm(weight_values, axis=0)
norms = lib.param(
name + '.g',
norm_values
)
normed_weight = weight * (norms / weight.norm(2, axis=0)).dimshuffle('x', 0)
result = T.dot(inputs, normed_weight)
else:
result = T.dot(inputs, weight)
if bias:
b = lib.param(
name + '.b',
numpy.zeros((output_dim,), dtype=theano.config.floatX)
)
result += b
result.name = name+".output"
if batch_size!=None:
return result.reshape((batch_size,hidden_dim,output_dim))
else:
return result
def Embedding(name, n_symbols, output_dim, indices):
vectors = lib.param(
name,
numpy.random.randn(
n_symbols,
output_dim
).astype(theano.config.floatX)
)
output_shape = tuple(list(indices.shape) + [output_dim])
return vectors[indices.flatten()].reshape(output_shape)
def __ConvLSTMStep(
name,
seq_len,
input_dim,
hidden_dim,
current_input,
last_hidden,
last_cell,
dilation_depth=10,
inp_bias_init=0.,
forget_bias_init=3.,
out_bias_init=0.,
g_bias_init=0.):
# X_t*(U^i, U^f, U^o, U^g)
dilations = [2**i for i in xrange(dilation_depth)]
prev_conv = current_input
last_cell_stack = T.concatenate((last_cell,last_cell),axis=1)
for i,value in enumerate(dilations):
#prev_conv = lib.ops.conv1d(name+".WaveNetConv%d"%(i+1),prev_conv,2,1,hidden_dim,input_dim,True,False,pad=(dilation,0),filter_dilation=(dilation,1))[:,:,:current_input.shape[2],:]
prev_conv,y = lib.ops.WaveNetConv1d("WaveNetBlock-%d"%(i+1),prev_conv,2,hidden_dim,input_dim,bias=True,batchnorm=False,dilation=value)
prev_conv = T.concatenate((prev_conv,last_hidden),axis=1)
prev_conv = lib.ops.conv1d(name+".ConvGates",prev_conv,1,1,4*hidden_dim,2*input_dim,True,False)
W_cell = lib.param(name+'.CellWeights',lasagne.init.HeNormal().sample((3*hidden_dim,seq_len,1)))
inp_forget = T.nnet.sigmoid(prev_conv[:,:2*hidden_dim] + W_cell[:2*hidden_dim]*last_cell_stack)
i_t = inp_forget[:,:hidden_dim]
f_t = inp_forget[:,hidden_dim:]
C_t = f_t*last_cell + i_t*T.tanh(prev_conv[:,2*hidden_dim:3*hidden_dim])
o_t = T.nnet.sigmoid(prev_conv[:,3*hidden_dim:]+W_cell[2*hidden_dim:]*C_t)
H_t = o_t*T.tanh(C_t)
return H_t,C_t
def Conv1D(name, input_dim, output_dim, filter_size, inputs, apply_biases=True):
"""
inputs.shape: (batch size, height, input_dim)
output.shape: (batch size, height, output_dim)
* performs valid convs
"""
def uniform(stdev, size):
"""uniform distribution with the given stdev and size"""
return numpy.random.uniform(
low=-stdev * numpy.sqrt(3),
high=stdev * numpy.sqrt(3),
size=size
).astype(theano.config.floatX)
filters = lib.param(
name+'.Filters',
uniform(
1./numpy.sqrt(input_dim * filter_size),
# output dim, input dim, height, width
(output_dim, input_dim, filter_size, 1)
)
)
# conv2d takes inputs as (batch size, input channels, height[?], width[?])
inputs = inputs.reshape((inputs.shape[0], inputs.shape[1], 1, inputs.shape[2]))
inputs = inputs.dimshuffle(0, 3, 1, 2)
result = T.nnet.conv2d(inputs, filters, border_mode='valid', filter_flip=False)
if apply_biases:
biases = lib.param(
name+'.Biases',
numpy.zeros(output_dim, dtype=theano.config.floatX)
)
result = result + biases[None, :, None, None]
result = result.dimshuffle(0, 2, 3, 1)
return result.reshape((result.shape[0], result.shape[1], result.shape[3]))
def big_frame_level_rnn(input_sequences, h0, reset):
"""
input_sequences.shape: (batch size, n big frames * BIG_FRAME_SIZE)
h0.shape: (batch size, N_BIG_GRUS, BIG_DIM)
reset.shape: ()
output[0].shape: (batch size, n frames, DIM)
output[1].shape: same as h0.shape
output[2].shape: (batch size, seq len, Q_LEVELS)
"""
learned_h0 = lib.param(
'BigFrameLevel.h0',
numpy.zeros((N_BIG_GRUS, BIG_DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_BIG_GRUS, BIG_DIM)
learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
frames = input_sequences.reshape((
input_sequences.shape[0],
input_sequences.shape[1] / BIG_FRAME_SIZE,
BIG_FRAME_SIZE
))
# Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2]
# (a reasonable range to pass as inputs to the RNN)
frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1)
frames *= lib.floatX(2)
gru0 = lib.ops.LowMemGRU('BigFrameLevel.GRU0', BIG_FRAME_SIZE, BIG_DIM, frames, h0=h0[:, 0])
grus = [gru0]
for i in xrange(1, N_BIG_GRUS):
gru = lib.ops.LowMemGRU('BigFrameLevel.GRU'+str(i), BIG_DIM, BIG_DIM, grus[-1], h0=h0[:, i])
grus.append(gru)
output = lib.ops.Linear(
'BigFrameLevel.Output',
BIG_DIM,
DIM * BIG_FRAME_SIZE / FRAME_SIZE,
grus[-1]
)
output = output.reshape((output.shape[0], output.shape[1] * BIG_FRAME_SIZE / FRAME_SIZE, DIM))
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
independent_preds = lib.ops.Linear(
'BigFrameLevel.IndependentPreds',
BIG_DIM,
Q_LEVELS * BIG_FRAME_SIZE,
grus[-1]
)
independent_preds = independent_preds.reshape((independent_preds.shape[0], independent_preds.shape[1] * BIG_FRAME_SIZE, Q_LEVELS))
return (output, last_hidden, independent_preds)
def frame_level_rnn(input_sequences, other_input, h0, reset):
"""
input_sequences.shape: (batch size, n frames * FRAME_SIZE)
other_input.shape: (batch size, n frames, DIM)
h0.shape: (batch size, N_GRUS, DIM)
reset.shape: ()
output.shape: (batch size, n frames * FRAME_SIZE, DIM)
"""
learned_h0 = lib.param(
'FrameLevel.h0',
numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM)
learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
frames = input_sequences.reshape((
input_sequences.shape[0],
input_sequences.shape[1] / FRAME_SIZE,
FRAME_SIZE
))
# Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2]
# (a reasonable range to pass as inputs to the RNN)
frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1)
frames *= lib.floatX(2)
gru_input = lib.ops.Linear('FrameLevel.InputExpand', FRAME_SIZE, DIM, frames) + other_input
gru0 = lib.ops.LowMemGRU('FrameLevel.GRU0', DIM, DIM, gru_input, h0=h0[:, 0])
grus = [gru0]
for i in xrange(1, N_GRUS):
gru = lib.ops.LowMemGRU('FrameLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i])
grus.append(gru)
output = lib.ops.Linear(
'FrameLevel.Output',
DIM,
FRAME_SIZE * DIM,
grus[-1],
initialization='he'
)
output = output.reshape((output.shape[0], output.shape[1] * FRAME_SIZE, DIM))
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
return (output, last_hidden)
def frame_level_rnn(input_sequences, h0, reset):
"""
input_sequences.shape: (batch size, n frames * FRAME_SIZE)
h0.shape: (batch size, N_GRUS, DIM)
reset.shape: ()
output.shape: (batch size, n frames * FRAME_SIZE, DIM)
"""
learned_h0 = lib.param(
'FrameLevel.h0',
numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM)
learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
frames = input_sequences.reshape((
input_sequences.shape[0],
input_sequences.shape[1] / FRAME_SIZE,
FRAME_SIZE
))
# Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2]
# (a reasonable range to pass as inputs to the RNN)
frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1)
frames *= lib.floatX(2)
gru0 = lib.ops.LowMemGRU('FrameLevel.GRU0', FRAME_SIZE, DIM, frames, h0=h0[:, 0])
grus = [gru0]
for i in xrange(1, N_GRUS):
gru = lib.ops.LowMemGRU('FrameLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i])
grus.append(gru)
output = lib.ops.Linear(
'FrameLevel.Output',
DIM,
FRAME_SIZE * DIM,
grus[-1],
initialization='he'
)
output = output.reshape((output.shape[0], output.shape[1] * FRAME_SIZE, DIM))
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
return (output, last_hidden)
def frame_level_rnn(input_sequences, h0, reset):
"""
input_sequences.shape: (batch size, n frames * FRAME_SIZE)
h0.shape: (batch size, N_GRUS, DIM)
reset.shape: ()
output.shape: (batch size, n frames * FRAME_SIZE, DIM)
"""
learned_h0 = lib.param(
'FrameLevel.h0',
numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM)
learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
frames = input_sequences.reshape((
input_sequences.shape[0],
input_sequences.shape[1] / FRAME_SIZE,
FRAME_SIZE
))
# Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2]
# (a reasonable range to pass as inputs to the RNN)
frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1)
frames *= lib.floatX(2)
gru0 = lib.ops.LowMemGRU('FrameLevel.GRU0', FRAME_SIZE, DIM, frames, h0=h0[:, 0])
grus = [gru0]
for i in xrange(1, N_GRUS):
gru = lib.ops.LowMemGRU('FrameLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i])
grus.append(gru)
output = lib.ops.Linear(
'FrameLevel.Output',
DIM,
FRAME_SIZE * DIM,
grus[-1],
initialization='he'
)
output = output.reshape((output.shape[0], output.shape[1] * FRAME_SIZE, DIM))
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
return (output, last_hidden)
def sample_level_rnn(input_sequences, h0, reset):
"""
input_sequences.shape: (batch size, seq len)
h0.shape: (batch size, N_GRUS, DIM)
reset.shape: ()
output.shape: (batch size, seq len, Q_LEVELS)
"""
learned_h0 = lib.param(
'SampleLevel.h0',
numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
# Embedded inputs
#################
# FRAME_SIZE = Q_LEVELS
# frames = lib.ops.Embedding('SampleLevel.Embedding', Q_LEVELS, Q_LEVELS, input_sequences)
# Real-valued inputs
####################
# 'frames' of size 1
FRAME_SIZE = 1
frames = input_sequences.reshape((
input_sequences.shape[0],
input_sequences.shape[1],
1
))
# # Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2]
# # (a reasonable range to pass as inputs to the RNN)
# frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1)
# frames *= lib.floatX(2)
gru0 = lib.ops.LowMemGRU('SampleLevel.GRU0', FRAME_SIZE, DIM, frames, h0=h0[:, 0])
# gru0 = T.nnet.relu(lib.ops.Linear('SampleLevel.GRU0FF', DIM, DIM, gru0, initialization='he'))
grus = [gru0]
for i in xrange(1, N_GRUS):
gru = lib.ops.LowMemGRU('SampleLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i])
# gru = T.nnet.relu(lib.ops.Linear('SampleLevel.GRU'+str(i)+'FF', DIM, DIM, gru, initialization='he'))
grus.append(gru)
# We apply the softmax later
output = lib.ops.Linear(
'Output',
N_GRUS*DIM,
2,
T.concatenate(grus, axis=2)
)
# output = lib.ops.Linear(
# 'Output',
# DIM,
# Q_LEVELS,
# grus[-1]
# )
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
return (output, last_hidden)
def frame_level_rnn(input_sequences, h0, reset):
"""
input_sequences.shape: (batch size, n frames * FRAME_SIZE)
h0.shape: (batch size, N_GRUS, DIM)
reset.shape: ()
output.shape: (batch size, n frames * FRAME_SIZE, DIM)
"""
learned_h0 = lib.param(
'FrameLevel.h0',
numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX)
)
learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM)
learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim)
h0 = theano.ifelse.ifelse(reset, learned_h0, h0)
frames = input_sequences.reshape((
input_sequences.shape[0],
input_sequences.shape[1] / FRAME_SIZE,
FRAME_SIZE
))
# Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2]
# (a reasonable range to pass as inputs to the RNN)
frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1)
frames *= lib.floatX(2)
gru0 = lib.ops.LowMemGRU('FrameLevel.GRU0', FRAME_SIZE, DIM, frames, h0=h0[:, 0])
grus = [gru0]
for i in xrange(1, N_GRUS):
gru = lib.ops.LowMemGRU('FrameLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i])
grus.append(gru)
output = lib.ops.Linear(
'FrameLevel.Output',
DIM,
FRAME_SIZE * DIM,
grus[-1],
initialization='he'
)
output = output.reshape((output.shape[0], output.shape[1] * FRAME_SIZE, DIM))
last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1)
return (output, last_hidden)
def conv1d(
name,
input,
input_dim,
output_dim,
filter_size,
init = 'glorot',
non_linearity = 'relu',
bias = True
):
import lasagne
inp = input.dimshuffle(0,2,1,'x')
if init == 'glorot':
initializer = lasagne.init.GlorotUniform()
elif init == 'he':
initializer = lasagne.init.HeUniform()
if non_linearity == 'gated':
num_filters = 2*output_dim
else:
num_filters = output_dim
W_shape = (num_filters, input_dim, filter_size, 1)
if bias:
bias_shape = (num_filters,)
W = lib.param(name+".W", initializer.sample(W_shape))
if bias:
b = lib.param(name+".b", lasagne.init.Constant(0.).sample(bias_shape))
conv_out = T.nnet.conv2d(
inp, W,
filter_flip= False,
border_mode = 'valid'
)
if bias:
conv_out = conv_out + b[None,:,None, None]
if non_linearity == 'gated':
activation = gated_non_linerity
elif non_linearity == 'relu':
activation = T.nnet.relu
elif non_linearity == 'elu':
activation = lambda x : T.switch( x >= 0., x, T.exp(x) - floatX(1.))
elif non_linearity == 'identity':
activation = lambda x: x
else:
raise NotImplementedError("{} non-linearity not implemented!".format(non_linearity))
output = conv_out
output = output.reshape((output.shape[0], output.shape[1], output.shape[2]))
output = output.dimshuffle(0,2,1)
return output
def DilatedConv1D(name, input_dim, output_dim, filter_size, inputs, dilation, mask_type=None, apply_biases=True):
"""
inputs.shape: (batch size, length, input_dim)
mask_type: None, 'a', 'b'
output.shape: (batch size, length, output_dim)
"""
def uniform(stdev, size):
"""uniform distribution with the given stdev and size"""
return numpy.random.uniform(
low=-stdev * numpy.sqrt(3),
high=stdev * numpy.sqrt(3),
size=size
).astype(theano.config.floatX)
filters_init = uniform(
1./numpy.sqrt(input_dim * filter_size),
# output dim, input dim, height, width
(output_dim, input_dim, filter_size, 1)
)
if mask_type is not None:
filters_init *= lib.floatX(numpy.sqrt(2.))
filters = lib.param(
name+'.Filters',
filters_init
)
if mask_type is not None:
mask = numpy.ones(
(output_dim, input_dim, filter_size, 1),
dtype=theano.config.floatX
)
center = filter_size//2
for i in xrange(filter_size):
if (i > center):
mask[:, :, i, :] = 0.
# if (mask_type=='a' and i == center):
# mask[:, :, center] = 0.
filters = filters * mask
inputs = inputs.reshape((inputs.shape[0], inputs.shape[1], 1, inputs.shape[2]))
# conv2d takes inputs as (batch size, input channels, height[?], width[?])
inputs = inputs.dimshuffle(0, 3, 1, 2)
result = T.nnet.conv2d(inputs, filters, border_mode='half', filter_flip=False, filter_dilation=(dilation, 1))
if apply_biases:
biases = lib.param(
name+'.Biases',
numpy.zeros(output_dim, dtype=theano.config.floatX)
)
result = result + biases[None, :, None, None]
result = result.dimshuffle(0, 2, 3, 1)
return result.reshape((result.shape[0], result.shape[1], result.shape[3]))
def Deconv2D(
name,
input_dim,
output_dim,
filter_size,
inputs,
he_init=True,
weightnorm=None,
):
"""
inputs: tensor of shape (batch size, num channels, height, width)
returns: tensor of shape (batch size, num channels, 2*height, 2*width)
"""
def uniform(stdev, size):
return np.random.uniform(
low=-stdev * np.sqrt(3),
high=stdev * np.sqrt(3),
size=size
).astype(theano.config.floatX)
filters_stdev = np.sqrt(1./(input_dim * filter_size**2))
filters_stdev *= 2. # Because of the stride
if he_init:
filters_stdev *= np.sqrt(2.)
filter_values = uniform(
filters_stdev,
(input_dim, output_dim, filter_size, filter_size)
)
filters = lib.param(
name+'.Filters',
filter_values
)
if weightnorm==None:
weightnorm = _default_weightnorm
if weightnorm:
norm_values = np.sqrt(np.sum(np.square(filter_values), axis=(0,2,3)))
norms = lib.param(
name + '.g',
norm_values
)
filters = filters * (norms / T.sqrt(T.sum(T.sqr(filters), axis=(0,2,3)))).dimshuffle('x',0,'x','x')
biases = lib.param(
name+'.Biases',
np.zeros(output_dim, dtype=theano.config.floatX)
)
pad = (filter_size-1)/2
result = _deconv2d(
inputs,
filters,
subsample=(2,2),
border_mode=(pad,pad),
)
result = result + biases[None, :, None, None]
# result = lib.debug.print_stats(name, result)
return result
def myGRU(name, input_dim, hidden_dim, inputs, h0=None):
#inputs.shape = (batch_size,N_FRAMES,FRAME_SIZE)
inputs = inputs.transpose(1,0,2)
weight_values = lasagne.init.GlorotUniform().sample((input_dim+hidden_dim,2*hidden_dim))
W1 = lib.param(
name+'.Gates.W',
weight_values
)
b1 = lib.param(
name+'.Gates.b',
np.ones(2*hidden_dim).astype(theano.config.floatX)
)
weight_values = lasagne.init.GlorotUniform().sample((input_dim+hidden_dim,hidden_dim))
W2 = lib.param(
name+'.Candidate.W',
weight_values
)
b2 = lib.param(
name+'.Candidate.b',
np.zeros(hidden_dim).astype(theano.config.floatX)
)
def step(x_t, h_tm1):
return recurrent_fn(
x_t,
h_tm1,
name,
input_dim,
hidden_dim,
W1,b1,W2,b2
)
outputs, _ = theano.scan(
step,
sequences=[inputs],
outputs_info=[h0],
)
out = outputs.dimshuffle(1,0,2)
out.name = name+'.output'
return out
def DiagonalLSTM(name, input_dim, inputs):
"""
inputs.shape: (batch size, height, width, input_dim)
outputs.shape: (batch size, height, width, DIM)
"""
inputs = Skew(inputs)
input_to_state = Conv2D(name+'.InputToState', input_dim, 4*DIM, 1, inputs, mask_type='b')
batch_size = inputs.shape[0]
c0_unbatched = lib.param(
name + '.c0',
numpy.zeros((HEIGHT, DIM), dtype=theano.config.floatX)
)
c0 = T.alloc(c0_unbatched, batch_size, HEIGHT, DIM)
h0_unbatched = lib.param(
name + '.h0',
numpy.zeros((HEIGHT, DIM), dtype=theano.config.floatX)
)
h0 = T.alloc(h0_unbatched, batch_size, HEIGHT, DIM)
def step_fn(current_input_to_state, prev_c, prev_h):
# all args have shape (batch size, height, DIM)
# TODO consider learning this padding
prev_h = T.concatenate([
T.zeros((batch_size, 1, DIM), theano.config.floatX),
prev_h
], axis=1)
state_to_state = Conv1D(name+'.StateToState', DIM, 4*DIM, 2, prev_h, apply_biases=False)
gates = current_input_to_state + state_to_state
o_f_i = T.nnet.sigmoid(gates[:,:,:3*DIM])
o = o_f_i[:,:,0*DIM:1*DIM]
f = o_f_i[:,:,1*DIM:2*DIM]
i = o_f_i[:,:,2*DIM:3*DIM]
g = T.tanh(gates[:,:,3*DIM:4*DIM])
new_c = (f * prev_c) + (i * g)
new_h = o * T.tanh(new_c)
return (new_c, new_h)
outputs, _ = theano.scan(
step_fn,
sequences=input_to_state.dimshuffle(2,0,1,3),
outputs_info=[c0, h0]
)
all_cs = outputs[0].dimshuffle(1,2,0,3)
all_hs = outputs[1].dimshuffle(1,2,0,3)
return Unskew(all_hs)
