def connect(self, inputs, mask, is_train):
""" is_train: A boolean tensor.
"""
max_length = inputs.shape[0]
batch_size = inputs.shape[1]
outputs_info = [tensor.alloc(numpy_floatX(0.), batch_size, self.hidden_dim),
tensor.alloc(numpy_floatX(0.), batch_size, self.hidden_dim)]
# Dropout mask sharing for variational dropout.
self.is_train = is_train
if self.recurrent_dropout_layer != None:
self.recurrent_dropout_layer.generate_mask([batch_size, self.hidden_dim], is_train)
inputs = tensor.dot(inputs, self.W) + self.b
rval, _ = theano.scan(self._step, # Scan function
sequences=[inputs, mask], # Input sequence
outputs_info=outputs_info,
name=_p(self.prefix, '_layers'),
n_steps=max_length) # scan steps
return rval[0]
python类alloc()的实例源码
def connect(self, inputs, mask, is_train):
max_length = inputs.shape[0]
batch_size = inputs.shape[1]
outputs_info = [tensor.alloc(numpy_floatX(0.), batch_size, self.hidden_dim),
tensor.alloc(numpy_floatX(0.), batch_size, self.hidden_dim)]
# Dropout layers
self.is_train = is_train
if self.recurrent_dropout_layer != None:
self.recurrent_dropout_layer.generate_mask([batch_size, self.hidden_dim], is_train)
proj_inputs = tensor.dot(inputs, self.W) + self.b
rval, _ = theano.scan(self._step, # Scan function
sequences=[inputs, proj_inputs, mask], # Input sequence
outputs_info=outputs_info,
name=_p(self.prefix, '_layers'),
n_steps=max_length) # scan steps
return rval[0]
def gru_layer(tparams, emb, options):
hiddenDimSize = options['hiddenDimSize']
timesteps = emb.shape[0]
if emb.ndim == 3: n_samples = emb.shape[1]
else: n_samples = 1
def stepFn(wx, h, U_gru):
uh = T.dot(h, U_gru)
r = T.nnet.sigmoid(_slice(wx, 0, hiddenDimSize) + _slice(uh, 0, hiddenDimSize))
z = T.nnet.sigmoid(_slice(wx, 1, hiddenDimSize) + _slice(uh, 1, hiddenDimSize))
h_tilde = T.tanh(_slice(wx, 2, hiddenDimSize) + r * _slice(uh, 2, hiddenDimSize))
h_new = z * h + ((1. - z) * h_tilde)
return h_new
Wx = T.dot(emb, tparams['W_gru']) + tparams['b_gru']
results, updates = theano.scan(fn=stepFn, sequences=[Wx], outputs_info=T.alloc(numpy_floatX(0.0), n_samples, hiddenDimSize), non_sequences=[tparams['U_gru']], name='gru_layer', n_steps=timesteps)
return results
def gru_layer(tparams, emb, layerIndex, hiddenDimSize, mask=None):
timesteps = emb.shape[0]
if emb.ndim == 3: n_samples = emb.shape[1]
else: n_samples = 1
W_rx = T.dot(emb, tparams['W_r_'+layerIndex])
W_zx = T.dot(emb, tparams['W_z_'+layerIndex])
Wx = T.dot(emb, tparams['W_'+layerIndex])
def stepFn(stepMask, wrx, wzx, wx, h):
r = T.nnet.sigmoid(wrx + T.dot(h, tparams['U_r_'+layerIndex]) + tparams['b_r_'+layerIndex])
z = T.nnet.sigmoid(wzx + T.dot(h, tparams['U_z_'+layerIndex]) + tparams['b_z_'+layerIndex])
h_tilde = T.tanh(wx + T.dot(r*h, tparams['U_'+layerIndex]) + tparams['b_'+layerIndex])
h_new = z * h + ((1. - z) * h_tilde)
h_new = stepMask[:, None] * h_new + (1. - stepMask)[:, None] * h
return h_new#, output, time
results, updates = theano.scan(fn=stepFn, sequences=[mask,W_rx,W_zx,Wx], outputs_info=T.alloc(numpy_floatX(0.0), n_samples, hiddenDimSize), name='gru_layer'+layerIndex, n_steps=timesteps)
return results
def gru_layer(tparams, emb, layerIndex, hiddenDimSize, mask=None):
timesteps = emb.shape[0]
if emb.ndim == 3: n_samples = emb.shape[1]
else: n_samples = 1
W_rx = T.dot(emb, tparams['W_r_'+layerIndex])
W_zx = T.dot(emb, tparams['W_z_'+layerIndex])
Wx = T.dot(emb, tparams['W_'+layerIndex])
def stepFn(stepMask, wrx, wzx, wx, h):
r = T.nnet.sigmoid(wrx + T.dot(h, tparams['U_r_'+layerIndex]) + tparams['b_r_'+layerIndex])
z = T.nnet.sigmoid(wzx + T.dot(h, tparams['U_z_'+layerIndex]) + tparams['b_z_'+layerIndex])
h_tilde = T.tanh(wx + T.dot(r*h, tparams['U_'+layerIndex]) + tparams['b_'+layerIndex])
h_new = z * h + ((1. - z) * h_tilde)
h_new = stepMask[:, None] * h_new + (1. - stepMask)[:, None] * h
return h_new
results, updates = theano.scan(fn=stepFn, sequences=[mask,W_rx,W_zx,Wx], outputs_info=T.alloc(numpy_floatX(0.0), n_samples, hiddenDimSize), name='gru_layer'+layerIndex, n_steps=timesteps)
return results
def set_output(self):
output_shape = self._output_shape
padding = self._padding
unpool_size = self._unpool_size
unpooled_output = tensor.alloc(0.0, # Value to fill the tensor
output_shape[0],
output_shape[1] + 2 * padding[0],
output_shape[2],
output_shape[3] + 2 * padding[1],
output_shape[4] + 2 * padding[2])
unpooled_output = tensor.set_subtensor(unpooled_output[:, padding[0]:output_shape[
1] + padding[0]:unpool_size[0], :, padding[1]:output_shape[3] + padding[1]:unpool_size[
1], padding[2]:output_shape[4] + padding[2]:unpool_size[2]],
self._prev_layer.output)
self._output = unpooled_output
def set_output(self):
padding = self._padding
input_shape = self._input_shape
if np.sum(self._padding) > 0:
padded_input = tensor.alloc(0.0, # Value to fill the tensor
input_shape[0],
input_shape[1] + 2 * padding[1],
input_shape[2],
input_shape[3] + 2 * padding[3],
input_shape[4] + 2 * padding[4])
padded_input = tensor.set_subtensor(
padded_input[:, padding[1]:padding[1] + input_shape[1], :, padding[3]:padding[3] +
input_shape[3], padding[4]:padding[4] + input_shape[4]],
self._prev_layer.output)
else:
padded_input = self._prev_layer.output
self._output = conv3d2d.conv3d(padded_input, self.W.val) + \
self.b.val.dimshuffle('x', 'x', 0, 'x', 'x')
def set_output(self):
padding = self._padding
input_shape = self._input_shape
padded_input = tensor.alloc(0.0, # Value to fill the tensor
input_shape[0],
input_shape[1] + 2 * padding[1],
input_shape[2],
input_shape[3] + 2 * padding[3],
input_shape[4] + 2 * padding[4])
padded_input = tensor.set_subtensor(padded_input[:, padding[1]:padding[1] + input_shape[
1], :, padding[3]:padding[3] + input_shape[3], padding[4]:padding[4] + input_shape[4]],
self._prev_layer.output)
fc_output = tensor.reshape(
tensor.dot(self._fc_layer.output, self.Wx.val), self._output_shape)
self._output = conv3d2d.conv3d(padded_input, self.Wh.val) + \
fc_output + self.b.val.dimshuffle('x', 'x', 0, 'x', 'x')
def set_output(self):
padding = self._padding
input_shape = self._input_shape
padded_input = tensor.alloc(0.0, # Value to fill the tensor
input_shape[0],
input_shape[1] + 2 * padding[1],
input_shape[2],
input_shape[3] + 2 * padding[3],
input_shape[4] + 2 * padding[4])
padded_input = tensor.set_subtensor(padded_input[:, padding[1]:padding[1] + input_shape[
1], :, padding[3]:padding[3] + input_shape[3], padding[4]:padding[4] + input_shape[4]],
self._prev_layer.output)
self._output = conv3d2d.conv3d(padded_input, self.W.val) + \
self.b.val.dimshuffle('x', 'x', 0, 'x', 'x')
def __call__(self, c01b):
"""
.. todo::
WRITEME
"""
half = self.n // 2
sq = T.sqr(c01b)
ch, r, c, b = c01b.shape
extra_channels = T.alloc(0., ch + 2*half, r, c, b)
sq = T.set_subtensor(extra_channels[half:half+ch,:,:,:], sq)
scale = self.k
for i in xrange(self.n):
scale += self.alpha * sq[i:i+ch,:,:,:]
scale = scale ** self.beta
return c01b / scale
def __call__(self, c01b):
"""
.. todo::
WRITEME
"""
half = self.n // 2
sq = T.sqr(c01b)
ch, r, c, b = c01b.shape
extra_channels = T.alloc(0., ch + 2*half, r, c, b)
sq = T.set_subtensor(extra_channels[half:half+ch,:,:,:], sq)
scale = self.k
for i in xrange(self.n):
scale += self.alpha * sq[i:i+ch,:,:,:]
scale = scale ** self.beta
return c01b / scale
def get_layer(self, x_in):
assert x_in.ndim == 2
n_steps = x_in.shape[0]
def __slice(x_, n, dim):
return x_[n * dim: (n + 1) * dim]
def __step(x_, h_, c_):
preact = T.dot(h_, self._params['U']) + x_ + self._params['b']
i = T.nnet.sigmoid(__slice(preact, 0, self._ydim))
f = T.nnet.sigmoid(__slice(preact, 1, self._ydim))
o = T.nnet.sigmoid(__slice(preact, 2, self._ydim))
c = T.tanh(__slice(preact, 3, self._ydim))
c = f * c_ + i * c
h = o * T.tanh(c)
return h, c
x_in = T.dot(x_in, self._params['W']) + self._params['b']
rval, updates = theano.scan(__step, sequences=x_in, go_backwards=self.go_backwards,
outputs_info=[T.alloc(np_floatX(0.), self._ydim),
T.alloc(np_floatX(0.), self._ydim)],
name='lstm_layers', n_steps=n_steps)
return reverse(rval[0]) if self.go_backwards else rval[0]
def rnn_ff(inps, dim, hidden, batSize, prefix, params, names):
Wx = theano.shared(randomMatrix(dim, hidden))
Wh = theano.shared(randomMatrix(hidden, hidden))
bh = theano.shared(numpy.zeros(hidden, dtype=theano.config.floatX))
#model.container['bi_h0'] = theano.shared(numpy.zeros(model.container['nh'], dtype=theano.config.floatX))
# bundle
params += [ Wx, Wh, bh ] #, model.container['bi_h0']
names += [ prefix + '_Wx', prefix + '_Wh', prefix + '_bh' ] #, 'bi_h0'
def recurrence(x_t, h_tm1):
h_t = T.nnet.sigmoid(T.dot(x_t, Wx) + T.dot(h_tm1, Wh) + bh)
return h_t
h, _ = theano.scan(fn=recurrence, \
sequences=inps, outputs_info=[T.alloc(0., batSize, hidden)], n_steps=inps.shape[0])
return h
def compute_emb(x, W):
def _step(xi, emb, W):
if prm.att_doc:
new_shape = (xi.shape[0], xi.shape[1], xi.shape[2], prm.dim_emb)
else:
new_shape = (xi.shape[0], xi.shape[1], prm.dim_emb)
out = W[xi.flatten()].reshape(new_shape).sum(-2)
return out / tensor.maximum(1., tensor.neq(xi,-1).astype('float32').sum(-1, keepdims=True))
if prm.att_doc:
emb_init = tensor.alloc(0., x.shape[1], x.shape[2], prm.dim_emb)
else:
emb_init = tensor.alloc(0., x.shape[1], prm.dim_emb)
(embs), scan_updates = theano.scan(_step,
sequences=[x],
outputs_info=[emb_init],
non_sequences=[W],
name='emb_scan',
n_steps=x.shape[0])
return embs
def call(self, x, mask=None):
X = x
half_n = self.n // 2
input_sqr = K.square(X)
if K._BACKEND == 'theano':
b, ch, r, c = X.shape
extra_channels = T.alloc(0., b, ch + 2*half_n, r, c)
input_sqr = T.set_subtensor(
extra_channels[:, half_n:half_n+ch, :, :], input_sqr)
elif K._BACKEND == 'tensorflow':
b, ch, r, c = K.int_shape(X)
up_dims = tf.pack([tf.shape(X)[0], half_n, r, c])
up = tf.fill(up_dims, 0.0)
middle = input_sqr
down_dims = tf.pack([tf.shape(X)[0], half_n, r, c])
down = tf.fill(down_dims, 0.0)
input_sqr = K.concatenate([up, middle, down], axis=1)
scale = self.k
norm_alpha = self.alpha / self.n
for i in range(self.n):
scale += norm_alpha * input_sqr[:, i:i+ch, :, :]
scale = scale ** self.beta
result = X / scale
return result
def gru_layer(tparams, emb, name, hiddenDimSize):
timesteps = emb.shape[0]
if emb.ndim == 3: n_samples = emb.shape[1]
else: n_samples = 1
def stepFn(wx, h, U_gru):
uh = T.dot(h, U_gru)
r = T.nnet.sigmoid(_slice(wx, 0, hiddenDimSize) + _slice(uh, 0, hiddenDimSize))
z = T.nnet.sigmoid(_slice(wx, 1, hiddenDimSize) + _slice(uh, 1, hiddenDimSize))
h_tilde = T.tanh(_slice(wx, 2, hiddenDimSize) + r * _slice(uh, 2, hiddenDimSize))
h_new = z * h + ((1. - z) * h_tilde)
return h_new
Wx = T.dot(emb, tparams['W_gru_'+name]) + tparams['b_gru_'+name]
results, updates = theano.scan(fn=stepFn, sequences=[Wx], outputs_info=T.alloc(numpy_floatX(0.0), n_samples, hiddenDimSize), non_sequences=[tparams['U_gru_'+name]], name='gru_layer', n_steps=timesteps)
return results
def gru_layer(tparams, emb, name, hiddenDimSize):
timesteps = emb.shape[0]
if emb.ndim == 3: n_samples = emb.shape[1]
else: n_samples = 1
def stepFn(wx, h, U_gru):
uh = T.dot(h, U_gru)
r = T.nnet.sigmoid(_slice(wx, 0, hiddenDimSize) + _slice(uh, 0, hiddenDimSize))
z = T.nnet.sigmoid(_slice(wx, 1, hiddenDimSize) + _slice(uh, 1, hiddenDimSize))
h_tilde = T.tanh(_slice(wx, 2, hiddenDimSize) + r * _slice(uh, 2, hiddenDimSize))
h_new = z * h + ((1. - z) * h_tilde)
return h_new
Wx = T.dot(emb, tparams['W_gru_'+name]) + tparams['b_gru_'+name]
results, updates = theano.scan(fn=stepFn, sequences=[Wx], outputs_info=T.alloc(numpy_floatX(0.0), n_samples, hiddenDimSize), non_sequences=[tparams['U_gru_'+name]], name='gru_layer', n_steps=timesteps)
return results
def test_gpujoin_gpualloc():
a = T.fmatrix('a')
a_val = numpy.asarray(numpy.random.rand(4, 5), dtype='float32')
b = T.fmatrix('b')
b_val = numpy.asarray(numpy.random.rand(3, 5), dtype='float32')
f = theano.function([a, b], T.join(0, T.zeros_like(a), T.ones_like(b)) + 4,
mode=mode_without_gpu)
f_gpu = theano.function([a, b], T.join(0, T.zeros_like(a), T.ones_like(b)),
mode=mode_with_gpu)
f_gpu2 = theano.function([a, b], T.join(0, T.zeros_like(a),
T.ones_like(b)) + 4,
mode=mode_with_gpu)
assert sum([node.op == T.alloc for node in f.maker.fgraph.toposort()]) == 2
assert sum([node.op == T.join for node in f.maker.fgraph.toposort()]) == 1
assert sum([isinstance(node.op, GpuAlloc)
for node in f_gpu.maker.fgraph.toposort()]) == 2
assert sum([node.op == gpu_join
for node in f_gpu.maker.fgraph.toposort()]) == 1
assert sum([isinstance(node.op, GpuAlloc)
for node in f_gpu2.maker.fgraph.toposort()]) == 2
assert sum([node.op == gpu_join
for node in f_gpu2.maker.fgraph.toposort()]) == 1
assert numpy.allclose(f(a_val, b_val), f_gpu2(a_val, b_val))
def local_gpua_alloc2(node):
"""
Join(axis, {Alloc or HostFromGPU}, ...) -> Join(axis, GpuAlloc, Alloc, ...)
Moves an alloc that is an input to join to the gpu.
"""
try:
get_context(None)
except ContextNotDefined:
# If there is no default context then we do not perform the move here.
return
if (isinstance(node.op, tensor.Alloc) and
all(c != 'output' and
c.op == tensor.join and
all(i.owner and
i.owner.op in [host_from_gpu, tensor.alloc]
for i in c.inputs[1:])
for c, idx in node.outputs[0].clients)):
return [host_from_gpu(gpu_alloc(None)(*node.inputs))]
def test0(self):
x = shared(self.rng.randn(3, 7))
a = tensor.alloc(x, 6, 7)
# It is a bad idea to have tensor.alloc return x directly,
# because the shape mismatch cannot be caught.
assert a.owner and isinstance(a.owner.op, tensor.Alloc)
f = function([], a, mode=mode_opt)
# The optimization should then be applied, and remove Alloc
assert ([node.op for node in f.maker.fgraph.toposort()]
== [deep_copy_op])
# In DebugMode, the shape mismatch should be detected
if isinstance(mode_opt, compile.DebugMode):
self.assertRaises(ValueError, f)
# No need to check_stack_trace as the optimization
# local_canonicalize_alloc only removes nodes.
def test2(self):
# Test that alloc never gets instantiated during optimization
mode = mode_opt.excluding('local_canonicalize_alloc')
x = tensor.matrix('x')
y = tensor.tile(x, (1,)*2)
f = function([x], [y], mode=mode)
op_classes = [node.op.__class__ for node in f.maker.fgraph.toposort()]
print(op_classes)
# We are supposed to test if tensr.Alloc is not in op_classes,
# but since the proper proper optimization is not currently
# implemented it will fail. Once the correct optimization is in place,
# we have to change the following we should not see tensor.Alloc
# in op_classes and we have to change the assert.
assert tensor.Alloc in op_classes
# The correct opt removes nodes, no need for check_stack_trace
def test_local_reshape_dimshuffle():
reshape_dimshuffle = out2in(local_dimshuffle_alloc)
x = tensor.vector('x')
out = tensor.alloc(x, 3, 2).dimshuffle('x', 'x', 0, 1)
g = FunctionGraph([x], [out])
reshape_dimshuffle(g)
l=theano.gof.PerformLinker()
l.accept(g)
f=l.make_function()
assert f([3, 4]).ndim == 4
topo = g.toposort()
assert any([not isinstance(x, DimShuffle) for x in topo])
def test_gpualloc():
'''
This tests tries to catch the scenario when, due to infer_shape,
the input of the alloc changes from tensor scalar to a constant
1. In this case the original constracted broadcastable pattern will
have a False for that dimension, but the new broadcastable pattern
that will be inserted by gpualloc will have a True since it knows the
dimension is 1 and therefore broadcastable.
'''
x = theano.shared(numpy.ones(3, dtype='float32'), 'x')
m = (x).dimshuffle(['x', 0])
v = tensor.alloc(1., *m.shape)
f = theano.function([],
v + x,
mode=mode_with_gpu.excluding(
"local_elemwise_alloc"))
l = f.maker.fgraph.toposort()
assert numpy.any([isinstance(y.op, cuda.GpuAlloc) for y in l])
def apply(self, sentences, init_hid=None):
"""
Parameters
----------
sentences: (length, batch, featuresdim)
Returns
-------
hs: (n_blocks, batch, hid_size)
"""
if sentences.ndim == 3:
batch_size = sentences.shape[1]
n_steps = sentences.shape[0]
else:
raise NotImplementedError
if init_hid is None:
init_hid = T.unbroadcast(T.alloc(numpy.float32(0.), batch_size, self.hid_size))
rval, updates = theano.scan(self._step_forward,
sequences=[sentences],
outputs_info=[init_hid],
n_steps=n_steps
)
self.hs = rval
return self.hs
def __Recurrent(name, hidden_dims, step_fn, inputs, non_sequences=[], h0s=None):
if not isinstance(inputs, list):
inputs = [inputs]
if not isinstance(hidden_dims, list):
hidden_dims = [hidden_dims]
if h0s is None:
h0s = [None]*len(hidden_dims)
for i in xrange(len(hidden_dims)):
if h0s[i] is None:
h0_unbatched = lib.param(
name + '.h0_' + str(i),
numpy.zeros((hidden_dims[i],), dtype=theano.config.floatX)
)
num_batches = inputs[0].shape[1]
h0s[i] = T.alloc(h0_unbatched, num_batches, hidden_dims[i])
h0s[i] = T.patternbroadcast(h0s[i], [False] * h0s[i].ndim)
outputs, _ = theano.scan(
step_fn,
sequences=inputs,
outputs_info=h0s,
non_sequences=non_sequences
)
return outputs
def extend_middle_dim(_2D, num):
"""
Gets a 2D tensor (A, B), outputs a 3D tensor (A, num, B)
:usage:
>>> TODO
"""
rval = _2D.dimshuffle((0, 'x', 1))
rval = T.alloc(rval, rval.shape[0], num, rval.shape[2])
return rval
def alloc_zeros_matrix(*dims):
return T.alloc(np.cast[theano.config.floatX](0.), *dims)
def alloc_ones_matrix(*dims):
return T.alloc(np.cast[theano.config.floatX](1.), *dims)
def localResponseNormalizationCrossChannel(incoming, alpha=1e-4,
k=2, beta=0.75, n=5):
"""
Implement the local response normalization cross the channels described
in <ImageNet Classification with Deep Convolutional Neural Networks>,
A.Krizhevsky et al. sec.3.3.
Reference of the code:
https://github.com/Lasagne/Lasagne/blob/master/lasagne/layers/
normalization.py
https://github.com/lisa-lab/pylearn2/blob/master/pylearn2/expr/normalize.py
Parameters:
incomping: The feature maps. (output of the convolution layer).
alpha: float scalar
k: float scalr
beta: float scalar
n: integer: number of adjacent channels. Must be odd.
"""
if n % 2 == 0:
raise NotImplementedError("Works only with odd n")
input_shape = incoming.shape
half_n = n // 2
input_sqr = T.sqr(incoming)
b, ch, r, c = input_shape
extra_channels = T.alloc(0., b, ch + 2*half_n, r, c)
input_sqr = T.set_subtensor(extra_channels[:, half_n:half_n+ch, :, :],
input_sqr)
scale = k
for i in range(n):
scale += alpha * input_sqr[:, i:i+ch, :, :]
scale = scale ** beta
return incoming / scale
def get_output_for(self, inputs, **kwargs):
vals, ref = inputs
N, _, H, W = ref.shape
yx = tt.stack(tt.mgrid[0:H, 0:W])[np.newaxis, :, :, :]
grid = tt.alloc(tt.cast(yx, "float32"), N, 2, H, W)
stacked = tt.concatenate([grid, ref], axis=1)
return super(BilateralFilterLayer, self).get_output_for(
[vals, stacked], **kwargs)
