def categorical_crossentropy(logit, y, mask, length_var, need_softmax=False):
logit_shp = logit.shape
# (n_samples, n_timesteps_f, n_labels)
# softmax, predict label prob
# (n_samples * n_timesteps_f, n_labels)
if need_softmax:
probs = T.nnet.softmax(logit.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]]))
else:
probs = logit.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]])
# (n_samples * n_timesteps_f)
y_flat = y.flatten()
# clip to avoid nan loss
probs = T.clip(probs, _EPSILON, 1.0 - _EPSILON)
loss = lasagne.objectives.categorical_crossentropy(probs, y_flat)
# (n_samples, n_timesteps_f)
loss = loss.reshape((logit_shp[0], logit_shp[1]))
loss = loss * mask
loss = T.sum(loss, axis=1) / length_var
probs = probs.reshape(logit_shp)
return loss, probs
python类sum()的实例源码
def binary_crossentropy(logit, y, mask, length_var):
logit_shp = logit.shape
# logit_shp[2] == 1
# n_labels is 1
# (n_samples, n_timesteps_f, n_labels)
# softmax, predict label prob
# (n_samples * n_timesteps_f, n_labels)
probs = T.nnet.sigmoid(logit.flatten())
# (n_samples * n_timesteps_f)
y_flat = y.flatten()
loss = lasagne.objectives.binary_crossentropy(probs, y_flat)
# (n_samples, n_timesteps_f)
loss = loss.reshape((logit_shp[0], logit_shp[1]))
loss = loss * mask
loss = T.sum(loss, axis=1) / length_var
# (n_samples, n_timesteps_f)
probs = probs.reshape([logit_shp[0], logit_shp[1]])
return loss, probs
def log_sum_exp(x, axis=1):
m = T.max(x, axis=axis)
return m+T.log(T.sum(T.exp(x-m.dimshuffle(0,'x')), axis=axis))
def softmax_loss(p_true, output_before_softmax):
output_before_softmax -= T.max(output_before_softmax, axis=1, keepdims=True)
if p_true.ndim==2:
return T.mean(T.log(T.sum(T.exp(output_before_softmax),axis=1)) - T.sum(p_true*output_before_softmax, axis=1))
else:
return T.mean(T.log(T.sum(T.exp(output_before_softmax),axis=1)) - output_before_softmax[T.arange(p_true.shape[0]),p_true])
def op_l2norm(s_x_, eps_=1e-6):
return T.sqrt(eps_+T.sum(T.sqr(s_x_)))
def op_cosine(s_u_, s_v_, flatten_=True, eps_=1e-6):
if flatten_:
s_u = s_u_.flatten()
s_v = s_v_.flatten()
return T.dot(s_u, s_v) / T.sqrt(eps_+T.sum(T.sqr(s_u))*T.sum(T.sqr(s_v)))
else:
s_u = s_u_
s_v = s_v_
T.sum(s_u*s_v, axis=-1)/T.sqrt(eps_+T.sum(T.sqr(s_u), axis=-1)*T.sum(T.sqr(s_v), axis=-1))
def compute_loss(output, num_samples, num_entries=6, gamma=500.0):
"""Compute the loss of a dataset, given the output of the DSSM.
Args:
output (:class:`lasagne.layers.Layer`): the output of the DSSM
num_samples (int): the number of samples in the dataset
num_entries (int): the number of compared papers in the DSSM structure
gamma (float): the coefficient applied in the softmax of the similarities
Returns:
theano.tensor.TensorType: the loss of the dataset
"""
assert (num_entries > 2)
assert (num_samples > 0)
# Post-NN operations to compute the loss
# First, we extract the first output of each bundle
mask = np.zeros(num_entries * num_samples)
mask[::num_entries] = 1
unmask = np.ones(num_entries * num_samples) - mask
cited = T.extra_ops.compress(mask, output, axis=0)
odocs = T.extra_ops.compress(unmask, output, axis=0)
# We duplicate each row 'x' num_entries-1 times
cited = T.extra_ops.repeat(cited, num_entries-1, axis=0)
# Then we compute element-wise product of x with each y, for each bundle
sims = T.sum(cited * odocs, axis=1)
# We reshape the similarities
sims = T.reshape(sims, (num_samples, num_entries-1))
sims = gamma * sims
# We take the softmax of each row
probs = T.nnet.softmax(sims)
# We compute the loss as the sum of element on the first column
loss_mask = np.zeros(num_entries-1)
loss_mask[0] = 1
loss = T.extra_ops.compress(loss_mask, probs, axis=1)
return -T.log(T.prod(loss))
def softmax(x):
e_x = T.exp(x - x.max(axis=0, keepdims=True))
out = e_x / e_x.sum(axis=0, keepdims=True)
return out
def get_norm(x):
x = np.array(x)
return np.sum(x * x)
def l2_reg(params):
return T.sum([T.sum(x ** 2) for x in params])
def l1_reg(params):
return T.sum([T.sum(abs(x)) for x in params])
def log_sum_exp(x, axis=1):
m = T.max(x, axis=axis)
return m+T.log(T.sum(T.exp(x-m.dimshuffle(0,'x')), axis=axis))
def __init__(self, incoming, b=lasagne.init.Constant(0.), g=lasagne.init.Constant(1.),
W=lasagne.init.Normal(0.05), train_g=False, init_stdv=1., nonlinearity=relu, **kwargs):
super(WeightNormLayer, self).__init__(incoming, **kwargs)
self.nonlinearity = nonlinearity
self.init_stdv = init_stdv
k = self.input_shape[1]
if b is not None:
self.b = self.add_param(b, (k,), name="b", regularizable=False)
if g is not None:
self.g = self.add_param(g, (k,), name="g", regularizable=False, trainable=train_g)
if len(self.input_shape)==4:
self.axes_to_sum = (0,2,3)
self.dimshuffle_args = ['x',0,'x','x']
else:
self.axes_to_sum = 0
self.dimshuffle_args = ['x',0]
# scale weights in layer below
incoming.W_param = incoming.W
#incoming.W_param.set_value(W.sample(incoming.W_param.get_value().shape))
if incoming.W_param.ndim==4:
if isinstance(incoming, Deconv2DLayer):
W_axes_to_sum = (0,2,3)
W_dimshuffle_args = ['x',0,'x','x']
else:
W_axes_to_sum = (1,2,3)
W_dimshuffle_args = [0,'x','x','x']
else:
W_axes_to_sum = 0
W_dimshuffle_args = ['x',0]
if g is not None:
incoming.W = incoming.W_param * (self.g/T.sqrt(1e-6 + T.sum(T.square(incoming.W_param),axis=W_axes_to_sum))).dimshuffle(*W_dimshuffle_args)
else:
incoming.W = incoming.W_param / T.sqrt(1e-6 + T.sum(T.square(incoming.W_param),axis=W_axes_to_sum,keepdims=True))
def __init__(self, incoming, num_kernels, dim_per_kernel=5, theta=lasagne.init.Normal(0.05),
log_weight_scale=lasagne.init.Constant(0.), b=lasagne.init.Constant(-1.), **kwargs):
super(MinibatchLayer, self).__init__(incoming, **kwargs)
self.num_kernels = num_kernels
num_inputs = int(np.prod(self.input_shape[1:]))
self.theta = self.add_param(theta, (num_inputs, num_kernels, dim_per_kernel), name="theta")
self.log_weight_scale = self.add_param(log_weight_scale, (num_kernels, dim_per_kernel), name="log_weight_scale")
self.W = self.theta * (T.exp(self.log_weight_scale)/T.sqrt(T.sum(T.square(self.theta),axis=0))).dimshuffle('x',0,1)
self.b = self.add_param(b, (num_kernels,), name="b")
def get_output_for(self, input, init=False, **kwargs):
if input.ndim > 2:
# if the input has more than two dimensions, flatten it into a
# batch of feature vectors.
input = input.flatten(2)
activation = T.tensordot(input, self.W, [[1], [0]])
abs_dif = (T.sum(abs(activation.dimshuffle(0,1,2,'x') - activation.dimshuffle('x',1,2,0)),axis=2)
+ 1e6 * T.eye(input.shape[0]).dimshuffle(0,'x',1))
if init:
mean_min_abs_dif = 0.5 * T.mean(T.min(abs_dif, axis=2),axis=0)
abs_dif /= mean_min_abs_dif.dimshuffle('x',0,'x')
self.init_updates = [(self.log_weight_scale, self.log_weight_scale-T.log(mean_min_abs_dif).dimshuffle(0,'x'))]
f = T.sum(T.exp(-abs_dif),axis=2)
if init:
mf = T.mean(f,axis=0)
f -= mf.dimshuffle('x',0)
self.init_updates.append((self.b, -mf))
else:
f += self.b.dimshuffle('x',0)
return T.concatenate([input, f], axis=1)
# Input Mixture of Gaussian Layer
def log_sum_exp(x, axis=1):
m = T.max(x, axis=axis)
return m+T.log(T.sum(T.exp(x-m.dimshuffle(0,'x')), axis=axis))
def __init__(self, incoming, b=lasagne.init.Constant(0.), g=lasagne.init.Constant(1.),
W=lasagne.init.Normal(0.05), train_g=False, init_stdv=1., nonlinearity=relu, **kwargs):
super(WeightNormLayer, self).__init__(incoming, **kwargs)
self.nonlinearity = nonlinearity
self.init_stdv = init_stdv
k = self.input_shape[1]
if b is not None:
self.b = self.add_param(b, (k,), name="b", regularizable=False)
if g is not None:
self.g = self.add_param(g, (k,), name="g", regularizable=False, trainable=train_g)
if len(self.input_shape)==4:
self.axes_to_sum = (0,2,3)
self.dimshuffle_args = ['x',0,'x','x']
else:
self.axes_to_sum = 0
self.dimshuffle_args = ['x',0]
# scale weights in layer below
incoming.W_param = incoming.W
#incoming.W_param.set_value(W.sample(incoming.W_param.get_value().shape))
if incoming.W_param.ndim==4:
if isinstance(incoming, Deconv2DLayer):
W_axes_to_sum = (0,2,3)
W_dimshuffle_args = ['x',0,'x','x']
else:
W_axes_to_sum = (1,2,3)
W_dimshuffle_args = [0,'x','x','x']
else:
W_axes_to_sum = 0
W_dimshuffle_args = ['x',0]
if g is not None:
incoming.W = incoming.W_param * (self.g/T.sqrt(1e-6 + T.sum(T.square(incoming.W_param),axis=W_axes_to_sum))).dimshuffle(*W_dimshuffle_args)
else:
incoming.W = incoming.W_param / T.sqrt(1e-6 + T.sum(T.square(incoming.W_param),axis=W_axes_to_sum,keepdims=True))
def __init__(self, incoming, num_kernels, dim_per_kernel=5, theta=lasagne.init.Normal(0.05),
log_weight_scale=lasagne.init.Constant(0.), b=lasagne.init.Constant(-1.), **kwargs):
super(MinibatchLayer, self).__init__(incoming, **kwargs)
self.num_kernels = num_kernels
num_inputs = int(np.prod(self.input_shape[1:]))
self.theta = self.add_param(theta, (num_inputs, num_kernels, dim_per_kernel), name="theta")
self.log_weight_scale = self.add_param(log_weight_scale, (num_kernels, dim_per_kernel), name="log_weight_scale")
self.W = self.theta * (T.exp(self.log_weight_scale)/T.sqrt(T.sum(T.square(self.theta),axis=0))).dimshuffle('x',0,1)
self.b = self.add_param(b, (num_kernels,), name="b")
def get_output_for(self, input, init=False, **kwargs):
if input.ndim > 2:
# if the input has more than two dimensions, flatten it into a
# batch of feature vectors.
input = input.flatten(2)
activation = T.tensordot(input, self.W, [[1], [0]])
abs_dif = (T.sum(abs(activation.dimshuffle(0,1,2,'x') - activation.dimshuffle('x',1,2,0)),axis=2)
+ 1e6 * T.eye(input.shape[0]).dimshuffle(0,'x',1))
if init:
mean_min_abs_dif = 0.5 * T.mean(T.min(abs_dif, axis=2),axis=0)
abs_dif /= mean_min_abs_dif.dimshuffle('x',0,'x')
self.init_updates = [(self.log_weight_scale, self.log_weight_scale-T.log(mean_min_abs_dif).dimshuffle(0,'x'))]
f = T.sum(T.exp(-abs_dif),axis=2)
if init:
mf = T.mean(f,axis=0)
f -= mf.dimshuffle('x',0)
self.init_updates.append((self.b, -mf))
else:
f += self.b.dimshuffle('x',0)
return T.concatenate([input, f], axis=1)
# Input Mixture of Gaussian Layer
def _read(w_read, memory):
# w_read : (nb_sample, memory_dim)
# memory : (nb_sample, memory_dim, memory_width)
# return dot(w_read, memory)
return T.sum(w_read[:, :, None] * memory, axis=1)
