def _mix_rbf_kernel(X, Y, sigma_list):
assert(X.size(0) == Y.size(0))
m = X.size(0)
Z = torch.cat((X, Y), 0)
ZZT = torch.mm(Z, Z.t())
diag_ZZT = torch.diag(ZZT).unsqueeze(1)
Z_norm_sqr = diag_ZZT.expand_as(ZZT)
exponent = Z_norm_sqr - 2 * ZZT + Z_norm_sqr.t()
K = 0.0
for sigma in sigma_list:
gamma = 1.0 / (2 * sigma**2)
K += torch.exp(-gamma * exponent)
return K[:m, :m], K[:m, m:], K[m:, m:], len(sigma_list)
python类exp()的实例源码
def test(self, dataset):
self.model.eval()
total_loss = 0
predictions = torch.zeros(len(dataset))
indices = torch.arange(1, dataset.num_classes + 1)
for idx in tqdm(range(len(dataset)),desc='Testing epoch ' + str(self.epoch) + ''):
ltree, lsent, rtree, rsent, label = dataset[idx]
linput, rinput = Var(lsent, volatile=True), Var(rsent, volatile=True)
target = Var(map_label_to_target(label, dataset.num_classes), volatile=True)
if self.args.cuda:
linput, rinput = linput.cuda(), rinput.cuda()
target = target.cuda()
output = self.model(ltree, linput, rtree, rinput)
loss = self.criterion(output, target)
total_loss += loss.data[0]
output = output.data.squeeze().cpu()
predictions[idx] = torch.dot(indices, torch.exp(output))
return total_loss / len(dataset), predictions
def decode(loc, priors, variances):
"""Decode locations from predictions using priors to undo
the encoding we did for offset regression at train time.
Args:
loc (tensor): location predictions for loc layers,
Shape: [num_priors,4]
priors (tensor): Prior boxes in center-offset form.
Shape: [num_priors,4].
variances: (list[float]) Variances of priorboxes
Return:
decoded bounding box predictions
"""
boxes = torch.cat((
priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
boxes[:, :2] -= boxes[:, 2:] / 2
boxes[:, 2:] += boxes[:, :2]
return boxes
def __call__(self, # type: ignore
logits: torch.Tensor,
mask: Optional[torch.Tensor] = None):
"""
Parameters
----------
logits : ``torch.Tensor``, required.
A tensor of unnormalized log probabilities of shape (batch_size, ..., num_classes).
mask: ``torch.Tensor``, optional (default = None).
A masking tensor of shape (batch_size, ...).
"""
# Get the data from the Variables.
logits, mask = self.unwrap_to_tensors(logits, mask)
if mask is None:
mask = torch.ones(logits.size()[:-1])
log_probs = torch.nn.functional.log_softmax(Variable(logits), dim=-1).data
probabilities = torch.exp(log_probs) * mask.unsqueeze(-1)
weighted_negative_likelihood = - log_probs * probabilities
entropy = weighted_negative_likelihood.sum(-1)
self._entropy += entropy.sum() / mask.sum()
self._count += 1
def logsumexp(tensor: torch.Tensor,
dim: int = -1,
keepdim: bool = False) -> torch.Tensor:
"""
A numerically stable computation of logsumexp. This is mathematically equivalent to
`tensor.exp().sum(dim, keep=keepdim).log()`. This function is typically used for summing log
probabilities.
Parameters
----------
tensor : torch.FloatTensor, required.
A tensor of arbitrary size.
dim : int, optional (default = -1)
The dimension of the tensor to apply the logsumexp to.
keepdim: bool, optional (default = False)
Whether to retain a dimension of size one at the dimension we reduce over.
"""
max_score, _ = tensor.max(dim, keepdim=keepdim)
if keepdim:
stable_vec = tensor - max_score
else:
stable_vec = tensor - max_score.unsqueeze(dim)
return max_score + (stable_vec.exp().sum(dim, keepdim=keepdim)).log()
def _gaussian(self, enc_output):
def latent_loss(mu, sigma):
pow_mu = mu * mu
pow_sigma = sigma * sigma
return 0.5 * torch.mean(pow_mu + pow_sigma - torch.log(pow_sigma) - 1)
mu = self._enc_mu(enc_output)
sigma = torch.exp(.5 * self._enc_log_sigma(enc_output))
self.latent_loss = latent_loss(mu, sigma)
weight = next(self.parameters()).data
std_z = Variable(weight.new(*sigma.size()), requires_grad=False)
std_z.data.copy_(torch.from_numpy(
np.random.normal(size=sigma.size())))
return mu + sigma * std_z
def forward(self, inputs, batch_size, hidden_cell=None):
if hidden_cell is None:
# then must init with zeros
if use_cuda:
hidden = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size).cuda())
cell = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size).cuda())
else:
hidden = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size))
cell = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size))
hidden_cell = (hidden, cell)
_, (hidden,cell) = self.lstm(inputs.float(), hidden_cell)
# hidden is (2, batch_size, hidden_size), we want (batch_size, 2*hidden_size):
hidden_forward, hidden_backward = torch.split(hidden,1,0)
hidden_cat = torch.cat([hidden_forward.squeeze(0), hidden_backward.squeeze(0)],1)
# mu and sigma:
mu = self.fc_mu(hidden_cat)
sigma_hat = self.fc_sigma(hidden_cat)
sigma = torch.exp(sigma_hat/2.)
# N ~ N(0,1)
z_size = mu.size()
if use_cuda:
N = Variable(torch.normal(torch.zeros(z_size),torch.ones(z_size)).cuda())
else:
N = Variable(torch.normal(torch.zeros(z_size),torch.ones(z_size)))
z = mu + sigma*N
# mu and sigma_hat are needed for LKL loss
return z, mu, sigma_hat
def __call__(self, x_mu):
"""
Args:
x_mu (FloatTensor/LongTensor or ndarray)
Returns:
x (FloatTensor or ndarray)
"""
mu = self.qc - 1.
if isinstance(x_mu, np.ndarray):
x = ((x_mu) / mu) * 2 - 1.
x = np.sign(x) * (np.exp(np.abs(x) * np.log1p(mu)) - 1.) / mu
elif isinstance(x_mu, (torch.Tensor, torch.LongTensor)):
if isinstance(x_mu, torch.LongTensor):
x_mu = x_mu.float()
mu = torch.FloatTensor([mu])
x = ((x_mu) / mu) * 2 - 1.
x = torch.sign(x) * (torch.exp(torch.abs(x) * torch.log1p(mu)) - 1.) / mu
return x
def test_do_propagation(self):
pyro.clear_param_store()
def model():
z = pyro.sample("z", Normal(10.0 * ng_ones(1), 0.0001 * ng_ones(1)))
latent_prob = torch.exp(z) / (torch.exp(z) + ng_ones(1))
flip = pyro.sample("flip", Bernoulli(latent_prob))
return flip
sample_from_model = model()
z_data = {"z": -10.0 * ng_ones(1)}
# under model flip = 1 with high probability; so do indirect DO surgery to make flip = 0
sample_from_do_model = poutine.trace(poutine.do(model, data=z_data))()
assert eq(sample_from_model, ng_ones(1))
assert eq(sample_from_do_model, ng_zeros(1))
def decode(loc, priors, variances):
"""Decode locations from predictions using priors to undo
the encoding we did for offset regression at train time.
Args:
loc (tensor): location predictions for loc layers,
Shape: [num_priors,4]
priors (tensor): Prior boxes in center-offset form.
Shape: [num_priors,4].
variances: (list[float]) Variances of priorboxes
Return:
decoded bounding box predictions
"""
boxes = torch.cat((
priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
boxes[:, :2] -= boxes[:, 2:] / 2
boxes[:, 2:] += boxes[:, :2]
return boxes
def decode(loc, priors, variances):
"""Decode locations from predictions using priors to undo
the encoding we did for offset regression at train time.
Args:
loc (tensor): location predictions for loc layers,
Shape: [num_priors,4]
priors (tensor): Prior boxes in center-offset form.
Shape: [num_priors,4].
variances: (list[float]) Variances of priorboxes
Return:
decoded bounding box predictions
"""
boxes = torch.cat((
priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
boxes[:, :2] -= boxes[:, 2:] / 2
boxes[:, 2:] += boxes[:, :2]
return boxes
def test(self, dataset):
self.model.eval()
self.embedding_model.eval()
loss = 0
accuracies = torch.zeros(len(dataset))
output_trees = []
outputs = []
for idx in tqdm(range(len(dataset)), desc='Testing epoch '+str(self.epoch)+''):
tree, sent, label = dataset[idx]
input = Var(sent, volatile=True)
target = Var(torch.LongTensor([int(label)]), volatile=True)
if self.args.cuda:
input = input.cuda()
target = target.cuda()
emb = F.torch.unsqueeze(self.embedding_model(input),1)
output, _, acc, tree = self.model(tree, emb)
err = self.criterion(output, target)
loss += err.data[0]
accuracies[idx] = acc
output_trees.append(tree)
outputs.append(tree.output_softmax.data.numpy())
# predictions[idx] = torch.dot(indices,torch.exp(output.data.cpu()))
return loss/len(dataset), accuracies, outputs, output_trees
def _forward_alg(self, feats):
# calculate in log domain
# feats is len(sentence) * tagset_size
# initialize alpha with a Tensor with values all equal to -10000.
init_alphas = torch.Tensor(1, self.tagset_size).fill_(-10000.)
init_alphas[0][self.tag_to_ix[START_TAG]] = 0.
forward_var = autograd.Variable(init_alphas)
if self.use_gpu:
forward_var = forward_var.cuda()
for feat in feats:
emit_score = feat.view(-1, 1)
tag_var = forward_var + self.transitions + emit_score
max_tag_var, _ = torch.max(tag_var, dim=1)
tag_var = tag_var - max_tag_var.view(-1, 1)
forward_var = max_tag_var + torch.log(torch.sum(torch.exp(tag_var), dim=1)).view(1, -1) # ).view(1, -1)
terminal_var = (forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]).view(1, -1)
alpha = log_sum_exp(terminal_var)
# Z(x)
return alpha
def _build_target(self, bbox_pred_np, gt_boxes, gt_classes, dontcare, iou_pred_np):
"""
:param bbox_pred: shape: (bsize, h x w, num_anchors, 4) : (sig(tx), sig(ty), exp(tw), exp(th))
"""
bsize = bbox_pred_np.shape[0]
targets = self.pool.map(_process_batch, ((bbox_pred_np[b], gt_boxes[b], gt_classes[b], dontcare[b], iou_pred_np[b]) for b in range(bsize)))
_boxes = np.stack(tuple((row[0] for row in targets)))
_ious = np.stack(tuple((row[1] for row in targets)))
_classes = np.stack(tuple((row[2] for row in targets)))
_box_mask = np.stack(tuple((row[3] for row in targets)))
_iou_mask = np.stack(tuple((row[4] for row in targets)))
_class_mask = np.stack(tuple((row[5] for row in targets)))
return _boxes, _ious, _classes, _box_mask, _iou_mask, _class_mask
def loss(self, x, samples):
_, proposal_output = self.forward(x, samples)
batch_size = len(samples)
means = proposal_output[:,0:self.mixture_components]
stds = proposal_output[:,self.mixture_components:2*self.mixture_components]
coeffs = proposal_output[:,2*self.mixture_components:3*self.mixture_components]
l = 0
for b in range(batch_size):
value = samples[b].value[0]
prior_min = samples[b].distribution.prior_min
prior_max = samples[b].distribution.prior_max
ll = 0
for c in range(self.mixture_components):
mean = means[b,c]
std = stds[b,c]
coeff = coeffs[b,c]
xi = (value - mean) / std
phi_min = 0.5 * (1 + util.erf(((prior_min - mean) / std) * util.one_over_sqrt_two))
phi_max = 0.5 * (1 + util.erf(((prior_max - mean) / std) * util.one_over_sqrt_two))
ll += coeff * util.one_over_sqrt_two_pi * torch.exp(-0.5 * xi * xi) / (std * (phi_max - phi_min))
l -= torch.log(ll + util.epsilon)
return l
def decode(loc, priors, variances):
"""Decode locations from predictions using priors to undo
the encoding we did for offset regression at train time.
Args:
loc (tensor): location predictions for loc layers,
Shape: [num_priors,4]
priors (tensor): Prior boxes in center-offset form.
Shape: [num_priors,4].
variances: (list[float]) Variances of priorboxes
Return:
decoded bounding box predictions
"""
boxes = torch.cat((
priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
boxes[:, :2] -= boxes[:, 2:] / 2
boxes[:, 2:] += boxes[:, :2]
return boxes
def _generate_pred_bbox(self, bbox_delta, anchors):
"""get predictions boxes from bbox_delta and anchors.
Args:
bbox_delta: (dcx, dcy, dw, dh)
shape:(H*W*num_anchor, 4)
anchor: (cx, cy, h, w)
shape:(H*W*num_anchor, 4)
Output:
output: (x_min, y_min, x_max, y_max)
"""
assert bbox_delta.dim() == anchors.dim(), "dim is not equal"
pred_xy = torch.sigmoid(bbox_delta[:, :2]) + anchors[:, :2]
pred_wh = torch.exp(bbox_delta[:, 2:]) * anchors[:, 2:]
pred_bbox = torch.cat((pred_xy, pred_wh), dim=1).contiguous()
# change (cx, xy, h, w) to (x_min, y_min, x_max, y_max)
pred_bbox[:, 0:2] = pred_bbox[:, 0:2] - pred_bbox[:, 2:4] / 2
pred_bbox[:, 2:4] = pred_bbox[:, 0:2] + pred_bbox[:, 2:4]
pred_bbox[:, 0::2] = pred_bbox[:, 0::2] / self.W
pred_bbox[:, 1::2] = pred_bbox[:, 1::2] / self.H
return pred_bbox
def decode(loc, priors, variances):
"""Decode locations from predictions using priors to undo
the encoding we did for offset regression at train time.
Args:
loc (tensor): location predictions for loc layers,
Shape: [num_priors,4]
priors (tensor): Prior boxes in center-offset form.
Shape: [num_priors,4].
variances: (list[float]) Variances of priorboxes
Return:
decoded bounding box predictions
"""
boxes = torch.cat((
priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
boxes[:, :2] -= boxes[:, 2:] / 2
boxes[:, 2:] += boxes[:, :2]
return boxes
def backward(self, grad_output):
z, log_phi_z = self.saved_tensors
log_phi_z_grad = z.new().resize_as_(z).zero_()
z_is_small = z.lt(-1)
z_is_not_small = 1 - z_is_small
if z_is_small.sum() > 0:
log_phi_z_grad[z_is_small] = torch.abs(self.denominator.div(self.numerator)).mul(math.sqrt(2 / math.pi))
exp = z[z_is_not_small].pow(2) \
.div(-2) \
.sub(log_phi_z[z_is_not_small]) \
.add(math.log(0.5))
log_phi_z_grad[z_is_not_small] = torch.exp(exp).mul(math.sqrt(2 / math.pi))
return log_phi_z_grad.mul(grad_output)
def logsumexp(x, dim=None):
"""
Args:
x: A pytorch tensor (any dimension will do)
dim: int or None, over which to perform the summation. `None`, the
default, performs over all axes.
Returns: The result of the log(sum(exp(...))) operation.
"""
if dim is None:
xmax = x.max()
xmax_ = x.max()
return xmax_ + numpy.log(torch.exp(x - xmax).sum())
else:
xmax, _ = x.max(dim, keepdim=True)
xmax_, _ = x.max(dim)
return xmax_ + torch.log(torch.exp(x - xmax).sum(dim))
def test(self, dataset):
self.model.eval()
loss = 0
predictions = torch.zeros(len(dataset))
indices = torch.arange(1, dataset.num_classes + 1)
for idx in tqdm(range(len(dataset)), desc='Testing epoch ' + str(self.epoch) + ''):
ltree, lsent, rtree, rsent, label = dataset[idx]
linput, rinput = Var(lsent, volatile=True), Var(rsent, volatile=True)
target = Var(map_label_to_target(label, dataset.num_classes), volatile=True)
if self.args.cuda:
linput, rinput = linput.cuda(), rinput.cuda()
target = target.cuda()
output = self.model(ltree, linput, rtree, rinput)
err = self.criterion(output, target)
loss += err.data[0]
predictions[idx] = torch.dot(indices, torch.exp(output.data.cpu()))
return loss / len(dataset), predictions
def softmax(input, dim=None, _stacklevel=3):
r"""Applies a softmax function.
Softmax is defined as:
:math:`softmax(x) = \frac{exp(x_i)}{\sum_j exp(x_j)}`
It is applied to all slices along dim, and will rescale them so that the elements
lie in the range `(0, 1)` and sum to 1.
See :class:`~torch.nn.Softmax` for more details.
Arguments:
input (Variable): input
dim (int): A dimension along which softmax will be computed.
.. note::
This function doesn't work directly with NLLLoss,
which expects the Log to be computed between the Softmax and itself.
Use log_softmax instead (it's faster and has better numerical properties).
"""
if dim is None:
dim = _get_softmax_dim('softmax', input.dim(), _stacklevel)
return torch._C._nn.softmax(input, dim)
def test_gamma_shape(self):
alpha = Variable(torch.exp(torch.randn(2, 3)), requires_grad=True)
beta = Variable(torch.exp(torch.randn(2, 3)), requires_grad=True)
alpha_1d = Variable(torch.exp(torch.randn(1)), requires_grad=True)
beta_1d = Variable(torch.exp(torch.randn(1)), requires_grad=True)
self.assertEqual(Gamma(alpha, beta).sample().size(), (2, 3))
self.assertEqual(Gamma(alpha, beta).sample_n(5).size(), (5, 2, 3))
self.assertEqual(Gamma(alpha_1d, beta_1d).sample_n(1).size(), (1, 1))
self.assertEqual(Gamma(alpha_1d, beta_1d).sample().size(), (1,))
self.assertEqual(Gamma(0.5, 0.5).sample().size(), (1,))
self.assertEqual(Gamma(0.5, 0.5).sample_n(1).size(), (1,))
def ref_log_prob(idx, x, log_prob):
a = alpha.data.view(-1)[idx]
b = beta.data.view(-1)[idx]
expected = scipy.stats.gamma.logpdf(x, a, scale=1 / b)
self.assertAlmostEqual(log_prob, expected, places=3)
self._check_log_prob(Gamma(alpha, beta), ref_log_prob)
# This is a randomized test.
def forward(self, x):
"""
A model for non-linear data that works off of mixing multiple Gaussian
distributions together. Uses linear projections of a given input to generate
a set of N Gaussian models' mixture components, means and standard deviations.
:param x: (num. samples, input dim.)
:return: Mixture components, means, and standard deviations
in the form (num. samples, num. mixtures)
"""
x = F.tanh(self.projection(x))
weights = F.softmax(self.weights_projection(x))
means = self.mean_projection(x)
stds = torch.exp(self.std_projection(x))
return weights, means, stds
def forward(self, y, weights, mean, std):
"""
Presents a maximum a-priori objective for a set of predicted means, mixture components,
and standard deviations to model a given ground-truth 'y'. Modeled using negative log
likelihood.
:param y: Non-linear target.
:param weights: Predicted mixture components.
:param mean: Predicted mixture means.
:param std: Predicted mixture standard deviations.
:return:
"""
normalization = 1.0 / ((2.0 * math.pi) ** 0.5)
gaussian_sample = (y.expand_as(mean) - mean) * torch.reciprocal(std)
gaussian_sample = normalization * torch.reciprocal(std) * torch.exp(-0.5 * gaussian_sample ** 2)
return -torch.mean(torch.log(torch.sum(weights * gaussian_sample, dim=1)))
def forward(self, last_state, states):
if len(states.size()) == 2: states = states.unsqueeze(0)
sequence_length, batch_size, state_dim = states.size()
transformed_last_state = last_state @ self.projection
transformed_last_state = transformed_last_state.expand(sequence_length, batch_size, self.encoder_dim)
transformed_last_state = transformed_last_state.transpose(0, 1).contiguous()
transformed_last_state = transformed_last_state.view(batch_size, -1)
states = states.transpose(0, 1).contiguous()
states = states.view(batch_size, -1)
energies = transformed_last_state * states
energies = energies.sum(dim=1)
if self.encoder_dim is not None:
attention_weights = torch.cat([torch.exp(energies[0]), F.softmax(energies[1:])], dim=0)
else:
attention_weights = F.softmax(energies)
return attention_weights
def logsumexp(x, axis=None, keepdims=False):
def _logsumexp(x, axis=axis, keepdims=keepdims):
y = torch.log(torch.sum(torch.exp(x), axis))
return y if keepdims else torch.squeeze(y, axis)
def _compute_output_shape(x, axis=axis, keepdims=keepdims):
if axis is None:
return ()
shape = list(_get_shape(x))
if keepdims:
shape[axis] = 1
else:
del shape[axis]
return tuple(shape)
return get_op(_logsumexp, output_shape=_compute_output_shape, arguments=[axis, keepdims])(x)
def train_epoch(model, data_iter, criterion, optimizer):
total_loss = 0.
total_words = 0.
for (data, target) in data_iter:#tqdm(
#data_iter, mininterval=2, desc=' - Training', leave=False):
data = Variable(data)
target = Variable(target)
if opt.cuda:
data, target = data.cuda(), target.cuda()
target = target.contiguous().view(-1)
pred = model.forward(data)
loss = criterion(pred, target)
total_loss += loss.data[0]
total_words += data.size(0) * data.size(1)
optimizer.zero_grad()
loss.backward()
optimizer.step()
data_iter.reset()
return math.exp(total_loss / total_words)
def eval_epoch(model, data_iter, criterion):
total_loss = 0.
total_words = 0.
for (data, target) in data_iter:#tqdm(
#data_iter, mininterval=2, desc=' - Training', leave=False):
data = Variable(data, volatile=True)
target = Variable(target, volatile=True)
if opt.cuda:
data, target = data.cuda(), target.cuda()
target = target.contiguous().view(-1)
pred = model.forward(data)
loss = criterion(pred, target)
total_loss += loss.data[0]
total_words += data.size(0) * data.size(1)
data_iter.reset()
return math.exp(total_loss / total_words)
def attn_window(self,h_dec):
params = self.dec_linear(h_dec)
gx_,gy_,log_sigma_2,log_delta,log_gamma = params.split(1,1) #21
# gx_ = Variable(torch.ones(4,1))
# gy_ = Variable(torch.ones(4, 1) * 2)
# log_sigma_2 = Variable(torch.ones(4, 1) * 3)
# log_delta = Variable(torch.ones(4, 1) * 4)
# log_gamma = Variable(torch.ones(4, 1) * 5)
gx = (self.A + 1) / 2 * (gx_ + 1) # 22
gy = (self.B + 1) / 2 * (gy_ + 1) # 23
delta = (max(self.A,self.B) - 1) / (self.N - 1) * torch.exp(log_delta) # 24
sigma2 = torch.exp(log_sigma_2)
gamma = torch.exp(log_gamma)
return self.filterbank(gx,gy,sigma2,delta),gamma
# correct
