def forward(self, x, lengths):
"""Handles variable size captions
"""
# Embed word ids to vectors
x = self.embed(x)
packed = pack_padded_sequence(x, lengths, batch_first=True)
# Forward propagate RNN
out, _ = self.rnn(packed)
# Reshape *final* output to (batch_size, hidden_size)
padded = pad_packed_sequence(out, batch_first=True)
I = torch.LongTensor(lengths).view(-1, 1, 1)
I = Variable(I.expand(x.size(0), 1, self.embed_size)-1).cuda()
out = torch.gather(padded[0], 1, I).squeeze(1)
# normalization in the joint embedding space
out = l2norm(out)
# take absolute value, used by order embeddings
if self.use_abs:
out = torch.abs(out)
return out
python类abs()的实例源码
def __call__(self, x):
"""
Args:
x (FloatTensor/LongTensor or ndarray)
Returns:
x_mu (LongTensor or ndarray)
"""
mu = self.qc - 1.
if isinstance(x, np.ndarray):
x_mu = np.sign(x) * np.log1p(mu * np.abs(x)) / np.log1p(mu)
x_mu = ((x_mu + 1) / 2 * mu + 0.5).astype(int)
elif isinstance(x, (torch.Tensor, torch.LongTensor)):
if isinstance(x, torch.LongTensor):
x = x.float()
mu = torch.FloatTensor([mu])
x_mu = torch.sign(x) * torch.log1p(mu *
torch.abs(x)) / torch.log1p(mu)
x_mu = ((x_mu + 1) / 2 * mu + 0.5).long()
return x_mu
def pack_to_matching_matrix(s1, s2, cat_only=[False, False]):
t1 = s1.size(0)
t2 = s2.size(0)
batch_size = s1.size(1)
d = s1.size(2)
expanded_p_s1 = s1.expand(t2, t1, batch_size, d)
expanded_p_s2 = s2.view(t2, 1, batch_size, d)
expanded_p_s2 = expanded_p_s2.expand(t2, t1, batch_size, d)
if not cat_only[0] and not cat_only[1]:
matrix = torch.cat((expanded_p_s1, expanded_p_s2), dim=3)
elif not cat_only[0] and cat_only[1]:
matrix = torch.cat((expanded_p_s1, expanded_p_s2, expanded_p_s1 * expanded_p_s2), dim=3)
else:
matrix = torch.cat((expanded_p_s1,
expanded_p_s2,
torch.abs(expanded_p_s1 - expanded_p_s2),
expanded_p_s1 * expanded_p_s2), dim=3)
# matrix = torch.cat((expanded_p_s1,
# expanded_p_s2), dim=3)
return matrix
def test_mu_law_companding(self):
sig = self.sig.clone()
quantization_channels = 256
sig = self.sig.numpy()
sig = sig / np.abs(sig).max()
self.assertTrue(sig.min() >= -1. and sig.max() <= 1.)
sig_mu = transforms.MuLawEncoding(quantization_channels)(sig)
self.assertTrue(sig_mu.min() >= 0. and sig.max() <= quantization_channels)
sig_exp = transforms.MuLawExpanding(quantization_channels)(sig_mu)
self.assertTrue(sig_exp.min() >= -1. and sig_exp.max() <= 1.)
sig = self.sig.clone()
sig = sig / torch.abs(sig).max()
self.assertTrue(sig.min() >= -1. and sig.max() <= 1.)
sig_mu = transforms.MuLawEncoding(quantization_channels)(sig)
self.assertTrue(sig_mu.min() >= 0. and sig.max() <= quantization_channels)
sig_exp = transforms.MuLawExpanding(quantization_channels)(sig_mu)
self.assertTrue(sig_exp.min() >= -1. and sig_exp.max() <= 1.)
def __call__(self, x_mu):
"""
Args:
x_mu (FloatTensor/LongTensor or ndarray)
Returns:
x (FloatTensor or ndarray)
"""
mu = self.qc - 1.
if isinstance(x_mu, np.ndarray):
x = ((x_mu) / mu) * 2 - 1.
x = np.sign(x) * (np.exp(np.abs(x) * np.log1p(mu)) - 1.) / mu
elif isinstance(x_mu, (torch.Tensor, torch.LongTensor)):
if isinstance(x_mu, torch.LongTensor):
x_mu = x_mu.float()
mu = torch.FloatTensor([mu])
x = ((x_mu) / mu) * 2 - 1.
x = torch.sign(x) * (torch.exp(torch.abs(x) * torch.log1p(mu)) - 1.) / mu
return x
def test_forward_backward(self):
import torch
import torch.nn.functional as F
from torch.autograd import Variable
from reid.loss import OIMLoss
criterion = OIMLoss(3, 3, scalar=1.0, size_average=False)
criterion.lut = torch.eye(3)
x = Variable(torch.randn(3, 3), requires_grad=True)
y = Variable(torch.range(0, 2).long())
loss = criterion(x, y)
loss.backward()
probs = F.softmax(x)
grads = probs.data - torch.eye(3)
abs_diff = torch.abs(grads - x.grad.data)
self.assertEquals(torch.log(probs).diag().sum(), -loss)
self.assertTrue(torch.max(abs_diff) < 1e-6)
def preProc2(x):
# Access the global variables
global P, expP, negExpP
P = P.type_as(x)
expP = expP.type_as(x)
negExpP = negExpP.type_as(x)
# Create a variable filled with -1. Second part of the condition
z = Variable(torch.zeros(x.size())).type_as(x)
absX = torch.abs(x)
cond1 = torch.gt(absX, negExpP)
cond2 = torch.le(absX, negExpP)
if (torch.sum(cond1) > 0).data.all():
x1 = torch.sign(x[cond1])
z[cond1] = x1
if (torch.sum(cond2) > 0).data.all():
x2 = x[cond2]*expP
z[cond2] = x2
return z
def _test_dropout(self, cls, input):
p = 0.2
input.fill_(1 - p)
module = cls(p)
input_var = Variable(input, requires_grad=True)
output = module(input_var)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
module = cls(p, True)
input_var = Variable(input.clone(), requires_grad=True)
output = module(input_var + 0)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
# Check that these don't raise errors
module.__repr__()
str(module)
def _test_dropout(self, cls, input):
p = 0.2
input.fill_(1 - p)
module = cls(p)
input_var = Variable(input, requires_grad=True)
output = module(input_var)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
module = cls(p, True)
input_var = Variable(input.clone(), requires_grad=True)
output = module(input_var + 0)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
# Check that these don't raise errors
module.__repr__()
str(module)
def forward(self, x, fast=False, unitTest=False):
start = time.time()
#Run gate
gates, expertInds = self.gate(x)
#Run experts
if unitTest:
vanilla, _ = self.vanillaExperts(x, gates, expertInds)
fast, _ = self.fastExperts(x, gates, expertInds)
return t.abs(vanilla - fast)
elif fast:
ret, cellTime = self.fastExperts(x, gates, expertInds)
else:
ret, cellTime = self.vanillaExperts(x, gates, expertInds)
forwardTime = time.time() - start
return ret, forwardTime, cellTime
def _test_dropout(self, cls, input):
p = 0.2
input.fill_(1 - p)
module = cls(p)
input_var = Variable(input, requires_grad=True)
output = module(input_var)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
module = cls(p, True)
input_var = Variable(input.clone(), requires_grad=True)
output = module(input_var + 0)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
# Check that these don't raise errors
module.__repr__()
str(module)
def test_AlphaDropout(self):
# generate random tensor with zero mean and unit std
input = torch.randn(5000)
mean = input.mean()
std = input.std()
for p in [0.2, 0.5, 0.8]:
module = nn.AlphaDropout(p)
input_var = Variable(input, requires_grad=True)
output = module(input_var)
# output mean should be close to input mean
self.assertLess(abs(output.data.mean() - mean), 0.1)
# output std should be close to input std
self.assertLess(abs(output.data.std() - std), 0.1)
output.backward(input)
def forward(self, logits, labels):
softmaxes = F.softmax(logits)
confidences, predictions = torch.max(softmaxes, 1)
accuracies = predictions.eq(labels)
ece = Variable(torch.zeros(1)).type_as(logits)
for bin_lower, bin_upper in zip(self.bin_lowers, self.bin_uppers):
# Calculated |confidence - accuracy| in each bin
in_bin = confidences.gt(bin_lower) * confidences.le(bin_upper)
prop_in_bin = in_bin.float().mean()
if prop_in_bin.data[0] > 0:
accuracy_in_bin = accuracies[in_bin].float().mean()
avg_confidence_in_bin = confidences[in_bin].mean()
ece += torch.abs(avg_confidence_in_bin- accuracy_in_bin) * prop_in_bin
return ece
def test_normal_gp_mll_forward():
covar = torch.Tensor([
[5, -3, 0],
[-3, 5, 0],
[0, 0, 2],
])
y = torch.randn(3)
actual = y.dot(covar.inverse().mv(y))
actual += math.log(np.linalg.det(covar.numpy()))
actual += math.log(2 * math.pi) * len(y)
actual *= -0.5
covarvar = Variable(covar)
yvar = Variable(y)
res = gpytorch.exact_gp_marginal_log_likelihood(covarvar, yvar)
assert(all(torch.abs(actual - res.data).div(res.data) < 0.1))
def test_normal_trace_log_det_quad_form_forward():
covar = torch.Tensor([
[5, -3, 0],
[-3, 5, 0],
[0, 0, 2],
])
mu_diffs = torch.Tensor([0, -1, 1])
chol_covar = torch.Tensor([
[1, -2, 0],
[0, 1, -2],
[0, 0, 1],
])
actual = mu_diffs.dot(covar.inverse().matmul(mu_diffs))
actual += math.log(np.linalg.det(covar.numpy()))
actual += (covar.inverse().matmul(chol_covar.t().matmul(chol_covar))).trace()
covarvar = Variable(covar)
chol_covarvar = Variable(chol_covar)
mu_diffsvar = Variable(mu_diffs)
res = gpytorch.trace_logdet_quad_form(mu_diffsvar, chol_covarvar, covarvar)
assert(all(torch.abs(actual - res.data).div(res.data) < 0.1))
def backward(self, grad_output):
z, log_phi_z = self.saved_tensors
log_phi_z_grad = z.new().resize_as_(z).zero_()
z_is_small = z.lt(-1)
z_is_not_small = 1 - z_is_small
if z_is_small.sum() > 0:
log_phi_z_grad[z_is_small] = torch.abs(self.denominator.div(self.numerator)).mul(math.sqrt(2 / math.pi))
exp = z[z_is_not_small].pow(2) \
.div(-2) \
.sub(log_phi_z[z_is_not_small]) \
.add(math.log(0.5))
log_phi_z_grad[z_is_not_small] = torch.exp(exp).mul(math.sqrt(2 / math.pi))
return log_phi_z_grad.mul(grad_output)
def _test_dropout(self, cls, input):
p = 0.2
input.fill_(1 - p)
module = cls(p)
input_var = Variable(input, requires_grad=True)
output = module(input_var)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
module = cls(p, True)
input_var = Variable(input.clone(), requires_grad=True)
output = module(input_var + 0)
self.assertLess(abs(output.data.mean() - (1 - p)), 0.05)
output.backward(input)
self.assertLess(abs(input_var.grad.data.mean() - (1 - p)), 0.05)
# Check that these don't raise errors
module.__repr__()
str(module)
def test_AlphaDropout(self):
# generate random tensor with zero mean and unit std
input = torch.randn(5000)
mean = input.mean()
std = input.std()
for p in [0.2, 0.5, 0.8]:
module = nn.AlphaDropout(p)
input_var = Variable(input, requires_grad=True)
output = module(input_var)
# output mean should be close to input mean
self.assertLess(abs(output.data.mean() - mean), 0.1)
# output std should be close to input std
self.assertLess(abs(output.data.std() - std), 0.1)
output.backward(input)
def forward(self, source_sentences, target_sentences):
"""
Supervised Learning of Universal Sentence Representations from Natural Language Inference Data
https://arxiv.org/abs/1705.02364
A Siamese text classification network made w/ the goal of creating sentence embeddings.
:param source_sentences: A tuple of Variable's representing padded sentence tensor batch
[seq. length, batch size, embed. size] and sentence lengths.
:param target_sentences: A tuple of Variable's representing padded sentence tensor batch
[seq. length, batch size, embed. size] and sentence lengths.
:return: Embedding. (batch size, # classes)
"""
u = self.encoder(source_sentences)
v = self.encoder(target_sentences)
features = torch.cat((u, v, torch.abs(u - v), u * v), 1)
return self.classifier(features)
def plot_examples(data_loader, model, epoch, plotter, ind = [0, 10, 20]):
# switch to evaluate mode
model.eval()
for i, (g, h, e, target) in enumerate(data_loader):
if i in ind:
subfolder_path = 'batch_' + str(i) + '_t_' + str(int(target[0][0])) + '/epoch_' + str(epoch) + '/'
if not os.path.isdir(args.plotPath + subfolder_path):
os.makedirs(args.plotPath + subfolder_path)
num_nodes = torch.sum(torch.sum(torch.abs(h[0, :, :]), 1) > 0)
am = g[0, 0:num_nodes, 0:num_nodes].numpy()
pos = h[0, 0:num_nodes, :].numpy()
plotter.plot_graph(am, position=pos, fig_name=subfolder_path+str(i) + '_input.png')
# Prepare input data
if args.cuda:
g, h, e, target = g.cuda(), h.cuda(), e.cuda(), target.cuda()
g, h, e, target = Variable(g), Variable(h), Variable(e), Variable(target)
# Compute output
model(g, h, e, lambda cls, id: plotter.plot_graph(am, position=pos, cls=cls,
fig_name=subfolder_path+ id))
def forward(self, input_1, input_2):
"""
:param : input_1
Size is (*, hidden_size)
:param input_2:
Size is (*, hidden_size)
:return:
Merged vectors, size is (*, 4*hidden size)
"""
assert input_1.size(-1) == input_2.size(-1)
mult_combined_vec = torch.mul(input_1, input_2)
diff_combined_vec = torch.abs(input_1 - input_2)
combined_vec = torch.cat((input_1,
input_2,
mult_combined_vec,
diff_combined_vec), input_1.dim()-1)
return combined_vec
def forward(self, images):
"""Extract image feature vectors."""
features = self.cnn(images)
# normalization in the image embedding space
features = l2norm(features)
# linear projection to the joint embedding space
features = self.fc(features)
# normalization in the joint embedding space
if not self.no_imgnorm:
features = l2norm(features)
# take the absolute value of the embedding (used in order embeddings)
if self.use_abs:
features = torch.abs(features)
return features
def simplified_topk(x, k):
''' Proof-of-concept implementation of simplified topk
Note all we neend the k-th largest vaule, thus an algorithm of log(n) complexity exists.
'''
original_size = None
if x.dim() > 2:
original_size = x.size()
x = x.view(x.size(0), -1)
ax = x.data.abs().sum(0).view(-1)
topk, ids = ax.topk(x.size(-1)-k, dim=0, largest=False)
y = x.clone()
# zero out small values
for id in ids:
y[:, id] = 0
if original_size:
y = y.view(original_size)
return y
def topk(x, k):
''' Proof-of-concept implementation of topk.
'''
original_size = None
if x.dim() > 2:
original_size = x.size()
x = x.view(x.size(0), -1)
ax = torch.abs(x.data)
topk, _ = ax.topk(k)
topk = topk[:, -1]
y = x.clone()
# zero out small values
y[ax < topk.repeat(x.size(-1), 1).transpose(0, 1)] = 0
if original_size:
y = y.view(original_size)
return y
def test_changing_model_reinitializes_optimizer(self, net, data):
# The idea is that we change the model using `set_params` to
# add parameters. Since the optimizer depends on the model
# parameters it needs to be reinitialized.
X, y = data
net.set_params(module__nonlin=F.relu)
net.fit(X, y)
net.set_params(module__nonlin=nn.PReLU())
assert isinstance(net.module_.nonlin, nn.PReLU)
d1 = net.module_.nonlin.weight.data.clone().cpu().numpy()
# make sure that we do not initialize again by making sure that
# the network is initialized and by using partial_fit.
assert net.initialized_
net.partial_fit(X, y)
d2 = net.module_.nonlin.weight.data.clone().cpu().numpy()
# all newly introduced parameters should have been trained (changed)
# by the optimizer after 10 epochs.
assert (abs(d2 - d1) > 1e-05).all()
def test_change_get_loss(self, net_cls, module_cls, data):
from skorch.utils import to_var
class MyNet(net_cls):
# pylint: disable=unused-argument
def get_loss(self, y_pred, y_true, X=None, training=False):
y_true = to_var(y_true, use_cuda=False)
loss_a = torch.abs(y_true.float() - y_pred[:, 1]).mean()
loss_b = ((y_true.float() - y_pred[:, 1]) ** 2).mean()
if training:
self.history.record_batch('loss_a', to_numpy(loss_a)[0])
self.history.record_batch('loss_b', to_numpy(loss_b)[0])
return loss_a + loss_b
X, y = data
net = MyNet(module_cls, max_epochs=1)
net.fit(X, y)
diffs = []
all_losses = net.history[
-1, 'batches', :, ('train_loss', 'loss_a', 'loss_b')]
diffs = [total - a - b for total, a, b in all_losses]
assert np.allclose(diffs, 0, atol=1e-7)
def allclose(x: T.FloatTensor,
y: T.FloatTensor,
rtol: float = 1e-05,
atol: float = 1e-08) -> bool:
"""
Test if all elements in the two tensors are approximately equal.
absolute(x - y) <= (atol + rtol * absolute(y))
Args:
x: A tensor.
y: A tensor.
rtol (optional): Relative tolerance.
atol (optional): Absolute tolerance.
returns:
bool: Check if all of the elements in the tensors are approximately equal.
"""
return tall(torch.abs(x - y).le((atol + rtol * torch.abs(y))))
def cost_matrix(x, y, p=2):
"Returns the matrix of $|x_i-y_j|^p$."
x_col = x.unsqueeze(1)
y_lin = y.unsqueeze(0)
c = torch.sum((torch.abs(x_col - y_lin)) ** p, 2)
return c
def forward(self, lvec, rvec):
mult_dist = torch.mul(lvec, rvec)
abs_dist = torch.abs(torch.add(lvec, -rvec))
vec_dist = torch.cat((mult_dist, abs_dist), 1)
out = F.sigmoid(self.wh(vec_dist))
out = F.log_softmax(self.wp(out))
return out
# putting the whole model together
def distance(self, A, B):
return torch.mean(torch.abs(A - B))
