def _mix_rbf_kernel(X, Y, sigma_list):
assert(X.size(0) == Y.size(0))
m = X.size(0)
Z = torch.cat((X, Y), 0)
ZZT = torch.mm(Z, Z.t())
diag_ZZT = torch.diag(ZZT).unsqueeze(1)
Z_norm_sqr = diag_ZZT.expand_as(ZZT)
exponent = Z_norm_sqr - 2 * ZZT + Z_norm_sqr.t()
K = 0.0
for sigma in sigma_list:
gamma = 1.0 / (2 * sigma**2)
K += torch.exp(-gamma * exponent)
return K[:m, :m], K[:m, m:], K[m:, m:], len(sigma_list)
python类cat()的实例源码
def forward(self, x, lengths, hidden):
# Basket Encoding
ub_seqs = [] # users' basket sequence
for user in x: # x shape (batch of user, time_step, indice of product) nested lists
embed_baskets = []
for basket in user:
basket = torch.LongTensor(basket).resize_(1, len(basket))
basket = basket.cuda() if self.config.cuda else basket # use cuda for acceleration
basket = self.encode(torch.autograd.Variable(basket)) # shape: 1, len(basket), embedding_dim
embed_baskets.append(self.pool(basket, dim = 1))
# concat current user's all baskets and append it to users' basket sequence
ub_seqs.append(torch.cat(embed_baskets, 1)) # shape: 1, num_basket, embedding_dim
# Input for rnn
ub_seqs = torch.cat(ub_seqs, 0).cuda() if self.config.cuda else torch.cat(ub_seqs, 0) # shape: batch_size, max_len, embedding_dim
packed_ub_seqs = torch.nn.utils.rnn.pack_padded_sequence(ub_seqs, lengths, batch_first=True) # packed sequence as required by pytorch
# RNN
output, h_u = self.rnn(packed_ub_seqs, hidden)
dynamic_user, _ = torch.nn.utils.rnn.pad_packed_sequence(output, batch_first=True) # shape: batch_size, max_len, embedding_dim
return dynamic_user, h_u
models.py 文件源码
项目：Structured-Self-Attentive-Sentence-Embedding
作者: ExplorerFreda
项目源码
文件源码
阅读 27
收藏 0
点赞 0
评论 0
def forward(self, inp, hidden):
outp = self.bilstm.forward(inp, hidden)[0]
size = outp.size() # [bsz, len, nhid]
compressed_embeddings = outp.view(-1, size[2]) # [bsz*len, nhid*2]
transformed_inp = torch.transpose(inp, 0, 1).contiguous() # [bsz, len]
transformed_inp = transformed_inp.view(size[0], 1, size[1]) # [bsz, 1, len]
concatenated_inp = [transformed_inp for i in range(self.attention_hops)]
concatenated_inp = torch.cat(concatenated_inp, 1) # [bsz, hop, len]
hbar = self.tanh(self.ws1(self.drop(compressed_embeddings))) # [bsz*len, attention-unit]
alphas = self.ws2(hbar).view(size[0], size[1], -1) # [bsz, len, hop]
alphas = torch.transpose(alphas, 1, 2).contiguous() # [bsz, hop, len]
penalized_alphas = alphas + (
-10000 * (concatenated_inp == self.dictionary.word2idx['<pad>']).float())
# [bsz, hop, len] + [bsz, hop, len]
alphas = self.softmax(penalized_alphas.view(-1, size[1])) # [bsz*hop, len]
alphas = alphas.view(size[0], self.attention_hops, size[1]) # [bsz, hop, len]
return torch.bmm(alphas, outp), alphas
def __call__(self, batch):
images, labels = zip(*batch)
imgH = self.imgH
imgW = self.imgW
if self.keep_ratio:
ratios = []
for image in images:
w, h = image.size
ratios.append(w / float(h))
ratios.sort()
max_ratio = ratios[-1]
imgW = int(np.floor(max_ratio * imgH))
imgW = max(imgH * self.min_ratio, imgW) # assure imgH >= imgW
transform = resizeNormalize((imgW, imgH))
images = [transform(image) for image in images]
images = torch.cat([t.unsqueeze(0) for t in images], 0)
return images, labels
def make_sprite(label_img, save_path):
import math
import torch
import torchvision
# this ensures the sprite image has correct dimension as described in
# https://www.tensorflow.org/get_started/embedding_viz
nrow = int(math.ceil((label_img.size(0)) ** 0.5))
# augment images so that #images equals nrow*nrow
label_img = torch.cat((label_img, torch.randn(nrow ** 2 - label_img.size(0), *label_img.size()[1:]) * 255), 0)
# Dirty fix: no pixel are appended by make_grid call in save_image (https://github.com/pytorch/vision/issues/206)
xx = torchvision.utils.make_grid(torch.Tensor(1, 3, 32, 32), padding=0)
if xx.size(2) == 33:
sprite = torchvision.utils.make_grid(label_img, nrow=nrow, padding=0)
sprite = sprite[:, 1:, 1:]
torchvision.utils.save_image(sprite, os.path.join(save_path, 'sprite.png'))
else:
torchvision.utils.save_image(label_img, os.path.join(save_path, 'sprite.png'), nrow=nrow, padding=0)
def log_img(x,name,iteration=0,nrow=8):
def log_img_final(x,name,iteration=0,nrow=8):
vutils.save_image(
x,
LOGDIR+name+'_'+str(iteration)+'.png',
nrow=nrow,
)
vis.images(
x.cpu().numpy(),
win=str(MULTI_RUN)+'-'+name,
opts=dict(caption=str(MULTI_RUN)+'-'+name+'_'+str(iteration)),
nrow=nrow,
)
if params['REPRESENTATION']==chris_domain.VECTOR:
x = vector2image(x)
x = x.squeeze(1)
if params['DOMAIN']=='2Dgrid':
if x.size()[1]==2:
log_img_final(x[:,0:1,:,:],name+'_b',iteration,nrow)
log_img_final(x[:,1:2,:,:],name+'_a',iteration,nrow)
x = torch.cat([x,x[:,0:1,:,:]],1)
log_img_final(x,name,iteration,nrow)
def forward(self, x, hint):
v = self.toH(hint)
x0 = self.to0(x)
x1 = self.to1(x0)
x2 = self.to2(x1)
x3 = self.to3(torch.cat([x2, v], 1))
x4 = self.to4(x3)
x = self.tunnel4(x4)
x = self.tunnel3(torch.cat([x, x3.detach()], 1))
x = self.tunnel2(torch.cat([x, x2.detach()], 1))
x = self.tunnel1(torch.cat([x, x1.detach()], 1))
x = F.tanh(self.exit(torch.cat([x, x0.detach()], 1)))
return x
def forward(self, x, hint):
v = self.toH(hint)
x0 = self.to0(x)
x1 = self.to1(x0)
x2 = self.to2(x1)
x3 = self.to3(torch.cat([x2, v], 1))
x4 = self.to4(x3)
x = self.tunnel4(x4)
x = self.tunnel3(torch.cat([x, x3.detach()], 1))
x = self.tunnel2(torch.cat([x, x2.detach()], 1))
x = self.tunnel1(torch.cat([x, x1.detach()], 1))
x = F.tanh(self.exit(torch.cat([x, x0.detach()], 1)))
return x
def forward(self, x, hint):
v = self.toH(hint)
x0 = self.to0(x)
x1 = self.to1(x0)
x2 = self.to2(x1)
x3 = self.to3(torch.cat([x2, v], 1))
x4 = self.to4(x3)
x = self.tunnel4(x4)
x = self.tunnel3(torch.cat([x, x3.detach()], 1))
x = self.tunnel2(torch.cat([x, x2.detach()], 1))
x = self.tunnel1(torch.cat([x, x1.detach()], 1))
x = F.tanh(self.exit(torch.cat([x, x0.detach()], 1)))
return x
def forward(self, x, hint):
v = self.toH(hint)
x0 = self.to0(x)
x1 = self.to1(x0)
x2 = self.to2(x1)
x3 = self.to3(torch.cat([x2, v], 1))
x4 = self.to4(x3)
x = self.tunnel4(x4)
x = self.tunnel3(torch.cat([x, x3.detach()], 1))
x = self.tunnel2(torch.cat([x, x2.detach()], 1))
x = self.tunnel1(torch.cat([x, x1.detach()], 1))
x = F.tanh(self.exit(torch.cat([x, x0.detach()], 1)))
return x
def forward(self, x, hint):
v = self.toH(hint)
x0 = self.to0(x)
x1 = self.to1(x0)
x2 = self.to2(x1)
x3 = self.to3(torch.cat([x2, v], 1))
x4 = self.to4(x3)
x = self.tunnel4(x4)
x = self.tunnel3(torch.cat([x, x3.detach()], 1))
x = self.tunnel2(torch.cat([x, x2.detach()], 1))
x = self.tunnel1(torch.cat([x, x1.detach()], 1))
x = F.tanh(self.exit(torch.cat([x, x0.detach()], 1)))
return x
def forward(self, x, hint):
v = self.toH(hint)
x0 = self.to0(x)
x1 = self.to1(x0)
x2 = self.to2(x1)
x3 = self.to3(torch.cat([x2, v], 1))
x4 = self.to4(x3)
x = self.tunnel4(x4)
x = self.tunnel3(torch.cat([x, x3.detach()], 1))
x = self.tunnel2(torch.cat([x, x2.detach()], 1))
x = self.tunnel1(torch.cat([x, x1.detach()], 1))
x = F.tanh(self.exit(torch.cat([x, x0.detach()], 1)))
return x
def forward(self, x, hint):
v = self.toH(hint)
x0 = self.to0(x)
x1 = self.to1(x0)
x2 = self.to2(x1)
x3 = self.to3(torch.cat([x2, v], 1))
x4 = self.to4(x3)
x = self.tunnel4(x4)
x = self.tunnel3(torch.cat([x, x3.detach()], 1))
x = self.tunnel2(torch.cat([x, x2.detach()], 1))
x = self.tunnel1(torch.cat([x, x1.detach()], 1))
x = F.tanh(self.exit(torch.cat([x, x0.detach()], 1)))
return x
def _loop(self):
done = False
total_reward, reward, iter = 0, 0, 0
self.state = self.env.reset()
while not done:
action = self.policy()
_state, reward, done, _ = self.env.step(action)
# if _state is terminal, state value is 0
v = 0 if done else self.state_value(_state)
delta = reward + self.gamma * v - self.state_value(self.state)
# \nabla_w v = s, since v = s^{\tim} w
self.state_value_weight += self.beta * delta * to_tensor(self.state).float()
# \pi(a) = x^{\top}(a)w, where x is feature and w is weight
# \nabla\ln\pi(a) = x(a)\sum_b \pi(b)x(b)
direction = self.feature(_state, action) - sum(
[self.softmax @ torch.cat([self.feature(_state, a).unsqueeze(0) for a in self.actions])])
self.weight += self.alpha * pow(self.gamma, iter) * delta * direction
total_reward += reward
self.state = _state
iter += 1
return total_reward
def _loop(self):
done = False
total_reward, reward, iter = 0, 0, 0
self.state = self.env.reset()
weight = self.weight
while not done:
action = self.policy()
_state, reward, done, _ = self.env.step(action)
# use current weight to generate an episode
# \pi(a) = x^{\top}(a)w, where x is feature and w is weight
# \nabla\ln\pi(a) = x(a)\sum_b \pi(b)x(b)
delta = reward - self.state_value(_state)
self.state_value_weight += self.beta * delta * to_tensor(_state).float()
direction = self.feature(_state, action) - sum(
[self.softmax @ torch.cat([self.feature(_state, a).unsqueeze(0) for a in self.actions])])
weight += self.alpha * pow(self.gamma, iter) * delta * direction
total_reward += reward
iter += 1
# update weight
self.weight = weight
return total_reward
def _loop(self):
done = False
total_reward, reward, iter = 0, 0, 0
self.state = self.env.reset()
weight = self.weight
while not done:
action = self.policy()
_state, reward, done, _ = self.env.step(action)
# use current weight to generate an episode
# \pi(a) = x^{\top}(a)w, where x is feature and w is weight
# \nabla\ln\pi(a) = x(a)\sum_b \pi(b)x(b)
direction = self.feature(_state, action) - sum(
[self.softmax @ torch.cat([self.feature(_state, a).unsqueeze(0) for a in self.actions])])
weight += self.alpha * pow(self.gamma, iter) * reward * direction
total_reward += reward
iter += 1
# update weight
self.weight = weight
return total_reward
def forward(self, mid_input, global_input):
w = mid_input.size()[2]
h = mid_input.size()[3]
global_input = global_input.unsqueeze(2).unsqueeze(2).expand_as(mid_input)
fusion_layer = torch.cat((mid_input, global_input), 1)
fusion_layer = fusion_layer.permute(2, 3, 0, 1).contiguous()
fusion_layer = fusion_layer.view(-1, 512)
fusion_layer = self.bn1(self.fc1(fusion_layer))
fusion_layer = fusion_layer.view(w, h, -1, 256)
x = fusion_layer.permute(2, 3, 0, 1).contiguous()
x = F.relu(self.bn2(self.conv1(x)))
x = self.upsample(x)
x = F.relu(self.bn3(self.conv2(x)))
x = F.relu(self.bn4(self.conv3(x)))
x = self.upsample(x)
x = F.sigmoid(self.bn5(self.conv4(x)))
x = self.upsample(self.conv5(x))
return x
def query(self, images):
if self.pool_size == 0:
return images
return_images = []
for image in images.data:
image = torch.unsqueeze(image, 0)
if self.num_imgs < self.pool_size:
self.num_imgs = self.num_imgs + 1
self.images.append(image)
return_images.append(image)
else:
p = random.uniform(0, 1)
if p > 0.5:
random_id = random.randint(0, self.pool_size-1)
tmp = self.images[random_id].clone()
self.images[random_id] = image
return_images.append(tmp)
else:
return_images.append(image)
return_images = Variable(torch.cat(return_images, 0))
return return_images
def default_collate(batch):
"Puts each data field into a tensor with outer dimension batch size"
if torch.is_tensor(batch[0]):
return torch.cat([t.view(1, *t.size()) for t in batch], 0)
elif isinstance(batch[0], int):
return torch.LongTensor(batch)
elif isinstance(batch[0], float):
return torch.DoubleTensor(batch)
elif isinstance(batch[0], str):
return batch
elif isinstance(batch[0], collections.Iterable):
# if each batch element is not a tensor, then it should be a tuple
# of tensors; in that case we collate each element in the tuple
transposed = zip(*batch)
return [default_collate(samples) for samples in transposed]
raise TypeError(("batch must contain tensors, numbers, or lists; found {}"
.format(type(batch[0]))))
def _numerical_jacobian(self, module, input, jacobian_input=True, jacobian_parameters=True):
output = self._forward(module, input)
output_size = output.nelement()
if jacobian_parameters:
param, d_param = self._get_parameters(module)
def fw(input):
out = self._forward(module, input)
if isinstance(out, Variable):
return out.data
return out
res = tuple()
# TODO: enable non-contig tests
input = contiguous(input)
if jacobian_input:
res += get_numerical_jacobian(fw, input, input),
if jacobian_parameters:
res += torch.cat(list(get_numerical_jacobian(fw, input, p) for p in param), 0),
return res
def test_cat(self):
SIZE = 10
# 2-arg cat
for dim in range(3):
x = torch.rand(13, SIZE, SIZE).transpose(0, dim)
y = torch.rand(17, SIZE, SIZE).transpose(0, dim)
res1 = torch.cat((x, y), dim)
self.assertEqual(res1.narrow(dim, 0, 13), x, 0)
self.assertEqual(res1.narrow(dim, 13, 17), y, 0)
# Check iterables
for dim in range(3):
x = torch.rand(13, SIZE, SIZE).transpose(0, dim)
y = torch.rand(17, SIZE, SIZE).transpose(0, dim)
z = torch.rand(19, SIZE, SIZE).transpose(0, dim)
res1 = torch.cat((x, y, z), dim)
self.assertEqual(res1.narrow(dim, 0, 13), x, 0)
self.assertEqual(res1.narrow(dim, 13, 17), y, 0)
self.assertEqual(res1.narrow(dim, 30, 19), z, 0)
self.assertRaises(ValueError, lambda: torch.cat([]))
def forward(self, non_rgb_state, rgb_state, h):
x = self.relu(self.conv1(rgb_state))
x = self.relu(self.conv2(x))
x = x.view(x.size(0), -1)
x = self.fc1(torch.cat((x, non_rgb_state), 1))
h = self.lstm(x, h) # h is (hidden state, cell state)
x = h[0]
policy1 = self.softmax(self.fc_actor1(x)).clamp(
max=1 - 1e-20) # Prevent 1s and hence NaNs
policy2 = self.softmax(self.fc_actor2(x)).clamp(max=1 - 1e-20)
policy3 = self.softmax(self.fc_actor3(x)).clamp(max=1 - 1e-20)
policy4 = self.softmax(self.fc_actor4(x)).clamp(max=1 - 1e-20)
policy5 = self.softmax(self.fc_actor5(x)).clamp(max=1 - 1e-20)
policy6 = self.softmax(self.fc_actor6(x)).clamp(max=1 - 1e-20)
V = self.fc_critic(x)
return (policy1, policy2, policy3, policy4, policy5, policy6), V, h
def encode(matched, priors, variances):
"""Encode the variances from the priorbox layers into the ground truth boxes
we have matched (based on jaccard overlap) with the prior boxes.
Args:
matched: (tensor) Coords of ground truth for each prior in point-form
Shape: [num_priors, 4].
priors: (tensor) Prior boxes in center-offset form
Shape: [num_priors,4].
variances: (list[float]) Variances of priorboxes
Return:
encoded boxes (tensor), Shape: [num_priors, 4]
"""
# dist b/t match center and prior's center
g_cxcy = (matched[:, :2] + matched[:, 2:])/2 - priors[:, :2]
# encode variance
g_cxcy /= (variances[0] * priors[:, 2:])
# match wh / prior wh
g_wh = (matched[:, 2:] - matched[:, :2]) / priors[:, 2:]
g_wh = torch.log(g_wh) / variances[1]
# return target for smooth_l1_loss
return torch.cat([g_cxcy, g_wh], 1) # [num_priors,4]
# Adapted from https://github.com/Hakuyume/chainer-ssd
def decode(loc, priors, variances):
"""Decode locations from predictions using priors to undo
the encoding we did for offset regression at train time.
Args:
loc (tensor): location predictions for loc layers,
Shape: [num_priors,4]
priors (tensor): Prior boxes in center-offset form.
Shape: [num_priors,4].
variances: (list[float]) Variances of priorboxes
Return:
decoded bounding box predictions
"""
boxes = torch.cat((
priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
boxes[:, :2] -= boxes[:, 2:] / 2
boxes[:, 2:] += boxes[:, :2]
return boxes
def query(self, images):
if self.pool_size == 0:
return images
return_images = []
for image in images.data:
image = torch.unsqueeze(image, 0)
if self.num_imgs < self.pool_size:
self.num_imgs = self.num_imgs + 1
self.images.append(image)
return_images.append(image)
else:
p = random.uniform(0, 1)
if p > 0.5:
random_id = random.randint(0, self.pool_size-1)
tmp = self.images[random_id].clone()
self.images[random_id] = image
return_images.append(tmp)
else:
return_images.append(image)
return_images = Variable(torch.cat(return_images, 0))
return return_images
def validate(val_loader, net, criterion):
net.eval()
batch_outputs = []
batch_labels = []
for vi, data in enumerate(val_loader, 0):
inputs, labels = data
inputs = Variable(inputs, volatile=True).cuda()
labels = Variable(labels.float(), volatile=True).cuda()
outputs = net(inputs)
batch_outputs.append(outputs)
batch_labels.append(labels)
batch_outputs = torch.cat(batch_outputs)
batch_labels = torch.cat(batch_labels)
val_loss = criterion(batch_outputs, batch_labels)
val_loss = val_loss.data[0]
print '--------------------------------------------------------'
print '[val_loss %.4f]' % val_loss
net.train()
return val_loss
def test_elmo_4D_input(self):
sentences = [[['The', 'sentence', '.'],
['ELMo', 'helps', 'disambiguate', 'ELMo', 'from', 'Elmo', '.']],
[['1', '2'], ['1', '2', '3', '4', '5', '6', '7']],
[['1', '2', '3', '4', '50', '60', '70'], ['The']]]
all_character_ids = []
for batch_sentences in sentences:
all_character_ids.append(self._sentences_to_ids(batch_sentences))
# (2, 3, 7, 50)
character_ids = torch.cat([ids.unsqueeze(1) for ids in all_character_ids], dim=1)
embeddings_4d = self.elmo(character_ids)
# Run the individual batches.
embeddings_3d = []
for char_ids in all_character_ids:
self.elmo._elmo_lstm._elmo_lstm.reset_states()
embeddings_3d.append(self.elmo(char_ids))
for k in range(3):
numpy.testing.assert_array_almost_equal(
embeddings_4d['elmo_representations'][0][:, k, :, :].data.numpy(),
embeddings_3d[k]['elmo_representations'][0].data.numpy()
)
def forward(self, input, last_context, last_hidden, encoder_outputs):
# input.size() = (B, 1), last_context.size() = (B, H), last_hidden.size() = (L, B, H), encoder_outputs.size() = (B, S, H)
# word_embedded.size() = (B, 1, H)
# print input.size()
word_embedded = self.embedding(input)
# rnn_input.size() = (B, 1, 2H), rnn_output.size() = (B, 1, H)
# print word_embedded.size(), last_context.unsqueeze(1).size()
rnn_input = torch.cat((word_embedded, last_context.unsqueeze(1)), -1)
rnn_output, hidden = self.gru(rnn_input, last_hidden)
rnn_output = rnn_output.squeeze(1) # B x S=1 x H -> B x H
# atten_weights.size() = (B, S)
attn_weights = self.attn(rnn_output, encoder_outputs)
context = attn_weights.unsqueeze(1).bmm(encoder_outputs).squeeze(1) # B x H
# TODO tanh?
# Final output layer (next word prediction) using the RNN hidden state and context vector
output = self.out(torch.cat((rnn_output, context), -1)) # B x V
# Return final output, hidden state, and attention weights (for visualization)
# output.size() = (B, V)
return output, context, hidden, attn_weights
def forward(self, x):
"""
Compute the forward pass of the composite transformation H(x),
where x is the concatenation of the current and all preceding
feature maps.
"""
if self.bottleneck:
out = self.conv1(F.relu(self.bn1(x)))
if self.p > 0:
out = F.dropout(out, p=self.p, training=self.training)
out = self.conv2(F.relu(self.bn2(out)))
if self.p > 0:
out = F.dropout(out, p=self.p, training=self.training)
else:
out = self.conv2(F.relu(self.bn2(x)))
if self.p > 0:
out = F.dropout(out, p=self.p, training=self.training)
return torch.cat((x, out), 1)
def forward(self, q, k, v, attn_mask):
d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
residual = q
bsz, len_q, d_model = q.size()
len_k, len_v = k.size(1), v.size(1)
def reshape(x):
"""[bsz, len, d_*] -> [n_head x (bsz*len) x d_*]"""
return x.repeat(n_head, 1, 1).view(n_head, -1, d_model)
q_s, k_s, v_s = map(reshape, [q, k, v])
q_s = torch.bmm(q_s, self.w_qs).view(-1, len_q, d_k)
k_s = torch.bmm(k_s, self.w_ks).view(-1, len_k, d_k)
v_s = torch.bmm(v_s, self.w_vs).view(-1, len_v, d_v)
outputs = self.attention(q_s, k_s, v_s, attn_mask.repeat(n_head, 1, 1))
outputs = torch.cat(torch.split(outputs, bsz, dim=0), dim=-1).view(-1, n_head*d_v)
outputs = F.dropout(self.w_o(outputs), p=self.dropout).view(bsz, len_q, -1)
return self.lm(outputs + residual)
