def classifier(self, xs):
"""
classify an image (or a batch of images)
:param xs: a batch of scaled vectors of pixels from an image
:return: a batch of the corresponding class labels (as one-hots)
"""
# use the trained model q(y|x) = categorical(alpha(x))
# compute all class probabilities for the image(s)
alpha = self.encoder_y.forward(xs)
# get the index (digit) that corresponds to
# the maximum predicted class probability
res, ind = torch.topk(alpha, 1)
# convert the digit(s) to one-hot tensor(s)
ys = Variable(torch.zeros(alpha.size()))
ys = ys.scatter_(1, ind, 1.0)
return ys
python类topk()的实例源码
def object_detection_gt_boxes(self, image_path, gt_boxes):
min_score = 1/150.
image = cv2.imread(image_path)
# print 'image.shape', image.shape
im_data, im_scales = self.get_image_blob_noscale(image)
gt_boxes[:, :4] = gt_boxes[:, :4] * im_scales[0]
# print 'im_data.shape', im_data.shape
# print 'im_scales', im_scales
im_info = np.array(
[[im_data.shape[1], im_data.shape[2], im_scales[0]]],
dtype=np.float32)
object_result = self(im_data, im_info, gt_boxes)[0]
cls_prob_object, bbox_object, object_rois = object_result[:]
prob_object = F.softmax(cls_prob_object)
prob = prob_object.cpu().data
top_5_cls = torch.topk(prob[:, 1:], 5, dim=1)
# print 'im_scales[0]', im_scales[0]
return top_5_cls[1].numpy()
def train(self):
self.loadData()
try:
self.load_state_dict(torch.load(self.model_path+'params.pkl'))
except Exception as e:
print(e)
print("No model!")
loss_track = []
for epoch in range(self.max_epoches):
start = time.time()
inputs, targets = self.next(1, shuffle=False)
loss, logits = self.step(inputs, targets, self.max_length)
loss_track.append(loss)
_,v = torch.topk(logits, 1)
pre = v.cpu().data.numpy().T.tolist()[0][0]
tar = targets.cpu().data.numpy().T.tolist()[0]
stop = time.time()
if epoch % self.show_epoch == 0:
print("-"*50)
print("epoch:", epoch)
print(" loss:", loss)
print(" target:%s\n output:%s" % (tar, pre))
print(" per-time:", (stop-start))
torch.save(self.state_dict(), self.model_path+'params.pkl')
def infer(self, input_variable):
input_length = input_variable.size()[0]
encoder_hidden = self.encoder.init_hidden()
encoder_outputs, encoder_hidden = self.encoder(input_variable, encoder_hidden)
decoder_input = Variable(torch.LongTensor([[SOS_token]]))
decoder_context = Variable(torch.zeros(1, self.decoder.hidden_size))
decoder_hidden = encoder_hidden
if USE_CUDA:
decoder_input = decoder_input.cuda()
decoder_context = decoder_context.cuda()
decoder_outputs = []
for i in range(self.max_length):
decoder_output, decoder_context, decoder_hidden, decoder_attention = self.decoder(decoder_input, decoder_context, decoder_hidden, encoder_outputs)
decoder_outputs.append(decoder_output.unsqueeze(0))
topv, topi = decoder_output.data.topk(1)
ni = topi[0][0]
decoder_input = Variable(torch.LongTensor([[ni]])) # Chosen word is next input
if USE_CUDA: decoder_input = decoder_input.cuda()
if ni == EOS_token: break
decoder_outputs = torch.cat(decoder_outputs, 0)
return decoder_outputs
def predict(self, x):
batch_size, dims = x.size()
query = F.normalize(self.query_proj(x), dim=1)
# Find the k-nearest neighbors of the query
scores = torch.matmul(query, torch.t(self.keys_var))
cosine_similarity, topk_indices_var = torch.topk(scores, self.top_k, dim=1)
# softmax of cosine similarities - embedding
softmax_score = F.softmax(self.softmax_temperature * cosine_similarity)
# retrive memory values - prediction
y_hat_indices = topk_indices_var.data[:, 0]
y_hat = self.values[y_hat_indices]
return y_hat, softmax_score
def _allocation(self, usage_vb, epsilon=1e-6):
"""
computes allocation by sorting usage, a = a_t[\phi_t[j]]
variables needed:
usage_vb: [batch_size x mem_hei]
-> indicating current memory usage, this is equal to u_t in
the paper when we only have one write head, but for
multiple write heads, one should update the usage while
iterating through the write heads to take into account the
allocation returned by this function
returns:
alloc_vb: [batch_size x num_write_heads x mem_hei]
"""
# ensure values are not too small prior to cumprod
usage_vb = epsilon + (1 - epsilon) * usage_vb
# NOTE: we sort usage in ascending order
sorted_usage_vb, indices_vb = torch.topk(usage_vb, k=self.mem_hei, dim=1, largest=False)
# to imitate tf.cumrprod(exclusive=True) https://discuss.pytorch.org/t/cumprod-exclusive-true-equivalences/2614/8
cat_sorted_usage_vb = torch.cat((Variable(torch.ones(self.batch_size, 1)).type(self.dtype), sorted_usage_vb), 1)[:, :-1]
# TODO: seems we have to wait for this PR: https://github.com/pytorch/pytorch/pull/1439
prod_sorted_usage_vb = fake_cumprod(cat_sorted_usage_vb)
# prod_sorted_usage_vb = torch.cumprod(cat_sorted_usage_vb, dim=1) # TODO: use this once the PR is ready
# alloc_weight_vb = (1 - sorted_usage_vb) * prod_sorted_usage_vb # equ. (1) # 0.1.12
alloc_weight_vb = (1 - sorted_usage_vb) * prod_sorted_usage_vb.squeeze() # equ. (1) # 0.2.0
_, indices_vb = torch.topk(indices_vb, k=self.mem_hei, dim=1, largest=False)
alloc_weight_vb = alloc_weight_vb.gather(1, indices_vb)
return alloc_weight_vb
def update(self, query, y, y_hat, y_hat_indices):
batch_size, dims = query.size()
# 1) Untouched: Increment memory by 1
self.age += 1
# Divide batch by correctness
result = torch.squeeze(torch.eq(y_hat, torch.unsqueeze(y.data, dim=1))).float()
incorrect_examples = torch.squeeze(torch.nonzero(1-result))
correct_examples = torch.squeeze(torch.nonzero(result))
incorrect = len(incorrect_examples.size()) > 0
correct = len(correct_examples.size()) > 0
# 2) Correct: if V[n1] = v
# Update Key k[n1] <- normalize(q + K[n1]), Reset Age A[n1] <- 0
if correct:
correct_indices = y_hat_indices[correct_examples]
correct_keys = self.keys[correct_indices]
correct_query = query.data[correct_examples]
new_correct_keys = F.normalize(correct_keys + correct_query, dim=1)
self.keys[correct_indices] = new_correct_keys
self.age[correct_indices] = 0
# 3) Incorrect: if V[n1] != v
# Select item with oldest age, Add random offset - n' = argmax_i(A[i]) + r_i
# K[n'] <- q, V[n'] <- v, A[n'] <- 0
if incorrect:
incorrect_size = incorrect_examples.size()[0]
incorrect_query = query.data[incorrect_examples]
incorrect_values = y.data[incorrect_examples]
age_with_noise = self.age + random_uniform((self.memory_size, 1), -self.age_noise, self.age_noise, cuda=True)
topk_values, topk_indices = torch.topk(age_with_noise, incorrect_size, dim=0)
oldest_indices = torch.squeeze(topk_indices)
self.keys[oldest_indices] = incorrect_query
self.values[oldest_indices] = incorrect_values
self.age[oldest_indices] = 0
def run_epoch(loader, model, criterion, optimizer, epoch=0, n_epochs=0, train=True):
time_meter = Meter(name='Time', cum=True)
loss_meter = Meter(name='Loss', cum=False)
error_meter = Meter(name='Error', cum=False)
if train:
model.train()
print('Training')
else:
model.eval()
print('Evaluating')
end = time.time()
for i, (input, target) in enumerate(loader):
if train:
model.zero_grad()
optimizer.zero_grad()
# Forward pass
input_var = Variable(input, volatile=(not train)).cuda(async=True)
target_var = Variable(target, volatile=(not train), requires_grad=False).cuda(async=True)
output_var = model(input_var)
loss = criterion(output_var, target_var)
# Backward pass
if train:
loss.backward()
optimizer.step()
optimizer.n_iters = optimizer.n_iters + 1 if hasattr(optimizer, 'n_iters') else 1
# Accounting
_, predictions_var = torch.topk(output_var, 1)
error = 1 - torch.eq(predictions_var, target_var).float().mean()
batch_time = time.time() - end
end = time.time()
# Log errors
time_meter.update(batch_time)
loss_meter.update(loss)
error_meter.update(error)
print(' '.join([
'%s: (Epoch %d of %d) [%04d/%04d]' % ('Train' if train else 'Eval',
epoch, n_epochs, i + 1, len(loader)),
str(time_meter),
str(loss_meter),
str(error_meter),
]))
return time_meter.value(), loss_meter.value(), error_meter.value()
def sample(vocab, video_feat, decoder, video_path, vid):
# ?????????????????
img_dir = os.path.join(visual_dir, str(vid))
if not os.path.exists(img_dir):
os.mkdir(img_dir)
frame_list = open_video(video_path)
if use_cuda:
video_feat = video_feat.cuda()
video_feat = video_feat.unsqueeze(0)
outputs, attens = decoder.sample(video_feat)
words = []
for i, token in enumerate(outputs.data.squeeze()):
if token == vocab('<end>'):
break
word = vocab.idx2word[token]
print(word)
words.append(word)
v, k = torch.topk(attens[i], 5)
# pair = zip(v.data[0], k.data[0])
# print(pair)
selected_id = k.data[0][0]
selected_frame = frame_list[selected_id]
cv2.imshow('Attend', selected_frame)
cv2.imwrite(os.path.join(img_dir, '%d_%d_%s.jpg' % (i, selected_id,
word)), selected_frame)
# ???????
sal = psal.get_saliency_rbd(selected_frame).astype('uint8')
cv2.imwrite(os.path.join(img_dir, '%d_%d_%s.jpg' % (i, selected_id,
'saliency')), sal)
binary_sal = psal.binarise_saliency_map(sal, method='adaptive')
I = binary_sal[:, :, np.newaxis]
binary_mask = np.concatenate((I, I, I), axis=2)
foreground_img = np.multiply(selected_frame, binary_mask).astype('uint8')
cv2.imwrite(os.path.join(img_dir, '%d_%d_%s.jpg' % (i, selected_id,
'foreground')), foreground_img)
k = cv2.waitKey(500)
if k == ord('n'):
return
caption = ' '.join(words)
print(caption)
def run_epoch(loader, model, criterion, optimizer, epoch=0, n_epochs=0, train=True):
time_meter = Meter(name='Time', cum=True)
loss_meter = Meter(name='Loss', cum=False)
error_meter = Meter(name='Error', cum=False)
if train:
model.train()
print('Training')
else:
model.eval()
print('Evaluating')
end = time.time()
for i, (input, target) in enumerate(loader):
if train:
model.zero_grad()
optimizer.zero_grad()
# Forward pass
input_var = Variable(input, volatile=(not train)).cuda(async=True)
target_var = Variable(target, volatile=(not train), requires_grad=False).cuda(async=True)
output_var = model(input_var)
loss = criterion(output_var, target_var)
# Backward pass
if train:
loss.backward()
optimizer.step()
optimizer.n_iters = optimizer.n_iters + 1 if hasattr(optimizer, 'n_iters') else 1
# Accounting
_, predictions_var = torch.topk(output_var, 1)
error = 1 - torch.eq(predictions_var, target_var).float().mean()
batch_time = time.time() - end
end = time.time()
# Log errors
time_meter.update(batch_time)
loss_meter.update(loss)
error_meter.update(error)
print(' '.join([
'%s: (Epoch %d of %d) [%04d/%04d]' % ('Train' if train else 'Eval',
epoch, n_epochs, i + 1, len(loader)),
str(time_meter),
str(loss_meter),
str(error_meter),
]))
return time_meter.value(), loss_meter.value(), error_meter.value()
def step(self, input_variable, target_variable, max_length):
teacher_forcing_ratio = 0.1
clip = 5.0
loss = 0 # Added onto for each word
self.encoder_optimizer.zero_grad()
self.decoder_optimizer.zero_grad()
input_length = input_variable.size()[0]
target_length = target_variable.size()[0]
encoder_hidden = self.encoder.init_hidden()
encoder_outputs, encoder_hidden = self.encoder(input_variable, encoder_hidden)
decoder_input = Variable(torch.LongTensor([[SOS_token]]))
decoder_context = Variable(torch.zeros(1, self.decoder.hidden_size))
decoder_hidden = encoder_hidden # Use last hidden state from encoder to start decoder
if USE_CUDA:
decoder_input = decoder_input.cuda()
decoder_context = decoder_context.cuda()
decoder_outputs = []
use_teacher_forcing = random.random() < teacher_forcing_ratio
use_teacher_forcing = True
if use_teacher_forcing:
for di in range(target_length):
decoder_output, decoder_context, decoder_hidden, decoder_attention = self.decoder(decoder_input, decoder_context, decoder_hidden, encoder_outputs)
loss += self.criterion(decoder_output, target_variable[di])
decoder_input = target_variable[di]
decoder_outputs.append(decoder_output.unsqueeze(0))
else:
for di in range(target_length):
decoder_output, decoder_context, decoder_hidden, decoder_attention = self.decoder(decoder_input, decoder_context, decoder_hidden, encoder_outputs)
loss += self.criterion(decoder_output, target_variable[di])
decoder_outputs.append(decoder_output.unsqueeze(0))
topv, topi = decoder_output.data.topk(1)
ni = topi[0][0]
decoder_input = Variable(torch.LongTensor([[ni]]))
if USE_CUDA: decoder_input = decoder_input.cuda()
if ni == EOS_token: break
loss.backward()
torch.nn.utils.clip_grad_norm(self.encoder.parameters(), clip)
torch.nn.utils.clip_grad_norm(self.decoder.parameters(), clip)
self.encoder_optimizer.step()
self.decoder_optimizer.step()
decoder_outputs = torch.cat(decoder_outputs, 0)
return loss.data[0] / target_length, decoder_outputs
def predict(self):
try:
self.load_state_dict(torch.load(self.model_path+'params.pkl'))
except Exception as e:
print(e)
print("No model!")
loss_track = []
# ????
str_to_vec = {}
with open("./data/enc.vocab") as enc_vocab:
for index,word in enumerate(enc_vocab.readlines()):
str_to_vec[word.strip()] = index
vec_to_str = {}
with open("./data/dec.vocab") as dec_vocab:
for index,word in enumerate(dec_vocab.readlines()):
vec_to_str[index] = word.strip()
while True:
input_strs = input("me > ")
# ??????
segement = jieba.lcut(input_strs)
input_vec = [str_to_vec.get(i, 3) for i in segement]
input_vec = self.make_infer_fd(input_vec)
# inference
if self.beam_search:
samples = self.beamSearchDecoder(input_vec)
for sample in samples:
outstrs = []
for i in sample[0]:
if i == 1:
break
outstrs.append(vec_to_str.get(i, "Un"))
print("ai > ", "".join(outstrs), sample[3])
else:
logits = self.infer(input_vec)
_,v = torch.topk(logits, 1)
pre = v.cpu().data.numpy().T.tolist()[0][0]
outstrs = []
for i in pre:
if i == 1:
break
outstrs.append(vec_to_str.get(i, "Un"))
print("ai > ", "".join(outstrs))
def beamSearchDecoder(self, input_variable):
input_length = input_variable.size()[0]
encoder_hidden = self.encoder.init_hidden()
encoder_outputs, encoder_hidden = self.encoder(input_variable, encoder_hidden)
decoder_input = Variable(torch.LongTensor([[SOS_token]]))
decoder_context = Variable(torch.zeros(1, self.decoder.hidden_size))
decoder_hidden = encoder_hidden
if USE_CUDA:
decoder_input = decoder_input.cuda()
decoder_context = decoder_context.cuda()
decoder_output, decoder_context, decoder_hidden, decoder_attention = self.decoder(decoder_input, decoder_context, decoder_hidden, encoder_outputs)
topk = decoder_output.data.topk(self.top_k)
samples = [[] for i in range(self.top_k)]
dead_k = 0
final_samples = []
for index in range(self.top_k):
topk_prob = topk[0][0][index]
topk_index = int(topk[1][0][index])
samples[index] = [[topk_index], topk_prob, 0, 0, decoder_context, decoder_hidden, decoder_attention, encoder_outputs]
for _ in range(self.max_length):
tmp = []
for index in range(len(samples)):
tmp.extend(self.beamSearchInfer(samples[index], index))
samples = []
# ???topk
df = pd.DataFrame(tmp)
df.columns = ['sequence', 'pre_socres', 'fin_scores', "ave_scores", "decoder_context", "decoder_hidden", "decoder_attention", "encoder_outputs"]
sequence_len = df.sequence.apply(lambda x:len(x))
df['ave_scores'] = df['fin_scores'] / sequence_len
df = df.sort_values('ave_scores', ascending=False).reset_index().drop(['index'], axis=1)
df = df[:(self.top_k-dead_k)]
for index in range(len(df)):
group = df.ix[index]
if group.tolist()[0][-1] == 1:
final_samples.append(group.tolist())
df = df.drop([index], axis=0)
dead_k += 1
print("drop {}, {}".format(group.tolist()[0], dead_k))
samples = df.values.tolist()
if len(samples) == 0:
break
if len(final_samples) < self.top_k:
final_samples.extend(samples[:(self.top_k-dead_k)])
return final_samples
def query(self, x, y, predict=False):
"""
Compute the nearest neighbor of the input queries.
Arguments:
x: A normalized matrix of queries of size (batch_size x key_dim)
y: A matrix of correct labels (batch_size x 1)
Returns:
y_hat, A (batch-size x 1) matrix
- the nearest neighbor to the query in memory_size
softmax_score, A (batch_size x 1) matrix
- A normalized score measuring the similarity between query and nearest neighbor
loss - average loss for memory module
"""
batch_size, dims = x.size()
query = F.normalize(self.query_proj(x), dim=1)
#query = F.normalize(torch.matmul(x, self.query_proj), dim=1)
# Find the k-nearest neighbors of the query
scores = torch.matmul(query, torch.t(self.keys_var))
cosine_similarity, topk_indices_var = torch.topk(scores, self.top_k, dim=1)
# softmax of cosine similarities - embedding
softmax_score = F.softmax(self.softmax_temperature * cosine_similarity)
# retrive memory values - prediction
topk_indices = topk_indices_var.detach().data
y_hat_indices = topk_indices[:, 0]
y_hat = self.values[y_hat_indices]
loss = None
if not predict:
# Loss Function
# topk_indices = (batch_size x topk)
# topk_values = (batch_size x topk x value_size)
# collect the memory values corresponding to the topk scores
batch_size, topk_size = topk_indices.size()
flat_topk = flatten(topk_indices)
flat_topk_values = self.values[topk_indices]
topk_values = flat_topk_values.resize_(batch_size, topk_size)
correct_mask = torch.eq(topk_values, torch.unsqueeze(y.data, dim=1)).float()
correct_mask_var = ag.Variable(correct_mask, requires_grad=False)
pos_score, pos_idx = torch.topk(torch.mul(cosine_similarity, correct_mask_var), 1, dim=1)
neg_score, neg_idx = torch.topk(torch.mul(cosine_similarity, 1-correct_mask_var), 1, dim=1)
# zero-out correct scores if there are no correct values in topk values
mask = 1.0 - torch.eq(torch.sum(correct_mask_var, dim=1), 0.0).float()
pos_score = torch.mul(pos_score, torch.unsqueeze(mask, dim=1))
#print(pos_score, neg_score)
loss = MemoryLoss(pos_score, neg_score, self.margin)
# Update memory
self.update(query, y, y_hat, y_hat_indices)
return y_hat, softmax_score, loss
def predict(dataset_iter=test_iter, dataset=test, data_name="test"):
print("Dataset: {}".format(data_name))
model.eval()
dataset_iter.init_epoch()
n_correct = 0
fname = "{}.txt".format(data_name)
results_file = open(os.path.join(results_path, fname), 'w')
n_retrieved = 0
fid = open(os.path.join(args.data_dir,"lineids_{}.txt".format(data_name)))
sent_id = [x.strip() for x in fid.readlines()]
for data_batch_idx, data_batch in enumerate(dataset_iter):
scores = model(data_batch)
if args.dataset == 'RelationPrediction':
n_correct += (torch.max(scores, 1)[1].view(data_batch.relation.size()).data == data_batch.relation.data).sum()
# Get top k
top_k_scores, top_k_indices = torch.topk(scores, k=args.hits, dim=1, sorted=True) # shape: (batch_size, k)
top_k_scores_array = top_k_scores.cpu().data.numpy()
top_k_indices_array = top_k_indices.cpu().data.numpy()
top_k_relatons_array = index2tag[top_k_indices_array]
for i, (relations_row, scores_row) in enumerate(zip(top_k_relatons_array, top_k_scores_array)):
index = (data_batch_idx * args.batch_size) + i
example = data_batch.dataset.examples[index]
for j, (rel, score) in enumerate(zip(relations_row, scores_row)):
if (rel == example.relation):
label = 1
n_retrieved += 1
else:
label = 0
results_file.write(
"{} %%%% {} %%%% {} %%%% {}\n".format( sent_id[index], rel, label, score))
else:
print("Wrong Dataset")
exit()
if args.dataset == 'RelationPrediction':
P = 1. * n_correct / len(dataset)
print("{} Precision: {:10.6f}%".format(data_name, 100. * P))
print("no. retrieved: {} out of {}".format(n_retrieved, len(dataset)))
retrieval_rate = 100. * n_retrieved / len(dataset)
print("{} Retrieval Rate {:10.6f}".format(data_name, retrieval_rate))
else:
print("Wrong dataset")
exit()
# run the model on the dev set and write the output to a file
