def accGradParameters(self, input, gradOutput, scale=1):
self.network.accGradParameters([input, self.partition], gradOutput, scale)
if self.bias is not None:
if self.buffer is None:
self.buffer = input.new()
self.buffer.resize_(gradOutput.size(1))
torch.mv(gradOutput.t(), self.addBuffer, out=self.buffer).mul_(scale)
self.gradBias.index_add_(
1, self.partition, self.buffer.view(1, self.buffer.nelement())
)
python类mv()的实例源码
def test_mv(self):
m1 = torch.randn(100, 100)
v1 = torch.randn(100)
res1 = torch.mv(m1, v1)
res2 = res1.clone().zero_()
for i, j in iter_indices(m1):
res2[i] += m1[i][j] * v1[j]
self.assertEqual(res1, res2)
def dot(x, y):
def _dot(X):
x, y = X
x_ndim = ndim(x)
y_ndim = ndim(y)
if x_ndim == 2 and y_ndim == 2:
return torch.mm(x, y)
if x_ndim == 2 and y_ndim == 1:
return torch.mv(x, y)
if x_ndim == 1 and y_ndim == 2:
return torch.mv(y, x)
if x_ndim == 1 and y_ndim == 1:
return torch.dot(x, y)
else:
raise Exception('Unsupported tensor ranks for dot operation : ' + str(x_ndim) + ' and ' + str(y_ndim) + '.')
def _compute_output_shape(X):
x, y = _get_shape(X[0]), _get_shape(X[1])
x_ndim = len(x)
y_ndim = len(y)
if x_ndim == 2 and y_ndim == 2:
return (x[0], y[1])
if x_ndim == 2 and y_ndim == 1:
return (x[0],)
if x_ndim == 1 and y_ndim == 2:
return (y[0],)
if x_ndim == 1 and y_ndim == 1:
return (0,)
return get_op(_dot, output_shape=_compute_output_shape)([x, y])
def test_back():
npr.seed(1)
nBatch, nz, neq, nineq = 1, 10, 1, 3
# nz, neq, nineq = 3,3,3
L = np.tril(np.random.randn(nz,nz)) + 2.*np.eye(nz,nz)
Q = L.dot(L.T)+1e-4*np.eye(nz)
G = 100.*npr.randn(nineq,nz)
A = 100.*npr.randn(neq,nz)
z0 = 1.*npr.randn(nz)
s0 = 100.*np.ones(nineq)
s0[:nineq//2] = 1e-6
# print(np.linalg.norm(L))
# print(np.linalg.norm(G))
# print(np.linalg.norm(A))
# print(np.linalg.norm(z0))
# print(np.linalg.norm(s0))
p = npr.randn(nBatch,nz)
# print(np.linalg.norm(p))
truez = npr.randn(nBatch,nz)
af = adact.AdactFunction()
zhat_0, nu_0, lam_0 = af.forward_single_np(p[0], L, G, A, z0, s0)
dl_dzhat_0 = zhat_0-truez[0]
S = Solver(L, A, G, z0, s0, 1e-8)
S.reinit(lam_0, zhat_0)
dp_0, dL_0, dG_0, dA_0, dz0_0, ds0_0 = af.backward_single_np_solver(
S, zhat_0, nu_0, lam_0, dl_dzhat_0, L, G, A, z0, s0)
# zhat_1, nu_1, lam_1 = af.forward_single_np(p[1], L, G, A, z0, s0)
# dl_dzhat_1 = zhat_1-truez[1]
# S.reinit(lam_1, zhat_1)
# dp_1, dL_1, dG_1, dA_1, dz0_1, ds0_1 = af.backward_single_np_solver(
# S, zhat_1, nu_1, lam_1, dl_dzhat_1, L, G, A, z0, s0)
p, L, G, A, z0, s0, truez = [torch.DoubleTensor(x) for x in [p, L, G, A, z0, s0, truez]]
Q = torch.mm(L, L.t())+0.001*torch.eye(nz).type_as(L)
if cuda:
p, L, Q, G, A, z0, s0, truez = [x.cuda() for x in [p, L, Q, G, A, z0, s0, truez]]
p, L, G, A, z0, s0 = [Variable(x) for x in [p, L, G, A, z0, s0]]
for x in [p, L, G, A, z0, s0]: x.requires_grad = True
# Q_LU, S_LU, R = aip.pre_factor_kkt_batch(Q, G, A, nBatch)
# b = torch.mv(A, z0) if neq > 0 else None
# h = torch.mv(G, z0)+s0
# zhat_b, nu_b, lam_b = aip.forward_batch(p, Q, G, A, b, h, Q_LU, S_LU, R)
zhats = af(p, L, G, A, z0, s0)
dl_dzhat = zhats.data - truez
zhats.backward(dl_dzhat)
dp, dL, dG, dA, dz0, ds0 = [x.grad.clone() for x in [p, L, G, A, z0, s0]]
def prof_instance(nz, neq, nineq, nBatch, cuda):
L = np.tril(npr.uniform(0,1, (nz,nz))) + np.eye(nz,nz)
G = npr.randn(nineq,nz)
A = npr.randn(neq,nz)
z0 = npr.randn(nz)
s0 = np.ones(nineq)
p = npr.randn(nBatch,nz)
p, L, G, A, z0, s0 = [torch.Tensor(x) for x in [p, L, G, A, z0, s0]]
Q = torch.mm(L, L.t())+0.001*torch.eye(nz).type_as(L)
if cuda:
p, L, Q, G, A, z0, s0 = [x.cuda() for x in [p, L, Q, G, A, z0, s0]]
b = torch.mv(A, z0) if neq > 0 else None
h = torch.mv(G, z0)+s0
af = adact.AdactFunction()
single_results = []
start = time.time()
U_Q, U_S, R = aip.pre_factor_kkt(Q, G, A)
for i in range(nBatch):
single_results.append(aip.forward_single(p[i], Q, G, A, b, h, U_Q, U_S, R))
single_time = time.time()-start
start = time.time()
Q_LU, S_LU, R = aip.pre_factor_kkt_batch(Q, G, A, nBatch)
zhat_b, nu_b, lam_b = aip.forward_batch(p, Q, G, A, b, h, Q_LU, S_LU, R)
batched_time = time.time()-start
zhat_diff = (single_results[0][0] - zhat_b[0]).norm()
lam_diff = (single_results[0][2] - lam_b[0]).norm()
eps = 0.1 # Pretty relaxed.
if zhat_diff > eps or lam_diff > eps:
print('===========')
print("Warning: Single and batched solutions might not match.")
print(" + zhat_diff: {}".format(zhat_diff))
print(" + lam_diff: {}".format(lam_diff))
print(" + (nz, neq, nineq, nBatch) = ({}, {}, {}, {})".format(
nz, neq, nineq, nBatch))
print('===========')
return single_time, batched_time
def test_functional_blas(self):
def compare(fn, *args):
unpacked_args = tuple(arg.data if isinstance(arg, Variable) else arg
for arg in args)
unpacked_result = fn(*unpacked_args)
packed_result = fn(*args).data
# if non-Variable torch function returns a scalar, compare to scalar
if not torch.is_tensor(unpacked_result):
assert packed_result.dim() == 1
assert packed_result.nelement() == 1
packed_result = packed_result[0]
self.assertEqual(packed_result, unpacked_result)
def test_blas_add(fn, x, y, z):
# Checks all signatures
compare(fn, x, y, z)
compare(fn, 0.5, x, y, z)
compare(fn, 0.5, x, 0.25, y, z)
def test_blas(fn, x, y):
compare(fn, x, y)
test_blas(torch.mm, Variable(torch.randn(2, 10)),
Variable(torch.randn(10, 4)))
test_blas_add(torch.addmm, Variable(torch.randn(2, 4)),
Variable(torch.randn(2, 10)), Variable(torch.randn(10, 4)))
test_blas(torch.bmm, Variable(torch.randn(4, 2, 10)),
Variable(torch.randn(4, 10, 4)))
test_blas_add(torch.addbmm, Variable(torch.randn(2, 4)),
Variable(torch.randn(4, 2, 10)), Variable(torch.randn(4, 10, 4)))
test_blas_add(torch.baddbmm, Variable(torch.randn(4, 2, 4)),
Variable(torch.randn(4, 2, 10)), Variable(torch.randn(4, 10, 4)))
test_blas(torch.mv, Variable(torch.randn(2, 10)),
Variable(torch.randn(10)))
test_blas_add(torch.addmv, Variable(torch.randn(2)),
Variable(torch.randn(2, 10)), Variable(torch.randn(10)))
test_blas(torch.ger, Variable(torch.randn(5)),
Variable(torch.randn(6)))
test_blas_add(torch.addr, Variable(torch.randn(5, 6)),
Variable(torch.randn(5)), Variable(torch.randn(6)))
test_blas(torch.matmul, Variable(torch.randn(6)), Variable(torch.randn(6)))
test_blas(torch.matmul, Variable(torch.randn(10, 4)), Variable(torch.randn(4)))
test_blas(torch.matmul, Variable(torch.randn(5)), Variable(torch.randn(5, 6)))
test_blas(torch.matmul, Variable(torch.randn(2, 10)), Variable(torch.randn(10, 4)))
test_blas(torch.matmul, Variable(torch.randn(5, 2, 10)), Variable(torch.randn(5, 10, 4)))
test_blas(torch.matmul, Variable(torch.randn(3, 5, 2, 10)), Variable(torch.randn(3, 5, 10, 4)))
test_blas(torch.matmul, Variable(torch.randn(3, 5, 2, 10)), Variable(torch.randn(10)))
test_blas(torch.matmul, Variable(torch.randn(10)), Variable(torch.randn(3, 5, 10, 4)))
def evaluate(data_source, batch_size=10, window=args.window):
# Turn on evaluation mode which disables dropout.
if args.model == 'QRNN': model.reset()
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
next_word_history = None
pointer_history = None
for i in range(0, data_source.size(0) - 1, args.bptt):
if i > 0: print(i, len(data_source), math.exp(total_loss / i))
data, targets = get_batch(data_source, i, evaluation=True, args=args)
output, hidden, rnn_outs, _ = model(data, hidden, return_h=True)
rnn_out = rnn_outs[-1].squeeze()
output_flat = output.view(-1, ntokens)
###
# Fill pointer history
start_idx = len(next_word_history) if next_word_history is not None else 0
next_word_history = torch.cat([one_hot(t.data[0], ntokens) for t in targets]) if next_word_history is None else torch.cat([next_word_history, torch.cat([one_hot(t.data[0], ntokens) for t in targets])])
#print(next_word_history)
pointer_history = Variable(rnn_out.data) if pointer_history is None else torch.cat([pointer_history, Variable(rnn_out.data)], dim=0)
#print(pointer_history)
###
# Built-in cross entropy
# total_loss += len(data) * criterion(output_flat, targets).data[0]
###
# Manual cross entropy
# softmax_output_flat = torch.nn.functional.softmax(output_flat)
# soft = torch.gather(softmax_output_flat, dim=1, index=targets.view(-1, 1))
# entropy = -torch.log(soft)
# total_loss += len(data) * entropy.mean().data[0]
###
# Pointer manual cross entropy
loss = 0
softmax_output_flat = torch.nn.functional.softmax(output_flat)
for idx, vocab_loss in enumerate(softmax_output_flat):
p = vocab_loss
if start_idx + idx > window:
valid_next_word = next_word_history[start_idx + idx - window:start_idx + idx]
valid_pointer_history = pointer_history[start_idx + idx - window:start_idx + idx]
logits = torch.mv(valid_pointer_history, rnn_out[idx])
theta = args.theta
ptr_attn = torch.nn.functional.softmax(theta * logits).view(-1, 1)
ptr_dist = (ptr_attn.expand_as(valid_next_word) * valid_next_word).sum(0).squeeze()
lambdah = args.lambdasm
p = lambdah * ptr_dist + (1 - lambdah) * vocab_loss
###
target_loss = p[targets[idx].data]
loss += (-torch.log(target_loss)).data[0]
total_loss += loss / batch_size
###
hidden = repackage_hidden(hidden)
next_word_history = next_word_history[-window:]
pointer_history = pointer_history[-window:]
return total_loss / len(data_source)
# Load the best saved model.
def test_functional_blas(self):
def compare(fn, *args):
unpacked_args = tuple(arg.data if isinstance(arg, Variable) else arg
for arg in args)
unpacked_result = fn(*unpacked_args)
packed_result = fn(*args).data
# if non-Variable torch function returns a scalar, compare to scalar
if not torch.is_tensor(unpacked_result):
assert packed_result.dim() == 1
assert packed_result.nelement() == 1
packed_result = packed_result[0]
self.assertEqual(packed_result, unpacked_result)
def test_blas_add(fn, x, y, z):
# Checks all signatures
compare(fn, x, y, z)
compare(fn, 0.5, x, y, z)
compare(fn, 0.5, x, 0.25, y, z)
def test_blas(fn, x, y):
compare(fn, x, y)
test_blas(torch.mm, Variable(torch.randn(2, 10)),
Variable(torch.randn(10, 4)))
test_blas_add(torch.addmm, Variable(torch.randn(2, 4)),
Variable(torch.randn(2, 10)), Variable(torch.randn(10, 4)))
test_blas(torch.bmm, Variable(torch.randn(4, 2, 10)),
Variable(torch.randn(4, 10, 4)))
test_blas_add(torch.addbmm, Variable(torch.randn(2, 4)),
Variable(torch.randn(4, 2, 10)), Variable(torch.randn(4, 10, 4)))
test_blas_add(torch.baddbmm, Variable(torch.randn(4, 2, 4)),
Variable(torch.randn(4, 2, 10)), Variable(torch.randn(4, 10, 4)))
test_blas(torch.mv, Variable(torch.randn(2, 10)),
Variable(torch.randn(10)))
test_blas_add(torch.addmv, Variable(torch.randn(2)),
Variable(torch.randn(2, 10)), Variable(torch.randn(10)))
test_blas(torch.ger, Variable(torch.randn(5)),
Variable(torch.randn(6)))
test_blas_add(torch.addr, Variable(torch.randn(5, 6)),
Variable(torch.randn(5)), Variable(torch.randn(6)))
test_blas(torch.matmul, Variable(torch.randn(6)), Variable(torch.randn(6)))
test_blas(torch.matmul, Variable(torch.randn(10, 4)), Variable(torch.randn(4)))
test_blas(torch.matmul, Variable(torch.randn(5)), Variable(torch.randn(5, 6)))
test_blas(torch.matmul, Variable(torch.randn(2, 10)), Variable(torch.randn(10, 4)))
test_blas(torch.matmul, Variable(torch.randn(5, 2, 10)), Variable(torch.randn(5, 10, 4)))
test_blas(torch.matmul, Variable(torch.randn(3, 5, 2, 10)), Variable(torch.randn(3, 5, 10, 4)))
test_blas(torch.matmul, Variable(torch.randn(3, 5, 2, 10)), Variable(torch.randn(10)))
test_blas(torch.matmul, Variable(torch.randn(10)), Variable(torch.randn(3, 5, 10, 4)))
