def make_batch(batch_size):
batch_idx = np.random.choice(len(data),batch_size)
batch_sequences = [data[idx] for idx in batch_idx]
strokes = []
lengths = []
indice = 0
for seq in batch_sequences:
len_seq = len(seq[:,0])
new_seq = np.zeros((Nmax,5))
new_seq[:len_seq,:2] = seq[:,:2]
new_seq[:len_seq-1,2] = 1-seq[:-1,2]
new_seq[:len_seq,3] = seq[:,2]
new_seq[(len_seq-1):,4] = 1
new_seq[len_seq-1,2:4] = 0
lengths.append(len(seq[:,0]))
strokes.append(new_seq)
indice += 1
if use_cuda:
batch = Variable(torch.from_numpy(np.stack(strokes,1)).cuda().float())
else:
batch = Variable(torch.from_numpy(np.stack(strokes,1)).float())
return batch, lengths
################################ adaptive lr
python类stack()的实例源码
def make_target(self, batch, lengths):
if use_cuda:
eos = Variable(torch.stack([torch.Tensor([0,0,0,0,1])]\
*batch.size()[1]).cuda()).unsqueeze(0)
else:
eos = Variable(torch.stack([torch.Tensor([0,0,0,0,1])]\
*batch.size()[1])).unsqueeze(0)
batch = torch.cat([batch, eos], 0)
mask = torch.zeros(Nmax+1, batch.size()[1])
for indice,length in enumerate(lengths):
mask[:length,indice] = 1
if use_cuda:
mask = Variable(mask.cuda()).detach()
else:
mask = Variable(mask).detach()
dx = torch.stack([Variable(batch.data[:,:,0])]*hp.M,2).detach()
dy = torch.stack([Variable(batch.data[:,:,1])]*hp.M,2).detach()
p1 = Variable(batch.data[:,:,2]).detach()
p2 = Variable(batch.data[:,:,3]).detach()
p3 = Variable(batch.data[:,:,4]).detach()
p = torch.stack([p1,p2,p3],2)
return mask,dx,dy,p
def __call__(self, tensor):
"""
Args:
tensor (Tensor): Tensor of audio of size (samples x channels)
Returns:
tensor (Tensor): n_mels x hops x channels (BxLxC), where n_mels is
the number of mel bins, hops is the number of hops, and channels
is unchanged.
"""
if librosa is None:
print("librosa not installed, cannot create spectrograms")
return tensor
L = []
for i in range(tensor.size(1)):
nparr = tensor[:, i].numpy() # (samples, )
sgram = librosa.feature.melspectrogram(
nparr, **self.kwargs) # (n_mels, hops)
L.append(sgram)
L = np.stack(L, 2) # (n_mels, hops, channels)
tensor = torch.from_numpy(L).type_as(tensor)
return tensor
def test_shapes(self):
dims = [
((1, 1), (2, 1, 1)), # broadcast first argument
((2, 1, 1), (1, 1)), # broadcast second argument
((2, 1, 1), (2, 1, 1)), # matrix stack sizes match
]
for dt, (dm1, dm2) in itertools.product(self.types, dims):
a = np.ones(dm1, dtype=dt)
b = np.ones(dm2, dtype=dt)
res = self.matmul(a, b)
assert_(res.shape == (2, 1, 1))
# vector vector returns scalars.
for dt in self.types:
a = np.ones((2,), dtype=dt)
b = np.ones((2,), dtype=dt)
c = self.matmul(a, b)
assert_(np.array(c).shape == ())
def train_priority(self , state , reward , action , state_next , done, batch_ISweight):
q , q_target = self.sess.run([self.q_value , self.q_target] ,
feed_dict={self.inputs_q : state , self.inputs_target : state_next } )
# DoubleDQN
if self.double:
q_next = self.sess.run(self.q_value , feed_dict={self.inputs_q : state_next})
action_best = np.argmax(q_next , axis = 1)
q_target_best = self.sess.run(self.q_target_action , feed_dict={self.action_best : action_best,
self.q_target : q_target})
else:
q_target_best = np.max(q_target , axis = 1) # dqn
q_target_best_mask = ( 1.0 - done) * q_target_best
target = reward + self.gamma * q_target_best_mask
batch_ISweight = np.stack([batch_ISweight , batch_ISweight] , axis = -1 )
loss, td_error, _ = self.sess.run([self.loss , self.td_error, self.train_op] ,
feed_dict={self.inputs_q: state , self.target:target , self.action:action, self.ISweight : batch_ISweight ,} )
return td_error
# self.loss_his.append(loss)
# ===============================================================
# A3C Agent
# ===============================================================
def stack_and_pad(values: List[Union[np.ndarray, int, float]], pad=0) -> np.ndarray:
"""Pads a list of numpy arrays so that they have equal dimensions, then stacks them."""
if isinstance(values[0], int) or isinstance(values[0], float):
return np.array(values)
dims = len(values[0].shape)
max_shape = [max(sizes) for sizes in zip(*[v.shape for v in values])]
padded_values = []
for value in values:
pad_width = [(0, max_shape[i] - value.shape[i])
for i in range(dims)]
padded_value = np.lib.pad(value, pad_width, mode='constant',
constant_values=pad)
padded_values.append(padded_value)
return np.stack(padded_values)
def join_embeddings(src_we, target_we):
"""joins and filters words not in common and produces two tensors"""
src_w = set(src_we.keys())
target_w = set(target_we.keys())
common_w = src_w & target_w
src_tensor = []
target_tensor = []
for w in common_w:
src_tensor.append(src_we[w])
target_tensor.append(target_we[w])
src_tensor = torch.Tensor(np.stack(src_tensor))
target_tensor = torch.Tensor(np.stack(target_tensor))
return src_tensor, target_tensor
def decode_bboxes(tcoords, anchors):
var_x, var_y, var_w, var_h = config['prior_variance']
t_x = tcoords[:, 0]*var_x
t_y = tcoords[:, 1]*var_y
t_w = tcoords[:, 2]*var_w
t_h = tcoords[:, 3]*var_h
a_w = anchors[:, 2]
a_h = anchors[:, 3]
a_x = anchors[:, 0]+a_w/2
a_y = anchors[:, 1]+a_h/2
x = t_x*a_w + a_x
y = t_y*a_h + a_y
w = tf.exp(t_w)*a_w
h = tf.exp(t_h)*a_h
x1 = tf.maximum(0., x - w/2)
y1 = tf.maximum(0., y - h/2)
x2 = tf.minimum(1., w + x1)
y2 = tf.minimum(1., h + y1)
return tf.stack([y1, x1, y2, x2], axis=1)
def encode_bboxes(proposals, gt):
prop_x = proposals[:, 0]
prop_y = proposals[:, 1]
prop_w = proposals[:, 2]
prop_h = proposals[:, 3]
gt_x = gt[:, 0]
gt_y = gt[:, 1]
gt_w = gt[:, 2]
gt_h = gt[:, 3]
diff_x = (gt_x + 0.5*gt_w - prop_x - 0.5*prop_w)/prop_w
diff_y = (gt_y + 0.5*gt_h - prop_y - 0.5*prop_h)/prop_h
if len(gt) > 0 and (np.min(gt_w/prop_w) < 1e-6 or np.min(gt_h/prop_h) < 1e-6):
print(np.min(gt_w), np.min(gt_h), np.min(gt_w/prop_w), np.max(gt_h/prop_h))
diff_w = np.log(gt_w/prop_w)
diff_h = np.log(gt_h/prop_h)
var_x, var_y, var_w, var_h = config['prior_variance']
x = np.stack([diff_x/var_x, diff_y/var_y, diff_w/var_w, diff_h/var_h],
axis=1)
return x
def batch(states, batch_size=None):
"""Combines a collection of state structures into a batch, padding if needed.
Args:
states: A collection of individual nested state structures.
batch_size: The desired final batch size. If the nested state structure
that results from combining the states is smaller than this, it will be
padded with zeros.
Returns:
A single state structure that results from stacking the structures in
`states`, with padding if needed.
Raises:
ValueError: If the number of input states is larger than `batch_size`.
"""
if batch_size and len(states) > batch_size:
raise ValueError('Combined state is larger than the requested batch size')
def stack_and_pad(*states):
stacked = np.stack(states)
if batch_size:
stacked.resize([batch_size] + list(stacked.shape)[1:])
return stacked
return tf_nest.map_structure(stack_and_pad, *states)
def second_order_nic(x):
"""
transform
x1 x2 ---> 1 x1 x2 x1x2 x1**2 x2**2
nic : no initial constant
"""
ones = np.ones(len(x))
x1 = x[:, 0]
x2 = x[:, 1]
x1_sqr = x1**2
x2_sqr = x2**2
x1x2 = x1 * x2
return np.stack([ones, x1, x2, x1x2, x1_sqr, x2_sqr], axis=1)
# STOCHASTIC GRADIENT DESCENT
def generate_quadratic_parameters(trials):
"""ax^2 + b"""
def transform(x):
"""
transform
x1 ---> 1 x1**2
"""
ones = np.ones(len(x))
x1 = x[:, 0]
x1_sqr = x1 ** 2
return np.stack([ones, x1_sqr], axis=1)
new_trials = [
DataML((training_set.z, training_set.y), transform)
for training_set in trials ]
weights = [ linear_percepton(training_set.z, training_set.y) for training_set in new_trials ]
return np.array(weights)
def recalculate_objects(pred_dict, image):
proposals = pred_dict['rpn_prediction']['proposals']
proposals_prob = pred_dict['classification_prediction']['rcnn']['cls_prob']
proposals_target = proposals_prob.argmax(axis=1) - 1
bbox_offsets = pred_dict[
'classification_prediction']['rcnn']['bbox_offsets']
bbox_offsets = bbox_offsets[proposals_target >= 0]
proposals = proposals[proposals_target >= 0]
proposals_target = proposals_target[proposals_target >= 0]
bbox_offsets_idx_pairs = np.stack(
np.array([
proposals_target * 4, proposals_target * 4 + 1,
proposals_target * 4 + 2, proposals_target * 4 + 3]), axis=1)
bbox_offsets = np.take(bbox_offsets, bbox_offsets_idx_pairs.astype(np.int))
bboxes = decode(proposals, bbox_offsets)
return bboxes, proposals_target
def next_batch(self, batch_size):
batches_ids = set()
while len(batches_ids) < batch_size:
h = random.randint(0, self.t_h-self.h)
w = random.randint(0, self.t_w-self.w)
d = random.randint(0, self.t_d-self.d)
batches_ids.add((h, w, d))
image_batches = []
label_batches = []
for h, w, d in batches_ids:
image_batches.append(
self.images[h:h+self.h, w:w+self.w, d:d+self.d])
label_batches.append(
self.labels[h:h+self.h, w:w+self.w, d:d+self.d])
images = np.expand_dims(np.stack(image_batches, axis=0), axis=-1)
images = np.transpose(images, (0, 3, 1, 2, 4))
labels = np.stack(label_batches, axis=0)
labels = np.transpose(labels, (0, 3, 1, 2))
return images, labels
def process(input_dir, output_dir, model_dir, resizing_size, gpu):
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3, visible_device_list=gpu)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)).as_default():
m = loader.LoadedModel(model_dir)
os.makedirs(output_dir, exist_ok=True)
input_filenames = glob(os.path.join(input_dir, '*.jpg')) + \
glob(os.path.join(input_dir, '*.png')) + \
glob(os.path.join(input_dir, '*.tif')) + \
glob(os.path.join(input_dir, '*.jp2'))
for path in tqdm(input_filenames):
img = Image.open(path).resize(resizing_size)
mat = np.asarray(img)
if len(mat.shape) == 2:
mat = np.stack([mat, mat, mat], axis=2)
predictions = m.predict(mat[None], prediction_key='labels')[0]
plt.imsave(os.path.join(output_dir, os.path.relpath(path, input_dir)), predictions)
def test_hexahedron():
val = quadpy.hexahedron.integrate(
lambda x: numpy.exp(x[0]),
quadpy.hexahedron.cube_points([0.0, 1.0], [0.0, 1.0], [0.0, 1.0]),
quadpy.hexahedron.Product(quadpy.line_segment.NewtonCotesClosed(3))
)
val = quadpy.hexahedron.integrate(
lambda x: [numpy.exp(x[0]), numpy.exp(x[1])],
numpy.stack([
quadpy.hexahedron.cube_points([0, 1], [0, 1], [0, 1]),
quadpy.hexahedron.cube_points([0, 1], [0, 1], [0, 1]),
quadpy.hexahedron.cube_points([0, 1], [0, 1], [0, 1]),
quadpy.hexahedron.cube_points([0, 1], [0, 1], [0, 1]),
quadpy.hexahedron.cube_points([0, 1], [0, 1], [0, 1]),
], axis=-2),
quadpy.hexahedron.Product(quadpy.line_segment.NewtonCotesClosed(3))
)
assert val.shape == (2, 5)
return
def test_quadrilateral():
quadpy.quadrilateral.integrate(
lambda x: numpy.exp(x[0]),
quadpy.quadrilateral.rectangle_points([0.0, 1.0], [0.0, 1.0]),
quadpy.quadrilateral.Stroud('C2 5-4')
)
val = quadpy.quadrilateral.integrate(
lambda x: [numpy.exp(x[0]), numpy.exp(x[1])],
numpy.stack([
quadpy.quadrilateral.rectangle_points([0.0, 1.0], [0.0, 1.0]),
quadpy.quadrilateral.rectangle_points([0.0, 1.0], [0.0, 1.0]),
quadpy.quadrilateral.rectangle_points([0.0, 1.0], [0.0, 1.0]),
quadpy.quadrilateral.rectangle_points([0.0, 1.0], [0.0, 1.0]),
quadpy.quadrilateral.rectangle_points([0.0, 1.0], [0.0, 1.0]),
], axis=-2),
quadpy.quadrilateral.Stroud('C2 3-1')
)
assert val.shape == (2, 5)
return
def test_tetrahedron():
quadpy.tetrahedron.integrate(
lambda x: numpy.exp(x[0]),
numpy.array([
[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]
], dtype=float),
quadpy.tetrahedron.ShunnHam(3)
)
quadpy.tetrahedron.integrate(
lambda x: [numpy.exp(x[0]), numpy.exp(x[1])],
numpy.stack([
[[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0], [0.0, 0, 1]],
[[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0], [0.0, 0, 1]],
[[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0], [0.0, 0, 1]],
[[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0], [0.0, 0, 1]],
], axis=-2),
quadpy.tetrahedron.ShunnHam(3)
)
return
def test_triangle():
quadpy.triangle.integrate(
lambda x: numpy.exp(x[0]),
numpy.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]]),
quadpy.triangle.Cubtri()
)
val = quadpy.triangle.integrate(
lambda x: [numpy.exp(x[0]), numpy.exp(x[1])],
numpy.stack([
[[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
[[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
[[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
[[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
[[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
], axis=-2),
quadpy.triangle.Cubtri()
)
assert val.shape == (2, 5)
return
def transform(xi, cube):
'''Transform the points `xi` from the reference cube to `cube`.
'''
# For d==2, the result used to be computed with
#
# out = (
# + outer(0.25*(1.0-xi[0])*(1.0-xi[1]), cube[0, 0])
# + outer(0.25*(1.0+xi[0])*(1.0-xi[1]), cube[1, 0])
# + outer(0.25*(1.0-xi[0])*(1.0+xi[1]), cube[0, 1])
# + outer(0.25*(1.0+xi[0])*(1.0+xi[1]), cube[1, 1])
# )
#
# This array of multiplications and additions is reminiscent of dot(), and
# indeed tensordot() can handle the situation. We just need to compute the
# `1+-xi` products and align them with `cube`.
one_mp_xi = numpy.stack([
0.5 * (1.0 - xi),
0.5 * (1.0 + xi),
], axis=1)
a = helpers.n_outer(one_mp_xi)
# TODO kahan tensordot
# <https://stackoverflow.com/q/45372098/353337>
d = xi.shape[0]
return numpy.tensordot(a, cube, axes=(range(d), range(d)))
