def analyse_data(input_dir):
shapes = []
relative_volumes = []
for folder in get_sub_folders(input_dir):
print(folder)
for sub_folder in get_sub_folders(os.path.join(input_dir, folder)):
image_type = get_image_type_from_folder_name(sub_folder)
# do not save the raw data (too heavy)
if image_type != '.OT':
continue
path = os.path.join(input_dir, folder, sub_folder)
filename = next(filename for filename in os.listdir(path) if get_extension(filename) == '.nii')
path = os.path.join(path, filename)
im = nib.load(path)
image = im.get_data()
shape = image.shape
shapes.append(shape)
relative_volumes.append(100 * np.sum(image) / np.cumprod(shape)[-1])
return shapes, relative_volumes
# train
python类cumprod()的实例源码
def test_basic(self):
ba = [1, 2, 10, 11, 6, 5, 4]
ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
for ctype in [np.int16, np.uint16, np.int32, np.uint32,
np.float32, np.float64, np.complex64, np.complex128]:
a = np.array(ba, ctype)
a2 = np.array(ba2, ctype)
if ctype in ['1', 'b']:
self.assertRaises(ArithmeticError, np.cumprod, a)
self.assertRaises(ArithmeticError, np.cumprod, a2, 1)
self.assertRaises(ArithmeticError, np.cumprod, a)
else:
assert_array_equal(np.cumprod(a, axis=-1),
np.array([1, 2, 20, 220,
1320, 6600, 26400], ctype))
assert_array_equal(np.cumprod(a2, axis=0),
np.array([[1, 2, 3, 4],
[5, 12, 21, 36],
[50, 36, 84, 180]], ctype))
assert_array_equal(np.cumprod(a2, axis=-1),
np.array([[1, 2, 6, 24],
[5, 30, 210, 1890],
[10, 30, 120, 600]], ctype))
def reshape_workaround(data, shape_out): # type: (TensorOp, Sequence[int]) -> TensorOp
"""Limited workaround for tensor reshape operation."""
shape_in = data.shape.lengths
if np.prod(shape_in) != np.prod(shape_out):
raise ValueError('Total size of input (%d) and output (%d) dimension mismatch.',
np.prod(shape_in), np.prod(shape_out))
ndims_out = len(shape_out)
if ndims_out == 1:
tensor = ng.flatten(data)
elif ndims_out == 2:
cumprods = list(np.cumprod(shape_in))
flatten_at_idx = cumprods.index(shape_out[0]) + 1
tensor = ng.flatten_at(data, flatten_at_idx)
else:
raise NotImplementedError('Reshape can only support flatten to 1d or 2d.')
return ng.cast_axes(tensor, make_pos_axes(shape_out))
def _decade_mortality_table(year,
url_template='https://www.ssa.gov/oact/NOTES/as120/LifeTables_Tbl_7_{}.html'):
assert int(year) % 10 == 0
url = url_template.format(year)
soup = BeautifulSoup(urlopen(url).read(), 'lxml')
table = soup.find('table', border=1)
rows = []
for row in table.find_all('tr'):
row_datum = [cell.text.strip() for cell in row.find_all('td')]
if len(row_datum) == 15 and row_datum[0] != '':
rows.append({
'year_of_birth': int(year),
'age': int(row_datum[0]),
'm_prob_survive_that_year': 1 - float(row_datum[1]),
'f_prob_survive_that_year': 1 - float(row_datum[9]),
})
df = pd.DataFrame(rows).sort_values(by='age')
for sex in 'mf':
df[sex + '_prob_alive'] = np.cumprod(df[sex + '_prob_survive_that_year']).astype(np.float64)
df['as_of_year'] = df['year_of_birth'] + df['age']
return df[['year_of_birth', 'as_of_year', 'm_prob_alive', 'f_prob_alive']]
def read_feature(filename):
"""Read feature dump by C3D
Parameters
----------
filename : str
Fullpath of file to read
Outputs
-------
x : ndarray
numpy array of features
Note: It accomplishes the same purpose of this code:
C3D/examples/c3d_feature_extraction/script/read_binary_blob.m
"""
s_parr, d_parr = array.array('i'), array.array('f')
with open(filename, 'r') as f:
s_parr.fromfile(f, 5)
s = np.array(s_parr)
m = np.cumprod(s)[-1]
d_parr.fromfile(f, m)
return s, np.array(d_parr)
def __init__(self, frame_sizes, n_rnn, dim, learn_h0, q_levels,
weight_norm):
super().__init__()
self.dim = dim
self.q_levels = q_levels
ns_frame_samples = map(int, np.cumprod(frame_sizes))
self.frame_level_rnns = torch.nn.ModuleList([
FrameLevelRNN(
frame_size, n_frame_samples, n_rnn, dim, learn_h0, weight_norm
)
for (frame_size, n_frame_samples) in zip(
frame_sizes, ns_frame_samples
)
])
self.sample_level_mlp = SampleLevelMLP(
frame_sizes[0], dim, q_levels, weight_norm
)
def test_basic(self):
ba = [1, 2, 10, 11, 6, 5, 4]
ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
for ctype in [np.int16, np.uint16, np.int32, np.uint32,
np.float32, np.float64, np.complex64, np.complex128]:
a = np.array(ba, ctype)
a2 = np.array(ba2, ctype)
if ctype in ['1', 'b']:
self.assertRaises(ArithmeticError, np.cumprod, a)
self.assertRaises(ArithmeticError, np.cumprod, a2, 1)
self.assertRaises(ArithmeticError, np.cumprod, a)
else:
assert_array_equal(np.cumprod(a, axis=-1),
np.array([1, 2, 20, 220,
1320, 6600, 26400], ctype))
assert_array_equal(np.cumprod(a2, axis=0),
np.array([[1, 2, 3, 4],
[5, 12, 21, 36],
[50, 36, 84, 180]], ctype))
assert_array_equal(np.cumprod(a2, axis=-1),
np.array([[1, 2, 6, 24],
[5, 30, 210, 1890],
[10, 30, 120, 600]], ctype))
def test_basic(self):
ba = [1, 2, 10, 11, 6, 5, 4]
ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
for ctype in [np.int16, np.uint16, np.int32, np.uint32,
np.float32, np.float64, np.complex64, np.complex128]:
a = np.array(ba, ctype)
a2 = np.array(ba2, ctype)
if ctype in ['1', 'b']:
self.assertRaises(ArithmeticError, np.cumprod, a)
self.assertRaises(ArithmeticError, np.cumprod, a2, 1)
self.assertRaises(ArithmeticError, np.cumprod, a)
else:
assert_array_equal(np.cumprod(a, axis=-1),
np.array([1, 2, 20, 220,
1320, 6600, 26400], ctype))
assert_array_equal(np.cumprod(a2, axis=0),
np.array([[1, 2, 3, 4],
[5, 12, 21, 36],
[50, 36, 84, 180]], ctype))
assert_array_equal(np.cumprod(a2, axis=-1),
np.array([[1, 2, 6, 24],
[5, 30, 210, 1890],
[10, 30, 120, 600]], ctype))
def addVariable(self, var_name, var_dims, kind='Var', **kwargs):
if var_name in self.var_dict:
print "error: %s is a duplicated variable"%var_name
var_name_parts = LPCompiler.var_name_regex.match(var_name)
new_var = {
'start' : self.total_var_length,
'dims' : var_dims,
'cumdims': np.cumprod([1]+var_dims),
'length': np.prod(var_dims),
'name' : var_name,
'idx' : var_name_parts.group('var').split(','),
'lbl' : var_name_parts.group('lbl').split(','),
'kind' : kind,
'min_value' : 0 if 'min_value' not in kwargs else kwargs['min_value']
}
self.var_dict[var_name] = new_var
if kind == 'Param':
for i in range(new_var['start'], new_var['start']+new_var['length']):
self.int_flag.append([i, 1])
self.total_var_length += new_var['length']
def parse_struc(img):
nbs = neighbors(img.shape)
acc = np.cumprod((1,)+img.shape[::-1][:-1])[::-1]
img = img.ravel()
pts = np.array(np.where(img==2))[0]
buf = np.zeros(131072, dtype=np.int64)
num = 10
nodes = []
for p in pts:
if img[p] == 2:
nds = fill(img, p, num, nbs, acc, buf)
num += 1
nodes.append(nds)
edges = []
for p in pts:
for dp in nbs:
if img[p+dp]==1:
edge = trace(img, p+dp, nbs, acc, buf)
edges.append(edge)
return nodes, edges
# use nodes and edges build a networkx graph
def test_basic(self):
ba = [1, 2, 10, 11, 6, 5, 4]
ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
for ctype in [np.int16, np.uint16, np.int32, np.uint32,
np.float32, np.float64, np.complex64, np.complex128]:
a = np.array(ba, ctype)
a2 = np.array(ba2, ctype)
if ctype in ['1', 'b']:
self.assertRaises(ArithmeticError, np.cumprod, a)
self.assertRaises(ArithmeticError, np.cumprod, a2, 1)
self.assertRaises(ArithmeticError, np.cumprod, a)
else:
assert_array_equal(np.cumprod(a, axis=-1),
np.array([1, 2, 20, 220,
1320, 6600, 26400], ctype))
assert_array_equal(np.cumprod(a2, axis=0),
np.array([[1, 2, 3, 4],
[5, 12, 21, 36],
[50, 36, 84, 180]], ctype))
assert_array_equal(np.cumprod(a2, axis=-1),
np.array([[1, 2, 6, 24],
[5, 30, 210, 1890],
[10, 30, 120, 600]], ctype))
def test_basic(self):
ba = [1, 2, 10, 11, 6, 5, 4]
ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
for ctype in [np.int16, np.uint16, np.int32, np.uint32,
np.float32, np.float64, np.complex64, np.complex128]:
a = np.array(ba, ctype)
a2 = np.array(ba2, ctype)
if ctype in ['1', 'b']:
self.assertRaises(ArithmeticError, np.cumprod, a)
self.assertRaises(ArithmeticError, np.cumprod, a2, 1)
self.assertRaises(ArithmeticError, np.cumprod, a)
else:
assert_array_equal(np.cumprod(a, axis=-1),
np.array([1, 2, 20, 220,
1320, 6600, 26400], ctype))
assert_array_equal(np.cumprod(a2, axis=0),
np.array([[1, 2, 3, 4],
[5, 12, 21, 36],
[50, 36, 84, 180]], ctype))
assert_array_equal(np.cumprod(a2, axis=-1),
np.array([[1, 2, 6, 24],
[5, 30, 210, 1890],
[10, 30, 120, 600]], ctype))
def cumprod(x, axis=None):
"""Return the cumulative product of the elements along a given axis.
Wraping of numpy.cumprod.
Parameters
----------
x
Input tensor variable.
axis
The axis along which the cumulative product is computed.
The default (None) is to compute the cumprod over the flattened array.
.. versionadded:: 0.7
"""
return CumprodOp(axis=axis)(x)
def summarizeVdToDocTopicCount(Vd):
''' Create DocTopicCount matrix from given stick-breaking parameters Vd
Returns
--------
DocTopicCount : 2D array, size D x K
'''
assert not np.any(np.isnan(Vd))
PRNG = np.random.RandomState(0)
DocTopicCount = np.zeros(Vd.shape)
for d in xrange(Vd.shape[0]):
N_d = 100 + 50 * PRNG.rand()
Pi_d = Vd[d, :].copy()
Pi_d[1:] *= np.cumprod(1.0 - Vd[d, :-1])
np.maximum(Pi_d, 1e-10, out=Pi_d)
Pi_d /= np.sum(Pi_d)
DocTopicCount[d, :] = N_d * Pi_d
return DocTopicCount
def rho2beta(rho, returnSize='K+1'):
''' Calculate probability for all components including remainder.
Returns
--------
beta : 1D array, size equal to 'K' or 'K+1', depending on returnSize
beta[k] := probability of topic k
'''
rho = np.asarray(rho, dtype=np.float64)
if returnSize == 'K':
beta = rho.copy()
beta[1:] *= np.cumprod(1 - rho[:-1])
else:
beta = np.append(rho, 1.0)
beta[1:] *= np.cumprod(1.0 - rho)
return beta
def discounts(self, periods_length, periods_num, libors, flat=False):
"""
Return the calculated discounts for each period, flat discounts don't include the spread:
if flat:
period_rate = (libors[i - 1]) / periods_num
else:
period_rate = (libors[i - 1] + spread) / periods_num
discounts[i] = discounts[i - 1] / (1 + period_rate)
:param periods_length:
:param periods_num:
:param libors:
:param flat: whether to calculate a flat discounts or not
"""
discounts = np.zeros(periods_length)
discounts[0] = 1
if flat:
rates = libors[:-1] + self.spread
else:
rates = libors[:-1]
discounts[1:] = np.cumprod(1 / (1 + (rates / periods_num)), dtype=float)
return discounts
def get_chunks(x, nbytes_desired):
nbytes = np.array(x).ravel()[0].nbytes
size_desired = nbytes_desired / nbytes
if size_desired >= x.size:
# desired chunk size is greater or equal than array size, thus we can include the whole array in a single chunk
return x.shape
s = x.shape[::-1]
cp = np.cumprod(s)
dim = np.argmax(cp >= size_desired)
s_dim_desired = size_desired / np.prod(s[:dim])
s_dim = np.round(s_dim_desired)
if s_dim < 1:
s_dim = 1
chunks = np.ones_like(s)
chunks[:dim] = s[:dim]
chunks[dim] = s_dim
result = tuple(chunks[::-1])
return result
def test_basic(self):
ba = [1, 2, 10, 11, 6, 5, 4]
ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
for ctype in [np.int16, np.uint16, np.int32, np.uint32,
np.float32, np.float64, np.complex64, np.complex128]:
a = np.array(ba, ctype)
a2 = np.array(ba2, ctype)
if ctype in ['1', 'b']:
self.assertRaises(ArithmeticError, np.cumprod, a)
self.assertRaises(ArithmeticError, np.cumprod, a2, 1)
self.assertRaises(ArithmeticError, np.cumprod, a)
else:
assert_array_equal(np.cumprod(a, axis=-1),
np.array([1, 2, 20, 220,
1320, 6600, 26400], ctype))
assert_array_equal(np.cumprod(a2, axis=0),
np.array([[1, 2, 3, 4],
[5, 12, 21, 36],
[50, 36, 84, 180]], ctype))
assert_array_equal(np.cumprod(a2, axis=-1),
np.array([[1, 2, 6, 24],
[5, 30, 210, 1890],
[10, 30, 120, 600]], ctype))
def test_allocation_weighting(self):
b, n = 5, 10
u = np.random.rand(b, n)
s = np.argsort(u, axis=1)
correct_alloc = np.zeros((b, n)).astype(np.float32)
for i in range(b):
cp = np.concatenate([[1], np.cumprod(u[i][s[i]])[:-1]])
correct_alloc[i][s[i]] = (1 - u[i][s[i]]) * cp
with self.test_session():
tf.global_variables_initializer().run()
Memory.memory_size = n
calculated_alloc = Memory.calculate_allocation_weighting(Memory, u).eval()
self.assertAllClose(correct_alloc, calculated_alloc)
def cumproduct(a, axis=None, dtype=None, out=None):
"""
Return the cumulative product over the given axis.
See Also
--------
cumprod : equivalent function; see for details.
"""
try:
cumprod = a.cumprod
except AttributeError:
return _wrapit(a, 'cumprod', axis, dtype, out)
return cumprod(axis, dtype, out)
def fill_diagonal(a, val, wrap=False):
"""Fills the main diagonal of the given array of any dimensionality.
For an array `a` with ``a.ndim > 2``, the diagonal is the list of
locations with indices ``a[i, i, ..., i]`` all identical. This function
modifies the input array in-place, it does not return a value.
Args:
a (cupy.ndarray): The array, at least 2-D.
val (scalar): The value to be written on the diagonal.
Its type must be compatible with that of the array a.
wrap (bool): If specified, the diagonal is "wrapped" after N columns.
This affects only tall matrices.
Examples
--------
>>> a = cupy.zeros((3, 3), int)
>>> cupy.fill_diagonal(a, 5)
>>> a
array([[5, 0, 0],
[0, 5, 0],
[0, 0, 5]])
.. seealso:: :func:`numpy.fill_diagonal`
"""
# The followings are imported from the original numpy
if a.ndim < 2:
raise ValueError('array must be at least 2-d')
end = None
if a.ndim == 2:
step = a.shape[1] + 1
if not wrap:
end = a.shape[1] * a.shape[1]
else:
if not numpy.alltrue(numpy.diff(a.shape) == 0):
raise ValueError('All dimensions of input must be of equal length')
step = 1 + numpy.cumprod(a.shape[:-1]).sum()
# Since the current cupy does not support a.flat,
# we use a.ravel() instead of a.flat
a.ravel()[:end:step] = val
def cumprod(a, axis=None, dtype=None, out=None):
"""Returns the cumulative product of an array along a given axis.
Args:
a (cupy.ndarray): Input array.
axis (int): Axis along which the cumulative product is taken. If it is
not specified, the input is flattened.
dtype: Data type specifier.
out (cupy.ndarray): Output array.
Returns:
cupy.ndarray: The result array.
.. seealso:: :func:`numpy.cumprod`
"""
return _cum_core(a, axis, dtype, out, _cumprod_kern, _cumprod_batch_kern)
# TODO(okuta): Implement diff
# TODO(okuta): Implement ediff1d
# TODO(okuta): Implement gradient
# TODO(okuta): Implement cross
# TODO(okuta): Implement trapz
def __init__(self, shape, dtype, buffer=None, offset=0, strides=None):
itemsize = dtype().itemsize
shape = tuple(np.array(shape).ravel().astype(np.uint64))
if strides is None:
# This magic came from http://stackoverflow.com/a/32874295
strides = itemsize * np.r_[1, np.cumprod(shape[::-1][:-1],
dtype=np.int64)][::-1]
self.shape = shape
self.dtype = dtype
self.buffer = buffer
self.offset = offset
self.strides = strides
self.base = None
self.flags = {'WRITEABLE': True,
'ALIGNED': buffer % (itemsize == 0
if buffer is not None
else True),
'OWNDATA': False,
'UPDATEIFCOPY': False,
'C_CONTIGUOUS': self.nbytes == strides[0] * shape[0],
'F_CONTIGUOUS': False,
'SPACE': 'cuda'}
class CTypes(object):
def __init__(self, parent):
self.parent = parent
@property
def data(self):
return self.parent.data
self.ctypes = CTypes(self)
if self.buffer is None:
self.buffer = raw_malloc(self.nbytes, space='cuda')
self.flags['OWNDATA'] = True
self.flags['ALIGNED'] = True
memset(self, 0)
else:
self.buffer += offset
def set_evaluation_feedback(self, feedbacks):
"""Set feedback for the last behavior.
Parameters
----------
feedbacks : list of float
feedback for each step or for the episode, depends on the problem
"""
visited_states = self.policy.visited_states
actions_taken = self.policy.actions_taken
n_steps = len(visited_states)
assert n_steps == len(feedbacks)
assert n_steps == len(actions_taken)
gammas = np.hstack(
((1,), np.cumprod(np.ones(n_steps - 1) * self.gamma)))
diff = 0.0
for t in range(n_steps):
s = visited_states[t]
a = actions_taken[t]
ret = sum(feedbacks[t:] * gammas[:n_steps - t])
self.returns[s][a].append(ret)
last_Q = self.Q[s][a]
self.Q[s][a] = np.mean(self.returns[s][a])
diff = max(diff, np.abs(last_Q - self.Q[s][a]))
self.done = any(feedbacks > 0) and diff < 1e-3
def discount(rewards, gamma, timestamps):
dt = np.diff(timestamps.squeeze())
x = rewards.squeeze()
g = np.power(gamma, dt)
y = np.zeros_like(x)
for n in range(len(y)):
y[n] = x[n] + np.sum(x[n + 1:] * np.cumprod(g[n:]))
return y
def __func__(arr):
arr = arr.copy()
arr[np.isnan(arr)] = 1
return np.cumprod(arr, dtype=float)
def cumproduct(a, axis=None, dtype=None, out=None):
"""
Return the cumulative product over the given axis.
See Also
--------
cumprod : equivalent function; see for details.
"""
try:
cumprod = a.cumprod
except AttributeError:
return _wrapit(a, 'cumprod', axis, dtype, out)
return cumprod(axis, dtype, out)
def __len__(self):
"""Returns the number of values handled by the :class:`MultiParameter`
instance. It is required, that the `shape` has been set beforehand,
which specifies the length in each dimension.
"""
return numpy.cumprod(self.shape)[-1]
def _create_prices(t):
last_average = 100 if t==0 else source.data['average'][-1]
returns = asarray(lognormal(mean.value, stddev.value, 1))
average = last_average * cumprod(returns)
high = average * exp(abs(gamma(1, 0.03, size=1)))
low = average / exp(abs(gamma(1, 0.03, size=1)))
delta = high - low
open = low + delta * uniform(0.05, 0.95, size=1)
close = low + delta * uniform(0.05, 0.95, size=1)
return open[0], high[0], low[0], close[0], average[0]
def _create_prices(t):
global last_average
returns = asarray(lognormal(mean, stddev, 1))
average = last_average * cumprod(returns)
last_average = average
high = average * exp(abs(gamma(1, 0.03, size=1)))
low = average / exp(abs(gamma(1, 0.03, size=1)))
delta = high - low
open = low + delta * uniform(0.05, 0.95, size=1)
close = low + delta * uniform(0.05, 0.95, size=1)
return open[0], high[0], low[0], close[0], average[0]
