def pack_samples(self, samples, dtype=None):
"""Pack samples into one integer per sample
Store one sample in a single integer instead of a list of
integers with length `len(self.nsoutdims)`. Example:
>>> p = pauli_mpp(nr_sites=2, local_dim=2)
>>> p.outdims
(6, 6)
>>> p.pack_samples(np.array([[0, 1], [1, 0], [1, 2], [5, 5]]))
array([ 1, 6, 8, 35])
"""
assert samples.ndim == 2
assert samples.shape[1] == len(self.nsoutdims)
samples = np.ravel_multi_index(samples.T, self.nsoutdims)
if dtype not in (True, False, None) and issubclass(dtype, np.integer):
info = np.iinfo(dtype)
assert samples.min() >= info.min
assert samples.max() <= info.max
samples = samples.astype(dtype)
return samples
python类integer()的实例源码
def __init__(self, config, model_dir, ob_shape_list):
self.model_dir = model_dir
self.cnn_format = config.cnn_format
self.memory_size = config.memory_size
self.actions = np.empty(self.memory_size, dtype = np.uint8)
self.rewards = np.empty(self.memory_size, dtype = np.integer)
# print(self.memory_size, config.screen_height, config.screen_width)
# self.screens = np.empty((self.memory_size, config.screen_height, config.screen_width), dtype = np.float16)
self.screens = np.empty([self.memory_size] + ob_shape_list, dtype = np.float16)
self.terminals = np.empty(self.memory_size, dtype = np.bool)
self.history_length = config.history_length
# self.dims = (config.screen_height, config.screen_width)
self.dims = tuple(ob_shape_list)
self.batch_size = config.batch_size
self.count = 0
self.current = 0
# pre-allocate prestates and poststates for minibatch
self.prestates = np.empty((self.batch_size, self.history_length) + self.dims, dtype = np.float16)
self.poststates = np.empty((self.batch_size, self.history_length) + self.dims, dtype = np.float16)
# self.prestates = np.empty((self.batch_size, self.history_length, self.dims), dtype = np.float16)
# self.poststates = np.empty((self.batch_size, self.history_length, self.dims), dtype = np.float16)
def test_auto_dtype_largeint(self):
# Regression test for numpy/numpy#5635 whereby large integers could
# cause OverflowErrors.
# Test the automatic definition of the output dtype
#
# 2**66 = 73786976294838206464 => should convert to float
# 2**34 = 17179869184 => should convert to int64
# 2**10 = 1024 => should convert to int (int32 on 32-bit systems,
# int64 on 64-bit systems)
data = TextIO('73786976294838206464 17179869184 1024')
test = np.ndfromtxt(data, dtype=None)
assert_equal(test.dtype.names, ['f0', 'f1', 'f2'])
assert_(test.dtype['f0'] == np.float)
assert_(test.dtype['f1'] == np.int64)
assert_(test.dtype['f2'] == np.integer)
assert_allclose(test['f0'], 73786976294838206464.)
assert_equal(test['f1'], 17179869184)
assert_equal(test['f2'], 1024)
def test_with_incorrect_minlength(self):
x = np.array([], dtype=int)
assert_raises_regex(TypeError, "an integer is required",
lambda: np.bincount(x, minlength="foobar"))
assert_raises_regex(ValueError, "must be positive",
lambda: np.bincount(x, minlength=-1))
assert_raises_regex(ValueError, "must be positive",
lambda: np.bincount(x, minlength=0))
x = np.arange(5)
assert_raises_regex(TypeError, "an integer is required",
lambda: np.bincount(x, minlength="foobar"))
assert_raises_regex(ValueError, "minlength must be positive",
lambda: np.bincount(x, minlength=-1))
assert_raises_regex(ValueError, "minlength must be positive",
lambda: np.bincount(x, minlength=0))
def test_allclose(self):
# Tests allclose on arrays
a = np.random.rand(10)
b = a + np.random.rand(10) * 1e-8
self.assertTrue(allclose(a, b))
# Test allclose w/ infs
a[0] = np.inf
self.assertTrue(not allclose(a, b))
b[0] = np.inf
self.assertTrue(allclose(a, b))
# Test allclose w/ masked
a = masked_array(a)
a[-1] = masked
self.assertTrue(allclose(a, b, masked_equal=True))
self.assertTrue(not allclose(a, b, masked_equal=False))
# Test comparison w/ scalar
a *= 1e-8
a[0] = 0
self.assertTrue(allclose(a, 0, masked_equal=True))
# Test that the function works for MIN_INT integer typed arrays
a = masked_array([np.iinfo(np.int_).min], dtype=np.int_)
self.assertTrue(allclose(a, a))
def _parse_fields(point):
output = []
for k, v in point['fields'].items():
k = escape(k, key_escape)
# noinspection PyUnresolvedReferences
if isinstance(v, bool):
output.append('{k}={v}'.format(k=k, v=str(v).upper()))
elif isinstance(v, (int, np.integer)):
output.append('{k}={v}i'.format(k=k, v=v))
elif isinstance(v, str):
output.append('{k}="{v}"'.format(k=k, v=v.translate(str_escape)))
elif v is None or np.isnan(v):
continue
else:
# Floats and other numerical formats go here.
# TODO: Add unit test
output.append('{k}={v}'.format(k=k, v=v))
return ','.join(output)
def get_subvolume(self, bounds):
if bounds.start is None or bounds.stop is None:
image_subvol = self.image_data
label_subvol = self.label_data
else:
image_subvol = self.image_data[
bounds.start[0]:bounds.stop[0],
bounds.start[1]:bounds.stop[1],
bounds.start[2]:bounds.stop[2]]
label_subvol = None
if np.issubdtype(image_subvol.dtype, np.integer):
raise ValueError('Sparse volume access does not support image data coercion.')
seed = bounds.seed
if seed is None:
seed = np.array(image_subvol.shape, dtype=np.int64) // 2
return Subvolume(image_subvol, label_subvol, seed, bounds.label_id)
def __init__(self, X_train, y_train, X_test, y_test, categorical=True):
self._x_train = X_train
self._x_test = X_test
# are the targets to be made one hot vectors
if categorical:
self._y_train = np_utils.to_categorical(y_train)
self._y_test = np_utils.to_categorical(y_test)
self._output_size = self._y_train.shape[1]
# handle sparse output classification
elif issubclass(y_train.dtype.type, np.integer):
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.max() + 1 # assume 0 based indexes
# not classification, just copy them
else:
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.shape[1]
def __init__(self, X_train, y_train, X_test, y_test, categorical=True):
self._x_train = X_train
self._x_test = X_test
# are the targets to be made one hot vectors
if categorical:
self._y_train = np_utils.to_categorical(y_train)
self._y_test = np_utils.to_categorical(y_test)
self._output_size = self._y_train.shape[1]
# handle sparse output classification
elif issubclass(y_train.dtype.type, np.integer):
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.max() + 1 # assume 0 based indexes
# not classification, just copy them
else:
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.shape[1]
def __init__(self, X_train, y_train, X_test, y_test, categorical=True):
self._x_train = X_train
self._x_test = X_test
# are the targets to be made one hot vectors
if categorical:
self._y_train = np_utils.to_categorical(y_train)
self._y_test = np_utils.to_categorical(y_test)
self._output_size = self._y_train.shape[1]
# handle sparse output classification
elif issubclass(y_train.dtype.type, np.integer):
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.max() + 1 # assume 0 based indexes
# not classification, just copy them
else:
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.shape[1]
def __init__(self, X_train, y_train, X_test, y_test, categorical=True):
self._x_train = X_train
self._x_test = X_test
# are the targets to be made one hot vectors
if categorical:
self._y_train = np_utils.to_categorical(y_train)
self._y_test = np_utils.to_categorical(y_test)
self._output_size = self._y_train.shape[1]
# handle sparse output classification
elif issubclass(y_train.dtype.type, np.integer):
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.max() + 1 # assume 0 based indexes
# not classification, just copy them
else:
self._y_train = y_train
self._y_test = y_test
self._output_size = self._y_train.shape[1]
def test_auto_dtype_largeint(self):
# Regression test for numpy/numpy#5635 whereby large integers could
# cause OverflowErrors.
# Test the automatic definition of the output dtype
#
# 2**66 = 73786976294838206464 => should convert to float
# 2**34 = 17179869184 => should convert to int64
# 2**10 = 1024 => should convert to int (int32 on 32-bit systems,
# int64 on 64-bit systems)
data = TextIO('73786976294838206464 17179869184 1024')
test = np.ndfromtxt(data, dtype=None)
assert_equal(test.dtype.names, ['f0', 'f1', 'f2'])
assert_(test.dtype['f0'] == np.float)
assert_(test.dtype['f1'] == np.int64)
assert_(test.dtype['f2'] == np.integer)
assert_allclose(test['f0'], 73786976294838206464.)
assert_equal(test['f1'], 17179869184)
assert_equal(test['f2'], 1024)
def test_with_incorrect_minlength(self):
x = np.array([], dtype=int)
assert_raises_regex(TypeError, "an integer is required",
lambda: np.bincount(x, minlength="foobar"))
assert_raises_regex(ValueError, "must be positive",
lambda: np.bincount(x, minlength=-1))
assert_raises_regex(ValueError, "must be positive",
lambda: np.bincount(x, minlength=0))
x = np.arange(5)
assert_raises_regex(TypeError, "an integer is required",
lambda: np.bincount(x, minlength="foobar"))
assert_raises_regex(ValueError, "minlength must be positive",
lambda: np.bincount(x, minlength=-1))
assert_raises_regex(ValueError, "minlength must be positive",
lambda: np.bincount(x, minlength=0))
def test_allclose(self):
# Tests allclose on arrays
a = np.random.rand(10)
b = a + np.random.rand(10) * 1e-8
self.assertTrue(allclose(a, b))
# Test allclose w/ infs
a[0] = np.inf
self.assertTrue(not allclose(a, b))
b[0] = np.inf
self.assertTrue(allclose(a, b))
# Test allclose w/ masked
a = masked_array(a)
a[-1] = masked
self.assertTrue(allclose(a, b, masked_equal=True))
self.assertTrue(not allclose(a, b, masked_equal=False))
# Test comparison w/ scalar
a *= 1e-8
a[0] = 0
self.assertTrue(allclose(a, 0, masked_equal=True))
# Test that the function works for MIN_INT integer typed arrays
a = masked_array([np.iinfo(np.int_).min], dtype=np.int_)
self.assertTrue(allclose(a, a))
def _changeArrayDType(img, dtype, **kwargs):
if dtype == 'noUint':
return toNoUintArray(img)
if issubclass(np.dtype(dtype).type, np.integer):
return toUIntArray(img, dtype, **kwargs)
return img.astype(dtype)
# def bitDepth(path, img=None):
# '''
# there are no python filetypes between 8bit and 16 bit
# so, to find out whether an image is 12 or 14 bit resolved
# we need to check actual file size and image shape
# '''
# if img is None:
# img = imread(img)
# size = os.path.getsize(path)*8
# print (size, img.size,8888888,img.shape, size/img.size)
# kh
# return size/img.size
def __init__(self, config, model_dir):
self.model_dir = model_dir
self.cnn_format = config.cnn_format
self.memory_size = config.memory_size
self.actions = np.empty(self.memory_size, dtype = np.uint8)
self.rewards = np.empty(self.memory_size, dtype = np.integer)
self.screens = np.empty((self.memory_size, config.screen_height, config.screen_width), dtype = np.float16)
self.terminals = np.empty(self.memory_size, dtype = np.bool)
self.history_length = config.history_length
self.dims = (config.screen_height, config.screen_width)
self.batch_size = config.batch_size
self.count = 0
self.current = 0
# pre-allocate prestates and poststates for minibatch
self.prestates = np.empty((self.batch_size, self.history_length) + self.dims, dtype = np.float16)
self.poststates = np.empty((self.batch_size, self.history_length) + self.dims, dtype = np.float16)
def _get_block(self, values, key_remainder=None):
item_block = None
for i, v in enumerate(values):
# Lists in the aggregate key index in tandem;
# so, index into those lists (the first list is `values`)
v_key_remainder = key_remainder
if isinstance(values, tuple) or isinstance(values, list):
if key_remainder is not None:
broadcasted_key_remainder = ()
for k in key_remainder:
if hasattr(k, '__len__') and len(k)==np.size(k):
broadcasted_key_remainder += (k[i],)
else:
broadcasted_key_remainder += (k,)
v_key_remainder = broadcasted_key_remainder
# Make a single read at an integer index of axis 0
elem = self._get_element(v, v_key_remainder)
if item_block is None:
item_block = np.zeros((len(values),)+elem.shape,
self.dtype)
item_block[i] = elem
return item_block
def fcn(self, data_in):
"""
If return list, [0] goes to original, [1] goes to affected
"""
inst_nrb_merge = _MergeNRBs(nrb_left=self.nrb_left,
nrb_right=self.nrb_right,
pix=self.parameters['pix_switchpt'],
left_side_scale=self.parameters['scale_left'])
if self.fullRange:
pix = _np.arange(self.wn.size, dtype=_np.integer)
else:
list_rng_pix = _find_nearest(self.wn, self.rng)[1]
pix = _np.arange(list_rng_pix[0],list_rng_pix[1]+1,
dtype=_np.integer)
nrb_merged = inst_nrb_merge.calculate()
kkd = _np.zeros(data_in.shape)
# Note: kk_widget.fcn return imag part
kkd[..., pix] = self.kk_widget.fcn([nrb_merged[pix], data_in[..., pix]])
return [_np.vstack((self.nrb_left, self.nrb_right, nrb_merged)),
kkd]
def __init__(self, path, size ,historySize, dims , batchSize):
self.size = size
self.dims = dims
# preallocate memory
self.actions = np.empty(self.size, dtype=np.uint8)
self.rewards = np.empty(self.size, dtype=np.integer)
self.screens = np.empty((self.size, self.dims[0], self.dims[1] ), dtype=np.uint8)
self.terminals = np.empty(self.size, dtype=np.bool)
self.history_length = historySize
self.batch_size = batchSize
self.buffer = np.zeros((self.batch_size, self.history_length) + self.dims, dtype=np.uint8)
self.count = 0
self.current = 0
# pre-allocate prestates and poststates for minibatch
self.prestates = np.empty([self.batch_size, self.history_length] + self.dims, dtype=np.uint8)
self.poststates = np.empty([self.batch_size, self.history_length] + self.dims, dtype=np.uint8)
def _prepare_mask(mask, label, erode=True):
fgmask = mask.copy()
if np.issubdtype(fgmask.dtype, np.integer):
if isinstance(label, string_types):
label = FSL_FAST_LABELS[label]
fgmask[fgmask != label] = 0
fgmask[fgmask == label] = 1
else:
fgmask[fgmask > .95] = 1.
fgmask[fgmask < 1.] = 0
if erode:
# Create a structural element to be used in an opening operation.
struc = nd.generate_binary_structure(3, 2)
# Perform an opening operation on the background data.
fgmask = nd.binary_opening(fgmask, structure=struc).astype(np.uint8)
return fgmask
def check_random_state(seed):
"""Turn seed into a np.random.RandomState instance.
If seed is None, return the RandomState singleton used by np.random.
If seed is an int, return a new RandomState instance seeded with seed.
If seed is already a RandomState instance, return it.
Otherwise raise ValueError.
"""
if seed is None or seed is np.random:
return np.random.mtrand._rand
if isinstance(seed, (int, np.integer)):
return np.random.RandomState(seed)
if isinstance(seed, np.random.RandomState):
return seed
raise ValueError('%r cannot be used to seed a numpy.random.RandomState'
' instance' % seed)
def check_random_state(seed):
"""Turn seed into a np.random.RandomState instance
If seed is None, return the RandomState singleton used by np.random.
If seed is an int, return a new RandomState instance seeded with seed.
If seed is already a RandomState instance, return it.
Otherwise raise ValueError.
"""
if seed is None or seed is np.random:
return np.random.mtrand._rand
if isinstance(seed, (numbers.Integral, np.integer)):
return np.random.RandomState(seed)
if isinstance(seed, np.random.RandomState):
return seed
raise ValueError('%r cannot be used to seed a numpy.random.RandomState'
' instance' % seed)
base.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 50
收藏 0
点赞 0
评论 0
def _can_reindex(self, indexer):
"""
*this is an internal non-public method*
Check if we are allowing reindexing with this particular indexer
Parameters
----------
indexer : an integer indexer
Raises
------
ValueError if its a duplicate axis
"""
# trying to reindex on an axis with duplicates
if not self.is_unique and len(indexer):
raise ValueError("cannot reindex from a duplicate axis")
nanops.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 28
收藏 0
点赞 0
评论 0
def unique1d(values):
"""
Hash table-based unique
"""
if np.issubdtype(values.dtype, np.floating):
table = _hash.Float64HashTable(len(values))
uniques = np.array(table.unique(_ensure_float64(values)),
dtype=np.float64)
elif np.issubdtype(values.dtype, np.datetime64):
table = _hash.Int64HashTable(len(values))
uniques = table.unique(_ensure_int64(values))
uniques = uniques.view('M8[ns]')
elif np.issubdtype(values.dtype, np.timedelta64):
table = _hash.Int64HashTable(len(values))
uniques = table.unique(_ensure_int64(values))
uniques = uniques.view('m8[ns]')
elif np.issubdtype(values.dtype, np.integer):
table = _hash.Int64HashTable(len(values))
uniques = table.unique(_ensure_int64(values))
else:
table = _hash.PyObjectHashTable(len(values))
uniques = table.unique(_ensure_object(values))
return uniques
internals.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 36
收藏 0
点赞 0
评论 0
def shift(self, periods, axis=0, mgr=None):
""" shift the block by periods """
N = len(self.values.T)
indexer = np.zeros(N, dtype=int)
if periods > 0:
indexer[periods:] = np.arange(N - periods)
else:
indexer[:periods] = np.arange(-periods, N)
new_values = self.values.to_dense().take(indexer)
# convert integer to float if necessary. need to do a lot more than
# that, handle boolean etc also
new_values, fill_value = com._maybe_upcast(new_values)
if periods > 0:
new_values[:periods] = fill_value
else:
new_values[periods:] = fill_value
return [self.make_block_same_class(new_values,
placement=self.mgr_locs)]
frame.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 51
收藏 0
点赞 0
评论 0
def to_sparse(self, fill_value=None, kind='block'):
"""
Convert to SparseDataFrame
Parameters
----------
fill_value : float, default NaN
kind : {'block', 'integer'}
Returns
-------
y : SparseDataFrame
"""
from pandas.core.sparse import SparseDataFrame
return SparseDataFrame(self._series, index=self.index,
columns=self.columns, default_kind=kind,
default_fill_value=fill_value)
frame.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 35
收藏 0
点赞 0
评论 0
def __getitem__(self, key):
# shortcut if we are an actual column
is_mi_columns = isinstance(self.columns, MultiIndex)
try:
if key in self.columns and not is_mi_columns:
return self._getitem_column(key)
except:
pass
# see if we can slice the rows
indexer = convert_to_index_sliceable(self, key)
if indexer is not None:
return self._getitem_slice(indexer)
if isinstance(key, (Series, np.ndarray, Index, list)):
# either boolean or fancy integer index
return self._getitem_array(key)
elif isinstance(key, DataFrame):
return self._getitem_frame(key)
elif is_mi_columns:
return self._getitem_multilevel(key)
else:
return self._getitem_column(key)
test_groupby.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 27
收藏 0
点赞 0
评论 0
def test_grouper_multilevel_freq(self):
# GH 7885
# with level and freq specified in a pd.Grouper
from datetime import date, timedelta
d0 = date.today() - timedelta(days=14)
dates = date_range(d0, date.today())
date_index = pd.MultiIndex.from_product(
[dates, dates], names=['foo', 'bar'])
df = pd.DataFrame(np.random.randint(0, 100, 225), index=date_index)
# Check string level
expected = df.reset_index().groupby([pd.Grouper(
key='foo', freq='W'), pd.Grouper(key='bar', freq='W')]).sum()
# reset index changes columns dtype to object
expected.columns = pd.Index([0], dtype='int64')
result = df.groupby([pd.Grouper(level='foo', freq='W'), pd.Grouper(
level='bar', freq='W')]).sum()
assert_frame_equal(result, expected)
# Check integer level
result = df.groupby([pd.Grouper(level=0, freq='W'), pd.Grouper(
level=1, freq='W')]).sum()
assert_frame_equal(result, expected)
test_tseries.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 36
收藏 0
点赞 0
评论 0
def test_floats(self):
arr = np.array([1., 2., 3., np.float64(4), np.float32(5)], dtype='O')
result = lib.infer_dtype(arr)
self.assertEqual(result, 'floating')
arr = np.array([1, 2, 3, np.float64(4), np.float32(5), 'foo'],
dtype='O')
result = lib.infer_dtype(arr)
self.assertEqual(result, 'mixed-integer')
arr = np.array([1, 2, 3, 4, 5], dtype='f4')
result = lib.infer_dtype(arr)
self.assertEqual(result, 'floating')
arr = np.array([1, 2, 3, 4, 5], dtype='f8')
result = lib.infer_dtype(arr)
self.assertEqual(result, 'floating')
test_indexing.py 文件源码
项目:PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 38
收藏 0
点赞 0
评论 0
def test_fancy_setitem_int_labels(self):
# integer index defers to label-based indexing
df = DataFrame(np.random.randn(10, 5), index=np.arange(0, 20, 2))
tmp = df.copy()
exp = df.copy()
tmp.ix[[0, 2, 4]] = 5
exp.values[:3] = 5
assert_frame_equal(tmp, exp)
tmp = df.copy()
exp = df.copy()
tmp.ix[6] = 5
exp.values[3] = 5
assert_frame_equal(tmp, exp)
tmp = df.copy()
exp = df.copy()
tmp.ix[:, 2] = 5
# tmp correctly sets the dtype
# so match the exp way
exp[2] = 5
assert_frame_equal(tmp, exp)