def get_table_and_array_for_set_of_dicts(dicts):
for j in range(len(dicts)):
if not j:
all_keys = np.array([i for i in dicts[j].keys()], dtype=np.uint64)
else:
all_keys = np.concatenate([all_keys, np.array([i for i in dicts[j].keys()], dtype=np.uint64)])
unique_keys = sorted(set(all_keys))
# The sorted is so that the index of 0 will always be 0
index_lookup_table = create_index_table(np.array(sorted([np.uint64(key) for key in unique_keys]), dtype=np.uint64))
array = np.zeros(shape=[len(dicts), len(unique_keys)], dtype=np.uint64)
for square_num, dict in enumerate(dicts):
for key, value in dict.items():
array[square_num][
khash_get(ffi.cast("void *", index_lookup_table), np.uint64(key), np.uint64(0))] = np.uint64(value)
return index_lookup_table, array
python类uint64()的实例源码
def attacks_mask(board_state, square):
bb_square = BB_SQUARES[square]
if bb_square & board_state.pawns:
if bb_square & board_state.occupied_w:
return BB_PAWN_ATTACKS[WHITE][square]
else:
return BB_PAWN_ATTACKS[BLACK][square]
elif bb_square & board_state.knights:
return BB_KNIGHT_ATTACKS[square]
elif bb_square & board_state.kings:
return BB_KING_ATTACKS[square]
else:
attacks = np.uint64(0)
if bb_square & board_state.bishops or bb_square & board_state.queens:
attacks = DIAG_ATTACK_ARRAY[square][
khash_get(DIAG_ATTACK_INDEX_LOOKUP_TABLE, BB_DIAG_MASKS[square] & board_state.occupied, 0)]
if bb_square & board_state.rooks or bb_square & board_state.queens:
attacks |= (RANK_ATTACK_ARRAY[square][
khash_get(RANK_ATTACK_INDEX_LOOKUP_TABLE, BB_RANK_MASKS[square] & board_state.occupied,0)] |
FILE_ATTACK_ARRAY[square][
khash_get(FILE_ATTACK_INDEX_LOOKUP_TABLE, BB_FILE_MASKS[square] & board_state.occupied, 0)])
return attacks
def _get_dtype_maps():
""" Get dictionaries to map numpy data types to ITK types and the
other way around.
"""
# Define pairs
tmp = [ (np.float32, 'MET_FLOAT'), (np.float64, 'MET_DOUBLE'),
(np.uint8, 'MET_UCHAR'), (np.int8, 'MET_CHAR'),
(np.uint16, 'MET_USHORT'), (np.int16, 'MET_SHORT'),
(np.uint32, 'MET_UINT'), (np.int32, 'MET_INT'),
(np.uint64, 'MET_ULONG'), (np.int64, 'MET_LONG') ]
# Create dictionaries
map1, map2 = {}, {}
for np_type, itk_type in tmp:
map1[np_type.__name__] = itk_type
map2[itk_type] = np_type.__name__
# Done
return map1, map2
def __init__(self, *args, **kwds):
import numpy
self.dst_types = [numpy.uint8, numpy.uint16, numpy.uint32]
try:
self.dst_types.append(numpy.uint64)
except AttributeError:
pass
pygame.display.init()
try:
unittest.TestCase.__init__(self, *args, **kwds)
self.sources = [self._make_src_surface(8),
self._make_src_surface(16),
self._make_src_surface(16, srcalpha=True),
self._make_src_surface(24),
self._make_src_surface(32),
self._make_src_surface(32, srcalpha=True)]
finally:
pygame.display.quit()
def _check_valid_data(self, data):
"""Checks that the incoming data is a 2 x #elements ndarray of ints.
Parameters
----------
data : :obj:`numpy.ndarray`
The data to verify.
Raises
------
ValueError
If the data is not of the correct shape or type.
"""
if data.dtype.type != np.int8 and data.dtype.type != np.int16 \
and data.dtype.type != np.int32 and data.dtype.type != np.int64 \
and data.dtype.type != np.uint8 and data.dtype.type != np.uint16 \
and data.dtype.type != np.uint32 and data.dtype.type != np.uint64:
raise ValueError('Must initialize image coords with a numpy int ndarray')
if data.shape[0] != 2:
raise ValueError('Illegal data array passed to image coords. Must have 2 coordinates')
if len(data.shape) > 2:
raise ValueError('Illegal data array passed to point cloud. Must have 1 or 2 dimensions')
def DecodeValues(block, values, encoded_values, bz, by, bx, nbits):
# get the number of values per 8 byte uint64
if (nbits > 0):
values_per_uint64 = 64 / nbits
ie = 0
for value in encoded_values:
for i in range(0, values_per_uint64):
lower_bits_to_remove = (
(values_per_uint64 - i - 1) * nbits
)
values[ie] = (
(value >> lower_bits_to_remove) % 2**nbits
)
ie += 1
ii = 0
# get the lookup table
for iw in range(0, bz):
for iv in range(0, by):
for iu in range(0, bx):
block[iw, iv, iu] = values[ii]
ii += 1
return block, values
def to_best_type(array):
'''Convert array to lowest possible bitrate.
'''
ui8 = np.iinfo(np.uint8)
ui8 = ui8.max
ui16 = np.iinfo(np.uint16)
ui16 = ui16.max
ui32 = np.iinfo(np.uint32)
ui32 = ui32.max
ui64 = np.iinfo(np.uint64)
ui64 = ui64.max
if array.max() <= ui64:
new_type = np.uint64
if array.max() <= ui32:
new_type = np.uint32
if array.max() <= ui16:
new_type = np.uint16
if array.max() <= ui8:
new_type = np.uint8
return array.astype(new_type)
def load_data(name='ac3', N=-1, prefix=None, gold=False):
'''Load data
'''
if not 'mri' in name:
if gold: filename = '~/compresso/data/' + name + '/gold/' + name + '_gold.h5'
else: filename = '~/compresso/data/' + name + '/rhoana/' + name + '_rhoana.h5'
with h5py.File(os.path.expanduser(filename), 'r') as hf:
output = np.array(hf['main'], dtype=np.uint64)
else:
filename = '~/compresso/data/MRI/' + name + '.h5'
with h5py.File(os.path.expanduser(filename), 'r') as hf:
output = np.array(hf['main'], dtype=np.uint64)
if (not N == -1):
output = output[0:N,:,:]
return output
def test_int(self):
for st, ut, s in [(np.int8, np.uint8, 8),
(np.int16, np.uint16, 16),
(np.int32, np.uint32, 32),
(np.int64, np.uint64, 64)]:
for i in range(1, s):
assert_equal(hash(st(-2**i)), hash(-2**i),
err_msg="%r: -2**%d" % (st, i))
assert_equal(hash(st(2**(i - 1))), hash(2**(i - 1)),
err_msg="%r: 2**%d" % (st, i - 1))
assert_equal(hash(st(2**i - 1)), hash(2**i - 1),
err_msg="%r: 2**%d - 1" % (st, i))
i = max(i - 1, 1)
assert_equal(hash(ut(2**(i - 1))), hash(2**(i - 1)),
err_msg="%r: 2**%d" % (ut, i - 1))
assert_equal(hash(ut(2**i - 1)), hash(2**i - 1),
err_msg="%r: 2**%d - 1" % (ut, i))
def _write(self, samples, keyvals):
"""Write new metadata to the Digital Metadata channel.
This function does no input checking, see `write` for that.
Parameters
----------
samples : 1-D numpy array of type uint64 sorted in ascending order
An array of sample indices, given in the number of samples since
the epoch (time_since_epoch*sample_rate).
keyvals : iterable of iterables same length as `samples`
Each element of this iterable corresponds to a sample in `samples`
and should be another iterable that produces (key, value) pairs to
write for that sample.
"""
grp_iter = self._sample_group_generator(samples)
for grp, keyval in zip(grp_iter, keyvals):
for key, val in keyval:
if val is not None:
grp.create_dataset(key, data=val)
def seed(self, seed=None):
"""Resets the state of the random number generator with a seed.
.. seealso::
:func:`cupy.random.seed` for full documentation,
:meth:`numpy.random.RandomState.seed`
"""
if seed is None:
try:
seed_str = binascii.hexlify(os.urandom(8))
seed = numpy.uint64(int(seed_str, 16))
except NotImplementedError:
seed = numpy.uint64(time.clock() * 1000000)
else:
seed = numpy.asarray(seed).astype(numpy.uint64, casting='safe')
curand.setPseudoRandomGeneratorSeed(self._generator, seed)
curand.setGeneratorOffset(self._generator, 0)
def get_random_state():
"""Gets the state of the random number generator for the current device.
If the state for the current device is not created yet, this function
creates a new one, initializes it, and stores it as the state for the
current device.
Returns:
RandomState: The state of the random number generator for the
device.
"""
dev = cuda.Device()
rs = _random_states.get(dev.id, None)
if rs is None:
seed = os.getenv('CUPY_SEED')
if seed is None:
seed = os.getenv('CHAINER_SEED')
if seed is not None:
seed = numpy.uint64(int(seed))
rs = RandomState(seed)
rs = _random_states.setdefault(dev.id, rs)
return rs
def test_dtype2(self, dtype):
dtype = numpy.dtype(dtype)
# randint does not support 64 bit integers
if dtype in (numpy.int64, numpy.uint64):
return
iinfo = numpy.iinfo(dtype)
size = (10000,)
x = random.randint(iinfo.min, iinfo.max + 1, size, dtype)
self.assertEqual(x.dtype, dtype)
self.assertLessEqual(iinfo.min, min(x))
self.assertLessEqual(max(x), iinfo.max)
# Lower bound check
with self.assertRaises(ValueError):
random.randint(iinfo.min - 1, iinfo.min + 10, size, dtype)
# Upper bound check
with self.assertRaises(ValueError):
random.randint(iinfo.max - 10, iinfo.max + 2, size, dtype)
def _all_day_bars_of(self, instrument):
i = self._index_of(instrument)
mongo_data = self._day_bars[i][instrument.order_book_id].find({}, {"_id": 0})
fields = mongo_data[0].keys()
fields.remove('date')
result = []
dtype = np.dtype(getType(i))
result = np.empty(shape=(mongo_data.count(),), dtype=dtype)
for f in fields:
bar_attr = []
mongo_data = self._day_bars[i][instrument.order_book_id].find({}, {"_id": 0})
for bar in mongo_data:
bar_attr.append(bar[f])
result[f] = np.array(bar_attr)
bar_attr = []
mongo_data = self._day_bars[i][instrument.order_book_id].find({}, {"_id": 0})
for bar in mongo_data:
bar_attr.append(np.array(bar['date']).astype(np.uint64) * 1000000)
result['datetime'] = np.array(bar_attr)
return result
def _all_day_bars_of(self, instrument):
i = self._index_of(instrument)
mongo_data = self._day_bars[i][instrument.order_book_id].find({}, {"_id": 0})
fields = mongo_data[0].keys()
fields.remove('date')
result = []
dtype = np.dtype(getType(i))
result = np.empty(shape=(mongo_data.count(),), dtype=dtype)
for f in fields:
bar_attr = []
mongo_data = self._day_bars[i][instrument.order_book_id].find({}, {"_id": 0})
for bar in mongo_data:
bar_attr.append(bar[f])
result[f] = np.array(bar_attr)
bar_attr = []
mongo_data = self._day_bars[i][instrument.order_book_id].find({}, {"_id": 0})
for bar in mongo_data:
bar_attr.append(np.array(bar['date']).astype(np.uint64) * 1000000)
result['datetime'] = np.array(bar_attr)
return result
def contains(self, order_book_id, dates):
try:
s, e = self._index[order_book_id]
except KeyError:
return [False] * len(dates)
def _to_dt_int(d):
if isinstance(d, (int, np.int64, np.uint64)):
if d > 100000000:
return int(d // 1000000)
else:
return d.year*10000 + d.month*100 + d.day
date_set = self._get_set(s, e)
return [(_to_dt_int(d) in date_set) for d in dates]
def numpy2string(dtype):
if dtype == np.int8: return 'i8'
elif dtype == np.int16: return 'i16'
elif dtype == np.int32: return 'i32'
elif dtype == np.int64: return 'i64'
elif dtype == np.uint8: return 'u8'
elif dtype == np.uint16: return 'u16'
elif dtype == np.uint32: return 'u32'
elif dtype == np.uint64: return 'u64'
elif dtype == np.float16: return 'f16'
elif dtype == np.float32: return 'f32'
elif dtype == np.float64: return 'f64'
elif dtype == np.float128: return 'f128'
elif dtype == np.complex64: return 'cf32'
elif dtype == np.complex128: return 'cf64'
elif dtype == np.complex256: return 'cf128'
else: raise TypeError("Unsupported dtype: " + str(dtype))
def make_array(shape=(1,), dtype=np.float32, shared=False, fill_val=None):
np_type_to_ctype = {np.float32: ctypes.c_float,
np.float64: ctypes.c_double,
np.bool: ctypes.c_bool,
np.uint8: ctypes.c_ubyte,
np.uint64: ctypes.c_ulonglong}
if not shared:
np_arr = np.empty(shape, dtype=dtype)
else:
numel = np.prod(shape)
arr_ctypes = sharedctypes.RawArray(np_type_to_ctype[dtype], numel)
np_arr = np.frombuffer(arr_ctypes, dtype=dtype, count=numel)
np_arr.shape = shape
if not fill_val is None:
np_arr[...] = fill_val
return np_arr
def dhash(img):
"""Compute a perceptual has of an image.
Algo explained here :
https://blog.bearstech.com/2014/07/numpy-par-lexemple-une-implementation-de-dhash.html
:param img: an image
:type img: numpy.ndarray
:return: a perceptual hash of img coded on 64 bits
:rtype: int
"""
TWOS = np.array([2 ** n for n in range(7, -1, -1)])
BIGS = np.array([256 ** n for n in range(7, -1, -1)], dtype=np.uint64)
img = rgb2grey(resize(img, (9, 8)))
h = np.array([0] * 8, dtype=np.uint8)
for i in range(8):
h[i] = TWOS[img[i] > img[i + 1]].sum()
return (BIGS * h).sum()
def get_rgb_mask(img, debug=False):
assert isinstance(img, numpy.ndarray), 'image must be a numpy array'
assert img.ndim == 3, 'skin detection can only work on color images'
logger.debug('getting rgb mask')
lower_thresh = numpy.array([45, 52, 108], dtype=numpy.uint8)
upper_thresh = numpy.array([255, 255, 255], dtype=numpy.uint8)
mask_a = cv2.inRange(img, lower_thresh, upper_thresh)
mask_b = 255 * ((img[:, :, 2] - img[:, :, 1]) / 20)
mask_c = 255 * ((numpy.max(img, axis=2) - numpy.min(img, axis=2)) / 20)
mask_d = numpy.bitwise_and(numpy.uint64(mask_a), numpy.uint64(mask_b))
# mask = numpy.zeros_like(mask_d, dtype=numpy.uint8)
msk_rgb = numpy.bitwise_and(numpy.uint64(mask_c), numpy.uint64(mask_d))
# msk_rgb = cv2.fromarray(mask_rgb)
msk_rgb[msk_rgb < 128] = 0
msk_rgb[msk_rgb >= 128] = 1
if debug:
scripts.display('input', img)
scripts.display('mask_rgb', msk_rgb)
return msk_rgb.astype(float)
def default(self, obj):
# convert dates and numpy objects in a json serializable format
if isinstance(obj, datetime):
return obj.strftime('%Y-%m-%dT%H:%M:%SZ')
elif isinstance(obj, date):
return obj.strftime('%Y-%m-%d')
elif type(obj) in (np.int_, np.intc, np.intp, np.int8, np.int16,
np.int32, np.int64, np.uint8, np.uint16,
np.uint32, np.uint64):
return int(obj)
elif type(obj) in (np.bool_,):
return bool(obj)
elif type(obj) in (np.float_, np.float16, np.float32, np.float64,
np.complex_, np.complex64, np.complex128):
return float(obj)
# Let the base class default method raise the TypeError
return json.JSONEncoder.default(self, obj)
def scan_genotypes(self, genotypes, sub_ids=None, db=None):
"""Pass through all genotypes and return only the indices of those that pass the filter.
:param genotypes: np.ndarray[uint64, dim=2]
:rtype: np.ndarray[uint64]"""
if self.shortcut:
return np.zeros(0)
N = len(genotypes)
if sub_ids is not None:
variant_ids = sub_ids
elif self.val == 'x_linked' and db:
variant_ids = genotypes_service(db).chrX
else:
variant_ids = np.asarray(range(1,N+1), dtype=np.uint64)
active_idx = np.asarray(self.ss.active_idx, dtype=np.uint16)
conditions = self.conditions_vector
is_and = self.merge_op == AND
if len(conditions) == 0:
passing = variant_ids
else:
passing = self.parallel_apply_bitwise(genotypes, variant_ids, conditions, active_idx, is_and)
return passing
def scan_genotypes_compound(self, genotypes, batches, parallel=True):
"""Scan the *genotypes* array for compounds. Variant ids are treated in batches,
- one list of variant_ids per gene."""
if self.shortcut:
passing, sources, pairs = np.zeros(0), {}, []
else:
N = len(genotypes)
active_idx = np.asarray(self.ss.active_idx, dtype=np.uint16)
batches = list(batches.items())
if parallel:
passing, sources, pairs = self.parallel_batches(genotypes, batches, active_idx, N)
else:
passing, sources, pairs = self.process_batches(genotypes, batches, active_idx, N)
passing = np.array(list(passing), dtype=np.uint64)
passing.sort()
return passing, sources, pairs
def test_int(self):
for st, ut, s in [(np.int8, np.uint8, 8),
(np.int16, np.uint16, 16),
(np.int32, np.uint32, 32),
(np.int64, np.uint64, 64)]:
for i in range(1, s):
assert_equal(hash(st(-2**i)), hash(-2**i),
err_msg="%r: -2**%d" % (st, i))
assert_equal(hash(st(2**(i - 1))), hash(2**(i - 1)),
err_msg="%r: 2**%d" % (st, i - 1))
assert_equal(hash(st(2**i - 1)), hash(2**i - 1),
err_msg="%r: 2**%d - 1" % (st, i))
i = max(i - 1, 1)
assert_equal(hash(ut(2**(i - 1))), hash(2**(i - 1)),
err_msg="%r: 2**%d" % (ut, i - 1))
assert_equal(hash(ut(2**i - 1)), hash(2**i - 1),
err_msg="%r: 2**%d - 1" % (ut, i))
def enhchr(indexes):
"""chromosome length vs. enhancer numbers on it"""
lens = np.array([len(sequences[keys[i]]) for i in range(24)], dtype=np.float)
nums = np.zeros((24,))
for index in indexes:
chrkey = index[0]
nums[keys.index(chrkey)] += 1
print "The length of 24 Chromosomes are \n{}".format(np.array(lens, dtype=np.uint64))
print "The number of enhancers on each chromosome are \n{}".format(np.array(nums, dtype=np.uint64))
ind = np.arange(24)
w = 0.35
fig, ax = plt.subplots()
rects1 = ax.bar(ind, lens / np.sum(lens), w, color='r')
rects2 = ax.bar(ind + w, nums / np.sum(nums), w, color='y')
ax.set_ylabel('Chrom Length & #Enhancers')
ax.set_xticks(ind + w)
ax.set_xticklabels(keys)
ax.legend((rects1[0], rects2[0]), ('Chrom Length (%)', '#Enahncers (%)'))
plt.show()
def _getchunk(self, z, y, x, zsize, ysize, xsize):
"""Internal function to retrieve data.
"""
data = None
# interface is the same for labels and raw arrays but the function is stateless
# and can benefit from extra compression possible in labels in some use cases
if self.dtype == ArrayDtype.uint8:
data = self.ns.get_array8bit3D(self.instancename, (zsize, ysize, xsize), (z, y, x), self.islabel3D)
elif self.dtype == ArrayDtype.uint16:
data = self.ns.get_array16bit3D(self.instancename, (zsize, ysize, xsize), (z, y, x), self.islabel3D)
elif self.dtype == ArrayDtype.uint32:
data = self.ns.get_array32bit3D(self.instancename, (zsize, ysize, xsize), (z, y, x), self.islabel3D)
elif self.dtype == ArrayDtype.uint64:
data = self.ns.get_array64bit3D(self.instancename, (zsize, ysize, xsize), (z, y, x), self.islabel3D)
else:
raise DicedException("Invalid datatype for array")
return data
def seed(self, seed=None):
"""Resets the state of the random number generator with a seed.
.. seealso::
:func:`cupy.random.seed` for full documentation,
:meth:`numpy.random.RandomState.seed`
"""
if seed is None:
try:
seed_str = binascii.hexlify(os.urandom(8))
seed = numpy.uint64(int(seed_str, 16))
except NotImplementedError:
seed = numpy.uint64(time.clock() * 1000000)
else:
seed = numpy.uint64(seed)
curand.setPseudoRandomGeneratorSeed(self._generator, seed)
curand.setGeneratorOffset(self._generator, 0)
test_constructors.py 文件源码
项目：PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 31
收藏 0
点赞 0
评论 0
def test_constructor_overflow_int64(self):
values = np.array([2 ** 64 - i for i in range(1, 10)],
dtype=np.uint64)
result = DataFrame({'a': values})
self.assertEqual(result['a'].dtype, object)
# #2355
data_scores = [(6311132704823138710, 273), (2685045978526272070, 23),
(8921811264899370420, 45),
(long(17019687244989530680), 270),
(long(9930107427299601010), 273)]
dtype = [('uid', 'u8'), ('score', 'u8')]
data = np.zeros((len(data_scores),), dtype=dtype)
data[:] = data_scores
df_crawls = DataFrame(data)
self.assertEqual(df_crawls['uid'].dtype, object)
test_multiarray.py 文件源码
项目：PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda
作者: SignalMedia
项目源码
文件源码
阅读 33
收藏 0
点赞 0
评论 0
def test_int(self):
for st, ut, s in [(np.int8, np.uint8, 8),
(np.int16, np.uint16, 16),
(np.int32, np.uint32, 32),
(np.int64, np.uint64, 64)]:
for i in range(1, s):
assert_equal(hash(st(-2**i)), hash(-2**i),
err_msg="%r: -2**%d" % (st, i))
assert_equal(hash(st(2**(i - 1))), hash(2**(i - 1)),
err_msg="%r: 2**%d" % (st, i - 1))
assert_equal(hash(st(2**i - 1)), hash(2**i - 1),
err_msg="%r: 2**%d - 1" % (st, i))
i = max(i - 1, 1)
assert_equal(hash(ut(2**(i - 1))), hash(2**(i - 1)),
err_msg="%r: 2**%d" % (ut, i - 1))
assert_equal(hash(ut(2**i - 1)), hash(2**i - 1),
err_msg="%r: 2**%d - 1" % (ut, i))
def centroid(im, labels, return_mm3=True):
"""
Centroid (center of mass, barycenter) of a list of labels.
:param im:
:param labels: list of labels, e.g. [3] or [2, 3, 45]
:param return_mm3: if true the answer is in mm if false in voxel indexes.
:return: list of centroids, one for each label in the input order.
"""
centers_of_mass = centroid_array(im.get_data(), labels)
ans = []
if return_mm3:
for cm in centers_of_mass:
if isinstance(cm, np.ndarray):
ans += [im.affine[:3, :3].dot(cm.astype(np.float64))]
else:
ans += [cm]
else:
for cm in centers_of_mass:
if isinstance(cm, np.ndarray): # else it is np.nan.
ans += [np.round(cm).astype(np.uint64)]
else:
ans += [cm]
return ans
