def safestr(obj, encoding='utf-8'):
r"""
Converts any given object to utf-8 encoded string.
>>> safestr('hello')
'hello'
>>> safestr(u'\u1234')
'\xe1\x88\xb4'
>>> safestr(2)
'2'
"""
if isinstance(obj, unicode):
return obj.encode(encoding)
elif isinstance(obj, str):
return obj
elif hasattr(obj, 'next'): # iterator
return itertools.imap(safestr, obj)
else:
return str(obj)
# for backward-compatibility
python类imap()的实例源码
def imap(self, func, iterable, chunksize=1):
'''
Equivalent of `itertools.imap()` -- can be MUCH slower than `Pool.map()`
'''
assert self._state == RUN
if chunksize == 1:
result = IMapIterator(self._cache)
self._taskqueue.put((((result._job, i, func, (x,), {})
for i, x in enumerate(iterable)), result._set_length))
return result
else:
assert chunksize > 1
task_batches = Pool._get_tasks(func, iterable, chunksize)
result = IMapIterator(self._cache)
self._taskqueue.put((((result._job, i, mapstar, (x,), {})
for i, x in enumerate(task_batches)), result._set_length))
return (item for chunk in result for item in chunk)
def imap_unordered(self, func, iterable, chunksize=1):
'''
Like `imap()` method but ordering of results is arbitrary
'''
assert self._state == RUN
if chunksize == 1:
result = IMapUnorderedIterator(self._cache)
self._taskqueue.put((((result._job, i, func, (x,), {})
for i, x in enumerate(iterable)), result._set_length))
return result
else:
assert chunksize > 1
task_batches = Pool._get_tasks(func, iterable, chunksize)
result = IMapUnorderedIterator(self._cache)
self._taskqueue.put((((result._job, i, mapstar, (x,), {})
for i, x in enumerate(task_batches)), result._set_length))
return (item for chunk in result for item in chunk)
def phase1(self): # Compute common names
a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
self.common = map(a.__getitem__, ifilter(b.__contains__, a))
self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b))
def json_dict_unicode_to_bytes(d, encoding='utf-8'):
""" Recursively convert dict keys and values to byte str
Specialized for json return because this only handles, lists, tuples,
and dict container types (the containers that the json module returns)
"""
if isinstance(d, unicode):
return d.encode(encoding)
elif isinstance(d, dict):
return OrderedDict(imap(json_dict_unicode_to_bytes, iteritems(d), repeat(encoding)))
elif isinstance(d, list):
return list(imap(json_dict_unicode_to_bytes, d, repeat(encoding)))
elif isinstance(d, tuple):
return tuple(imap(json_dict_unicode_to_bytes, d, repeat(encoding)))
else:
return d
def json_dict_bytes_to_unicode(d, encoding='utf-8'):
""" Recursively convert dict keys and values to byte str
Specialized for json return because this only handles, lists, tuples,
and dict container types (the containers that the json module returns)
"""
if isinstance(d, bytes):
return unicode(d, encoding)
elif isinstance(d, dict):
return dict(imap(json_dict_bytes_to_unicode, iteritems(d), repeat(encoding)))
elif isinstance(d, list):
return list(imap(json_dict_bytes_to_unicode, d, repeat(encoding)))
elif isinstance(d, tuple):
return tuple(imap(json_dict_bytes_to_unicode, d, repeat(encoding)))
else:
return d
def get_memberships(self, obj):
"""
Retrieve all nodes where user is a member.
Returns an iterable of objects containing the name and id,
plus a custom field is_recoverable for each workspace.
A workspace is recoverable if it is share among any other user
and its membership status is MemberStatusField.STATUS_MEMBER
:return :dict {'workspace_id': int, 'workspace_name': str,
'is_recoverable': bool}
"""
nodes = Node.objects.all_for_user(obj.created_by)
return imap(
lambda node:
{'workspace_id': node.id, 'workspace_name': node.name,
'is_recoverable': LostKey.objects.is_recoverable(
node.id, obj.created_by)},
nodes)
def get_bc_grouped_pair_iter(bam):
""" Yields (bc, pair_iter)
where pair_iter yields (AugmentedFastqHeader, (read1, read2)) for the barcode """
wrap_header = lambda pair: (cr_fastq.AugmentedFastqHeader(pair[0].qname), pair)
get_barcode = lambda hdr_pair: hdr_pair[0].get_tag(cr_constants.PROCESSED_BARCODE_TAG)
return itertools.groupby(
itertools.imap(wrap_header, get_pair_iter(bam)),
key=get_barcode)
def _set(self, i, success_result):
success, result = success_result
if success:
self._value[i*self._chunksize:(i+1)*self._chunksize] = result
self._number_left -= 1
if self._number_left == 0:
if self._callback:
self._callback(self._value)
del self._cache[self._job]
self._cond.acquire()
try:
self._ready = True
self._cond.notify()
finally:
self._cond.release()
else:
self._success = False
self._value = result
del self._cache[self._job]
self._cond.acquire()
try:
self._ready = True
self._cond.notify()
finally:
self._cond.release()
#
# Class whose instances are returned by `Pool.imap()`
#
def phase1(self): # Compute common names
a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
self.common = map(a.__getitem__, ifilter(b.__contains__, a))
self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b))
def nsmallest(n, iterable, key=None):
"""Find the n smallest elements in a dataset.
Equivalent to: sorted(iterable, key=key)[:n]
"""
# Short-cut for n==1 is to use min() when len(iterable)>0
if n == 1:
it = iter(iterable)
head = list(islice(it, 1))
if not head:
return []
if key is None:
return [min(chain(head, it))]
return [min(chain(head, it), key=key)]
# When n>=size, it's faster to use sorted()
try:
size = len(iterable)
except (TypeError, AttributeError):
pass
else:
if n >= size:
return sorted(iterable, key=key)[:n]
# When key is none, use simpler decoration
if key is None:
it = izip(iterable, count()) # decorate
result = _nsmallest(n, it)
return map(itemgetter(0), result) # undecorate
# General case, slowest method
in1, in2 = tee(iterable)
it = izip(imap(key, in1), count(), in2) # decorate
result = _nsmallest(n, it)
return map(itemgetter(2), result) # undecorate
def iter_stereo_frames(self, *args, **kwargs):
return self.iteritems(*args, **kwargs)
# class BumblebeeStereoDatasetReader:
# def __init__(self, directory):
# bfiles = read_dir(directory, pattern='*.bumblebee', recursive=False)
# self.dataset = DatasetReader(process_cb=lambda x: read_bumblebee(x), files=bfiles)
# self.iter_stereo_frames = lambda : imap(lambda x: self.split_stereo(x), self.dataset.iteritems())
# def split_stereo(self, im):
# h = im.shape[0]/2
# return im[:h], im[h:]
def __init__(self, directory, max_files=20000):
"""
SUN RGB-D Dataset reader
Note: First run find . | grep seg.mat > annotations.txt (in SUNRGBD folder)
@params directory: SUNRGBD directory listing with image/*.png, and seg.mat files
"""
self.directory_ = os.path.expanduser(directory)
with open(os.path.join(self.directory_, 'image.txt')) as f:
rgb_files = f.read().splitlines()
with open(os.path.join(self.directory_, 'depth.txt')) as f:
depth_files = f.read().splitlines()
assert(len(rgb_files) == len(depth_files))
self.rgb_files_ = [os.path.join(self.directory_, fn) for fn in fnmatch.filter(rgb_files,'*mit_*')][:max_files]
self.depth_files_ = [os.path.join(self.directory_, fn) for fn in fnmatch.filter(depth_files,'*mit_*')][:max_files]
self.label_files_ = [ os.path.join(
os.path.split(
os.path.split(fn)[0])[0], 'seg.mat') for fn in self.rgb_files_ ]
if not len(self.rgb_files_):
raise RuntimeError('{} :: Failed to load dataset'.format(self.__class__.__name__))
print('{} :: Loading {} image/depth/segmentation pairs'.format(self.__class__.__name__, len(self.rgb_files_)))
self.rgb_ = imap(lambda fn: self._pad_image(cv2.imread(fn, cv2.CV_LOAD_IMAGE_COLOR)), self.rgb_files_)
self.depth_ = imap(lambda fn: self._pad_image(cv2.imread(fn, -1)), self.depth_files_)
self.labels_ = imap(self._process_label, self.label_files_)
# self.target_hash_ = {item.encode('utf8'): idx+1
# for idx, item in enumerate(loadmat('data/sun3d/seg37list.mat', squeeze_me=True)['seg37list'])}
# self.target_unhash_ = {v:k for k,v in self.target_hash_.iteritems()}
# self.target_hash_ = SUNRGBDDataset.target_hash
# self.target_unhash_ = SUNRGBDDataset.target_unhash
# @property
# def target_unhash(self):
# return self.objects_.target_unhash
# @property
# def target_hash(self):
# return self.objects_.target_hash
def itervalues_for_key(self, key, inds=None, verbose=False):
if key not in self.keys:
raise RuntimeError('Key %s not found in dataset. keys: %s' % (key, self.keys))
return imap(self.unpack, self.get_node(key).iterrows())
def __repr__(self):
""" Converts the value of this instance to its string representation.
The value of this ConfigurationSettings instance is represented as a string of comma-separated
:code:`(name, value)` pairs.
:return: String representation of this instance
"""
definitions = type(self).configuration_setting_definitions
settings = imap(
lambda setting: repr((setting.name, setting.__get__(self), setting.supporting_protocols)), definitions)
return '[' + ', '.join(settings) + ']'
def __str__(self):
""" Converts the value of this instance to its string representation.
The value of this ConfigurationSettings instance is represented as a string of comma-separated
:code:`name=value` pairs. Items with values of :const:`None` are filtered from the list.
:return: String representation of this instance
"""
text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems()))
return text
# region Methods
def iteritems(self):
definitions = type(self).configuration_setting_definitions
version = self.command.protocol_version
return ifilter(
lambda (name, value): value is not None, imap(
lambda setting: (setting.name, setting.__get__(self)), ifilter(
lambda setting: setting.is_supported_by_protocol(version), definitions)))
def iteritems(self):
iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
version = self.command.protocol_version
if version == 1:
if self.required_fields is None:
iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
else:
iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
if self.distributed:
iteritems = imap(
lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
return iteritems
# endregion
def __init__(self, command):
definitions = type(command).option_definitions
item_class = Option.Item
OrderedDict.__init__(self, imap(lambda (name, option): (option.name, item_class(command, option)), definitions))
def validate_configuration_setting(specification, name, value):
if not isinstance(value, specification.type):
if isinstance(specification.type, type):
type_names = specification.type.__name__
else:
type_names = ', '.join(imap(lambda t: t.__name__, specification.type))
raise ValueError('Expected {} value, not {}={}'.format(type_names, name, repr(value)))
if specification.constraint and not specification.constraint(value):
raise ValueError('Illegal value: {}={}'.format(name, repr(value)))
return value
