def multithread(fn, args=[[]], pool_type=Pool,
processes=_cpus, maxtasksperchild=1,
chunksize=1):
'''Multithread method using a Pool. Not inherently threadsafe.
For threadsafe operations, use Managers or Locks.
Args must be wrapped in their own iterator, as starmap is used for
multiple arguments.
Returns an iterator of the results'''
def helper(pool):
return pool.starmap(fn, args, chunksize=chunksize)
# ThreadPools do not take a maxtasksperchild argument,
# so we need to conditionally construct a pool
if type(pool_type) is Pool:
with pool_type(processes, maxtasksperchild=maxtasksperchild) as pool:
results = helper(pool)
else:
with pool_type(processes) as pool:
results = helper(pool)
return results
评论列表
文章目录