parallel.py 文件源码

def preduce(func, iterable, args = tuple(), kwargs = dict(), processes = 1):
    """
    Parallel application of the reduce function, with keyword arguments.

    Parameters
    ----------
    func : callable
        Function to be applied to every element of `iterable`.
    iterable : iterable
        Iterable of items to be reduced. Generators are consumed.
    args : tuple
        Positional arguments of `function`.
    kwargs : dictionary, optional
        Keyword arguments of `function`.
    processes : int or None, optional
        Number of processes to use. If `None`, maximal number of processes
        is used. Default is one.

    Returns
    -------
    reduced : object

    Notes
    -----
    If `processes` is 1, `preduce` is equivalent to functools.reduce with the
    added benefit of using `args` and `kwargs`, but `initializer` is not supported.
    """
    func = partial(func, *args, **kwargs)

    if processes == 1:
        return reduce(func, iterable)

    with Pool(processes) as pool:
        if isinstance(iterable, Sized):
            chunksize = max(1, int(len(iterable)/pool._processes))
        else:
            chunksize = 1

        # Some reductions are order-sensitive
        res = pool.imap(partial(reduce, func), tuple(chunked(iterable, chunksize)))
        return reduce(func, res)