scrape_wiki.py 文件源码

def for_genre(genre,num):
    pool = ThreadPool(multiprocessing.cpu_count()-1)
    nums = list(range(1,num))
    results = pool.starmap(soupit,zip(nums,itertools.repeat(genre)))
    pool.close()
    pool.join()

    #build up the list of urls with the results of all the sub-processes that succeeded in a single list
    new_results = []
    for j in results:
        if j:
            for i in j:
                new_results.append(i)

    pool = ThreadPool(multiprocessing.cpu_count()-1)
    pool.starmap(dwnld,zip(enumerate(new_results),itertools.repeat(genre)))
    pool.close
    pool.close()