extractseeds.py 文件源码

def manager(dbs):
    tasks = mp.cpu_count()-1

    #que to store the db tasks , outque to store result for each cpu
    que = mp.Queue()
    outque = mp.Queue()
    for db in dbs:
        que.put(db)

    lock = mp.Lock()
    plist = []
    for i in xrange(tasks):
        p = mp.Process(target=worker, args=(que, lock, i+1, outque))
        p.start()
        plist.append(p)

    for p in plist:
        p.join()

    #here we got all extract tasks done
    #then merge it and insert into redis