LocationCrawler.py 文件源码

def scrapeCitiesFromList(cityList, dirPrefix, threadCount, scrapers):
    """scrapes locations in cities in cityList to file in dirPrefix, spawns threadCount threads"""
    threads = []
    i = 0
    while(True):
        if(threading.active_count() <= threadCount):
            if(i == len(cityList)):
                break
            for s in scrapers:
                if(not s.inUse):
                    threads.append(ScrapeThread(target=scrapeCityToFile, args=(dirPrefix, cityList[i], s)))
                    threads[-1].start()
                    i += 1
                    break
        time.sleep(1)

    for t in threads:
        t.join()