aioWebScraper.py 文件源码

def wait_with_progress(urlList, concurency = 30, timeout = 120, rawResults = False, cloudflare = False, headers = None):
    sem = asyncio.Semaphore(concurency)
    # Client session worker
    headers = headers or {}
    headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36 vulners.com/bot'})
    if cloudflare:
        sessionClient = CloudflareScraper
    else:
        sessionClient = aiohttp.ClientSession
    urlToResultDict = {}
    with sessionClient(connector=aiohttp.TCPConnector(verify_ssl=False), headers=headers) as session:
        coros = [parseUrl(url = d, semaphore = sem, session = session, timeout = timeout, rawResults=rawResults) for d in urlList]
        for f in tqdm.tqdm(asyncio.as_completed(coros), total=len(coros)):
            result = yield from f
            urlToResultDict.update(result)
    return urlToResultDict