tenda_cn_crawler.py 文件源码

def main():
    global executor
    try:
        session = requests.Session()
        executor = ThreadPoolExecutor()
        os.makedirs(dlDir, exist_ok=True)
        with open('tenda_filelist.csv', 'w') as fout:
            cw = csv.writer(fout)
            cw.writerow(['model', 'fver', 'fname', 'furl', 'fdate', 'fsize', 'sha1', 'md5'])
        walkFiles('http://www.tendacn.com/en/service/download-cata-11.html')
        walkFiles('http://tendacn.com/en/service/download-cata-11-2.html')
        walkFiles('http://www.tendacn.com/en/service/download-cata-11-3.html')
    except BaseException as ex:
        traceback.print_exc()
    finally:
        print('Wait for exeuctor shuddown')
        executor.shutdown(True)