launch_crawler.py 文件源码

def launch_crawlers(crawler_class, exclusion=None):
    settings = get_settings()
    configure_logging(settings=settings)
    launcher = CrawlerRunner(settings)
    crawlers = launcher.spider_loader.list()
    crawlers = list([c for c in crawlers if c.__contains__(crawler_class)])
    if exclusion:
        for c in settings.get(exclusion, []):
            crawlers.remove(c)

    try:
        for crawler in crawlers:
            launcher.crawl(crawler)
        d = launcher.join()
        d.addBoth(lambda _: reactor.stop())
        reactor.run()
        return True
    except Exception as e:
        launch_logger.error('(????)????? | ?????:\n{excep}'
                            .format(excep=e))
        return False