engine.py 文件源码

def __init__(self, spider):
        self.spider = spider
        self.logger = spider.logger
        self.scheduler = Scheduler(spider)
        self.settings = spider.settings
        max_request_size = self.settings["MAX_REQUEST_SIZE"]
        self.dynamic = self.settings["DYNAMIC_CRAWL"]
        if self.dynamic:
            module_path = DRIVER_MODULE
            module = import_module(module_path)
            init_kwargs = self.settings['DRIVER_INIT_KWARGS']
            self.driver = getattr(module,
                                  self.settings.get('DRIVER').title())(**init_kwargs)
        else:
            self.driver = None
        self.driver_sem = BoundedSemaphore(1)
        self.downloader = Downloader(spider, self.driver, self.driver_sem)
        self.pool = Pool(size=max_request_size)