download.py 文件源码

def process_request(self, request, spider):
        if self.use_selenium(request.url):
            if self.use_proxy():
                if self._count > 20:
                    self.update_driver()
                    self._count = 0
                    log.info('update driver')
            yield HtmlResponse(request.url, encoding='utf-8', body=self.driver.page_source.encode('utf8'))