def _response(self, _, driver, spider):
body = driver.execute_script('return document.documentElement.innerHTML')
if body.startswith(
"<head></head>"): # selenium ????http??????,???????????????,????body??????<head></head>???,??????
body = driver.execute_script('return document.documentElement.textContent')
url = driver.current_url
respcls = responsetypes.from_args(url=url, body=body[:100].encode('utf-8'))
response = respcls(url=url, body=body, encoding='utf-8')
response_failed = getattr(spider, 'response_failed', None)
if response_failed and callable(response_failed) and response_failed(response, driver):
driver.quit()
return defer.fail(Failure())
else:
self.queue.put(driver) # ?driver????queue
return defer.succeed(response) # ??response??
评论列表
文章目录
