def fetch(self, url):
"""Load a webpage and read return the body as plaintext."""
self.logger.info("{url}: loading...".format(**locals()))
try:
with aiohttp.Timeout(self.page_load_timeout, loop=self.loop):
async with self.session.get(url,
allow_redirects=True,
headers=self.headers) as resp:
if resp.status != 200:
self.logger.warning("{url} was not reachable. HTTP "
"error code {resp.status} was "
"returned".format(**locals()))
raise SorterResponseCodeError
self.logger.info("{url}: loaded "
"successfully.".format(**locals()))
return await resp.text()
except asyncio.TimeoutError:
self.logger.warning("{url}: timed out after "
"{self.page_load_timeout}.".format(**locals()))
raise SorterTimeoutError
except (aiosocks.errors.SocksError,
aiohttp.errors.ServerDisconnectedError,
aiohttp.errors.ClientResponseError) as exc:
self.logger.warning("{url} was not reachable: "
"{exc}".format(**locals()))
raise SorterConnectionError
except aiohttp.errors.ClientOSError as exception_msg:
if "SSL" in exception_msg:
self.logger.warning("{url}: certificate error (probably due to "
"use of a self-signed "
"cert.".format(**locals()))
raise SorterCertError
else:
raise
except (ssl.CertificateError, aiohttp.errors.ClientOSError):
self.logger.warning("{url}: certificate error (probably due to "
"use of a self-signed "
"cert.".format(**locals()))
raise SorterCertError
评论列表
文章目录
