def fetch_decode(url, encoding=None):
""" Fetch url and decode. """
try:
req = g.opener.open(url)
except HTTPError as e:
if e.getcode() == 503:
time.sleep(.5)
return fetch_decode(url, encoding)
else:
raise
ct = req.headers['content-type']
if encoding:
return req.read().decode(encoding)
elif "charset=" in ct:
dbg("charset: %s", ct)
encoding = re.search(r"charset=([\w-]+)\s*(:?;|$)", ct).group(1)
return req.read().decode(encoding)
else:
dbg("encoding unknown")
return req.read()
评论列表
文章目录