def open (self, filepart = None, data = None, do_soup = True) :
filepart = filepart or self.nextfile
url = '/'.join (('%s:/' % self.args.protocol, self.args.host, filepart))
if data :
data = data.encode ('ascii')
rq = Request (url, data)
self.f = f = self.opener.open (rq, timeout = 10)
if do_soup :
# Sometimes we get incomplete read. So we read everything
# the server sent us and hope this is ok. Note: This means
# we cannot pass the file to BeautifulSoup but need to read
# everything here.
try:
page = f.read ()
except IncompleteRead as e:
page = e.partial
self.soup = BeautifulSoup (page, "lxml")
self.purl = f.geturl ()
self.info = f.info ()
# end def open
评论列表
文章目录