crawl.py 文件源码

def handle(self, url):
        tries = 0
        while tries < self.max_tries:
            try:
                response = await self.session.get(
                    url, allow_redirects=False)
                break
            except aiohttp.ClientError:
                pass
            tries += 1
        try:
            doc = await self.fetch_etree(response)
            if is_root_url(url):
                print('root:{}'.format(url))
                self.parse_root_etree(doc)
            else:
                print('second level:{}'.format(url))
                self.parse_second_etree(doc, url)
        finally:
            await response.release()