textrank.py 文件源码

def from_url(cls, url, tokenizer):
        headers = {
            'User-Agent': ' '.join([
                'Mozilla/5.0 (X11; Linux x86_64)',
                'AppleWebKit/537.11 (KHTML, like Gecko)',
                'Chrome/23.0.1271.64 Safari/537.11'
            ]),
            'Accept': ','.join([
                'text/html', 'application/xhtml+xml', 'application/xml;q=0.9',
                '*/*;q=0.8'
            ]),
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding': 'none',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive'
        }
        session = Session()

        session.mount('http://', HTTPAdapter(max_retries=2))
        session.mount('https://', HTTPAdapter(max_retries=2))

        cookies = CookieJar()
        request = Request(method='GET',
                          url=url,
                          headers=headers,
                          cookies=cookies)
        prepare = session.prepare_request(request)
        response = session.send(prepare, verify=True)

        if response.status_code != requests.codes.ok:
            response.raise_for_status()

        return cls(response.text, tokenizer, url)