handle_html.py 文件源码

def downloader_html(self,url):
        '''
        :param url: ??????url
        :return  ???????,????????????0
        '''
        try:
            print url
            BAIDU_UA = 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)'#????UA
            headers = {'User-Agent': BAIDU_UA}  ##????????User-Agent ?UA?????????????????
            data = requests.get(url, headers=headers) ##??????????????????
            html_parser = HTMLParser.HTMLParser()
            data = html_parser.unescape(data.text)
            return data
        except:
            print '?????????',url
            return '0'