html_downloader.py 文件源码

def download(self, url, retry_count=3, headers=None, proxies=None, data=None):
        '''
        :param url: ????? URL ??
        :param retry_count: ?? url ????????
        :param headers: http header={'X':'x', 'X':'x'}
        :param proxies: ???? proxies={"https": "http://12.112.122.12:3212"}
        :param data: ?? urlencode(post_data) ? POST ??
        :return: ?????? None
        '''
        if headers:
            self.request_session.headers.update(headers)
        try:
            if data:
                content = self.request_session.post(url, data, proxies=proxies).content
            else:
                content = self.request_session.get(url, proxies=proxies).content
        except (ConnectionError, Timeout) as e:
            print('Downloader download ConnectionError or Timeout:' + str(e))
            content = None
            if retry_count > 0:
                self.download(url, retry_count - 1, headers, proxies, data)
        except Exception as e:
            print('Downloader download Exception:' + str(e))
            content = None
        return content