requests_downloader.py 文件源码

def __init__(self, loginer=None, use_proxy=False):
        self.loginer = loginer
        self.use_proxy = use_proxy
        if use_proxy:
            self.proxy_pool = ProxyPool()
            if len(self.proxy_pool) == 0:
                self.use_proxy = False
        self._cookies = None

        self._headers = dict()
        self._headers[
            "User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"
        self._headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
        self._headers["Accept-Encoding"] = "gzip, deflate, sdch"
        self._headers["Accept-Language"] = "zh-CN,zh;q=0.8"
        self._request_retry = HTTPAdapter(max_retries=3)

        cookie_dict = dict()
        self._cookies = cookie_dict