Opener.py 文件源码

python
阅读 26 收藏 0 点赞 0 评论 0

项目:DistributeCrawler 作者: SmallHedgehog 项目源码 文件源码
def __init__(self, cookie_filename=None, user_agent=None, timeout=None, **kwargs):
        try:
            import mechanize
        except ImportError:
            raise DependencyNotInstalledError('mechanize')

        if user_agent is None:
            user_agent = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)'

        self.browser = mechanize.Browser()
        self.cj = cookielib.LWPCookieJar()
        if cookie_filename is not None:
            self.cj.load(cookie_filename)
        self.browser.set_cookiejar(self.cj)
        self.browser.set_handle_equiv(True)
        self.browser.set_handle_gzip(True)
        self.browser.set_handle_redirect(True)
        self.browser.set_handle_referer(True)
        self.browser.set_handle_robots(False)
        self.browser.addheaders = [
            ('User-agnet', user_agent)
        ]

        if timeout is None:
            # self._default_timout = mechanize._sockettimeout._GLOBAL_DEFAULT_TIMEOUT
            self._default_timout = 5
        else:
            self._default_timout = timeout
评论列表
文章目录


问题


面经


文章

微信
公众号

扫码关注公众号