MechanizeCrawler.py 文件源码

def __init__(self, headers = {}, debug = True, p = ''):
        self.timeout = 10  
        self.br = mechanize.Browser() #???br
        self.cj = cookielib.LWPCookieJar()
        self.br.set_cookiejar(self.cj)#??cookie
        self.br.set_handle_equiv(True)#????http equiv
        self.br.set_handle_gzip(True)#??????
        self.br.set_handle_redirect(True)#???????
        self.br.set_handle_referer(True)#??????referer
        self.br.set_handle_robots(False)#????robots??
        self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
        self.br.addheaders = [('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36')]
        self.debug = debug
        #debug?????????????
        if self.debug:
            self.br.set_debug_http(True)
            self.br.set_debug_redirects(True) 
            self.br.set_debug_responses(True)
        #headers
        for keys in headers.keys():
            self.br.addheaders += [(key, headers[key]), ]
        #proxy
        if len(p) > 0 and p != 'None' and p != None and p != 'NULL':
            self.br.set_proxies({'http': p})