spider.py 文件源码

def __init__(self,proxyHost = ""):
        #???headers
        self.headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36',
                        'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
                        'Accept-Encoding':'en-us',
                        'Connection':'keep-alive',
                        'Referer':'http://www.baidu.com/'}

        self.proxyHeaders = [('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36'),
                             ('Accept-Charset','ISO-8859-1,utf-8;q=0.7,*;q=0.3'),
                             ('Accept-Encoding','en-us'),
                             ('Connection','keep-alive'),
                             ('Referer','http://www.baidu.com/')]

        self.cookies = urllib2.HTTPCookieProcessor()
        self.req_timeout = 5

        self.proxyHost = {"http":proxyHost}