main.py 文件源码

def getPage(self, url, postdata=None, headers=None):
        try:
            cookie = cookielib.CookieJar()
            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
            if postdata is None:
                if headers is None:
                    request = urllib2.Request(url)
                else:
                    request = urllib2.Request(url, headers=headers)
            else:
                if headers is not None:
                    request = urllib2.Request(url, postdata, headers)
            page = opener.open(request)
            redirect_url = opener.open(request).geturl()  # ?????url
            soup = BeautifulSoup(page, 'html.parser')
            return page, soup, redirect_url
        # ???????
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                print u"????,????", e.reason
                return None

    # ?????