toolClass.py 文件源码

def getPage (self, url, requestHeader = []) :
        resultFormate = StringIO.StringIO()

        fakeIp = self.fakeIp()
        requestHeader.append('CLIENT-IP:' + fakeIp)
        requestHeader.append('X-FORWARDED-FOR:' + fakeIp)

        try:
            curl = pycurl.Curl()
            curl.setopt(pycurl.URL, url.strip())
            curl.setopt(pycurl.ENCODING, 'gzip,deflate')
            curl.setopt(pycurl.HEADER, 1)
            curl.setopt(pycurl.TIMEOUT, 120)
            curl.setopt(pycurl.SSL_VERIFYPEER, 0)   
            curl.setopt(pycurl.SSL_VERIFYHOST, 0)
            curl.setopt(pycurl.HTTPHEADER, requestHeader)
            curl.setopt(pycurl.WRITEFUNCTION, resultFormate.write)
            curl.perform()
            headerSize = curl.getinfo(pycurl.HEADER_SIZE)
            curl.close()

            header = resultFormate.getvalue()[0 : headerSize].split('\r\n')
            body = resultFormate.getvalue()[headerSize : ]
        except Exception, e:
            header = ''
            body = ''

        return header, body