python类FOLLOWLOCATION的实例源码

InstagramRegistration.py 文件源码 项目:Instagram-API 作者: danleyb2 项目源码 文件源码 阅读 27 收藏 0 点赞 0 评论 0
def request(self, endpoint, post=None):
        buffer = BytesIO()

        ch = pycurl.Curl()
        ch.setopt(pycurl.URL, Constants.API_URL + endpoint)
        ch.setopt(pycurl.USERAGENT, self.userAgent)
        ch.setopt(pycurl.WRITEFUNCTION, buffer.write)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.HEADER, True)
        ch.setopt(pycurl.VERBOSE, False)
        ch.setopt(pycurl.COOKIEFILE, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat"))
        ch.setopt(pycurl.COOKIEJAR, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat"))

        if post is not None:
            ch.setopt(pycurl.POST, True)
            ch.setopt(pycurl.POSTFIELDS, post)

        if self.proxy:
            ch.setopt(pycurl.PROXY, self.proxyHost)
            if self.proxyAuth:
                ch.setopt(pycurl.PROXYUSERPWD, self.proxyAuth)

        ch.perform()
        resp = buffer.getvalue()
        header_len = ch.getinfo(pycurl.HEADER_SIZE)
        header = resp[0: header_len]
        body = resp[header_len:]

        ch.close()

        if self.debug:
            print("REQUEST: " + endpoint)
            if post is not None:
                if not isinstance(post, list):
                    print("DATA: " + str(post))
            print("RESPONSE: " + body)

        return [header, json_decode(body)]
httpadapter.py 文件源码 项目:searx-stats2 作者: dalf 项目源码 文件源码 阅读 18 收藏 0 点赞 0 评论 0
def get_connection():
    # pycurl initialization
    h = pycurl.Curl()

    # follow redirects
    h.setopt(pycurl.FOLLOWLOCATION, False)

    # enable compression
    h.setopt(pycurl.ENCODING, 'gzip, deflate')

    # certifi
    h.setopt(pycurl.CAINFO, certifi.where())

    # no signal
    h.setopt(pycurl.NOSIGNAL, 1)

    # certificate informations
    h.setopt(pycurl.OPT_CERTINFO, 1)

    return h
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 19 收藏 0 点赞 0 评论 0
def test_post(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", post=True, curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.POST: True,
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"})
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 25 收藏 0 点赞 0 评论 0
def test_post_data(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", post=True, data="data", curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options[pycurl.READFUNCTION](), b"data")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.POST: True,
                          pycurl.POSTFIELDSIZE: 4,
                          pycurl.READFUNCTION: Any(),
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"})
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 17 收藏 0 点赞 0 评论 0
def test_cainfo(self):
        curl = CurlStub(b"result")
        result = fetch("https://example.com", cainfo="cainfo", curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"https://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.CAINFO: b"cainfo",
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"})
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 26 收藏 0 点赞 0 评论 0
def test_headers(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com",
                       headers={"a": "1", "b": "2"}, curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.HTTPHEADER: ["a: 1", "b: 2"],
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"})
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 17 收藏 0 点赞 0 评论 0
def test_pycurl_insecure(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com/get-ca-cert", curl=curl,
                       insecure=True)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com/get-ca-cert",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.SSL_VERIFYPEER: False,
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"})
HTTPRequest.py 文件源码 项目:download-manager 作者: thispc 项目源码 文件源码 阅读 21 收藏 0 点赞 0 评论 0
def initHandle(self):
        """ sets common options to curl handle """
        self.c.setopt(pycurl.FOLLOWLOCATION, 1)
        self.c.setopt(pycurl.MAXREDIRS, 5)
        self.c.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.c.setopt(pycurl.NOSIGNAL, 1)
        self.c.setopt(pycurl.NOPROGRESS, 1)
        if hasattr(pycurl, "AUTOREFERER"):
            self.c.setopt(pycurl.AUTOREFERER, 1)
        self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
        self.c.setopt(pycurl.LOW_SPEED_TIME, 30)
        self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5)

        #self.c.setopt(pycurl.VERBOSE, 1)

        self.c.setopt(pycurl.USERAGENT,
            "Mozilla/5.0 (Windows NT 6.1; Win64; x64;en; rv:5.0) Gecko/20110619 Firefox/5.0")
        if pycurl.version_info()[7]:
            self.c.setopt(pycurl.ENCODING, "gzip, deflate")
        self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*",
                                          "Accept-Language: en-US,en",
                                          "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
                                          "Connection: keep-alive",
                                          "Keep-Alive: 300",
                                          "Expect:"])
asynclient.py 文件源码 项目:xtdpy 作者: psycofdj 项目源码 文件源码 阅读 21 收藏 0 点赞 0 评论 0
def __init__(self, p_request, p_timeoutMs = 1000, p_curlOpts=None):
    if isinstance(p_request, str):
      p_request = HTTPRequest(p_url=p_request)
    self.m_request   = p_request
    self.m_timeoutMs = p_timeoutMs
    self.m_response  = None
    self.m_handle    = None
    self.m_data      = None
    self.m_headers   = None
    self.m_handle    = pycurl.Curl()
    self.m_opts      = p_curlOpts
    if p_curlOpts is None:
      self.m_opts = {}

    self.cleanup()
    self._init_opt()
    self._init_url()
    self._init_method()
    self._init_headers()

    self.m_handle.setopt(pycurl.USERAGENT,      self.m_request.m_agent)
    self.m_handle.setopt(pycurl.HEADERFUNCTION, self._read_header)
    if self.m_timeoutMs:
      self.m_handle.setopt(pycurl.TIMEOUT_MS, self.m_timeoutMs)
    self.m_handle.setopt(pycurl.FOLLOWLOCATION, True)
zoomeye.py 文件源码 项目:ZPoc 作者: zidier215 项目源码 文件源码 阅读 29 收藏 0 点赞 0 评论 0
def _get_url(self, url):
        if self.API_TOKEN == None:
            logging.error('none token') # 3 For ERROR level
            return
        try:
            c = pycurl.Curl()
            c.setopt(pycurl.CAINFO, certifi.where())
            c.setopt(pycurl.URL, url)
            b = StringIO.StringIO()
            c.setopt(pycurl.WRITEFUNCTION, b.write)
            c.setopt(pycurl.USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)")
            c.setopt(pycurl.HTTPHEADER, ['Authorization: JWT %s' % self.API_TOKEN.encode()])
            c.setopt(pycurl.CUSTOMREQUEST, "GET")
            c.setopt(pycurl.FOLLOWLOCATION, 1)
            c.perform()
            result = b.getvalue()
            logging.debug('result')
        except Exception as e:
            logging.error(e.message)
            logging.error('go error')
            pass
        return result
weixin.py 文件源码 项目:hzlgithub 作者: hzlRises 项目源码 文件源码 阅读 26 收藏 0 点赞 0 评论 0
def Curl(url,headers):
    while 1:
        try:
            c = pycurl.Curl()
            c.setopt(pycurl.REFERER, 'http://weixin.sogou.com/')
            c.setopt(pycurl.FOLLOWLOCATION, True)
            c.setopt(pycurl.MAXREDIRS,5)
            c.setopt(pycurl.CONNECTTIMEOUT, 60)
            c.setopt(pycurl.TIMEOUT,120)
            c.setopt(pycurl.ENCODING, 'gzip,deflate')
            c.fp = StringIO.StringIO()  
            c.setopt(pycurl.URL, url)
            c.setopt(pycurl.HTTPHEADER,headers)
            c.setopt(c.WRITEFUNCTION, c.fp.write)
            c.perform()
            html = c.fp.getvalue()
            if '??????' in html:
                print u'??????,??10??'
                time.sleep(600)
            else:
                return html
        except Exception, e:
            print url,'curl(url)',e
            continue 
#????????
pycurl.py 文件源码 项目:hzlgithub 作者: hzlRises 项目源码 文件源码 阅读 19 收藏 0 点赞 0 评论 0
def getHtml(url,headers):
    c = pycurl.Curl()                           #??curl????????
    c.setopt(pycurl.URL, url)                   #??????URL
    c.setopt(pycurl.FOLLOWLOCATION, True)       #????????
    c.setopt(pycurl.MAXREDIRS,5)                #?????????
    c.setopt(pycurl.CONNECTTIMEOUT, 60)         #??????
    c.setopt(pycurl.TIMEOUT,120)                #????
    c.setopt(pycurl.ENCODING, 'gzip,deflate')   #??gzip???????????????????gzip?????????gzip??????
    c.fp = StringIO.StringIO()                  #??StringIO??
    c.setopt(pycurl.HTTPHEADER,headers)         #?????
    c.setopt(pycurl.POST, 1)                    #??get
    c.setopt(pycurl.POSTFIELDS, data)           #??POST??
    c.setopt(c.WRITEFUNCTION, c.fp.write)       #???????
    c.perform()                                 #?? 
    html = c.fp.getvalue()                      #?????
    return html
xiala_loop.py 文件源码 项目:hzlgithub 作者: hzlRises 项目源码 文件源码 阅读 17 收藏 0 点赞 0 评论 0
def curl(url, debug=False, **kwargs):
        while 1:
                try:
                        s = StringIO.StringIO()
                        c = pycurl.Curl()
                        c.setopt(pycurl.URL, url)
                        c.setopt(pycurl.REFERER, url)
                        c.setopt(pycurl.FOLLOWLOCATION, True)
                        c.setopt(pycurl.TIMEOUT, 60)
                        c.setopt(pycurl.ENCODING, 'gzip')
                        c.setopt(pycurl.USERAGENT, 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36')
                        c.setopt(pycurl.NOSIGNAL, True)
                        c.setopt(pycurl.WRITEFUNCTION, s.write)
                        for k, v in kwargs.iteritems():
                                c.setopt(vars(pycurl)[k], v)
                        c.perform()
                        c.close()
                        return s.getvalue()
                except:
                        if debug:
                                raise
                        continue
zoomeye.py 文件源码 项目:pymsf 作者: s0m30ne 项目源码 文件源码 阅读 25 收藏 0 点赞 0 评论 0
def searchIP(self, query, pages, queue, STOP_ME):
        if self.API_TOKEN == None:
            print "please config your API_TOKEN"
            sys.exit()
        for page in range(1, pages+1):
            b = StringIO.StringIO()
            c = pycurl.Curl()
            c.setopt(pycurl.URL, "%s?query=%s&page=%s" % (self.API_URL, query, page))
            c.setopt(pycurl.WRITEFUNCTION, b.write)
            c.setopt(pycurl.FOLLOWLOCATION, 1)
            c.setopt(pycurl.CUSTOMREQUEST, "GET")
            c.setopt(pycurl.HTTPHEADER, ['Authorization: JWT %s' % self.API_TOKEN.encode()])
            c.perform()
            hosts = json.loads(b.getvalue())
            for host in hosts['matches']:
                queue.put(host["ip"])
        STOP_ME[0] = True
restClient.py 文件源码 项目:recipebook 作者: dpapathanasiou 项目源码 文件源码 阅读 23 收藏 0 点赞 0 评论 0
def get (url, user_agent=UA, referrer=None):
    """Make a GET request of the url using pycurl and return the data
    (which is None if unsuccessful)"""

    data = None
    databuffer = StringIO()

    curl = pycurl.Curl()
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 5)
    curl.setopt(pycurl.TIMEOUT, 8)
    curl.setopt(pycurl.WRITEFUNCTION, databuffer.write)
    curl.setopt(pycurl.COOKIEFILE, '')
    if user_agent:
        curl.setopt(pycurl.USERAGENT, user_agent)
    if referrer is not None:
        curl.setopt(pycurl.REFERER, referrer)
    try:
        curl.perform()
        data = databuffer.getvalue()
    except Exception:
        pass
    curl.close()

    return data
Checkpoint.py 文件源码 项目:Instagram-API 作者: danleyb2 项目源码 文件源码 阅读 31 收藏 0 点赞 0 评论 0
def request(self, endpoint, headers=None, post=None, first=True):
        buffer = BytesIO()

        ch = pycurl.Curl()

        ch.setopt(pycurl.URL, endpoint)
        ch.setopt(pycurl.USERAGENT, self.userAgent)
        ch.setopt(pycurl.WRITEFUNCTION, buffer.write)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.HEADER, True)
        if headers:
            ch.setopt(pycurl.HTTPHEADER, headers)

        ch.setopt(pycurl.VERBOSE, self.debug)
        ch.setopt(pycurl.SSL_VERIFYPEER, False)
        ch.setopt(pycurl.SSL_VERIFYHOST, False)
        ch.setopt(pycurl.COOKIEFILE, self.settingsPath + self.username + '-cookies.dat')
        ch.setopt(pycurl.COOKIEJAR, self.settingsPath + self.username + '-cookies.dat')

        if post:
            import urllib
            ch.setopt(pycurl.POST, len(post))
            ch.setopt(pycurl.POSTFIELDS, urllib.urlencode(post))

        ch.perform()
        resp = buffer.getvalue()
        header_len = ch.getinfo(pycurl.HEADER_SIZE)
        header = resp[0: header_len]
        body = resp[header_len:]
        ch.close()

        if self.debug:
            import urllib
            print("REQUEST: " + endpoint)
            if post is not None:
                if not isinstance(post, list):
                    print('DATA: ' + urllib.unquote_plus(json.dumps(post)))
            print("RESPONSE: " + body + "\n")

        return [header, json_decode(body)]
__init__.py 文件源码 项目:ceiba-dl 作者: lantw44 项目源码 文件源码 阅读 21 收藏 0 点赞 0 评论 0
def __init__(self, api_cookies, web_cookies, cipher=None, api_args={'api': '1'},
        api_url='https://ceiba.ntu.edu.tw/course/f03067/app/login.php',
        file_url='https://ceiba.ntu.edu.tw',
        web_url='https://ceiba.ntu.edu.tw'):

        self.logger = logging.getLogger(__name__)
        self.curl = pycurl.Curl()
        self.api_cookie = ';'.join(map(lambda x: '{}={}'.format(*x), api_cookies.items()))
        self.web_cookie = ';'.join(map(lambda x: '{}={}'.format(*x), web_cookies.items()))
        self.api_args = api_args
        self.api_url = api_url
        self.file_url = file_url
        self.web_url = web_url
        self.api_cache = None
        self.web_cache = dict()
        if not cipher:
            tls_backend = pycurl.version_info()[5].split('/')[0]
            if tls_backend == 'OpenSSL':
                cipher = 'AES128-SHA'
            elif tls_backend == 'GnuTLS':
                cipher = 'AES128-SHA'
            elif tls_backend == 'NSS':
                cipher = 'rsa_aes_128_sha'
            else:
                assert False, 'TLS ?? {} ????'.format(tls_backend)
        self.curl.setopt(pycurl.USE_SSL, pycurl.USESSL_ALL)
        self.curl.setopt(pycurl.SSL_CIPHER_LIST, cipher)
        self.curl.setopt(pycurl.PROTOCOLS, pycurl.PROTO_HTTPS)
        self.curl.setopt(pycurl.REDIR_PROTOCOLS, pycurl.PROTO_HTTPS)
        self.curl.setopt(pycurl.DEFAULT_PROTOCOL, 'https')
        self.curl.setopt(pycurl.FOLLOWLOCATION, False)
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 25 收藏 0 点赞 0 评论 0
def test_basic(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"})
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 18 收藏 0 点赞 0 评论 0
def test_pycurl_follow_true(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", curl=curl,
                       follow=True)
        self.assertEqual(result, b"result")
        self.assertEqual(1, curl.options[pycurl.FOLLOWLOCATION])
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 18 收藏 0 点赞 0 评论 0
def test_pycurl_follow_false(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", curl=curl,
                       follow=False)
        self.assertEqual(result, b"result")
        self.assertNotIn(pycurl.FOLLOWLOCATION, curl.options.keys())
test_fetch.py 文件源码 项目:landscape-client 作者: CanonicalLtd 项目源码 文件源码 阅读 19 收藏 0 点赞 0 评论 0
def test_create_curl(self):
        curls = []

        def pycurl_Curl():
            curl = CurlStub(b"result")
            curls.append(curl)
            return curl
        Curl = pycurl.Curl
        try:
            pycurl.Curl = pycurl_Curl
            result = fetch("http://example.com")
            curl = curls[0]
            self.assertEqual(result, b"result")
            self.assertEqual(curl.options,
                             {pycurl.URL: b"http://example.com",
                              pycurl.FOLLOWLOCATION: 1,
                              pycurl.MAXREDIRS: 5,
                              pycurl.CONNECTTIMEOUT: 30,
                              pycurl.LOW_SPEED_LIMIT: 1,
                              pycurl.LOW_SPEED_TIME: 600,
                              pycurl.NOSIGNAL: 1,
                              pycurl.WRITEFUNCTION: Any(),
                              pycurl.DNS_CACHE_TIMEOUT: 0,
                              pycurl.ENCODING: b"gzip,deflate"})
        finally:
            pycurl.Curl = Curl
test.py 文件源码 项目:redcap_deployment 作者: ctsit 项目源码 文件源码 阅读 17 收藏 0 点赞 0 评论 0
def get(self, URL, FOLLOWLOCATION = False):
        self.URL = URL
        buffer = StringIO()
        c = pycurl.Curl()
        c.setopt(c.URL, URL)
        c.setopt(c.WRITEDATA, buffer)
        c.setopt(pycurl.FOLLOWLOCATION, FOLLOWLOCATION)
        c.perform()
        c.close()
        return buffer.getvalue().replace('\r\n', '').replace('\n', '')
HTTPRequest.py 文件源码 项目:download-manager 作者: thispc 项目源码 文件源码 阅读 25 收藏 0 点赞 0 评论 0
def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False):
        """ load and returns a given page """

        self.setRequestContext(url, get, post, referer, cookies, multipart)

        self.header = ""

        self.c.setopt(pycurl.HTTPHEADER, self.headers)

        if just_header:
            self.c.setopt(pycurl.FOLLOWLOCATION, 0)
            self.c.setopt(pycurl.NOBODY, 1)
            self.c.perform()
            rep = self.header

            self.c.setopt(pycurl.FOLLOWLOCATION, 1)
            self.c.setopt(pycurl.NOBODY, 0)

        else:
            self.c.perform()
            rep = self.getResponse()

        self.c.setopt(pycurl.POSTFIELDS, "")
        self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL)
        self.code = self.verifyHeader()

        self.addCookies()

        if decode:
            rep = self.decodeResponse(rep)

        return rep
RequestCore.py 文件源码 项目:AppBackend 作者: 540871129 项目源码 文件源码 阅读 21 收藏 0 点赞 0 评论 0
def handle_request(self):
        curl_handle = pycurl.Curl()
        # set default options.
        curl_handle.setopt(pycurl.URL, self.request_url)
        curl_handle.setopt(pycurl.REFERER, self.request_url)
        curl_handle.setopt(pycurl.USERAGENT, self.useragent)
        curl_handle.setopt(pycurl.TIMEOUT, self.curlopts['TIMEOUT'])
        curl_handle.setopt(pycurl.CONNECTTIMEOUT, self.curlopts['CONNECTTIMEOUT'])
        curl_handle.setopt(pycurl.HEADER, True)
        #curl_handle.setopt(pycurl.VERBOSE, 1)
        curl_handle.setopt(pycurl.FOLLOWLOCATION, 1)
        curl_handle.setopt(pycurl.MAXREDIRS, 5)
        if(self.request_headers and len(self.request_headers) > 0):
            tmplist = list()
            for(key, value) in self.request_headers.items():
                tmplist.append(key + ':' + value)
            curl_handle.setopt(pycurl.HTTPHEADER, tmplist)
        #??????POST
        curl_handle.setopt(pycurl.HTTPPROXYTUNNEL, 1)
        curl_handle.setopt(pycurl.POSTFIELDS, self.request_body)

        response = StringIO.StringIO()
        curl_handle.setopt(pycurl.WRITEFUNCTION, response.write)

        try:
            curl_handle.perform()
        except pycurl.error as error:
            raise ChannelException(error, 5)

        self.response_code = curl_handle.getinfo(curl_handle.HTTP_CODE)
        header_size = curl_handle.getinfo(curl_handle.HEADER_SIZE)
        resp_str = response.getvalue()
        self.response_headers = resp_str[0 : header_size]
        self.response_body = resp_str[header_size : ]

        response.close()
        curl_handle.close()
zoomeye.py 文件源码 项目:ZPoc 作者: zidier215 项目源码 文件源码 阅读 24 收藏 0 点赞 0 评论 0
def _login(self):
        try:
            c = pycurl.Curl()
            c.setopt(pycurl.CAINFO, certifi.where())
            c.setopt(pycurl.URL, self.url)
            b = StringIO.StringIO()
            c.setopt(pycurl.USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)")
            c.setopt(pycurl.WRITEFUNCTION, b.write)
            c.setopt(pycurl.FOLLOWLOCATION, 1)
            c.setopt(pycurl.MAXREDIRS, 5)
            c.setopt(pycurl.CUSTOMREQUEST, "POST")
            c.setopt(pycurl.POSTFIELDS, self.post_data)
            c.perform()
            if b.getvalue():
                logging.info('success login') # For INFO level
                self.API_TOKEN = json.loads(b.getvalue())["access_token"]
                self.save_token()
            else:
                logging.warning('success fail,get null result') #2 For WARNING level
            logging.debug(self.API_TOKEN)
            b.close()
            c.close()
        except pycurl.E_HTTP_POST_ERROR:
            logging.error(str(pycurl.E_HTTP_POST_ERROR))
        except Exception as e:
            logging.error('please check your password or username')
            logging.error(e.message) #3 For ERROR level
            pass
sourceHelper.py 文件源码 项目:VWGen 作者: qazbnm456 项目源码 文件源码 阅读 21 收藏 0 点赞 0 评论 0
def __init__(self):
        self.c = pycurl.Curl()
        self.c.setopt(pycurl.TIMEOUT, 5)
        self.c.setopt(pycurl.FOLLOWLOCATION, True)
FengChao.py 文件源码 项目:hzlgithub 作者: hzlRises 项目源码 文件源码 阅读 17 收藏 0 点赞 0 评论 0
def getKeyword(i):#??json
    try:
        time.sleep(1)           
        headers = [ 
        'Host:fengchao.baidu.com',
        'User-Agent: %s' %getUA(),
        'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding: gzip, deflate',
        'Referer: http://fengchao.baidu.com/nirvana/main.html?userid=8048066',
        'Connection: keep-alive',
        'COOKIE:%s' %COOKIE,
        ]
        post = urllib.urlencode({
            'params': '{"entry":"kr_station","query":"%s","querytype":1,"pageNo":1,"pageSize":300}' % keyword_list[i],
            'path': 'jupiter/GET/kr/word',
            'token': TOKEN,
            'userid': USERID,
        })
        url = 'http://fengchao.baidu.com/nirvana/request.ajax?path=jupiter/GET/kr/word'
        c = pycurl.Curl()
#       c.setopt(pycurl.PROXY, getRandomAlbIp())                            
        c.setopt(pycurl.URL, url)                   
        c.setopt(pycurl.FOLLOWLOCATION, True)       
        c.setopt(pycurl.MAXREDIRS,5)                
        c.setopt(pycurl.CONNECTTIMEOUT, 20)         
        c.setopt(pycurl.TIMEOUT,20)                 
        c.setopt(pycurl.ENCODING, 'gzip,deflate')   
        c.fp = StringIO.StringIO()                  
        c.setopt(pycurl.HTTPHEADER,headers)         
        c.setopt(pycurl.POST, 1)                
        c.setopt(pycurl.POSTFIELDS, post)           
        c.setopt(c.WRITEFUNCTION, c.fp.write)       
        c.perform()
#       mutex.acquire()#??
        jsonData = c.fp.getvalue()              
        analyseJsonData(i,jsonData)
#       mutex.release()#??
    except Exception,e:
        print e
        pass
polproxy.py 文件源码 项目:polproxy 作者: kevinlekiller 项目源码 文件源码 阅读 16 收藏 0 点赞 0 评论 0
def curlRequest(self, url, headers = False, post = False, returnHeaders=True):
        ch = pycurl.Curl()
        ch.setopt(pycurl.URL, url)
        hdrs = [
                "Host: poloniex.com",
                "Connection: close",
                "User-Agent: Mozilla/5.0 (CLI; Linux x86_64) polproxy",
                "accept: application/json"
        ]
        if post != False:
            ch.setopt(pycurl.POSTFIELDS, post)
            hdrs = hdrs + ["content-type: application/x-www-form-urlencoded", "content-length: " + str(len(post))]
        if headers != False:
            hdrs = hdrs + headers
        ch.setopt(pycurl.HTTPHEADER, hdrs)
        ch.setopt(pycurl.SSL_VERIFYHOST, 0)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.CONNECTTIMEOUT, 5)
        ch.setopt(pycurl.TIMEOUT, 5)
        ret = BytesIO()
        if returnHeaders:
            ch.setopt(pycurl.HEADERFUNCTION, ret.write)
        ch.setopt(pycurl.WRITEFUNCTION, ret.write)
        try:
            ch.perform()
        except:
            return ""
        ch.close()
        return ret.getvalue().decode("ISO-8859-1")
request.py 文件源码 项目:pyload-requests 作者: pyload 项目源码 文件源码 阅读 52 收藏 0 点赞 0 评论 0
def init_handle(self):
        """
        Sets common options to curl handle.
        """
        self.setopt(pycurl.FOLLOWLOCATION, 1)
        self.setopt(pycurl.MAXREDIRS, 5)
        self.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.setopt(pycurl.NOSIGNAL, 1)
        self.setopt(pycurl.NOPROGRESS, 1)
        if hasattr(pycurl, "AUTOREFERER"):
            self.setopt(pycurl.AUTOREFERER, 1)
        self.setopt(pycurl.SSL_VERIFYPEER, 0)
        # Interval for low speed, detects connection loss, but can abort dl if
        # hoster stalls the download
        self.setopt(pycurl.LOW_SPEED_TIME, 45)
        self.setopt(pycurl.LOW_SPEED_LIMIT, 5)

        # do not save the cookies
        self.setopt(pycurl.COOKIEFILE, '')
        self.setopt(pycurl.COOKIEJAR, '')

        # self.setopt(pycurl.VERBOSE, 1)

        self.setopt(
            pycurl.USERAGENT,
            'Mozilla/5.0 (Windows NT 10.0; Win64; rv:53.0) '
            'Gecko/20100101 Firefox/53.0')
        if pycurl.version_info()[7]:
            self.setopt(pycurl.ENCODING, 'gzip,deflate')

        self.headers.update(
            {'Accept': "*/*",
             'Accept-Language': "en-US,en",
             'Accept-Charset': "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
             'Connection': "keep-alive",
             'Keep-Alive': "300",
             'Expect': ""})
mimir.py 文件源码 项目:Mimir 作者: NullArray 项目源码 文件源码 阅读 19 收藏 0 点赞 0 评论 0
def hosts():
    a = StringIO.StringIO()

    # Options for PyCurl
    opts = ['X-HoneyDb-ApiId: ' + DB_API_ID, 'X-HoneyDb-ApiKey: ' + DB_API_KEY]
    c.setopt(pycurl.HTTPHEADER, (opts))
    c.setopt(pycurl.FOLLOWLOCATION, 1)

    c.setopt(pycurl.URL, "https://riskdiscovery.com/honeydb/api/bad-hosts")
    c.setopt(c.WRITEDATA, a)

    try:
        c.perform()
    except Exception as e:
        print "\n[" + t.red("!") + "]Critical. An error was raised with the following message"  
        print e

    os.system("clear")
    print "\n\n[" + t.green("+") + "]Retrieved Bad Hosts, formatting..."
    time.sleep(1)

    response_h = json.loads(a. getvalue())
    pprint(response_h)

    format = json.dumps(response_h, indent = 2)

    with open('hosts.log', 'ab') as outfile:
        outfile.write(format)
        outfile.close()

    print "\n\nResults saved to 'hosts.log' in the current directory"


问题


面经


文章

微信
公众号

扫码关注公众号