def request(self, url, method, body, headers):
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
if 'proxy_host' in self.proxy:
c.setopt(pycurl.PROXY, self.proxy['proxy_host'])
if 'proxy_port' in self.proxy:
c.setopt(pycurl.PROXYPORT, self.proxy['proxy_port'])
if 'proxy_user' in self.proxy:
c.setopt(pycurl.PROXYUSERPWD, "%(proxy_user)s:%(proxy_pass)s" % self.proxy)
self.buf = StringIO()
c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
#c.setopt(pycurl.READFUNCTION, self.read)
#self.body = StringIO(body)
#c.setopt(pycurl.HEADERFUNCTION, self.header)
if self.cacert:
c.setopt(c.CAINFO, self.cacert)
c.setopt(pycurl.SSL_VERIFYPEER, self.cacert and 1 or 0)
c.setopt(pycurl.SSL_VERIFYHOST, self.cacert and 2 or 0)
c.setopt(pycurl.CONNECTTIMEOUT, self.timeout / 6)
c.setopt(pycurl.TIMEOUT, self.timeout)
if method == 'POST':
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, body)
if headers:
hdrs = ['%s: %s' % (k, v) for k, v in headers.items()]
log.debug(hdrs)
c.setopt(pycurl.HTTPHEADER, hdrs)
c.perform()
c.close()
return {}, self.buf.getvalue()
python类HTTPHEADER的实例源码
def request(self, url, method, body, headers):
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
if 'proxy_host' in self.proxy:
c.setopt(pycurl.PROXY, self.proxy['proxy_host'])
if 'proxy_port' in self.proxy:
c.setopt(pycurl.PROXYPORT, self.proxy['proxy_port'])
if 'proxy_user' in self.proxy:
c.setopt(pycurl.PROXYUSERPWD, "%(proxy_user)s:%(proxy_pass)s" % self.proxy)
self.buf = StringIO()
c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
#c.setopt(pycurl.READFUNCTION, self.read)
#self.body = StringIO(body)
#c.setopt(pycurl.HEADERFUNCTION, self.header)
if self.cacert:
c.setopt(c.CAINFO, self.cacert)
c.setopt(pycurl.SSL_VERIFYPEER, self.cacert and 1 or 0)
c.setopt(pycurl.SSL_VERIFYHOST, self.cacert and 2 or 0)
c.setopt(pycurl.CONNECTTIMEOUT, self.timeout / 6)
c.setopt(pycurl.TIMEOUT, self.timeout)
if method == 'POST':
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, body)
if headers:
hdrs = ['%s: %s' % (k, v) for k, v in headers.items()]
log.debug(hdrs)
c.setopt(pycurl.HTTPHEADER, hdrs)
c.perform()
c.close()
return {}, self.buf.getvalue()
def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False):
""" load and returns a given page """
self.setRequestContext(url, get, post, referer, cookies, multipart)
self.header = ""
self.c.setopt(pycurl.HTTPHEADER, self.headers)
if just_header:
self.c.setopt(pycurl.FOLLOWLOCATION, 0)
self.c.setopt(pycurl.NOBODY, 1)
self.c.perform()
rep = self.header
self.c.setopt(pycurl.FOLLOWLOCATION, 1)
self.c.setopt(pycurl.NOBODY, 0)
else:
self.c.perform()
rep = self.getResponse()
self.c.setopt(pycurl.POSTFIELDS, "")
self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL)
self.code = self.verifyHeader()
self.addCookies()
if decode:
rep = self.decodeResponse(rep)
return rep
def handle_free(self, pyfile):
data = dict(re.findall(r'(md5|file): "(.*?)"', self.data))
if len(data) != 2:
self.error(_("No file ID"))
self.req.http.c.setopt(
pycurl.HTTPHEADER,
["X-Requested-With: XMLHttpRequest"])
self.req.http.lastURL = pyfile.url
self.load(
"http://www.wrzuc.to/ajax/server/prepair",
post={
'md5': data['md5']})
self.req.http.lastURL = pyfile.url
self.data = self.load(
"http://www.wrzuc.to/ajax/server/download_link",
post={
'file': data['file']})
data.update(
re.findall(
r'"(download_link|server_id)":"(.*?)"',
self.data))
if len(data) != 4:
self.error(_("No download URL"))
self.link = "http://%s.wrzuc.to/pobierz/%s" % (
data['server_id'], data['download_link'])
def handle_free(self, pyfile):
self.req.http.lastURL = pyfile.url
self.req.http.c.setopt(
pycurl.HTTPHEADER,
["X-Requested-With: XMLHttpRequest"])
jsvars = self.get_json_response("https://rapidu.net/ajax.php",
get={'a': "getLoadTimeToDownload"},
post={'_go': ""})
if str(jsvars['timeToDownload']) == "stop":
t = (24 * 60 * 60) - (int(time.time()) %
(24 * 60 * 60)) + time.altzone
self.log_info(_("You've reach your daily download transfer"))
# @NOTE: check t in case of not synchronised clock
self.retry(10, 10 if t < 1 else None, _("Try tomorrow again"))
else:
self.wait(int(jsvars['timeToDownload']) - int(time.time()))
self.captcha = ReCaptcha(pyfile)
response, challenge = self.captcha.challenge(self.RECAPTCHA_KEY)
jsvars = self.get_json_response("https://rapidu.net/ajax.php",
get={'a': "getCheckCaptcha"},
post={'_go': "",
'captcha1': challenge,
'captcha2': response,
'fileId': self.info['pattern']['ID']})
if jsvars['message'] == "success":
self.link = jsvars['url']
def send_to_transmission(self, url):
transmission_rpc_url = self.config.get('rpc_url')
client_request_id = self.classname + \
"".join(random.choice('0123456789ABCDEF') for _i in range(4))
req = get_request()
try:
response = self.load(transmission_rpc_url,
post=json.dumps({'arguments': {'filename': url},
'method': 'torrent-add',
'tag': client_request_id}),
req=req)
except Exception, e:
if isinstance(e, BadHeader) and e.code == 409:
headers = dict(
re.findall(
r'(?P<name>.+?): (?P<value>.+?)\r?\n',
req.header))
session_id = headers['X-Transmission-Session-Id']
req.c.setopt(
pycurl.HTTPHEADER, [
"X-Transmission-Session-Id: %s" %
session_id])
try:
response = self.load(transmission_rpc_url,
post=json.dumps({'arguments': {'filename': url},
'method': 'torrent-add',
'tag': client_request_id}),
req=req)
res = json.loads(response)
if "result" in res:
self.log_debug("Result: %s" % res['result'])
except Exception, e:
self.log_error(e)
else:
self.log_error(e)
def send(self, event, msg, key):
req = get_request()
req.c.setopt(pycurl.HTTPHEADER, ["Access-Token: %s" % str(key)])
self.load("https://api.pushbullet.com/v2/pushes",
post={'type': 'note',
'title': event,
'message': msg},
req=req)
def _init_headers(self):
l_headers = [ "%s: %s" % (x,y) for x,y in self.m_request.m_headers.items() ]
self.m_handle.setopt(pycurl.HTTPHEADER, l_headers)
def handle_request(self):
curl_handle = pycurl.Curl()
# set default options.
curl_handle.setopt(pycurl.URL, self.request_url)
curl_handle.setopt(pycurl.REFERER, self.request_url)
curl_handle.setopt(pycurl.USERAGENT, self.useragent)
curl_handle.setopt(pycurl.TIMEOUT, self.curlopts['TIMEOUT'])
curl_handle.setopt(pycurl.CONNECTTIMEOUT, self.curlopts['CONNECTTIMEOUT'])
curl_handle.setopt(pycurl.HEADER, True)
#curl_handle.setopt(pycurl.VERBOSE, 1)
curl_handle.setopt(pycurl.FOLLOWLOCATION, 1)
curl_handle.setopt(pycurl.MAXREDIRS, 5)
if(self.request_headers and len(self.request_headers) > 0):
tmplist = list()
for(key, value) in self.request_headers.items():
tmplist.append(key + ':' + value)
curl_handle.setopt(pycurl.HTTPHEADER, tmplist)
#??????POST
curl_handle.setopt(pycurl.HTTPPROXYTUNNEL, 1)
curl_handle.setopt(pycurl.POSTFIELDS, self.request_body)
response = StringIO.StringIO()
curl_handle.setopt(pycurl.WRITEFUNCTION, response.write)
try:
curl_handle.perform()
except pycurl.error as error:
raise ChannelException(error, 5)
self.response_code = curl_handle.getinfo(curl_handle.HTTP_CODE)
header_size = curl_handle.getinfo(curl_handle.HEADER_SIZE)
resp_str = response.getvalue()
self.response_headers = resp_str[0 : header_size]
self.response_body = resp_str[header_size : ]
response.close()
curl_handle.close()
def request(self, url, method, body, headers):
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
if 'proxy_host' in self.proxy:
c.setopt(pycurl.PROXY, self.proxy['proxy_host'])
if 'proxy_port' in self.proxy:
c.setopt(pycurl.PROXYPORT, self.proxy['proxy_port'])
if 'proxy_user' in self.proxy:
c.setopt(pycurl.PROXYUSERPWD, "%(proxy_user)s:%(proxy_pass)s" % self.proxy)
self.buf = StringIO()
c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
#c.setopt(pycurl.READFUNCTION, self.read)
#self.body = StringIO(body)
#c.setopt(pycurl.HEADERFUNCTION, self.header)
if self.cacert:
c.setopt(c.CAINFO, self.cacert)
c.setopt(pycurl.SSL_VERIFYPEER, self.cacert and 1 or 0)
c.setopt(pycurl.SSL_VERIFYHOST, self.cacert and 2 or 0)
c.setopt(pycurl.CONNECTTIMEOUT, self.timeout)
c.setopt(pycurl.TIMEOUT, self.timeout)
if method == 'POST':
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, body)
if headers:
hdrs = ['%s: %s' % (k, v) for k, v in headers.items()]
log.debug(hdrs)
c.setopt(pycurl.HTTPHEADER, hdrs)
c.perform()
c.close()
return {}, self.buf.getvalue()
def getPage (self, url, requestHeader = []) :
resultFormate = StringIO.StringIO()
fakeIp = self.fakeIp()
requestHeader.append('CLIENT-IP:' + fakeIp)
requestHeader.append('X-FORWARDED-FOR:' + fakeIp)
try:
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url.strip())
curl.setopt(pycurl.ENCODING, 'gzip,deflate')
curl.setopt(pycurl.HEADER, 1)
curl.setopt(pycurl.TIMEOUT, 120)
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
curl.setopt(pycurl.HTTPHEADER, requestHeader)
curl.setopt(pycurl.WRITEFUNCTION, resultFormate.write)
curl.perform()
headerSize = curl.getinfo(pycurl.HEADER_SIZE)
curl.close()
header = resultFormate.getvalue()[0 : headerSize].split('\r\n')
body = resultFormate.getvalue()[headerSize : ]
except Exception, e:
header = ''
body = ''
return header, body
def transfer(ipaddr, username, password, commandfile, trackprogress):
#transfers commandfile to camera
storage = StringIO()
c = pycurl.Curl()
c.setopt(c.URL, 'http://' + ipaddr + '/admin/remoteconfig')
c.setopt(c.POST, 1)
c.setopt(c.CONNECTTIMEOUT, 5)
c.setopt(c.TIMEOUT, TIMEOUT)
filesize = os.path.getsize(commandfile)
f = open(commandfile, 'rb')
c.setopt(c.FAILONERROR, True)
c.setopt(pycurl.POSTFIELDSIZE, filesize)
c.setopt(pycurl.READFUNCTION, FileReader(f).read_callback)
if trackprogress:
c.setopt(pycurl.NOPROGRESS, 0)
c.setopt(pycurl.PROGRESSFUNCTION, progresscallback)
starttime = time.time()
else:
c.setopt(pycurl.NOPROGRESS, 1)
c.setopt(c.WRITEFUNCTION, storage.write)
c.setopt(pycurl.HTTPHEADER, ["application/x-www-form-urlencoded"])
c.setopt(c.VERBOSE, VERBOSE)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
c.setopt(pycurl.USERPWD, username + ':' + password)
try:
c.perform()
except pycurl.error, error:
errno, errstr = error
print 'An error occurred: ', errstr
return False, ''
c.close()
content = storage.getvalue()
f.close()
return True, content
def transfer(ipaddr, username, password, commandfile):
#transfers commandfile to camera
storage = StringIO()
c = pycurl.Curl()
c.setopt(c.URL, 'http://' + ipaddr + '/admin/remoteconfig')
c.setopt(c.POST, 1)
c.setopt(c.CONNECTTIMEOUT, 5)
c.setopt(c.TIMEOUT, 60)
filesize = os.path.getsize(commandfile)
f = open(commandfile, 'rb')
c.setopt(c.FAILONERROR, True)
c.setopt(pycurl.POSTFIELDSIZE, filesize)
c.setopt(pycurl.READFUNCTION, FileReader(f).read_callback)
c.setopt(c.WRITEFUNCTION, storage.write)
c.setopt(pycurl.HTTPHEADER, ["application/x-www-form-urlencoded"])
c.setopt(c.VERBOSE, 0)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
c.setopt(pycurl.USERPWD, username + ':' + password)
try:
c.perform()
except pycurl.error, error:
errno, errstr = error
print 'An error occurred: ', errstr
return False, ''
c.close()
content = storage.getvalue()
f.close()
return True, content
# ***************************************************************
# *** Main program ***
# ***************************************************************
def setup_curl_for_get(c, p, data_buf, headers=None, share=None):
setup_curl_basic(c, p, data_buf, headers, share)
httpheader = p.get('httpheader')
if httpheader:
# c.setopt(pycurl.HEADER, p.get('header', 1))
c.setopt(c.HTTPHEADER, httpheader)
return c
def getKeyword(i):#??json
try:
time.sleep(1)
headers = [
'Host:fengchao.baidu.com',
'User-Agent: %s' %getUA(),
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding: gzip, deflate',
'Referer: http://fengchao.baidu.com/nirvana/main.html?userid=8048066',
'Connection: keep-alive',
'COOKIE:%s' %COOKIE,
]
post = urllib.urlencode({
'params': '{"entry":"kr_station","query":"%s","querytype":1,"pageNo":1,"pageSize":300}' % keyword_list[i],
'path': 'jupiter/GET/kr/word',
'token': TOKEN,
'userid': USERID,
})
url = 'http://fengchao.baidu.com/nirvana/request.ajax?path=jupiter/GET/kr/word'
c = pycurl.Curl()
# c.setopt(pycurl.PROXY, getRandomAlbIp())
c.setopt(pycurl.URL, url)
c.setopt(pycurl.FOLLOWLOCATION, True)
c.setopt(pycurl.MAXREDIRS,5)
c.setopt(pycurl.CONNECTTIMEOUT, 20)
c.setopt(pycurl.TIMEOUT,20)
c.setopt(pycurl.ENCODING, 'gzip,deflate')
c.fp = StringIO.StringIO()
c.setopt(pycurl.HTTPHEADER,headers)
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, post)
c.setopt(c.WRITEFUNCTION, c.fp.write)
c.perform()
# mutex.acquire()#??
jsonData = c.fp.getvalue()
analyseJsonData(i,jsonData)
# mutex.release()#??
except Exception,e:
print e
pass
def request(self, url, method, body, headers):
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
if 'proxy_host' in self.proxy:
c.setopt(pycurl.PROXY, self.proxy['proxy_host'])
if 'proxy_port' in self.proxy:
c.setopt(pycurl.PROXYPORT, self.proxy['proxy_port'])
if 'proxy_user' in self.proxy:
c.setopt(pycurl.PROXYUSERPWD, "%(proxy_user)s:%(proxy_pass)s" % self.proxy)
self.buf = StringIO()
c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
#c.setopt(pycurl.READFUNCTION, self.read)
#self.body = StringIO(body)
#c.setopt(pycurl.HEADERFUNCTION, self.header)
if self.cacert:
c.setopt(c.CAINFO, self.cacert)
c.setopt(pycurl.SSL_VERIFYPEER, self.cacert and 1 or 0)
c.setopt(pycurl.SSL_VERIFYHOST, self.cacert and 2 or 0)
c.setopt(pycurl.CONNECTTIMEOUT, self.timeout / 6)
c.setopt(pycurl.TIMEOUT, self.timeout)
if method == 'POST':
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, body)
if headers:
hdrs = ['%s: %s' % (k, v) for k, v in headers.items()]
log.debug(hdrs)
c.setopt(pycurl.HTTPHEADER, hdrs)
c.perform()
c.close()
return {}, self.buf.getvalue()
def curlRequest(self, url, headers = False, post = False, returnHeaders=True):
ch = pycurl.Curl()
ch.setopt(pycurl.URL, url)
hdrs = [
"Host: poloniex.com",
"Connection: close",
"User-Agent: Mozilla/5.0 (CLI; Linux x86_64) polproxy",
"accept: application/json"
]
if post != False:
ch.setopt(pycurl.POSTFIELDS, post)
hdrs = hdrs + ["content-type: application/x-www-form-urlencoded", "content-length: " + str(len(post))]
if headers != False:
hdrs = hdrs + headers
ch.setopt(pycurl.HTTPHEADER, hdrs)
ch.setopt(pycurl.SSL_VERIFYHOST, 0)
ch.setopt(pycurl.FOLLOWLOCATION, True)
ch.setopt(pycurl.CONNECTTIMEOUT, 5)
ch.setopt(pycurl.TIMEOUT, 5)
ret = BytesIO()
if returnHeaders:
ch.setopt(pycurl.HEADERFUNCTION, ret.write)
ch.setopt(pycurl.WRITEFUNCTION, ret.write)
try:
ch.perform()
except:
return ""
ch.close()
return ret.getvalue().decode("ISO-8859-1")
