def feed():
b = StringIO.StringIO()
# Options for PyCurl
opts = ['X-HoneyDb-ApiId: ' + DB_API_ID, 'X-HoneyDb-ApiKey: ' + DB_API_KEY]
c.setopt(pycurl.HTTPHEADER, (opts))
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.URL, "https://riskdiscovery.com/honeydb/api/twitter-threat-feed")
c.setopt(c.WRITEDATA, b)
try:
c.perform()
except Exception as e:
print "\n[" + t.red("!") + "]Critical. An error was raised with the following message"
print e
os.system("clear")
print "\n\n[" + t.green("+") + "]Retrieved Threat Feed, formatting..."
time.sleep(1)
response_f = json.loads(b. getvalue())
pprint(response_f)
format = json.dumps(response_f, indent = 2)
with open('feed.log', 'ab') as outfile:
outfile.write(format)
outfile.close()
print "\n\nResults saved to 'feed.log' in the current directory"
python类FOLLOWLOCATION的实例源码
def _check_version(self):
try:
c = pycurl.Curl()
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
c.setopt(pycurl.FOLLOWLOCATION, 1)
data = getJsonFromURL(self.VERSION_URL, handle=c, max_attempts=3)
if data:
msgr.send_object(VersionInfo(data))
# except pycurl.error as e:
# pass
except Exception as e:
logger.error('Failed checking for version updates. Unexpected error: {}'.format(e))
def check_mzURL(mz_server, file_name):
'''Checks if an mzURL actually exists.
mz_server should be the base URL of the server
file_name is the name of the specific file (without its extension)
'''
if mz_server[-1] == '/':
mz_server = mz_server[:-1]
# Handle to libcurl object
crl = pycurl.Curl()
# set some general options
crl.setopt(pycurl.FOLLOWLOCATION, True)
crl.setopt(pycurl.URL, str(mz_server + '/files.txt'))
output = cStringIO.StringIO()
crl.setopt(pycurl.WRITEFUNCTION, output.write)
try:
for i in range(5):
#print 'check mzurl %d' % i
crl.perform()
if output.getvalue():
break
except pycurl.error, e:
return False
for f in output.getvalue().splitlines():
if os.path.splitext(f)[0].lower() == file_name.lower():
return True
else:
return False
def __init__(self, data_file, verbose=False, **kwargs):
self.file_type = 'mzurl'
# strip off the final slash, if it exists
if data_file[-1] == '/':
data_file = data_file[:-1]
# Likewise, html or other madness.
if any([data_file.lower().endswith(x) for x in ['html', 'raw', 'wiff']]):
data_file = ".".join(data_file.split(".")[:-1])
self.data_file = data_file # actually a URL to a file
self.verbose = verbose
self._scans = None # cache of scan_info results for the whole file
# A string with the name and path of an appropriate temp file
# (varies by platform)
fd, self.cookie_file_name = tempfile.mkstemp(text=True)
os.close(fd)
# Handle to libcurl object
self.crl = pycurl.Curl()
# set some general options
self.crl.setopt(pycurl.COOKIEFILE, self.cookie_file_name)
self.crl.setopt(pycurl.COOKIEJAR, self.cookie_file_name)
self.crl.setopt(pycurl.FOLLOWLOCATION, True)
self.crl.setopt(pycurl.VERBOSE, verbose)
self.output = cStringIO.StringIO()
self.crl.setopt(pycurl.WRITEFUNCTION, self.output.write)
# how would you store an info file?
#if os.path.exists(data_file + '.mzi'):
#self._info_file = data_file + '.mzi'
#info_fh = open(self._info_file)
#self._info_scans = cPickle.load(info_fh)
#info_fh.close()
#else:
#self._info_file = None
def getToken(self):
user_auth = '{"username": "%s","password": "%s"}' % (self.USERNAME, self.PASSWORD)
b = StringIO.StringIO()
c = pycurl.Curl()
c.setopt(pycurl.URL, "http://api.zoomeye.org/user/login")
c.setopt(pycurl.WRITEFUNCTION, b.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.CUSTOMREQUEST, "POST")
c.setopt(pycurl.POSTFIELDS, user_auth)
c.perform()
ReturnData = json.loads(b.getvalue())
self.API_TOKEN = ReturnData['access_token']
b.close()
c.close()
def curl_common_init(buf):
handle = pycurl.Curl()
handle.setopt(pycurl.WRITEDATA, buf)
handle.setopt(pycurl.HEADERFUNCTION, curl_hdr)
handle.setopt(pycurl.DEBUGFUNCTION, curl_debug)
handle.setopt(pycurl.USERPWD, '{}:{}'.format(_g.conf._user,_g.conf._pass))
handle.setopt(pycurl.FOLLOWLOCATION, True)
# avoid FTP CWD for fastest directory transversal
handle.setopt(pycurl.FTP_FILEMETHOD, pycurl.FTPMETHOD_NOCWD)
# we always set this flag and let the logging module
# handle filtering.
handle.setopt(pycurl.VERBOSE, True)
# use ipv4 for VPNs
handle.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
handle.setopt(pycurl.USE_SSL, True)
handle.setopt(pycurl.SSL_VERIFYPEER, False)
# XXX
handle.setopt(pycurl.SSL_VERIFYHOST, 0)
return handle
def update_quotes():
url = "http://finance.yahoo.com/d/quotes.csv?s={}&f=sc1p2oghl1".format(
"+".join(stocks.keys())
)
buf = cStringIO.StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, buf.write)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.perform()
data = buf.getvalue()
data = data.replace('"', "")
quote_lines = data.split("\n")
new_quotes = []
for line in quote_lines:
fields = line.split(",")
if len(fields) == 7:
new_quotes.append({
'symbol': fields[0],
'change': fields[1],
'pctchange': fields[2],
'open': fields[3],
'low': fields[4],
'high': fields[5],
'last': fields[6]
})
new_quotes.sort(key=lambda x: quotes_order(x))
if len(quotes) == DATA_POINTS:
quotes.pop(DATA_POINTS-1)
quotes.insert(0, new_quotes)
def __init__(self, handle=None, options=None):
self.setCurlHandle(handle)
if options == None:
options = Options()
self.setOptions(options)
# To start with, disable FOLLOWLOCATION since we'll handle it
self._handle.setopt(pycurl.FOLLOWLOCATION, False)
# Force IPv4, since this class isn't yet compatible with IPv6
self._handle.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
def __init__(self, base_url="", fakeheaders=[]):
self.handle = pycurl.Curl()
# These members might be set.
self.set_url(base_url)
self.verbosity = 0
self.fakeheaders = fakeheaders
# Nothing past here should be modified by the caller.
self.payload = None
self.payload_io = BytesIO()
self.hrd = ""
# Verify that we've got the right site; harmless on a non-SSL connect.
self.set_option(pycurl.SSL_VERIFYHOST, 2)
# Follow redirects in case it wants to take us to a CGI...
self.set_option(pycurl.FOLLOWLOCATION, 1)
self.set_option(pycurl.MAXREDIRS, 5)
self.set_option(pycurl.NOSIGNAL, 1)
# Setting this option with even a nonexistent file makes libcurl
# handle cookie capture and playback automatically.
self.set_option(pycurl.COOKIEFILE, "/dev/null")
# Set timeouts to avoid hanging too long
self.set_timeout(30)
# Use password identification from .netrc automatically
self.set_option(pycurl.NETRC, 1)
self.set_option(pycurl.WRITEFUNCTION, self.payload_io.write)
def header_callback(x):
self.hdr += x.decode('ascii')
self.set_option(pycurl.HEADERFUNCTION, header_callback)
def to_pycurl_object(c, req):
c.setopt(pycurl.MAXREDIRS, 5)
c.setopt(pycurl.WRITEFUNCTION, req.body_callback)
c.setopt(pycurl.HEADERFUNCTION, req.header_callback)
c.setopt(pycurl.NOSIGNAL, 1)
c.setopt(pycurl.SSL_VERIFYPEER, False)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
c.setopt(pycurl.URL,req.completeUrl)
if req.getConnTimeout():
c.setopt(pycurl.CONNECTTIMEOUT, req.getConnTimeout())
if req.getTotalTimeout():
c.setopt(pycurl.TIMEOUT, req.getTotalTimeout())
authMethod, userpass = req.getAuth()
if authMethod or userpass:
if authMethod == "basic":
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
elif authMethod == "ntlm":
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_NTLM)
elif authMethod == "digest":
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
c.setopt(pycurl.USERPWD, userpass)
c.setopt(pycurl.HTTPHEADER, req.getHeaders())
if req.method == "POST":
c.setopt(pycurl.POSTFIELDS, req.postdata)
if req.method != "GET" and req.method != "POST":
c.setopt(pycurl.CUSTOMREQUEST, req.method)
if req.method == "HEAD":
c.setopt(pycurl.NOBODY, True)
if req.followLocation:
c.setopt(pycurl.FOLLOWLOCATION, 1)
proxy = req.getProxy()
if proxy != None:
c.setopt(pycurl.PROXY, proxy)
if req.proxytype=="SOCKS5":
c.setopt(pycurl.PROXYTYPE,pycurl.PROXYTYPE_SOCKS5)
elif req.proxytype=="SOCKS4":
c.setopt(pycurl.PROXYTYPE,pycurl.PROXYTYPE_SOCKS4)
req.delHeader("Proxy-Connection")
return c
def getOneGameODD(game):
resultStr = ''
try:
oddURL = 'http://27.45.161.37:8072/phone/1x2.aspx?ID=' + str(game.soccerID) + '&an=iosQiuTan&av=5.9&from=2&lang=0&subversion=1'
# print oddURL
except:
pass
c = pycurl.Curl()
c.setopt(pycurl.URL, oddURL)
b = StringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, b.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
c.perform()
resultStr = b.getvalue().decode('utf8')
if resultStr != '':
array = resultStr.split('!')
companys = []
for unit in array:
# print unit.decode('utf-8')
company = BetCompany()
company.league = game.leauge;
company.result = game.soccer
company.homeSoccer = game.allHome
company.friendSoccer = game.allFriend
company.soccerGameId = game.soccerID
unitArray = unit.split('^')
try:
company.companyTitle = unitArray[0].encode('utf-8')
company.orignal_winOdd = float(unitArray[2])
company.orignal_drawOdd = float(unitArray[3])
company.orignal_loseOdd = float(unitArray[4])
company.winOdd = float(unitArray[5])
company.drawOdd = float(unitArray[6])
company.loseOdd = float(unitArray[7])
except IndexError as e:
print e
print unitArray
if company.companyTitle in ['????', '10BET', 'bet 365', 'bwin', 'Interwetten', 'SB', '??', '??', '????', '????', '??','Oddset','SNAI','ManbetX']:
companys.append(company)
if company.companyTitle == '??':
game.orignal_aomenOdd = (company.orignal_winOdd, company.orignal_drawOdd, company.orignal_loseOdd)
game.now_aomenOdd = (company.winOdd, company.drawOdd, company.loseOdd)
return companys
else:
return []
def to_pycurl_object(c, req):
c.setopt(pycurl.MAXREDIRS, 5)
c.setopt(pycurl.WRITEFUNCTION, req.body_callback)
c.setopt(pycurl.HEADERFUNCTION, req.header_callback)
c.setopt(pycurl.NOSIGNAL, 1)
c.setopt(pycurl.SSL_VERIFYPEER, False)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
c.setopt(pycurl.URL,req.completeUrl)
if req.getConnTimeout():
c.setopt(pycurl.CONNECTTIMEOUT, req.getConnTimeout())
if req.getTotalTimeout():
c.setopt(pycurl.TIMEOUT, req.getTotalTimeout())
authMethod, userpass = req.getAuth()
if authMethod or userpass:
if authMethod == "basic":
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
elif authMethod == "ntlm":
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_NTLM)
elif authMethod == "digest":
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
c.setopt(pycurl.USERPWD, userpass)
c.setopt(pycurl.HTTPHEADER, req.getHeaders())
if req.method == "POST":
c.setopt(pycurl.POSTFIELDS, req.postdata)
if req.method != "GET" and req.method != "POST":
c.setopt(pycurl.CUSTOMREQUEST, req.method)
if req.method == "HEAD":
c.setopt(pycurl.NOBODY, True)
if req.followLocation:
c.setopt(pycurl.FOLLOWLOCATION, 1)
proxy = req.getProxy()
if proxy != None:
c.setopt(pycurl.PROXY, proxy)
if req.proxytype=="SOCKS5":
c.setopt(pycurl.PROXYTYPE,pycurl.PROXYTYPE_SOCKS5)
elif req.proxytype=="SOCKS4":
c.setopt(pycurl.PROXYTYPE,pycurl.PROXYTYPE_SOCKS4)
req.delHeader("Proxy-Connection")
return c
def load(self, url, get={}, post={}, referer=True, cookies=True,
just_header=False, multipart=False, decode=False):
"""
Load and returns a given page.
"""
self.set_request_context(url, get, post, referer, cookies, multipart)
# TODO: use http/rfc message instead
self.header = ""
if "header" in self.options:
# TODO
# print("custom header not implemented")
self.setopt(pycurl.HTTPHEADER, self.options['header'])
if just_header:
self.setopt(pycurl.FOLLOWLOCATION, 0)
self.setopt(pycurl.NOBODY, 1) # TODO: nobody= no post?
# overwrite HEAD request, we want a common request type
if post:
self.setopt(pycurl.CUSTOMREQUEST, 'POST')
else:
self.setopt(pycurl.CUSTOMREQUEST, 'GET')
try:
self.c.perform()
rep = self.header
finally:
self.setopt(pycurl.FOLLOWLOCATION, 1)
self.setopt(pycurl.NOBODY, 0)
self.unsetopt(pycurl.CUSTOMREQUEST)
else:
self.c.perform()
rep = self.get_response()
self.setopt(pycurl.POSTFIELDS, '')
self.last_url = safequote(url)
self.last_effective_url = self.c.getinfo(pycurl.EFFECTIVE_URL)
if self.last_effective_url:
self.last_url = self.last_effective_url
self.code = self.verify_header()
if cookies:
self.parse_cookies()
if decode:
rep = self.decode_response(rep)
return rep
def now_playing_update_thread(self, playlist_id, playlist_end, media_id, media_end, show_name):
if not obplayer.Config.setting('sync_url'):
return
postfields = {}
postfields['id'] = obplayer.Config.setting('sync_device_id')
postfields['pw'] = obplayer.Config.setting('sync_device_password')
postfields['playlist_id'] = playlist_id
postfields['media_id'] = media_id
postfields['show_name'] = show_name
if playlist_end != '':
postfields['playlist_end'] = int(round(playlist_end))
else:
postfields['playlist_end'] = ''
if media_end != '':
postfields['media_end'] = int(round(media_end))
else:
postfields['media_end'] = ''
curl = pycurl.Curl()
enc_postfields = urllib.urlencode(postfields)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.USERAGENT, 'OpenBroadcaster Player')
curl.setopt(pycurl.URL, obplayer.Config.setting('sync_url') + '?action=now_playing')
curl.setopt(pycurl.HEADER, False)
curl.setopt(pycurl.POST, True)
curl.setopt(pycurl.POSTFIELDS, enc_postfields)
#curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.LOW_SPEED_LIMIT, 10)
curl.setopt(pycurl.LOW_SPEED_TIME, 60)
curl.setopt(pycurl.NOPROGRESS, 0)
curl.setopt(pycurl.PROGRESSFUNCTION, self.curl_progress)
try:
curl.perform()
except:
obplayer.Log.log("exception in NowPlayingUpdate thread", 'error')
obplayer.Log.log(traceback.format_exc(), 'error')
curl.close()
#
# Request sync data from web application.
# This is used by sync (with request_type='schedule') and sync_priority_broadcasts (with request_type='emerg').
# Function outputs XML response from server.
#
