def download(self, url, retry_count=3, headers=None, proxy=None, data=None):
if url is None:
return None
try:
req = request.Request(url, headers=headers, data=data)
cookie = cookiejar.CookieJar()
cookie_process = request.HTTPCookieProcessor(cookie)
opener = request.build_opener()
if proxy:
proxies = {urlparse(url).scheme: proxy}
opener.add_handler(request.ProxyHandler(proxies))
content = opener.open(req).read()
except error.URLError as e:
print('HtmlDownLoader download error:', e.reason)
content = None
if retry_count > 0:
if hasattr(e, 'code') and 500 <= e.code < 600:
#??? HTTPError ??? HTTP CODE ? 5XX ???????????????????
return self.download(url, retry_count-1, headers, proxy, data)
return content
python类HTTPCookieProcessor()的实例源码
def __init__(self, timeout=None, proxy=None, cacert=None, sessions=False):
if (timeout is not None) and not self.supports_feature('timeout'):
raise RuntimeError('timeout is not supported with urllib2 transport')
if proxy:
raise RuntimeError('proxy is not supported with urllib2 transport')
if cacert:
raise RuntimeError('cacert is not support with urllib2 transport')
handlers = []
if ((sys.version_info[0] == 2 and sys.version_info >= (2,7,9)) or
(sys.version_info[0] == 3 and sys.version_info >= (3,2,0))):
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
handlers.append(urllib2.HTTPSHandler(context=context))
if sessions:
handlers.append(urllib2.HTTPCookieProcessor(CookieJar()))
opener = urllib2.build_opener(*handlers)
self.request_opener = opener.open
self._timeout = timeout
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'loaded cookies file')
except Exception as msg:
ScholarUtils.log('warn', 'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'loaded cookies file')
except Exception as msg:
ScholarUtils.log('warn', 'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def cookie_friendly_download(referer_url, file_url, store_dir='.', timeout=1000):
from http.cookiejar import CookieJar
from urllib import request
cj = CookieJar()
cp = request.HTTPCookieProcessor(cj)
opener = request.build_opener(cp)
with opener.open(referer_url) as fin:
fin.headers.items()
import os
from os import path
with opener.open(file_url, timeout=timeout) as fin:
file_bin = fin.read()
filename = fin.headers['Content-Disposition']
filename = filename.split(';')[-1].split('=')[1]
os.makedirs(store_dir, exist_ok=True)
with open(path.join(store_dir, filename), mode='wb') as fout:
fout.write(file_bin)
return path.join(store_dir, filename)
def __init__(self, timeout=None, proxy=None, cacert=None, sessions=False):
if (timeout is not None) and not self.supports_feature('timeout'):
raise RuntimeError('timeout is not supported with urllib2 transport')
if proxy:
raise RuntimeError('proxy is not supported with urllib2 transport')
if cacert:
raise RuntimeError('cacert is not support with urllib2 transport')
handlers = []
if ((sys.version_info[0] == 2 and sys.version_info >= (2,7,9)) or
(sys.version_info[0] == 3 and sys.version_info >= (3,2,0))):
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
handlers.append(urllib2.HTTPSHandler(context=context))
if sessions:
handlers.append(urllib2.HTTPCookieProcessor(CookieJar()))
opener = urllib2.build_opener(*handlers)
self.request_opener = opener.open
self._timeout = timeout
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'loaded cookies file')
except Exception as msg:
ScholarUtils.log('warn', 'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'loaded cookies file')
except Exception as msg:
ScholarUtils.log('warn', 'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def index(request):
if request.method == "GET":
try:
ssl._create_default_https_context = ssl._create_unverified_context
opener = wdf_urllib.build_opener(
wdf_urllib.HTTPCookieProcessor(CookieJar()))
wdf_urllib.install_opener(opener)
except:
pass
uuid = getUUID()
url = 'https://login.weixin.qq.com/qrcode/' + uuid
params = {
't': 'webwx',
'_': int(time.time()),
}
request = getRequest(url=url, data=urlencode(params))
response = wdf_urllib.urlopen(request)
context = {
'uuid': uuid,
'response': response.read(),
'delyou': '',
}
return render_to_response('index.html', context)
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
print "Using cookie file"
ScholarUtils.log('info', 'loaded cookies file')
except Exception as msg:
print "Ignoring cookie file: %s" % msg
ScholarUtils.log('warn', 'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def __get_cookies(self, req):
cookies = cookiejar.CookieJar()
handler = request.HTTPCookieProcessor(cookies)
opener = request.build_opener(handler)
try:
with opener.open(req) as f:
if f.code == 200:
pattern = re.compile(r"<input.*?type='hidden'.*?name='csrfmiddlewaretoken'.*?value='(.*?)'.*>")
try:
self.csrfmiddlewaretoken = pattern.search(f.read().decode("utf-8")).group(1)
print("Achieved cookies and csrfmiddlewaretoken sucessfully")
except:
print("Achieved cookies sucessfully")
return cookies
else:
print("Lost cookies")
except error.URLError as e:
if hasattr(e, "reason"):
print ("We failed to reach a server. Please check your url and read the Reason")
print ("Reason: {}".format(e.reason))
elif hasattr(e, "code"):
print("The server couldn't fulfill the request.")
print("Error code: {}".format(e.code))
exit()
def login(self, username, pwd, cookie_file):
""""
Login with use name, password and cookies.
(1) If cookie file exists then try to load cookies;
(2) If no cookies found then do login
"""
# If cookie file exists then try to load cookies
if os.path.exists(cookie_file):
try:
cookie_jar = cookielib.LWPCookieJar(cookie_file)
cookie_jar.load(ignore_discard=True, ignore_expires=True)
loaded = 1
except cookielib.LoadError:
loaded = 0
LOG.info('Loading cookies error')
# install loaded cookies for urllib2
if loaded:
cookie_support = urllib2.HTTPCookieProcessor(cookie_jar)
opener = urllib2.build_opener(cookie_support,
urllib2.HTTPHandler)
urllib2.install_opener(opener)
LOG.info('Loading cookies success')
return 1
else:
return self.do_login(username, pwd, cookie_file)
else: # If no cookies found
return self.do_login(username, pwd, cookie_file)
def save_cookie(self, text, cookie_file=CONF.cookie_file):
cookie_jar2 = cookielib.LWPCookieJar()
cookie_support2 = urllib2.HTTPCookieProcessor(cookie_jar2)
opener2 = urllib2.build_opener(cookie_support2, urllib2.HTTPHandler)
urllib2.install_opener(opener2)
if six.PY3:
text = text.decode('gbk')
p = re.compile('location\.replace\(\'(.*?)\'\)')
# ???httpfox??????????????
# location.replace('http://weibo.com ?????????
# ?????????????# ????login_url?? ??????re?????
# p = re.compile('location\.replace\(\B'(.*?)'\B\)')
# ??? ??????? re?????\'???????
try:
# Search login redirection URL
login_url = p.search(text).group(1)
data = urllib2.urlopen(login_url).read()
# Verify login feedback, check whether result is TRUE
patt_feedback = 'feedBackUrlCallBack\((.*)\)'
p = re.compile(patt_feedback, re.MULTILINE)
feedback = p.search(data).group(1)
feedback_json = json.loads(feedback)
if feedback_json['result']:
cookie_jar2.save(cookie_file,
ignore_discard=True,
ignore_expires=True)
return 1
else:
return 0
except:
return 0
def login(self, username, pwd, cookie_file):
""""
Login with use name, password and cookies.
(1) If cookie file exists then try to load cookies;
(2) If no cookies found then do login
"""
# If cookie file exists then try to load cookies
if os.path.exists(cookie_file):
try:
cookie_jar = cookielib.LWPCookieJar(cookie_file)
cookie_jar.load(ignore_discard=True, ignore_expires=True)
loaded = 1
except cookielib.LoadError:
loaded = 0
print('Loading cookies error')
#install loaded cookies for urllib2
if loaded:
cookie_support = urllib2.HTTPCookieProcessor(cookie_jar)
opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
urllib2.install_opener(opener)
print('Loading cookies success')
return 1
else:
return self.do_login(username, pwd, cookie_file)
else: #If no cookies found
return self.do_login(username, pwd, cookie_file)
def build_opener():
cookie = http.cookiejar.CookieJar()
cookie_processor = request.HTTPCookieProcessor(cookie)
opener = request.build_opener(cookie_processor)
opener.addheaders = [("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"),
("Referer", "https://passport.weibo.cn"),
("Origin", "https://passport.weibo.cn"),
("Host", "passport.weibo.cn")]
request.install_opener(opener)
#??
def build_opener():
cookie = http.cookiejar.CookieJar()
cookie_processor = request.HTTPCookieProcessor(cookie)
opener = request.build_opener(cookie_processor)
opener.addheaders = [("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0"),
("Referer", "http://cn.v2ex.com/signin"),
("Origin", "http://cn.v2ex.com"),
("Host", "cn.v2ex.com")]
request.install_opener(opener)
def build_opener():
cookie = http.cookiejar.CookieJar()
cookie_processor = request.HTTPCookieProcessor(cookie)
opener = request.build_opener(cookie_processor)
opener.addheaders = [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36"),
("Referer", "https://wx.qq.com/"),
("Origin", "https://wx.qq.com/"),
("Host", "wx.qq.com")]
request.install_opener(opener)
#??uuid
def build_opener():
cookie = http.cookiejar.CookieJar()
cookie_processor = request.HTTPCookieProcessor(cookie)
opener = request.build_opener(cookie_processor)
opener.addheaders = [("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"),
("Referer", "https://www.zhihu.com/"),
("Origin", "https://www.zhihu.com/"),
("Host", "www.zhihu.com")]
request.install_opener(opener)
def create_cookie_opener(self):
'''
????Cookie
:return: ????????opener
'''
cookie = cookiejar.CookieJar()
cookie_process = request.HTTPCookieProcessor(cookie)
opener = request.build_opener(cookie_process)
return opener
def __init__(self):
self.cj = MozillaCookieJar()
self.opener = request.build_opener(
request.HTTPCookieProcessor(self.cj))
self.threadPool = Pool(processes=10)
def download_image(url, output_dir, image, creds):
cookies = HTTPCookieProcessor()
opener = build_opener(cookies)
install_opener(opener)
data = urlopen("https://ers.cr.usgs.gov").read().decode('utf-8')
m = re.search(r'<input .*?name="csrf_token".*?value="(.*?)"', data)
if m:
token = m.group(1)
else:
print("Error : CSRF_Token not found")
params = urlencode(dict(username=creds['account'], password=creds['passwd'], csrf_token=token))
params = params.encode('ascii')
request = Request("https://ers.cr.usgs.gov/login", params, headers={})
f = urlopen(request)
data = f.read().decode('utf-8')
f.close()
if data.find('You must sign in as a registered user to download data or place orders for USGS EROS products') > 0:
print("Authentification failed")
req = urlopen(url)
uri = req.url
response = requests.get(uri)
if response.status_code == 200:
with open(os.path.join(output_dir, image), 'wb') as f:
for chunk in response.iter_content(chunk_size=1024 * 1024 * 8):
f.write(chunk)
elif response.status_code > 399:
print('Code {}'.format(response.status_code))
raise BadRequestsResponse(Exception)
def __init__(self):
self.cookiejar = CookieJar()
self._cookie_processor = HTTPCookieProcessor(self.cookiejar)
self.form = None
self.url = "http://0.0.0.0:8080/"
self.path = "/"
self.status = None
self.data = None
self._response = None
self._forms = None
def request(url, data={}, headers={}, timeout=10):
""" Returns a file-like object to the given URL.
"""
if cookies is not None:
f = urllib2.HTTPCookieProcessor(cookies)
f = urllib2.build_opener(f)
else:
f = urllib2.build_opener()
try:
f = f.open(Request(url, urlencode(data) if data else None, headers), timeout=timeout)
except URLError as e:
status = getattr(e, 'code', None) # HTTPError
if status == 401:
raise Forbidden
if status == 403:
raise Forbidden
if status == 404:
raise NotFound
if status == 420:
raise TooManyRequests
if status == 429:
raise TooManyRequests
raise e
except socket.error as e:
if 'timed out' in repr(e.args):
raise Timeout
else:
raise e
log.info(url)
return f
def __init__(self, username="admin", password="admin", hostname="192.168.1.1"):
self.username = username
self.password = password
self.hostname = hostname
self.baseurl = "https://" + self.hostname
try:
self.gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
except AttributeError:
try:
self.gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1)
except AttributeError:
self.gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
self.cj = CookieJar()
self.opener = build_opener(HTTPCookieProcessor(self.cj), HTTPSHandler(context=self.gcontext))
self.loginresponse = self._login()
def __init__(self):
super(WebService, self).__init__()
self.cache = defaultdict(defaultdict)
self._cookie = CookieJar()
self._opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(self._cookie))
self.query_interval = 1
def __init__(self, begin_date_str):
self.__base_url = "http://xk.autoisp.shu.edu.cn"
monday = datetime.strptime(begin_date_str, '%Y.%m.%d').date()
tuesday = monday + timedelta(days=1)
wednesday = tuesday + timedelta(days=1)
thursday = wednesday + timedelta(days=1)
friday = thursday + timedelta(days=1)
self.__weekday_table = {'?': monday, '?': tuesday, '?': wednesday, '?': thursday, '?': friday}
self.__course_time_table = (time(hour=8, minute=0, tzinfo=timezone("Asia/Shanghai")),
time(hour=8, minute=55, tzinfo=timezone("Asia/Shanghai")),
time(hour=10, minute=0, tzinfo=timezone("Asia/Shanghai")),
time(hour=10, minute=55, tzinfo=timezone("Asia/Shanghai")),
time(hour=12, minute=10, tzinfo=timezone("Asia/Shanghai")),
time(hour=13, minute=5, tzinfo=timezone("Asia/Shanghai")),
time(hour=14, minute=10, tzinfo=timezone("Asia/Shanghai")),
time(hour=15, minute=5, tzinfo=timezone("Asia/Shanghai")),
time(hour=16, minute=0, tzinfo=timezone("Asia/Shanghai")),
time(hour=16, minute=55, tzinfo=timezone("Asia/Shanghai")),
time(hour=18, minute=0, tzinfo=timezone("Asia/Shanghai")),
time(hour=18, minute=55, tzinfo=timezone("Asia/Shanghai")),
time(hour=19, minute=50, tzinfo=timezone("Asia/Shanghai")))
# Create a opener with the ability to record cookies.
self.__opener = build_opener(HTTPCookieProcessor(CookieJar()))
self.__validate_img_path = os.path.join(os.getcwd(), "validate_code.jpg")
self.term_index = 0
self.terms_and_ports = []
ports = (80, 8080)
for port in ports:
url = '%s:%d' % (self.__base_url, port)
data = self.__opener.open(url).read().decode('utf-8')
term = re.search(pattern=r'<div style="color: Red; font-size: 26px; text-align: center;">(.+?)</div>',
string=data).group(1)
self.terms_and_ports.append((term, port))
def __init__ (self, args) :
self.modulus = None
self.exponent = None
self.args = args
self.jar = j = LWPCookieJar ()
self.has_cookies = False
if self.args.cookiefile :
self.has_cookies = True
try :
j.load (self.args.cookiefile, ignore_discard = True)
except IOError :
self.has_cookies = False
self.opener = build_opener (HTTPCookieProcessor (j))
self.nextfile = args.file
# end def __init__
