def get(self, width, height):
url = self.request.get('url')
key = '{}:{}:{}'.format(url, width, height)
height, width = [int(height), int(width)]
cached = Photo.get_by_id(key)
if cached is None:
img = urlfetch.fetch(url).content
output = images.resize(img, width=width, height=height)
cache = Photo(id = key, image = output)
cache.put()
else:
output = cached.image
self.response.headers['Content-Type'] = 'image/png'
self.response.headers['Cache-Control'] = 'max-age=3600'
self.response.write(output)
python类fetch()的实例源码
def request(self, method, url, body, headers):
# Calculate the absolute URI, which fetch requires
netloc = self.host
if self.port:
netloc = '%s:%s' % (self.host, self.port)
absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
try:
response = fetch(absolute_uri, payload=body, method=method,
headers=headers, allow_truncated=False, follow_redirects=False,
deadline=self.timeout,
validate_certificate=self.validate_certificate)
self.response = ResponseDict(response.headers)
self.response['status'] = str(response.status_code)
self.response['reason'] = httplib.responses.get(response.status_code, 'Ok')
self.response.status = response.status_code
setattr(self.response, 'read', lambda : response.content)
# Make sure the exceptions raised match the exceptions expected.
except InvalidURLError:
raise socket.gaierror('')
except (DownloadError, ResponseTooLargeError, SSLCertificateError):
raise httplib.HTTPException()
def request(self, method, url, body, headers):
# Calculate the absolute URI, which fetch requires
netloc = self.host
if self.port:
netloc = '%s:%s' % (self.host, self.port)
absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
try:
response = fetch(absolute_uri, payload=body, method=method,
headers=headers, allow_truncated=False, follow_redirects=False,
deadline=self.timeout,
validate_certificate=self.validate_certificate)
self.response = ResponseDict(response.headers)
self.response['status'] = str(response.status_code)
self.response['reason'] = httplib.responses.get(response.status_code, 'Ok')
self.response.status = response.status_code
setattr(self.response, 'read', lambda : response.content)
# Make sure the exceptions raised match the exceptions expected.
except InvalidURLError:
raise socket.gaierror('')
except (DownloadError, ResponseTooLargeError, SSLCertificateError):
raise httplib.HTTPException()
def request(self, method, url, body, headers):
# Calculate the absolute URI, which fetch requires
netloc = self.host
if self.port:
netloc = '%s:%s' % (self.host, self.port)
absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
try:
response = fetch(absolute_uri, payload=body, method=method,
headers=headers, allow_truncated=False, follow_redirects=False,
deadline=self.timeout,
validate_certificate=self.validate_certificate)
self.response = ResponseDict(response.headers)
self.response['status'] = str(response.status_code)
self.response['reason'] = httplib.responses.get(response.status_code, 'Ok')
self.response.status = response.status_code
setattr(self.response, 'read', lambda : response.content)
# Make sure the exceptions raised match the exceptions expected.
except InvalidURLError:
raise socket.gaierror('')
except (DownloadError, ResponseTooLargeError, SSLCertificateError):
raise httplib.HTTPException()
def request(self, method, url, body, headers):
# Calculate the absolute URI, which fetch requires
netloc = self.host
if self.port:
netloc = '%s:%s' % (self.host, self.port)
absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
try:
response = fetch(absolute_uri, payload=body, method=method,
headers=headers, allow_truncated=False, follow_redirects=False,
deadline=self.timeout,
validate_certificate=self.validate_certificate)
self.response = ResponseDict(response.headers)
self.response['status'] = response.status_code
setattr(self.response, 'read', lambda : response.content)
# Make sure the exceptions raised match the exceptions expected.
except InvalidURLError:
raise socket.gaierror('')
except (DownloadError, ResponseTooLargeError, SSLCertificateError):
raise httplib.HTTPException()
def request(self, method, url, headers, post_data=None):
try:
result = urlfetch.fetch(
url=url,
method=method,
headers=headers,
# Google App Engine doesn't let us specify our own cert bundle.
# However, that's ok because the CA bundle they use recognizes
# api.ubivar.com.
validate_certificate=self._verify_ssl_certs,
deadline=self._deadline,
payload=post_data
)
except urlfetch.Error as e:
self._handle_request_error(e, url)
return result.content, result.status_code, result.headers
def getAddress(longitude, latitude):
gsp_key = "gps-" + str(longitude) + "," + str(latitude)
resultData = memcache.get(key=gsp_key)
if resultData == None:
url = "https://maps.googleapis.com/maps/api/geocode/json?language=ja&sensor=false&key=" + const.GOOGLE_API_KEY + "&latlng=" + str(
longitude) + "," + str(latitude)
logging.debug(url)
result = urlfetch.fetch(
url=url,
method=urlfetch.GET,
headers={
}
)
if result.status_code == 200:
logging.debug(result.content)
else:
logging.debug(result.content)
jsonstr = result.content
jsonobj = json.loads(jsonstr)
if len(jsonobj["results"]) > 0:
memcache.set(key=gsp_key, value=jsonobj, time=3600)
resultData = jsonobj;
else:
logging.debug(resultData)
return resultData["results"]
def getUserProfine(mid):
# midstr= ','.join(mids)
url = "https://api.line.me/v2/bot/profile/"+mid
result = urlfetch.fetch(
url=url,
method=urlfetch.GET,
headers={
'Authorization': 'Bearer '+const.ChannelAccessToken
}
)
if result.status_code == 200:
logging.debug(result.content)
else:
logging.debug(result.content)
jsonstr = result.content
jsonobj = json.loads(jsonstr)
return jsonobj
def _appengine_fetch(self, uri, params, method):
if method == 'GET':
uri = self._build_get_uri(uri, params)
try:
httpmethod = getattr(urlfetch, method)
except AttributeError:
raise NotImplementedError(
"Google App Engine does not support method '%s'" % method)
authstring = base64.encodestring('%s:%s' % (self.auth_id, self.auth_token))
authstring = authstring.replace('\n', '')
r = urlfetch.fetch(url=uri, payload=urllib.urlencode(params),
method=httpmethod,
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'Basic %s' % authstring})
if r.status_code >= 300:
raise HTTPErrorAppEngine("HTTP %s: %s" % \
(r.status_code, r.content))
return r.content
def _appengine_fetch(self, uri, params, method):
if method == 'GET':
uri = self._build_get_uri(uri, params)
try:
httpmethod = getattr(urlfetch, method)
except AttributeError:
raise NotImplementedError(
"Google App Engine does not support method '%s'" % method)
authstring = base64.encodestring('%s:%s' % (self.auth_id, self.auth_token))
authstring = authstring.replace('\n', '')
r = urlfetch.fetch(url=uri, payload=urllib.urlencode(params),
method=httpmethod,
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'Basic %s' % authstring})
if r.status_code >= 300:
raise HTTPErrorAppEngine("HTTP %s: %s" % \
(r.status_code, r.content))
return r.content
def _appengine_fetch(self, uri, params, method):
if method == 'GET':
uri = self._build_get_uri(uri, params)
try:
httpmethod = getattr(urlfetch, method)
except AttributeError:
raise NotImplementedError(
"Google App Engine does not support method '%s'" % method)
authstring = base64.encodestring('%s:%s' % (self.auth_id, self.auth_token))
authstring = authstring.replace('\n', '')
r = urlfetch.fetch(url=uri, payload=urllib.urlencode(params),
method=httpmethod,
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'Basic %s' % authstring})
if r.status_code >= 300:
raise HTTPErrorAppEngine("HTTP %s: %s" % \
(r.status_code, r.content))
return r.content
def pushtxn(raw_tx):
'''Insight send raw tx API'''
url = PUSH_TX_URL
payload = urllib.urlencode({
"rawtx": raw_tx
})
result = urlfetch.fetch(url,
method=urlfetch.POST,
payload=payload
)
if result.status_code == 200:
j = json.loads(result.content)
txid = j.get('txid')
return txid, raw_tx
else:
msg = 'Error accessing insight API:'+str(result.status_code)+" "+str(result.content)
ErrorNotification.new(msg)
return None, msg
def request(self, method, url, body, headers):
# Calculate the absolute URI, which fetch requires
netloc = self.host
if self.port:
netloc = '%s:%s' % (self.host, self.port)
absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
try:
response = fetch(absolute_uri, payload=body, method=method,
headers=headers, allow_truncated=False, follow_redirects=False,
deadline=self.timeout,
validate_certificate=self.validate_certificate)
self.response = ResponseDict(response.headers)
self.response['status'] = str(response.status_code)
self.response['reason'] = httplib.responses.get(response.status_code, 'Ok')
self.response.status = response.status_code
setattr(self.response, 'read', lambda : response.content)
# Make sure the exceptions raised match the exceptions expected.
except InvalidURLError:
raise socket.gaierror('')
except (DownloadError, ResponseTooLargeError, SSLCertificateError):
raise httplib.HTTPException()
def SetExpressCheckout(self, amount):
params = {
'METHOD' : "SetExpressCheckout",
'NOSHIPPING' : 1,
'PAYMENTACTION' : 'Authorization',
'RETURNURL' : self.returnurl,
'CANCELURL' : self.cancelurl,
'AMT' : amount,
'CURRENCYCODE': self.currency
}
params_string = self.signature + urllib.urlencode(params)
response = urlfetch.fetch(self.API_ENDPOINT, params_string.encode("UTF-8"),"POST",deadline=10).content.decode("UTF-8")
response_token = ""
for token in response.split('&'):
if token.find("TOKEN=") != -1:
response_token = token[ (token.find("TOKEN=")+6):]
return response_token
def DoExpressCheckoutPayment(self, token, payer_id, amt):
params = {
'METHOD' : "DoExpressCheckoutPayment",
'PAYMENTACTION' : 'Sale',
'RETURNURL' : self.returnurl,
'CANCELURL' : self.cancelurl,
'TOKEN' : token,
'AMT' : amt,
'PAYERID' : payer_id,
'CURRENCYCODE': self.currency,
}
params_string = self.signature + urllib.urlencode(params)
response = urlfetch.fetch(self.API_ENDPOINT, params_string.encode("UTF-8"),"POST",deadline=10).content.decode("UTF-8")
response_tokens = {}
for token in response.split('&'):
response_tokens[token.split("=")[0]] = token.split("=")[1]
for key in list(response_tokens.keys()):
response_tokens[key] = urllib.unquote(response_tokens[key])
return response_tokens
def exportItems( module, target, importKey, startCursor, endCursor):
Skel = skeletonByKind( module )
query = Skel().all().cursor( startCursor, endCursor )
for item in query.run(250):
flatItem = DbTransfer.genDict( item )
formFields = {
"e": pickle.dumps(flatItem).encode("HEX"),
"key": importKey
}
result = urlfetch.fetch( url=target,
payload=urllib.urlencode(formFields),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if startCursor == endCursor:
try:
utils.sendEMailToAdmins("Export of kind %s finished" % module,
"ViUR finished to export kind %s to %s.\n" % (module, target))
except: #OverQuota, whatever
pass
# --- import ---
def __init__(self, league_id, start_date, end_date):
"""
Args:
league_id (string): a valid league_id from constants.py
start_date (string): date string in iso 8601 format
end_date (string): date string in iso 8601 format
"""
self.league = league_id
self.start_date = start_date
self.end_date = end_date
logging.info("Requesting %s for %s until %s" % (self.league, self.start_date, self.end_date))
response = urlfetch.fetch(self.url)
self.response_dict = json.loads(response.content)
self.game_list = self.parse_game_list()
def _request(self):
"""Sends the request
Returns:
list
"""
if not memcache.add(type(self).__name__, True, 3):
time.sleep(3)
logging.info('Scraping %s' % (type(self).__name__))
url = "https://www.usatoday.com/sports/mlb/sagarin/2017/team/"
response = urlfetch.fetch(url)
try:
ratings = self._scrape(response.content)
except (AttributeError, IndexError) as e:
logging.exception(e)
ratings = []
return ratings
def fetch(self, at_milliseconds):
"""
Tries to fetch the thumbnail from the cache.
If it is not there, tries to retrieve it from YouTube
and caches the result if successful
Returns:
either the thumbnail or the default thumbnail image
whether the thumbnail fetch was successful or not
"""
cached_key = self.create_key(at_milliseconds)
image_content = self.fetch_cached_thumbnail(cached_key)
if not image_content:
image_content, found = self.fetch_thumbnail_from_youtube(
at_milliseconds)
if found:
self.cache_thumbnail(image_content, cached_key)
else:
found = True
return image_content, found
def fetch_thumbnail_from_youtube(self, at_milliseconds):
"""
Retrieves the YT thumbnail.
Returns the raw image content with the
"""
url = \
"http://img.youtube.com/vd?id={yt_id}&ats={at_milliseconds}".format(
yt_id=self.youtube_id, at_milliseconds=at_milliseconds)
retries = 0
content = None
found = False
while retries <= self.max_fetch_retries:
result = urlfetch.fetch(url, deadline=50)
if result.status_code in (200, 404,):
content = result.content
if result.status_code == 200:
found = True
break
else:
retries += 1
return content, found
def get(self):
auth_token, _ = app_identity.get_access_token(
'https://www.googleapis.com/auth/cloud-platform')
logging.info(
'Using token {} to represent identity {}'.format(
auth_token, app_identity.get_service_account_name()))
response = urlfetch.fetch(
'https://www.googleapis.com/storage/v1/b?project={}'.format(
app_identity.get_application_id()),
method=urlfetch.GET,
headers={
'Authorization': 'Bearer {}'.format(auth_token)
}
)
if response.status_code != 200:
raise Exception(
'Call failed. Status code {}. Body {}'.format(
response.status_code, response.content))
result = json.loads(response.content)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(result, indent=2))
def check_language(text):
if len(text) > MAX_LENGTH:
logging.info("trimming text from %s to %s", len(text), MAX_LENGTH)
text = text[:MAX_LENGTH]
base_url = 'https://www.googleapis.com/language/translate/v2/detect'
params = {'key': API_KEY, 'q': text}
form_data = urls.urlencode(params)
if urllib2_fallback:
request = urllib2.Request(base_url, form_data, {'X-HTTP-Method-Override': 'GET'})
response_content = urllib2.urlopen(request).read()
else:
result = urlfetch.fetch(url=base_url, payload=form_data, method=urlfetch.POST, headers={'X-HTTP-Method-Override': 'GET'})
if result.status_code != 200:
error = "result status code is %s for content %s" % (result.status_code, result.content)
logging.error(error)
raise Exception("Error in translation: %s" % error)
response_content = result.content
json_content = json.loads(response_content)
real_results = json_content['data']['detections'][0][0]
logging.info("text classification returned %s", real_results)
if real_results['confidence'] > 0.10:
return real_results['language']
else:
return None
def request(self, method, url, body, headers):
# Calculate the absolute URI, which fetch requires
netloc = self.host
if self.port:
netloc = '%s:%s' % (self.host, self.port)
absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
try:
response = fetch(absolute_uri, payload=body, method=method,
headers=headers, allow_truncated=False, follow_redirects=False,
deadline=self.timeout,
validate_certificate=self.validate_certificate)
self.response = ResponseDict(response.headers)
self.response['status'] = str(response.status_code)
self.response['reason'] = httplib.responses.get(response.status_code, 'Ok')
self.response.status = response.status_code
setattr(self.response, 'read', lambda : response.content)
# Make sure the exceptions raised match the exceptions expected.
except InvalidURLError:
raise socket.gaierror('')
except (DownloadError, ResponseTooLargeError, SSLCertificateError):
raise httplib.HTTPException()
def list_bucket_files(bucket_name, prefix, max_keys=1000):
"""Returns a listing of of a bucket that matches the given prefix."""
scope = config.GoogleApiScope('devstorage.read_only')
bucket_url = config.GsBucketURL(bucket_name)
url = bucket_url + '?'
query = [('max-keys', max_keys)]
if prefix:
query.append(('prefix', prefix))
url += urllib.urlencode(query)
auth_token, _ = app_identity.get_access_token(scope)
result = urlfetch.fetch(url, method=urlfetch.GET, headers={
'Authorization': 'OAuth %s' % auth_token,
'x-goog-api-version': '2'})
if result and result.status_code == 200:
doc = xml.dom.minidom.parseString(result.content)
return [node.childNodes[0].data for node in doc.getElementsByTagName('Key')]
raise BackupValidationError('Request to Google Cloud Storage failed')
def get_gs_object(bucket_name, path):
"""Returns a listing of of a bucket that matches the given prefix."""
scope = config.GoogleApiScope('devstorage.read_only')
bucket_url = config.GsBucketURL(bucket_name)
url = bucket_url + path
auth_token, _ = app_identity.get_access_token(scope)
result = urlfetch.fetch(url, method=urlfetch.GET, headers={
'Authorization': 'OAuth %s' % auth_token,
'x-goog-api-version': '2'})
if result and result.status_code == 200:
return result.content
if result and result.status_code == 403:
raise BackupValidationError(
'Requested path %s is not accessible/access denied' % url)
if result and result.status_code == 404:
raise BackupValidationError('Requested path %s was not found' % url)
raise BackupValidationError('Error encountered accessing requested path %s' %
url)
def fetch(self):
logging.debug("Fetching google calendar data")
self.build_service('calendar', 'v3')
timeMin = self.date_dt.isoformat() + 'Z'
timeMax = self.next_date_dt.isoformat() + 'Z'
results = self.service.events().list(calendarId='primary',
maxResults=self.limit,
timeMin=timeMin,
timeMax=timeMax).execute()
if results:
items = [
Item(
svc=SERVICE.GCAL,
title=r.get('summary'),
details=r.get('description'),
id=r.get('id'),
type=SERVICE.EVENT).json() for r in results.get(
'items',
[])]
return items
return []
def get_access_token(force=False):
"""Tries to obtain access token from memcache and, if it fails,
obtains a new set and stores in memcache.
See https://dev.twitter.com/oauth/application-only.
Deleting the memcache key `access_token` will trigger a token refresh.
"""
token = memcache.get('access_token')
if force or token is None:
logging.warning('Needed to fetch access_token')
encoded_key = urllib.quote_plus(CUSTOMER_KEY)
encoded_secret = urllib.quote_plus(CUSTOMER_SECRET)
encoded_credentials = base64.b64encode(
"{}:{}".format(encoded_key, encoded_secret))
response = urlfetch.fetch(
'https://api.twitter.com/oauth2/token',
payload='grant_type=client_credentials',
method=urlfetch.POST,
headers={'Authorization': 'Basic ' + encoded_credentials})
if response.status_code == urlfetch.httplib.OK:
response_data = json.loads(response.content)
token = response_data['access_token']
memcache.set('access_token', token, 2592000) # 30 days
return token
def authenticated_get(
url, customer_key=CUSTOMER_KEY, customer_secret=CUSTOMER_SECRET):
"""Performs an authenticated GET to the given URL, returns the response's
content.
See https://dev.twitter.com/oauth/application-only
"""
token = get_access_token()
response = urlfetch.fetch(
url,
method=urlfetch.GET,
headers={'Authorization': 'Bearer ' + token})
if response.status_code == urlfetch.httplib.OK:
return response.content
elif response.status_code == urlfetch.httplib.UNAUTHORIZED:
return response.content # User is probably suspended
else:
message = 'Url ' + url + ' returned ' + response.content
logging.warning(message)
raise urlfetch.httplib.HTTPException(message)
def urlread(url, data=None, headers=None):
if data is not None:
if headers is None:
headers = {"Content-type": "application/x-www-form-urlencoded"}
method = urlfetch.POST
else:
if headers is None:
headers = {}
method = urlfetch.GET
result = urlfetch.fetch(url, method=method,
payload=data, headers=headers)
if result.status_code == 200:
return result.content
else:
raise urllib2.URLError("fetch error url=%s, code=%d" % (url, result.status_code))
def request(self, method, url, body, headers):
# Calculate the absolute URI, which fetch requires
netloc = self.host
if self.port:
netloc = '%s:%s' % (self.host, self.port)
absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
try:
response = fetch(absolute_uri, payload=body, method=method,
headers=headers, allow_truncated=False, follow_redirects=False,
deadline=self.timeout,
validate_certificate=self.validate_certificate)
self.response = ResponseDict(response.headers)
self.response['status'] = str(response.status_code)
self.response['reason'] = httplib.responses.get(response.status_code, 'Ok')
self.response.status = response.status_code
setattr(self.response, 'read', lambda : response.content)
# Make sure the exceptions raised match the exceptions expected.
except InvalidURLError:
raise socket.gaierror('')
except (DownloadError, ResponseTooLargeError, SSLCertificateError):
raise httplib.HTTPException()
