def parse_category(response: aiohttp.Response):
'''
response can be parsed to a json-object.
The field `Count` stands for how many emojis this category have.
The field `title` is the name of this category.
'''
js = await response.json()
if not js.get("Count", 0):
return
category = js.get("title", "").strip() # name of folder no blank space
if not category:
return
confirmDir(category)
# emojis contained by `picList`
for pic in js.get("picList", []):
src = pic.get("link", "") # emoji's download link
eid = pic.get("eid", "") # emoji's id
if not src:
continue
# name for saving
name = str(eid) + splitext(src)[1] if eid else basename(src)
dst = join(os.curdir, category, name)
await ss.download(src, dst)
python类Response()的实例源码
def process_connect(self, message: RawRequestMessage, payload) -> aiohttp.Response:
# TODO: implement.
pass
def process_normally(self, message: RawRequestMessage, payload) -> aiohttp.Response:
"""Process request normally."""
req_data = payload if not isinstance(payload, EmptyStreamReader) else None
# Request from a host.
try:
async with aiohttp.ClientSession(headers=message.headers, loop=self._loop) as session:
async with session.request(message.method, message.path,
data=req_data,
allow_redirects=False) as host_resp: # type: aiohttp.ClientResponse
client_res = aiohttp.Response(
self.writer, host_resp.status, http_version=message.version)
# Process host response headers.
for name, value in host_resp.headers.items():
if name == hdrs.CONTENT_ENCODING:
continue
if name == hdrs.CONTENT_LENGTH:
continue
if name == hdrs.TRANSFER_ENCODING:
if value.lower() == 'chunked':
client_res.enable_chunked_encoding()
client_res.add_header(name, value)
# Send headers to the client.
client_res.send_headers()
# Send a payload.
while True:
chunk = await host_resp.content.read(self._chunk_size)
if not chunk:
break
client_res.write(chunk)
if client_res.chunked or client_res.autochunked():
await client_res.write_eof()
return client_res
except aiohttp.ClientResponseError:
self.log_debug("CANCELLED {!s} {!r}.".format(message.method, message.path))
raise
def process_parallel(self, message: RawRequestMessage, payload) -> aiohttp.Response:
"""Try process a request parallel. Returns True in case of processed parallel, otherwise False."""
# Checking the opportunity of parallel downloading.
if message.method != hdrs.METH_GET or not need_file_to_parallel(message.path):
return None
head = await self.get_file_head(message.path)
if head is None:
return None
accept_ranges = head.get(hdrs.ACCEPT_RANGES)
if not accept_ranges or accept_ranges.lower() != 'bytes':
return None
content_length = head.get(hdrs.CONTENT_LENGTH)
if content_length is None:
return None
content_length = int(content_length)
if content_length <= 0 or content_length < DEFAULT_PART_SIZE:
return None
# All checks pass, start a parallel downloading.
self.log_debug("PARALLEL GET {!r} [{!s} bytes].".format(message.path, content_length))
# Get additional file info.
content_type = head.get(hdrs.CONTENT_TYPE)
# Prepare a response to a client.
client_res = aiohttp.Response(self.writer, 200, http_version=message.version)
client_res.add_header(hdrs.CONTENT_LENGTH, str(content_length))
if content_type:
client_res.add_header(hdrs.CONTENT_TYPE, content_type)
client_res.add_header(PARALLELS_HEADER, str(self._parallels))
client_res.send_headers()
try:
await self._cached_downloader.download(message.path, head, lambda chunk: client_res.write(chunk))
client_res.write_eof()
except Exception as exc:
self.log_debug("CANCELLED PARALLEL GET {!r}. Caused by exception: {!r}.".format(message.path, exc))
raise
return client_res
def response_to_proxy_response(self, response, response_content):
proxy_response = aiohttp.Response(self.writer, response.status, http_version=response.version)
# Copy response headers, except for Content-Encoding header,
# since unfortunately aiohttp transparently decodes content.
proxy_response_headers = [(name, value)
for name, value
in response.headers.items() if name not in ('CONTENT-ENCODING',)]
proxy_response.add_headers(*proxy_response_headers)
proxy_response.send_headers()
proxy_response.write(response_content)
yield from proxy_response.write_eof()
def post(self, address, message, headers):
self.logger.debug("HTTP Post to %s:\n%s", address, message)
with aiohttp.Timeout(self.operation_timeout):
response = await self.session.post(
address, data=message, headers=headers)
self.logger.debug(
"HTTP Response from %s (status: %d):\n%s",
address, response.status, await response.read())
return response
def new_response(self, response):
"""Convert an aiohttp.Response object to a requests.Response object"""
new = Response()
new._content = await response.read()
new.status_code = response.status
new.headers = response.headers
new.cookies = response.cookies
new.encoding = response.charset
return new
def start_response(self, status, headers, exc_info=None):
if exc_info:
try:
if self.status:
raise exc_info[1]
finally:
exc_info = None
status_code = int(status.split(' ', 1)[0])
self.status = status
resp = self.response = aiohttp.Response(
self.writer, status_code,
self.message.version, self.message.should_close)
resp.HOP_HEADERS = self.HOP_HEADERS
for name, value in headers:
resp.add_header(name, value)
if resp.has_chunked_hdr:
resp.enable_chunked_encoding()
# send headers immediately for websocket connection
if status_code == 101 and resp.upgrade and resp.websocket:
resp.send_headers()
else:
resp._send_headers = True
return self.response.write
def handle_request(self, message, payload):
"""Handle a single HTTP request.
Subclass should override this method. By default it always
returns 404 response.
:param message: Request headers
:type message: aiohttp.protocol.HttpRequestParser
:param payload: Request payload
:type payload: aiohttp.streams.FlowControlStreamReader
"""
now = self._loop.time()
response = aiohttp.Response(
self.writer, 404, http_version=message.version, close=True)
body = b'Page Not Found!'
response.add_header(hdrs.CONTENT_TYPE, 'text/plain')
response.add_header(hdrs.CONTENT_LENGTH, str(len(body)))
response.send_headers()
response.write(body)
drain = response.write_eof()
self.keep_alive(False)
self.log_access(message, None, response, self._loop.time() - now)
return drain
def logged(response: aiohttp.Response):
print(response.url)
def handle_error(self, status=500, message=None, payload=None, exc=None, headers=None, reason=None):
"""Handle errors.
Returns http response with specific status code. Logs additional
information. It always closes current connection."""
now = self._loop.time()
try:
if self._request_handler is None:
# client has been disconnected during writing.
return ()
if status == 500:
self.log_exception("Error handling request")
try:
if reason is None or reason == '':
reason, msg = RESPONSES[status]
else:
msg = reason
except KeyError:
status = 500
reason, msg = '???', ''
if self._debug and exc is not None:
try:
tb = traceback.format_exc()
tb = html_escape(tb)
msg += '<br><h2>Traceback:</h2>\n<pre>{}</pre>'.format(tb)
except:
pass
html = DEFAULT_ERROR_MESSAGE.format(
status=status, reason=reason, message=msg).encode('utf-8')
response = aiohttp.Response(self.writer, status, close=True)
response.add_header(hdrs.CONTENT_TYPE, 'text/html; charset=utf-8')
response.add_header(hdrs.CONTENT_LENGTH, str(len(html)))
if headers is not None:
for name, value in headers:
response.add_header(name, value)
response.send_headers()
response.write(html)
# disable CORK, enable NODELAY if needed
self.writer.set_tcp_nodelay(True)
drain = response.write_eof()
self.log_access(message, None, response, self._loop.time() - now)
return drain
finally:
self.keep_alive(False)
def handle_error(self, status=500, message=None,
payload=None, exc=None, headers=None, reason=None):
"""Handle errors.
Returns HTTP response with specific status code. Logs additional
information. It always closes current connection."""
now = self._loop.time()
try:
if self.transport is None:
# client has been disconnected during writing.
return ()
if status == 500:
self.log_exception("Error handling request")
try:
if reason is None or reason == '':
reason, msg = RESPONSES[status]
else:
msg = reason
except KeyError:
status = 500
reason, msg = '???', ''
if self.debug and exc is not None:
try:
tb = traceback.format_exc()
tb = html_escape(tb)
msg += '<br><h2>Traceback:</h2>\n<pre>{}</pre>'.format(tb)
except:
pass
html = DEFAULT_ERROR_MESSAGE.format(
status=status, reason=reason, message=msg).encode('utf-8')
response = aiohttp.Response(self.writer, status, close=True)
response.add_header(hdrs.CONTENT_TYPE, 'text/html; charset=utf-8')
response.add_header(hdrs.CONTENT_LENGTH, str(len(html)))
if headers is not None:
for name, value in headers:
response.add_header(name, value)
response.send_headers()
response.write(html)
# disable CORK, enable NODELAY if needed
self.writer.set_tcp_nodelay(True)
drain = response.write_eof()
self.log_access(message, None, response, self._loop.time() - now)
return drain
finally:
self.keep_alive(False)
