def test_create_kernel_url():
mock_resp = asynctest.MagicMock(spec=aiohttp.ClientResponse)
mock_resp.status = 201
mock_resp.json = asynctest.MagicMock()
mock_req_obj = asynctest.MagicMock(spec=Request)
mock_req_obj.asend.return_value = mock_resp
with asynctest.patch('ai.backend.client.kernel.Request',
return_value=mock_req_obj) as mock_req_cls:
await Kernel.get_or_create('python')
mock_req_cls.assert_called_once_with(
'POST', '/kernel/create', mock.ANY)
mock_req_obj.asend.assert_called_once_with()
mock_req_obj.asend.return_value.json.assert_called_once_with()
python类ClientResponse()的实例源码
def test_get(self):
async with self.client.get(format="some/format") as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
async with self.client.get(format="some/format", graph=IRI("foo")) \
as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
def request(self, url, **kwargs):
if self.timeout is not None:
kwargs.setdefault('request_timeout', self.timeout)
if 'timeout' in kwargs:
timeout = kwargs.pop('timeout')
kwargs.setdefault('request_timeout', timeout)
kwargs.setdefault('headers', {})
kwargs['headers'].setdefault('content-type', 'application/json')
kwargs.setdefault('method', 'GET')
response = await self.client.fetch(url, **kwargs)
if isinstance(response, ClientResponse):
response.raise_for_status()
return response
def _go(self, file_path, check_resp: Callable[[ClientResponse], None] = None):
""" Make a test request to the web server through the proxy."""
expected_content = TEST_WEB_SERVER_FILES.get(file_path)
expected_content_length = len(expected_content)
connector = aiohttp.ProxyConnector(proxy=PROXY_ADDRESS, loop=self.loop)
async with aiohttp.client.ClientSession(connector=connector, loop=self.loop) as session:
url = create_host_url(file_path)
async with session.get(url) as resp: # type: ClientResponse
self.assertEqual(resp.status, 200)
if resp.headers.get(hdrs.TRANSFER_ENCODING) != 'chunked':
self.assertEqual(resp.headers.get(hdrs.CONTENT_LENGTH), str(expected_content_length))
content = await resp.read()
self.assertEqual(content, expected_content)
if check_resp:
check_resp(resp)
await asyncio.sleep(1, loop=self.loop) # Wait a little bit before closing the session.
def get(
self, url, *, allow_redirects=True, **kwargs) -> ClientResponse:
"""
Make HTTP GET request
:param url: Request URL, str or URL
:param allow_redirects: If set to False, do not follow redirects.
True by default (optional).
:param kwargs: In order to modify inner request parameters,
provide kwargs.
:return: a client response object.
:raises: HTTPStatusError if status code isn't 200
"""
query_url = url
if 'params' in kwargs:
query_url += get_query_string(kwargs['params'])
self.logger.log(logging.INFO, 'Sending GET request to ' + query_url)
r = await self.session.get(
url, allow_redirects=allow_redirects, **kwargs)
return self.return_response(r, r.status, url)
def create_response(method, url, content, loop=None):
loop = loop or asyncio.get_event_loop()
response = aiohttp.ClientResponse(method.lower(), URL(url))
def side_effect(*args, **kwargs):
fut = loop.create_future()
if isinstance(content, str):
fut.set_result(content.encode())
else:
fut.set_result(content)
return fut
response.content = mock.Mock()
response.content.read.side_effect = side_effect
return response
def mock_session(response, session=None, mock_object=None):
"""
:param aiohttp.ClientSession session:
:param aiohttp.ClientResponse|list[aiohttp.ClientResponse] response:
"""
session = session or aiohttp.ClientSession()
request = session._request
session.mock = mock_object or mock.Mock()
if isinstance(response, (list, tuple)):
session.mock.side_effect = response
else:
session.mock.return_value = response
async def _request(*args, **kwargs):
return session.mock(*args, **kwargs)
try:
with mock.patch.object(session, '_request') as request_mock:
request_mock.side_effect = _request
yield session
finally:
delattr(session, 'mock')
def _handle_fb_response(self, response: aiohttp.ClientResponse):
"""
Check that Facebook was OK with the API call we just made and raise
an exception if it failed.
"""
ok = response.status == 200
if not ok:
# noinspection PyBroadException
try:
error = (await response.json())['error']['message']
except Exception:
error = '(nothing)'
raise PlatformOperationError('Facebook says: "{}"'
.format(error))
def _heartbeat(self, video_id: str, text: str) -> None:
"""
??????????????????????????????????????
:param video_id:
:param text:
"""
try:
self.logger.debug("?????XML: %s", text)
api_url = self.glossary[video_id][KeyDmc.API_URL]
# 1?????????????????????????
waiting = (self.glossary[video_id][KeyDmc.HEARTBEAT] / 1000) - 5
companion = self._extract_session_tag(text)
self.logger.debug("????XML: %s", companion)
session_id = self._extract_session_id_xml(text)
await asyncio.sleep(waiting)
async with self.session.post(
url=api_url + "/" + session_id,
params={"_format": "xml", "_method": "PUT"},
data=companion
) as response: # type: aiohttp.ClientResponse
res_text = await response.text()
await self._heartbeat(video_id, res_text)
except asyncio.CancelledError:
pass
def download_pixiv_image(self, image_url: str) -> bytes:
"""
Downloads an image from Pixiv.
Pixiv disables hotlinking or downloading the images directly without a Referer [sic] header with the correct
location. This method automatically provides it.
:param image_url: The image URL to get.
:return: The bytes of the image.
"""
headers = {
"Referer": "http://spapi.pixiv.net/",
"User-Agent": 'PixivIOSApp/6.0.9 (iOS 9.3.3; iPhone8,1)'
}
async with self.sess.get(image_url, headers=headers) as r:
assert isinstance(r, aiohttp.ClientResponse)
if r.status != 200:
raise PixivError("Failed to download image {}".format(image_url))
return await r.read()
def test_create_kernel_return_id_only():
mock_resp = asynctest.MagicMock(spec=aiohttp.ClientResponse)
mock_resp.status = 201
mock_resp.json = lambda: {'kernelId': 'mock_kernel_id'}
mock_req_obj = asynctest.MagicMock(spec=Request)
mock_req_obj.asend.return_value = mock_resp
with asynctest.patch('ai.backend.client.kernel.Request',
return_value=mock_req_obj) as mock_req_cls:
k = await Kernel.get_or_create('python')
assert k.kernel_id == mock_resp.json()['kernelId']
def test_get(self):
async with self.client.get(format="some/format") as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string, "default")
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
async with self.client.get(format="some/format", graph=IRI("foo")) \
as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
def setUpAsync(self):
self.response = ClientResponse('get', URL('/'))
self.response.status = 200
self.mock_client = Mock()
self.mock_client.fetch = CoroutineMock(return_value=self.response)
self.session = Session()
self.session.client = self.mock_client
def process_normally(self, message: RawRequestMessage, payload) -> aiohttp.Response:
"""Process request normally."""
req_data = payload if not isinstance(payload, EmptyStreamReader) else None
# Request from a host.
try:
async with aiohttp.ClientSession(headers=message.headers, loop=self._loop) as session:
async with session.request(message.method, message.path,
data=req_data,
allow_redirects=False) as host_resp: # type: aiohttp.ClientResponse
client_res = aiohttp.Response(
self.writer, host_resp.status, http_version=message.version)
# Process host response headers.
for name, value in host_resp.headers.items():
if name == hdrs.CONTENT_ENCODING:
continue
if name == hdrs.CONTENT_LENGTH:
continue
if name == hdrs.TRANSFER_ENCODING:
if value.lower() == 'chunked':
client_res.enable_chunked_encoding()
client_res.add_header(name, value)
# Send headers to the client.
client_res.send_headers()
# Send a payload.
while True:
chunk = await host_resp.content.read(self._chunk_size)
if not chunk:
break
client_res.write(chunk)
if client_res.chunked or client_res.autochunked():
await client_res.write_eof()
return client_res
except aiohttp.ClientResponseError:
self.log_debug("CANCELLED {!s} {!r}.".format(message.method, message.path))
raise
def get_file_head(self, url: str) -> Optional[CIMultiDictProxy]:
"""Make a HEAD request to get a 'content-length' and 'accept-ranges' headers."""
self.log_debug('Getting a HEAD for url: {!s}.'.format(url))
try:
async with aiohttp.ClientSession(loop=self._loop) as session:
async with session.request(hdrs.METH_HEAD, url) as res: # type: aiohttp.ClientResponse
return res.headers
except Exception as exc:
self.log_debug("Could not get a HEAD for the {!r}. Error: {!r}.".format(url, exc))
return None
def test_normal_get(self):
def check_resp(resp: aiohttp.ClientResponse):
self.assertEqual(resp.headers.get(paraproxio.PARALLELS_HEADER), None)
self.loop.run_until_complete(self._go(SMALL_FILE_PATH, check_resp))
def request(self, *, url: str, method: str, data=None) -> Tuple[int, Any]:
if data is None:
data = {}
if method not in {'POST', 'PUT'}:
data = None
body = json.dumps(data) if data is not None else None
full_url = self.prefix + url
log.info('request', url=strip_auth(full_url), method=method, body=body)
response: ClientResponse = await self.session.request(
url=full_url,
method=method,
data=body
)
response_body = await response.read()
try:
data = json.loads(response_body)
except JSONDecodeError as exc:
log.error('json-decode', body=response_body)
data = {
'error': '!internal',
'message': str(exc),
'stacktrace': ''
}
wrap_screen(data)
log.info('response', url=strip_auth(full_url), method=method, body=body, response=response, data=data)
return response.status, data
def delayed_send(self, *args, **kwargs):
req = self.req
if self.delay and self.delay > 0:
#sync_sleep(self.delay)
_ = yield from asyncio.sleep(self.delay)
t = req.loop.time()
print("sending at {}".format(t), flush=True)
conn = next(iter(args)) # first arg is connection
try:
delayed_resp = self.orig_send(*args, **kwargs)
except Exception as e:
return aiohttp.ClientResponse(req.method, req.url)
return delayed_resp
def error_handler(self, response: aiohttp.ClientResponse, method):
"""Called if the response's status is not 200.
Raises the appropriate error to the status code.
Parameters
----------
response : dict
The response obj, used for grabbing status/JSON.
method
The method this response was used with.
Raises
-------
Any
"""
error_message = None
error_json = await response.json()
if error_json is not None:
error_message = error_json['status']['message']
if response.status == 400:
try:
raise errors.BadRequest(error_json)
except errors.BadRequest as e:
log.error(e)
elif response.status == 403:
raise errors.UnAuthorized
elif response.status == 404:
raise errors.EmptyResponse(message=error_message)
elif response.status == 422:
raise errors.InactivePlayer
elif response.status == 429:
cooldown = int(response.headers.get("Retry-After"))
log.error(" 429 - Rate limited for {0} seconds on method {1}".format(cooldown, method))
self.handlers[method].cool_down = cooldown
await self.handlers[method].cool_down_handler()
elif response.status in [500, 502, 503, 504]:
raise errors.ServiceUnavailable
def _retrieve_info(self, video_id: str) -> Dict:
interval = self.interval
backoff = self.backoff
attempt = max(0, self.retries) + 1
url = URL.URL_Watch + video_id
self.logger.debug("_worker: %s", locals())
async with asyncio.Semaphore(self.__parallel_limit):
st = 0
while attempt > 0:
attempt -= 1
async with self.session.get(url) as response: # type: aiohttp.ClientResponse
self.logger.debug("Video ID: %s, Status Code: %s", video_id, response.status)
if response.status == 200:
info_data = await response.text()
return self._junction(info_data)
# ?????????400?????????
elif 400 <= response.status < 500:
response.raise_for_status()
elif 500 <= response.status < 600:
await asyncio.sleep(interval/2)
print(Err.waiting_for_permission)
await asyncio.sleep(interval/2)
interval *= backoff
st = response.status
else:
st = response.status
break
raise aiohttp.errors.HttpProcessingError(
code=st, message=Err.connection_timeout.format(video_id))
def _first_nego_xml(self, video_id: str) -> str:
payload = self._make_param_xml(self.glossary[video_id])
self.logger.debug("Attempting to first negotiation of %s", video_id)
self.logger.debug("This is the posting XML: %s", payload)
async with self.session.post(
url=self.glossary[video_id][KeyDmc.API_URL],
params={"_format": "xml"},
data=payload,
) as response: # type: aiohttp.ClientResponse
return await response.text()
def _first_nego_json(self, video_id: str) -> str: # pragma: no cover
payload = self._make_param_json(self.glossary[video_id])
self.logger.debug("Attempting to first negotiation of %s", video_id)
self.logger.debug("This is the posting JSON: %s", payload)
async with self.session.post(
url=self.glossary[video_id][KeyDmc.API_URL],
params={"_format": "json"},
data=payload,
) as response: # type: aiohttp.ClientResponse
return await response.text()
def retriever(self, data: str, url: str) -> str:
self.logger.debug("Posting Parameters: %s", data)
async with asyncio.Semaphore(self.__parallel_limit):
async with self.session.post(url=url, data=data) as resp: # type: aiohttp.ClientResponse
return await resp.text()
def download(self) -> str:
if self._state != NOT_STARTED:
return self._state
# Prepare an empty buffer file.
await self._loop.run_in_executor(None, self._create_buffer_file)
try:
# Create client session for downloading a file part from a host.
async with aiohttp.ClientSession(loop=self._loop, headers=self._headers) as session:
# Request a host for a file part.
async with session.request('GET', self._url) as res: # type: aiohttp.ClientResponse
if res.status != 206:
raise WrongResponseError('Expected status code 206, but {!s} ({!s}) received.',
res.status,
res.reason)
hrh = res.headers # type: CIMultiDictProxy
# TODO: check headers.
# Read content by chunks and write to the buffer file.
if self._state == NOT_STARTED:
self._state = DOWNLOADING
while self._state is DOWNLOADING:
with aiohttp.Timeout(self._chunk_download_timeout, loop=self._loop):
chunk = await res.content.read(self._chunk_size)
self._bytes_downloaded += len(chunk)
self._debug("Read ({!s} bytes). Downloaded: {!s} of {!s} bytes. [{:.2%}]".format(
len(chunk), self._bytes_downloaded, self._length,
self._bytes_downloaded / self._length))
if not chunk:
self._state = DOWNLOADED
break
await self._write_chunk(chunk)
await self._flush_and_release()
if self._state != DOWNLOADED:
res.close() # Close the response if not downloaded.
except aiohttp.ServerDisconnectedError as exc:
self._debug('Server disconnected error: {!r}.'.format(exc))
self.cancel()
except WrongResponseError as exc:
self._debug('Wrong response error: {!r}.'.format(exc))
self.cancel()
except asyncio.TimeoutError:
self._debug('Timeout.')
self.cancel()
except Exception as exc:
self._debug('Unexpected exception: {!r}.'.format(exc))
self.cancel()
finally:
return self._state
