def test_single_proxy(self, proxy):
"""
text one proxy, if valid, put them to usable_proxies.
"""
try:
async with aiohttp.ClientSession() as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('Testing', proxy)
async with session.get(self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) as response:
if response.status == 200:
self._conn.put(proxy)
print('Valid proxy', proxy)
except (ProxyConnectionError, TimeoutError, ValueError):
print('Invalid proxy', proxy)
except (ServerDisconnectedError, ClientResponseError,ClientConnectorError) as s:
print(s)
pass
python类ClientResponseError()的实例源码
def get_picture_urls(dates, verbose=False):
semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
tasks = [get_picture_url(date, semaphore) for date in dates]
urls = []
count = 0
# get results as jobs are done
for job in asyncio.as_completed(tasks, timeout=GLOBAL_TIMEOUT):
try:
url = yield from job
except NoPictureForDate as exc:
if verbose:
print('*** {!r} ***'.format(exc))
continue
except aiohttp.ClientResponseError as exc:
print('****** {!r} ******'.format(exc))
continue
count += 1
if verbose:
print(format(count, '3d'), end=' ')
print(url.split('/')[-1])
else:
print(url)
urls.append(url)
return urls
def stream_pty(self):
request = Request('GET', '/stream/kernel/{}/pty'.format(self.kernel_id))
try:
sess, ws = await request.connect_websocket()
except aiohttp.ClientResponseError as e:
raise BackendClientError(e.code, e.message)
return StreamPty(self.kernel_id, sess, ws)
def test_stream_pty_raises_error_with_abnormal_status(mocker):
mock_req_obj = asynctest.MagicMock(spec=Request)
mock_exception = aiohttp.ClientResponseError(
None, None, code=400,
message='emulated-handshake-error')
mock_req_obj.connect_websocket = \
asynctest.MagicMock(side_effect=mock_exception)
with asynctest.patch('ai.backend.client.asyncio.kernel.Request',
return_value=mock_req_obj) as mock_req_cls:
with pytest.raises(BackendClientError):
await Kernel('mykernel').stream_pty()
def tearDown(self):
try:
self.loop.run_until_complete(self.client.delete())
except aiohttp.ClientResponseError as exc:
if exc.code != 404:
raise
self.loop.run_until_complete(self.client.close())
self.loop.close()
def test_checks_for_bad_response(self):
self.response.status = 404
with self.assertRaises(ClientResponseError):
await self.session.request('/')
def sendToTelegram(self):
session = SessionManager.get()
TELEGRAM_BASE_URL = "https://api.telegram.org/bot{token}/sendVenue".format(token=conf.TELEGRAM_BOT_TOKEN)
title = self.name
try:
minutes, seconds = divmod(self.tth, 60)
description = 'Expires at: {} ({:.0f}m{:.0f}s left)'.format(self.expire_time, minutes, seconds)
except AttributeError:
description = "It'll expire between {} & {}.".format(self.min_expire_time, self.max_expire_time)
try:
title += ' ({}/{}/{})'.format(self.attack, self.defense, self.stamina)
except AttributeError:
pass
payload = {
'chat_id': conf.TELEGRAM_CHAT_ID,
'latitude': self.coordinates[0],
'longitude': self.coordinates[1],
'title' : title,
'address' : description,
}
try:
async with session.post(TELEGRAM_BASE_URL, data=payload) as resp:
self.log.info('Sent a Telegram notification about {}.', self.name)
return True
except ClientResponseError as e:
self.log.error('Error {} from Telegram: {}', e.code, e.message)
except ClientError as e:
self.log.error('{} during Telegram notification.', e.__class__.__name__)
except CancelledError:
raise
except Exception:
self.log.exception('Exception caught in Telegram notification.')
return False
def hook_post(self, w, session, payload, headers={'content-type': 'application/json'}):
try:
async with session.post(w, json=payload, timeout=4, headers=headers) as resp:
return True
except ClientResponseError as e:
self.log.error('Error {} from webook {}: {}', e.code, w, e.message)
except (TimeoutError, ServerTimeoutError):
self.log.error('Response timeout from webhook: {}', w)
except ClientError as e:
self.log.error('{} on webhook: {}', e.__class__.__name__, w)
except CancelledError:
raise
except Exception:
self.log.exception('Error from webhook: {}', w)
return False
def process_normally(self, message: RawRequestMessage, payload) -> aiohttp.Response:
"""Process request normally."""
req_data = payload if not isinstance(payload, EmptyStreamReader) else None
# Request from a host.
try:
async with aiohttp.ClientSession(headers=message.headers, loop=self._loop) as session:
async with session.request(message.method, message.path,
data=req_data,
allow_redirects=False) as host_resp: # type: aiohttp.ClientResponse
client_res = aiohttp.Response(
self.writer, host_resp.status, http_version=message.version)
# Process host response headers.
for name, value in host_resp.headers.items():
if name == hdrs.CONTENT_ENCODING:
continue
if name == hdrs.CONTENT_LENGTH:
continue
if name == hdrs.TRANSFER_ENCODING:
if value.lower() == 'chunked':
client_res.enable_chunked_encoding()
client_res.add_header(name, value)
# Send headers to the client.
client_res.send_headers()
# Send a payload.
while True:
chunk = await host_resp.content.read(self._chunk_size)
if not chunk:
break
client_res.write(chunk)
if client_res.chunked or client_res.autochunked():
await client_res.write_eof()
return client_res
except aiohttp.ClientResponseError:
self.log_debug("CANCELLED {!s} {!r}.".format(message.method, message.path))
raise
def _mal_fetch(session, kind, query, username, password):
"""Returns a bs4 tag or a string.
session is an aiohttp.ClientSession
kind should be either anime or manga
query is self-explanatory
username is self-explanatory
password is self-explanatory
"""
auth = aiohttp.BasicAuth(username, password)
query = urllib.parse.quote(query)
url = BASE_URL_MYANIMELIST_SEARCH.format(kind, query)
try: # This is gross, but MAL doesn't respond nicely.
async with session.request("GET", url, auth=auth) as response:
if response.status == 200:
xml = await response.text()
soup = BeautifulSoup(xml)
entry = soup.find("entry")
return entry
else:
message = "Could not reach MyAnimeList. x.x"
return message
except aiohttp.ClientResponseError:
message = ("No results found. Make sure you use spaces (e.g. "
"`one piece`, not `onepiece`). Also make sure to spell things right.")
return message
def fetch(self, url, retry=3):
'''?? URL?????? HTML ??'''
try:
start_time = self.loop.time()
response = await aiohttp.request('GET', url)
time_used = self.loop.time() - start_time
except (TimeoutError, aiohttp.ClientResponseError) as e:
# ??
if retry > 0:
retry -= 1
await asyncio.sleep(1)
return await self.fetch(url, retry)
else:
time_used = self.loop.time() - start_time
logger.error('USE %6.3f s STAT: 500 URL: %s (%s)'
% (time_used, url, e))
return ''
except Exception as e:
time_used = self.loop.time() - start_time
logger.error('USE %6.3f s STAT: 500 URL: %s (%s)'
% (time_used, url, e))
return ''
if not (200 <= response.status < 300):
logger.error('USE %6.3f s STAT: %s URL: %s'
% (time_used, response.status, url))
# ?? html ??????
body = await response.read()
try:
return body.decode('utf-8')
except UnicodeDecodeError:
try:
return body.decode('gbk')
except UnicodeDecodeError:
return body
def __post(self, data):
with aiohttp.Timeout(self.timeout, loop=self.loop):
try:
response = await self.__session.post(str(self.__url), data=data, headers=self.__headers)
except aiohttp.ClientError as e:
log.debug('Caught during POST request: %r', e)
raise ConnectionError(str(self.url))
else:
if response.status == CSRF_ERROR_CODE:
# Send request again with CSRF header
self.__headers[CSRF_HEADER] = response.headers[CSRF_HEADER]
log.debug('Setting CSRF header: %s = %s',
CSRF_HEADER, response.headers[CSRF_HEADER])
await response.release()
return await self.__post(data)
elif response.status == AUTH_ERROR_CODE:
await response.release()
log.debug('Authentication failed')
raise AuthError(str(self.url))
else:
try:
answer = await response.json()
except aiohttp.ClientResponseError as e:
text = textwrap.shorten(await response.text(),
50, placeholder='...')
raise RPCError('Server sent malformed JSON: {}'.format(text))
else:
return answer
def _local_request(self, method, uri, cookies=None, *args,
**kwargs):
if self._loop is None:
self._loop = asyncio.get_event_loop()
if uri.startswith(('http:', 'https:', 'ftp:', 'ftps://' '//')):
url = uri
else:
url = 'http://{host}:{port}{uri}'.format(
host=HOST, port=PORT, uri=uri)
conn = DelayableTCPConnector(pre_request_delay=self._request_delay,
verify_ssl=False, loop=self._loop)
async with aiohttp.ClientSession(cookies=cookies, connector=conn,
loop=self._loop) as session:
# Insert a delay after creating the connection
# But before sending the request.
async with getattr(session, method.lower())(
url, *args, **kwargs) as response:
try:
response.text = await response.text()
except UnicodeDecodeError:
response.text = None
try:
response.json = await response.json()
except (JSONDecodeError,
UnicodeDecodeError,
aiohttp.ClientResponseError):
response.json = None
response.body = await response.read()
return response
def _local_request(self, method, uri, cookies=None, *args, **kwargs):
import aiohttp
if uri.startswith(('http:', 'https:', 'ftp:', 'ftps://' '//')):
url = uri
else:
url = 'http://{host}:{port}{uri}'.format(
host=HOST, port=self.port, uri=uri)
logger.info(url)
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(
cookies=cookies, connector=conn) as session:
async with getattr(
session, method.lower())(url, *args, **kwargs) as response:
try:
response.text = await response.text()
except UnicodeDecodeError as e:
response.text = None
try:
response.json = await response.json()
except (JSONDecodeError,
UnicodeDecodeError,
aiohttp.ClientResponseError):
response.json = None
response.body = await response.read()
return response
def send_message(self, payload):
payload['disable_notification'] = True
try:
resp = await self.session.post(self._get_uri('sendMessage'), data=payload)
result = await self._raise_for_response(resp)
log.info("sent: %s", result)
except aiohttp.ClientResponseError as ex:
log.error("Message send failed %s", ex, exc_info=True)
def send_photo(self, payload):
payload['disable_notification'] = True
try:
resp = await self.session.post(self._get_uri('sendPhoto'), data=payload)
result = await self._raise_for_response(resp)
log.info("sent: %s", result)
except aiohttp.ClientResponseError as ex:
log.error("Message send failed %s", ex, exc_info=True)
def get_file(self, file_id):
try:
resp = await self.session.get(self._get_uri('getFile'), params={'file_id': file_id})
result = await self._raise_for_response(resp)
log.info("File: %s", result)
except aiohttp.ClientResponseError as ex:
log.error("Failed to retreive file: %s", ex, exc_info=True)
return None
return result
def get_data(self, category, name):
try:
with aiohttp.ClientSession(auth=self.auth,headers=self.header) as session:
async with session.get('https://myanimelist.net/api/{}/search.xml?q={}'.format(category, name),headers=self.header) as resp:
return (await resp.read())
except aiohttp.ClientResponseError: # some reason, mal dont return API, just nothing... causing error
return None
def _local_request(self, method, uri, cookies=None, *args,
**kwargs):
request_keepalive = kwargs.pop('request_keepalive',
Config.KEEP_ALIVE_TIMEOUT)
if uri.startswith(('http:', 'https:', 'ftp:', 'ftps://' '//')):
url = uri
else:
url = 'http://{host}:{port}{uri}'.format(
host=HOST, port=PORT, uri=uri)
do_kill_session = kwargs.pop('end_session', False)
if self._session:
session = self._session
else:
if self._tcp_connector:
conn = self._tcp_connector
else:
conn = ReuseableTCPConnector(verify_ssl=False,
loop=self._loop,
keepalive_timeout=
request_keepalive)
self._tcp_connector = conn
session = aiohttp.ClientSession(cookies=cookies,
connector=conn,
loop=self._loop)
self._session = session
async with getattr(session, method.lower())(
url, *args, **kwargs) as response:
try:
response.text = await response.text()
except UnicodeDecodeError:
response.text = None
try:
response.json = await response.json()
except (JSONDecodeError,
UnicodeDecodeError,
aiohttp.ClientResponseError):
response.json = None
response.body = await response.read()
if do_kill_session:
session.close()
self._session = None
return response
