def test_single_proxy(self, proxy):
"""
text one proxy, if valid, put them to usable_proxies.
"""
try:
async with aiohttp.ClientSession() as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('Testing', proxy)
async with session.get(self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) as response:
if response.status == 200:
self._conn.put(proxy)
print('Valid proxy', proxy)
except (ProxyConnectionError, TimeoutError, ValueError):
print('Invalid proxy', proxy)
except (ServerDisconnectedError, ClientResponseError,ClientConnectorError) as s:
print(s)
pass
python类ServerDisconnectedError()的实例源码
def read_stream(app_name, auth_token):
while True:
stream_url = yield from get_stream_url(app_name, auth_token)
print('Reading stream: %s' % stream_url)
log = b''
with aiohttp.ClientSession() as session:
response = yield from session.get(stream_url)
while True:
try:
chunk = yield from response.content.read(1)
except aiohttp.ServerDisconnectedError:
break
if not chunk:
break
if chunk == b'\n':
try:
yield from write_to_queue(log)
except ValueError as e:
print(str(e))
log = b''
else:
log += chunk
def target_fetch(client, url):
"""
:param client: aiohttp client
:param url: target url
:return: text
"""
with async_timeout.timeout(30):
try:
headers = {'user-agent': get_random_user_agent()}
async with client.get(url, headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
try:
text = await response.read()
except aiohttp.ServerDisconnectedError as e:
LOGGER.exception(e)
text = None
return text
except Exception as e:
LOGGER.exception(e)
return None
def longpoll_processor(self):
await self.init_long_polling()
session = aiohttp.ClientSession(loop=self.loop)
while True:
try:
self.longpoll_request = session.get(self.server, params=self.values)
resp = await self.longpoll_request
except aiohttp.ClientOSError:
session = aiohttp.ClientSession(loop=self.loop)
except (asyncio.TimeoutError, aiohttp.ServerDisconnectedError):
self.logger.warning("Long polling server doesn't respond. Changing server")
await self.init_long_polling()
continue
try:
events = json.loads(await resp.text())
except ValueError:
continue
failed = events.get('failed')
if failed:
err_num = int(failed)
if err_num == 1: # 1 - update timestamp
self.values['ts'] = events['ts']
elif err_num in (2, 3): # 2, 3 - new data for long polling
await self.init_long_polling(err_num)
continue
self.values['ts'] = events['ts']
for event in events['updates']:
asyncio.ensure_future(self.process_longpoll_event(event))
def clear_channel(self, channel):
current_index = None
last_index = None
num_errors = 0
while current_index != -1:
if num_errors >= 5:
break
try:
async for message in self.logs_from(channel, before=current_index):
current_index = message
await self.delete_message(message)
except ServerDisconnectedError:
try:
async for message in self.logs_from(channel, before=current_index):
current_index = message
await self.delete_message(message)
except Exception:
num_errors += 1
continue
except Exception:
num_errors += 1
continue
if last_index == current_index:
break
last_index = current_index
def test_disconnect(self, mock_book, mock_connect):
mock_connect.return_value.aenter.receive_str = CoroutineMock()
mock_connect.return_value.aenter.send_json = CoroutineMock()
mock_book.return_value = {'bids': [], 'asks': [], 'sequence': 1}
messages_expected = [
json.dumps({
"type": "done",
"side": "sell",
"order_id": "4eef1226-4b38-422c-a5b1-56def7107f9a",
"reason": "canceled",
"product_id": "ETH-USD",
"price": "2601.76000000",
"remaining_size": "3.09000000",
"sequence": 2,
"time": "2017-06-25T11:23:14.775000Z"
}),
aiohttp.ServerDisconnectedError('error'),
json.dumps({
"type": "done",
"side": "sell",
"order_id": "4eef1226-4b38-422c-a5b1-56def7107f9a",
"reason": "canceled",
"product_id": "ETH-USD",
"price": "2601.76000000",
"remaining_size": "3.09000000",
"sequence": 2,
"time": "2017-06-25T11:23:14.775000Z"
})
]
mock_connect.return_value.aenter.receive_str.side_effect = \
messages_expected
async with gdax.orderbook.OrderBook() as orderbook:
message = await orderbook.handle_message()
assert message == json.loads(messages_expected[0])
message = await orderbook.handle_message()
assert message is None
message = await orderbook.handle_message()
assert message == json.loads(messages_expected[2])
def download(self) -> str:
if self._state != NOT_STARTED:
return self._state
# Prepare an empty buffer file.
await self._loop.run_in_executor(None, self._create_buffer_file)
try:
# Create client session for downloading a file part from a host.
async with aiohttp.ClientSession(loop=self._loop, headers=self._headers) as session:
# Request a host for a file part.
async with session.request('GET', self._url) as res: # type: aiohttp.ClientResponse
if res.status != 206:
raise WrongResponseError('Expected status code 206, but {!s} ({!s}) received.',
res.status,
res.reason)
hrh = res.headers # type: CIMultiDictProxy
# TODO: check headers.
# Read content by chunks and write to the buffer file.
if self._state == NOT_STARTED:
self._state = DOWNLOADING
while self._state is DOWNLOADING:
with aiohttp.Timeout(self._chunk_download_timeout, loop=self._loop):
chunk = await res.content.read(self._chunk_size)
self._bytes_downloaded += len(chunk)
self._debug("Read ({!s} bytes). Downloaded: {!s} of {!s} bytes. [{:.2%}]".format(
len(chunk), self._bytes_downloaded, self._length,
self._bytes_downloaded / self._length))
if not chunk:
self._state = DOWNLOADED
break
await self._write_chunk(chunk)
await self._flush_and_release()
if self._state != DOWNLOADED:
res.close() # Close the response if not downloaded.
except aiohttp.ServerDisconnectedError as exc:
self._debug('Server disconnected error: {!r}.'.format(exc))
self.cancel()
except WrongResponseError as exc:
self._debug('Wrong response error: {!r}.'.format(exc))
self.cancel()
except asyncio.TimeoutError:
self._debug('Timeout.')
self.cancel()
except Exception as exc:
self._debug('Unexpected exception: {!r}.'.format(exc))
self.cancel()
finally:
return self._state
def update_user_data():
""" Go through all registered members playing osu!, and update their data. """
global osu_tracking
# Go through each member playing and give them an "old" and a "new" subsection
# for their previous and latest user data
for member_id, profile in osu_config.data["profiles"].items():
member = discord.utils.find(lambda m: check_playing(m, member_id), client.get_all_members())
# If the member is not playing anymore, remove them from the tracking data
if not member:
if member_id in osu_tracking:
del osu_tracking[member_id]
continue
mode = get_mode(member_id).value
try:
user_data = await api.get_user(u=profile, type="id", m=mode)
except ServerDisconnectedError:
continue
except asyncio.TimeoutError:
logging.warning("Timed out when retrieving osu! info from {} ({})".format(member, profile))
continue
# Sleep after using get_user as to not put too much strain on the API at once
await asyncio.sleep(.2)
# Just in case something goes wrong, we skip this member (these things are usually one-time occurrences)
if user_data is None:
logging.info("Could not retrieve osu! info from {} ({})".format(member, profile))
continue
# User is already tracked
if member_id in osu_tracking:
# Move the "new" data into the "old" data of this user
osu_tracking[member_id]["old"] = osu_tracking[member_id]["new"]
else:
# If this is the first time, update the user's list of scores for later
user_scores = await api.get_user_best(u=profile, type="id", limit=score_request_limit, m=mode)
osu_tracking[member_id] = dict(member=member, scores=user_scores)
# Update the "new" data
osu_tracking[member_id]["new"] = user_data
osu_tracking[member_id]["new"]["ripple"] = True if api.ripple_pattern.match(profile) else False
def _send(self, payload, processor):
'''Send a payload to be converted to JSON.
Handles temporary connection issues. Daemon reponse errors
are raise through DaemonError.
'''
def log_error(error):
self.down = True
now = time.time()
prior_time = self.last_error_time
if now - prior_time > 60:
self.last_error_time = now
if prior_time and self.failover():
secs = 0
else:
self.logger.error('{} Retrying occasionally...'
.format(error))
data = json.dumps(payload)
secs = 1
max_secs = 4
while True:
try:
result = await self._send_data(data)
if not isinstance(result, tuple):
result = processor(result)
if self.down:
self.down = False
self.last_error_time = 0
self.logger.info('connection restored')
return result
log_error('HTTP error code {:d}: {}'
.format(result[0], result[1]))
except asyncio.TimeoutError:
log_error('timeout error.')
except aiohttp.ServerDisconnectedError:
log_error('disconnected.')
except self.ClientHttpProcessingError:
log_error('HTTP error.')
except self.ClientPayloadError:
log_error('payload encoding error.')
except aiohttp.ClientConnectionError:
log_error('connection problem - is your daemon running?')
except self.DaemonWarmingUpError:
log_error('starting up checking blocks.')
except (asyncio.CancelledError, DaemonError):
raise
except Exception:
self.log_error(traceback.format_exc())
await asyncio.sleep(secs)
secs = min(max_secs, secs * 2, 1)
def handle_message(self):
try:
message = await self._recv()
except aiohttp.ServerDisconnectedError as exc:
logging.error(
f'Error: Exception: f{exc}. Re-initializing websocket.')
await self.__aexit__(None, None, None)
await self.__aenter__()
return
msg_type = message['type']
if msg_type == 'error':
raise OrderBookError(f'Error: {message["message"]}')
if msg_type == 'subscriptions':
return # must filter out here because the subscriptions message does not have a product_id key
product_id = message['product_id']
assert self._sequences[product_id] is not None
sequence = message['sequence']
if sequence <= self._sequences[product_id]:
# ignore older messages (e.g. before order book initialization
# from getProductOrderBook)
return message
elif sequence > self._sequences[product_id] + 1:
logging.error(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequences[product_id]))
await self.__aexit__(None, None, None)
await self.__aenter__()
return
if msg_type == 'open':
self.add(product_id, message)
elif msg_type == 'done' and 'price' in message:
self.remove(product_id, message)
elif msg_type == 'match':
self.match(product_id, message)
elif msg_type == 'change':
self.change(product_id, message)
elif msg_type == 'heartbeat':
pass
elif msg_type == 'received':
pass
elif msg_type == 'done':
pass
else:
raise OrderBookError(f'unknown message type {msg_type}')
self._sequences[product_id] = sequence
return message
