def __init__(self, agent_addr, *, pool_size=100, block_time=3600, max_fail_times=2,
update_interval=300, auth=None, params=None, timeout=20, loop=None):
"""
ProxyPool constructor.
agent_addr - Proxy agent address.
pool_size - (optional) The size of the pool.
block_time - (optional) Time for blocking a proxy.
max_fail_times - (optional) The maximum acceptable number of the continuous failure of a proxy.
update_interval - (optional) Time interval to update the proxy list from proxy agent.
auth - (optional) Http Basic Auth tuple.
params - (optional) Prameters dictionary be sent in the query.
timeout - (optional) Timeout when connects proxy agent.
loop - (optional) Event loop.
"""
self.agent_addr = agent_addr
if loop is None:
self.asyn = False
self.loop = asyncio.new_event_loop()
else:
self.asyn = True
self.loop = loop
self.auth = auth
if self.auth is not None:
if isinstance(self.auth, tuple):
self.auth = aiohttp.BasicAuth(*self.auth)
elif not isinstance(self.auth, aiohttp.BasicAuth):
raise TypeError('The type of "auth" must be tuple or aiohttp.BasicAuth')
self.params = params or {}
self.timeout = timeout
self.update_interval = update_interval
self._last_update = 0
self._update_lock = asyncio.Lock(loop=self.loop)
self.max_fail_times = max_fail_times
self._proxies = {}
self._pool = PriorityQueue(pool_size)
self._pool_p = PriorityQueue(pool_size)
self._pool_n = PriorityQueue(pool_size)
self.backup_size = pool_size * 5
self._backup = PriorityQueue(self.backup_size)
self._backup_p = PriorityQueue(self.backup_size)
self._backup_n = PriorityQueue(self.backup_size)
self.block_time = block_time
self._trash = {}
self._block_queue = deque()
评论列表
文章目录
