def get_proxy(q,msg_queue):
bloom.clear()
times = 0
while True:
try:
num = db_zcount()
log.debug("PID:%d db current ips %d------" % (os.getpid(),num))
while num > MIN_NUM:
time.sleep(REFRESH_WEB_SITE_TIMEER)
times += 1
if times == REFRESH_BF:
bloom.clear()
times = 0
log.debug("PID:%d refresh bloom filter" % os.getpid())
msg_queue.put("OK")
t1 = time.time()
event = []
for key,value in URL_PATTERN.items():
event.append(gevent.spawn(worker,value,q))
gevent.joinall(event)
t2 = time.time()
t = REFRESH_WEB_SITE_TIMEER - (t2 - t1)
times += 1
if t > 0:
time.sleep(t)
log.debug("PID:%d proxy sleep end------" % os.getpid())
if times == REFRESH_BF:
bloom.clear()
times = 0
log.debug("PID:%d refresh bloom filter" % os.getpid())
except Exception as e:
log.error("PID:%d proxy error:%s" % os.getpid(),e.message())
评论列表
文章目录