proxy.py 文件源码

def worker(pattern,q):
    try:
        num = pattern["page_range"]
        for i in range(len(pattern["url"])):
            index = pattern["url"][i].find("%d")
            if index == -1:
                get_and_check(pattern["url"][i],pattern,q)
                gevent.sleep(10)
                continue
            for j in range(1,num+1):
                url = pattern["url"][i] % j
                #log.debug("PID:%d url:%s" % (os.getpid(),url))
                get_and_check(url,pattern,q)
                gevent.sleep(10)
    except Exception as e:
        log.error("PID:%d proxy error:%s " % (os.getpid(),e))