IPSpider.py 文件源码

def run(self):
        while(True):
            logging.info("[+] Spider start runing")
            # print '[+] ', 'Spider start runing'
            spawns = []
            # ??????ip???
            # select * from ip_table
            db = DBHelper()
            ids = db.getIds()
            # self.db_ip_num.value = len(ids)
            # print selected_id
            # exit()
            logging.info("[+] db save ip: {0}".format(len(ids)))
            # print '[+] ', 'db save ip:%d' % len(ids)
            if len(ids) < config.MINNUM:
                logging.info("[+] now ip num < MINNUM start spider")
                # print '[+] ', 'now ip num < MINNUM start spider...'
                for parser in config.parser_list:
                    if ids:
                        selected_id = random.choice(ids)
                        ip = db.getIp(selected_id)
                    else:
                        ip = ''
                    spawns.append(gevent.spawn(self.spider, parser, ip))
                    if len(spawns) >= config.MAX_DOWNLOAD_CONCURRENT:
                        gevent.joinall(spawns)
                        spawns = []
                gevent.joinall(spawns)
            else:
                logging.info("[+] now ip num meet the requirement,wait check again...'")
                # print '[+] ', 'now ip num meet the requirement,wait check again...', '#'

            logging.info("[+] sleep now")
            # print "sleep now"
            time.sleep(config.CHECK_INTERVAL)