excrawler.py 文件源码

def doCrawlNewest(self):
        flag = True
        while flag:
            futures = []
            #??????5?
            with ThreadPoolExecutor(max_workers = 5) as executor:
                for i in range(self.context['currentPage'],self.context['currentPage'] + 5):
                    futures.append(executor.submit(self.getMangaDataByPage,i))
            wait(futures)
            #??????
            for f in futures:    
                gmetadata = f.result()
                for data in gmetadata:
                    #????????
                    if int(data['posted']) <= self.context['newestPosted']:
                        info("info","excrawler.crawlNewest","???????????? posted:" + data['posted'])
                        setConfig("app","new_context","")
                        flag = False
                        exit()
                    #??
                    if int(data['posted']) < self.context['currentPosted']:
                        self.db.insertEromanga(data)
                        self.context['currentPosted'] = int(data['posted'])
                self.context['currentPage'] += 1