excrawler.py 文件源码

def doCrawl(self):
        flag = True
        while flag:
            futures = []
            #??????5?
            with ThreadPoolExecutor(max_workers = 5) as executor:
                for i in range(self.context['currentPage'],self.context['currentPage'] + 5):
                    futures.append(executor.submit(self.getMangaDataByPage,i))
            wait(futures)
            #??????
            for f in futures:    
                gmetadata = f.result()
                #??????
                if gmetadata == None:
                    flag = False
                    break
                for data in gmetadata:
                    #??
                    if int(data['posted']) < self.context['currentPosted']:
                        self.db.insertEromanga(data)
                        self.context['currentPosted'] = int(data['posted'])
                self.context['currentPage'] += 1