pipelines.py 文件源码

def process_item(self, item, spider):
        if isinstance(item, AirbnbItem):
            self.room_count += 1
            if self.room_count > 100000:
                self.room_count = 0
                self.room_file_count += 1
                self.spider_closed(spider, mode=1)
                self.spider_opened(spider, mode=1)
            self.exporter_room.export_item(item)
        elif isinstance(item, UserItem):
            self.user_count += 1
            if self.user_count > 100000:
                self.user_count = 0
                self.user_file_count += 1
                self.spider_closed(spider, mode=2)
                self.spider_opened(spider, mode=2)
            self.exporter_user.export_item(item)
        else:
            logger.info('Some error happened!')