pipelines.py 文件源码

def open_spider(self, spider):
        self.connection = pymongo.MongoClient(
            settings['MONGODB_SERVER'],
            settings['MONGODB_PORT']
        )
        self.db = self.connection[settings['MONGODB_DB']]
        self.collection = self.db[settings['MONGODB_COLLECTION']]

        log.msg('Load nid from MongoDB database!',
                level=log.DEBUG, spider=spider)
        self.itemlist = set()
        for i in self.collection.find():
            self.itemlist.add(i['nid'])