pipelines.py 文件源码

def open_spider(self, spider):
        logging.warning('??spider')
        try:
            self.client = pymongo.MongoClient(self.mongo_uri)
            self.db = self.client[self.mongo_db]
        except ValueError:
            logging.error('???????')
        # ?????????????????
        if self.mongo_col not in self.db.collection_names():
            self.db[self.mongo_col].create_index(
                [('created_at', pymongo.DESCENDING)])
            self.db[self.mongo_col].create_index(
                [('admin', pymongo.ASCENDING)], sparse=True)
            self.db[self.mongo_col].create_index(
                [('price', pymongo.ASCENDING)], sparse=True)
            self.db[self.mongo_col].create_index(
                [('mblogid', pymongo.ASCENDING)], unique=True)
        else:
            # ????????????????????
            recent_row = list(self.db[self.mongo_col].find({'title': {'$ne': None}}, projection=['created_at'],
                                                           limit=1, sort=[('created_at', pymongo.DESCENDING)]))
            if recent_row:
                self.recent = recent_row[0]['created_at']  # ????
            logging.warning("???????????%s" % (
                self.recent + datetime.timedelta(hours=8)).__str__())