spider_engine.py 文件源码

def load_conf(self):
        spider_conf = {}
        conn = pymongo.Connection(self.setting.CONF_MONGO_HOST, self.setting.CONF_MONGO_PORT)
        row = conn.taobao.spider.find_one({"name":self.spider})
        spider_conf = dict(row)
        open("models/get_seed_%s.py" % str(spider_conf["_id"]), "w").write(spider_conf["get_seed"].encode("utf-8"))

        spider_conf["navi_models"] = []
        for navi in spider_conf["navi_list"]: 
            navi_row = conn.taobao.spider_navi.find_one({"_id":navi})
            open("models/navi_%s.py" % str(navi_row["_id"]), "w").write(navi_row["python_code"].encode("utf-8"))
            spider_conf["navi_models"].append(dict(navi_row))
        redis_queue = conn.taobao.redis_queue.find({"_id":spider_conf["_id"]})
        spider_conf["redis_queue"] = list(redis_queue)
        return spider_conf