tagger.py 文件源码

def __init__(self, model_path, word_dim=None, caps_dim=None, suffix_dim=None):
        self.model_path = model_path
        if word_dim is None:
            # use as supertagger
            with open(os.path.join(model_path, "tagger_defs.txt")) as defs_file:
                defs = json.load(defs_file)
            self.word_dim = defs["word_dim"]
            self.caps_dim = defs["caps_dim"]
            self.suffix_dim = defs["suffix_dim"]
        else:
            # training
            self.word_dim = word_dim
            self.caps_dim = caps_dim
            self.suffix_dim = suffix_dim

        self.words = read_model_defs(os.path.join(model_path, "words.txt"))
        self.suffixes = read_model_defs(os.path.join(model_path, "suffixes.txt"))
        self.caps = read_model_defs(os.path.join(model_path, "caps.txt"))
        self.targets = read_model_defs(os.path.join(model_path, "target.txt"))

        # self.unk_word = self.words["*UNKNOWN*"]
        self.unk_suffix = self.suffixes["UNK"]

        in_dim = 7 * (self.word_dim + self.caps_dim + self.suffix_dim)
        super(EmbeddingTagger, self).__init__(
                emb_word=L.EmbedID(len(self.words), self.word_dim),
                emb_caps=L.EmbedID(len(self.caps), self.caps_dim),
                emb_suffix=L.EmbedID(len(self.suffixes), self.suffix_dim),
                linear=L.Linear(in_dim, len(self.targets)),
                )