test_vocab.py 文件源码

def test_without_counts(self):
        vocab_list = ["Hello", ".", "?"]
        vocab_file = create_temporary_vocab_file(vocab_list)

        vocab_to_id_table, id_to_vocab_table, _, vocab_size = \
            vocabulary.create_vocabulary_lookup_table(vocab_file.name)

        self.assertEqual(vocab_size, 6)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            sess.run(tf.tables_initializer())

            ids = vocab_to_id_table.lookup(
                tf.convert_to_tensor(["Hello", ".", "?", "??", "xxx"]))
            ids = sess.run(ids)
            self.assertAllEqual(ids, [0, 1, 2, 3, 3])

            words = id_to_vocab_table.lookup(
                tf.convert_to_tensor(
                    [0, 1, 2, 3], dtype=tf.int64))
            words = sess.run(words)
            self.assertAllEqual(
                np.char.decode(words.astype("S"), "utf-8"),
                ["Hello", ".", "?", "UNK"])