test_tokenizer.py 文件源码

def test_invertibility_on_random_strings(self):
        for _ in xrange(1000):
            s = u"".join(six.unichr(random.randint(0, 65535))
                         for _ in xrange(10))
            self.assertEqual(s, self.tokenizer.decode(self.tokenizer.encode(s)))