test_tokenize.py 文件源码

def test_error(self):
        from cStringIO import StringIO

        stream = StringIO("FOOBAR{")

        lex = lexer.Lexer()
        lex.bufsize = 4
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 2)
        self.assertEqual(tokens[1][0], Error)