tokenizer.py 文件源码

def __call__(self, text):
        for match in re.finditer(self.regexp, text):
            name = match.lastgroup
            value = match.group(0)
            span = match.span()
            rule = self.mapping[name]
            token = rule(value, span)
            yield token