tokenizer.py 文件源码

def compile(self, rules):
        mapping = {}
        patterns = []
        for rule in rules:
            name = 'rule_{id}'.format(id=id(rule))
            pattern = r'(?P<{name}>{pattern})'.format(
                name=name,
                pattern=rule.pattern
            )
            mapping[name] = rule
            patterns.append(pattern)

        pattern = '|'.join(patterns)
        regexp = re.compile(pattern, re.UNICODE | re.IGNORECASE)
        return regexp, mapping