lexer.py 文件源码

def simple_lexer(rules):
    """
    A very simple lexer factory based on a recipe on Python's regex module.
    """

    # This is a simplified version of the techique described at
    # https://docs.python.org/3/library/re.html#writing-a-lexer

    regex = '|'.join(r'(?P<%s>%s)' % item for item in rules)
    regex += r'|(?P<whitespace>\s+)|(?P<error>.+)'
    regex = re.compile(regex)

    def lexer(expr):
        for match in re.finditer(regex, expr):
            typ = match.lastgroup
            value = match.group(typ)

            if typ == 'whitespace':
                continue
            elif typ == 'error':
                raise SyntaxError('invalid value: %r' % value)

            yield Token(typ, value)

    lexer.which = 'simple'
    return lexer