working_with_tokens.py 文件源码

def persist_highlighting(tokens):
    """
    Given a stream of tokens, yield tokens with additional
    START_HIGHLIGHT and END_HIGHLIGHT tokens inserted to persist
    highlighting across tokens with a newline '\n' as text.

    """
    should_be_highlighting = False
    is_highlighting = False
    for token in tokens:
        token_type = get_token_type(token)
        if token_type == 'START_HIGHLIGHT':
            assert not should_be_highlighting, 'Multiple attempts to start highlighting'
            should_be_highlighting = True
        elif token_type == 'END_HIGHLIGHT':
            assert should_be_highlighting, 'Attempt to end highlighting while not highlighting'
            should_be_highlighting = False
        else:
            if get_token_text(token) == '\n':
                if is_highlighting:
                    yield ('END_HIGHLIGHT', '')
                    is_highlighting = False
            elif is_highlighting is not should_be_highlighting:
                if should_be_highlighting:
                    yield ('START_HIGHLIGHT', '')
                else:
                    yield ('END_HIGHLIGHT', '')
                is_highlighting = should_be_highlighting
            yield token