working_with_tokens.py 文件源码

def convert_to_lines(tokens):
    """
    Given a stream of tokens, yield lines as strings.

    Each output string is guaranteed to end with a newline.

    """
    line = []
    for token in tokens:
        text = get_token_text(token)
        line.append(text)
        if text == '\n':
            yield ''.join(line)
            line = []
    if line:
        line.append('\n')
        yield ''.join(line)