build_dictionary.py 文件源码

def compare(tokens, trans_tokens):
    pairs = []

    same_len = len(tokens) == len(trans_tokens)
    consecutive = False

    for a, b in zip(tokens, trans_tokens):
        if a != b:
            if consecutive and not same_len:
                break

            if not consecutive:
                consecutive = True

            pairs.append((a, b))
        else:
            consecutive = False

    return pairs