tokchars.py 文件源码

def open_f(filename):
    if filename.endswith('.bz2'):
        return bz2.open(filename, 'r')
    else: # assume it's normal tzt
        return open(filename, 'r')


# prints to stdout for piping into kenlm