tokenizer.py 文件源码

def cutandsplit(s):
    for ln in filterlist(splitsentence(stripblank(s))):
        l = RE_BRACKETS.sub(brcksub, ln.strip())
        if notchinese(l):
            continue
        yield ' '.join(cut(l.replace('?', '“').replace('?', '”').replace('?', '‘').replace('?', '’').lstrip(tailpunct).rstrip(headpunct)))