def tokenize(string, lower=True):
if lower:
return nltk.wordpunct_tokenize(string.lower().strip())
else:
return nltk.wordpunct_tokenize(string.strip())
微信
公众号
扫码关注公众号