def tokenize(self, text):
"""
Performs tokenization in addition to normalization.
"""
return self.normalize(nltk.wordpunct_tokenize(text))
微信
公众号
扫码关注公众号