def tokenize(text, filter_stopwords=False, lowercase=True):
words = wordpunct_tokenize(text)
if filter_stopwords:
words = [w for w in words if w not in STOPWORDS]
return words
微信
公众号
扫码关注公众号