language_model.py 文件源码

def __init__(self, text):
        self.text = text
        self.tokens = nltk.word_tokenize(text)
        self.lowercase_tokens = [t.lower() for t in self.tokens]
        self.alpha_tokens = [t for t in self.lowercase_tokens if t.isalpha()]