preprocess_data.py 文件源码

def tokenize_documents(documents):
    for document in documents:
        text = document.text
        tokenized_doc = []
        for sent in nltk.sent_tokenize(text):
            tokenized_doc += nltk.word_tokenize(sent)
        document.text = tokenized_doc