word_tokenizers.py 文件源码

def tokenize(self, sentence):
        """
        Given a string, tokenize it into words (with the conventional notion
        of word).

        Parameters
        ----------
        sentence: str
            The string to tokenize.

        Returns
        -------
        tokenized_sentence: List[str]
            The tokenized representation of the string, as a list of tokens.
        """
        return nltk.word_tokenize(sentence.lower())