utils.py 文件源码

def remove_punc(string):
    # '''Description: This function takes in a string of descriptions and return a tokenized string without punctuation
    #   Parameters: String of descriptions
    #   Output: Tokenized string with punctuation removed'''
    tokenizer = RegexpTokenizer(r'\w+')
    tokens = tokenizer.tokenize(string)
    return " ".join(tokens)