Any.py 文件源码

def cutwords_jieba(self,sentence,userdict='dict/userdict.txt',stopwords='dict/stopwords.txt'):
        stropw = []
        if userdict:
            jieba.load_userdict(userdict)
            stropw = [line.strip() for line in open(stopwords,'r',encoding='utf-8').readlines()]

        frequency = defaultdict(int)
        l = list(jieba.cut(sentence))
        for t in l:
            frequency[t] += 1

        texts = [token for token in frequency if frequency[token] > 0]

        rtexts = list(set(texts)-set(stropw))
        return rtexts