def add_items(self, sentence_li):
"""Add new items to the tok2emb dictionary from a given text."""
for sen in sentence_li:
sent_toks = sent_tokenize(sen)
word_toks = [word_tokenize(el) for el in sent_toks]
tokens = [val for sublist in word_toks for val in sublist]
tokens = [el for el in tokens if el != '']
for tok in tokens:
if self.tok2emb.get(tok) is None:
self.tok2emb[tok] = self.fasttext_model[tok]
评论列表
文章目录