def read_line_eos_noums(self,
path):
"""
Generator.
Similar as the function read_line_eos from
the text_mani module. The only diference here
is that we keep track of all the noums.
:type path: str
"""
for line in open(path):
if len(list(self.all_noums)) <= self.max_noums:
blob = TextBlob(line)
noums = set(blob.noun_phrases)
self.all_noums = self.all_noums.union(noums)
for word in line.split():
yield word
yield '<eos>'
评论列表
文章目录