def train_ranker(options):
lemmatizer = WordNetLemmatizer()
words, answers, good_lfs, bad_lfs = load_lf_train(options.data_dir)
r = ranker.LogLinear(options.word_dim, options.embedding_file, options.stopwords_file)
trainer = optimizers[options.optimizer](r.model)
sents = 0
total_loss = 0.0
train_size = len(words)
i = 0
for epoch in range(options.epochs):
for word, answer, good_lf, bad_lf in iter_lf_train(words, answers, good_lfs, bad_lfs):
if len(good_lf) == 0:
continue
lemma = [lemmatizer.lemmatize(w) for w in word]
loss = r.train(word, lemma, good_lf, bad_lf)
sents += 1
if loss is not None:
total_loss += loss.scalar_value()
loss.backward()
trainer.update()
e = float(i) / train_size
if i % options.print_every == 0:
print('epoch {}: loss per sentence: {}'.format(e, total_loss / sents))
sents = 0
total_loss = 0.0
i += 1
print ('saving model...')
save_as = '%s/epoch%03d.ranker' % (options.result_dir, epoch)
r.save_model(save_as)
评论列表
文章目录