def test():
"""Test the translation model."""
nltk.download('punkt')
with tf.Session() as sess:
model = create_model(sess, True)
model.batch_size = 1 # We decode one sentence at a time.
# Load vocabularies.
src_lang_vocab_path = PATH_TO_DATA_FILES + FLAGS.src_lang + "_mapping%d.txt" % FLAGS.src_lang_vocab_size
dst_lang_vocab_path = PATH_TO_DATA_FILES + FLAGS.dst_lang + "_mapping%d.txt" % FLAGS.dst_lang_vocab_size
src_lang_vocab, _ = data_utils.initialize_vocabulary(src_lang_vocab_path)
_, rev_dst_lang_vocab = data_utils.initialize_vocabulary(dst_lang_vocab_path)
weights = [0.25, 0.25, 0.25, 0.25]
first_lang_file = open(generate_src_lang_sentences_file_name(FLAGS.src_lang))
second_lang_file = open(generate_src_lang_sentences_file_name(FLAGS.dst_lang))
total_bleu_value = 0.0
computing_bleu_iterations = 0
for first_lang_raw in first_lang_file:
second_lang_gold_raw = second_lang_file.readline()
# Get token-ids for the input sentence.
token_ids = data_utils.sentence_to_token_ids(tf.compat.as_bytes(first_lang_raw), src_lang_vocab)
# Which bucket does it belong to?
try:
bucket_id = min([b for b in xrange(len(_buckets))
if _buckets[b][0] > len(token_ids)])
except ValueError:
continue
# Get a 1-element batch to feed the sentence to the model.
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
{bucket_id: [(token_ids, [])]}, bucket_id)
# Get output logits for the sentence.
_, _, output_logits = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True)
# This is a greedy decoder - outputs are just argmaxes of output_logits.
outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]
# If there is an EOS symbol in outputs, cut them at that point.
if data_utils.EOS_ID in outputs:
outputs = outputs[:outputs.index(data_utils.EOS_ID)]
# Print out sentence corresponding to outputs.
model_tran_res = " ".join([tf.compat.as_str(rev_dst_lang_vocab[output]) for output in outputs])
second_lang_gold_tokens = word_tokenize(second_lang_gold_raw)
model_tran_res_tokens = word_tokenize(model_tran_res)
try:
current_bleu_value = sentence_bleu([model_tran_res_tokens], second_lang_gold_tokens, weights)
total_bleu_value += current_bleu_value
computing_bleu_iterations += 1
except ZeroDivisionError:
pass
if computing_bleu_iterations % 10 == 0:
print("BLEU value after %d iterations: %.2f"
% (computing_bleu_iterations, total_bleu_value / computing_bleu_iterations))
final_bleu_value = total_bleu_value / computing_bleu_iterations
print("Final BLEU value after %d iterations: %.2f" % (computing_bleu_iterations, final_bleu_value))
return
评论列表
文章目录
