def _epoch_log(self, file, num_epoch, train_accuracy, dev_accuracy, average_loss):
"""
Log epoch
:param file:
:param num_epoch:
:param train_accuracy:
:param dev_accuracy:
:param average_loss:
:return:
"""
with open(file, "a") as f:
f.write("epoch: %d, train_accuracy: %f, dev_accuracy: %f, average_loss: %f\n" % (num_epoch, train_accuracy, dev_accuracy, average_loss))
python类write()的实例源码
def test(self, data_iterator, is_log=False):
tqdm.write("Testing...")
total = 0
correct = 0
file = os.path.join(self._result_log_base_path, "test_" + self._curr_time + ".log")
for i in tqdm(range(data_iterator.batch_per_epoch)):
batch = data_iterator.get_batch()
predictions, feed_dict = self._test_model.predict(batch)
predictions = self._session.run(predictions, feed_dict=feed_dict)
correct += self._check_predictions(
predictions=predictions,
ground_truth=batch.ground_truth
)
total += batch.size
if is_log:
self.log(
file=file,
batch=batch,
predictions=predictions
)
accuracy = float(correct)/float(total)
tqdm.write("test_acc: %f" % accuracy)
return accuracy
def _epoch_log(self, file, num_epoch, train_accuracy, dev_accuracy, average_loss):
"""
Log epoch
:param file:
:param num_epoch:
:param train_accuracy:
:param dev_accuracy:
:param average_loss:
:return:
"""
with open(file, "a") as f:
f.write("epoch: %d, train_accuracy: %f, dev_accuracy: %f, average_loss: %f\n" % (num_epoch, train_accuracy, dev_accuracy, average_loss))
def test(self, data_iterator, is_log=False):
tqdm.write("Testing...")
total = 0
correct = 0
file = os.path.join(self._result_log_base_path, "test_" + self._curr_time + ".log")
for i in tqdm(range(data_iterator.batch_per_epoch)):
batch = data_iterator.get_batch()
predictions, feed_dict = self._test_model.predict(batch)
predictions = self._session.run(predictions, feed_dict=feed_dict)
correct += self._check_predictions(
predictions=predictions,
ground_truth=batch.ground_truth
)
total += batch.size
if is_log:
self.log(
file=file,
batch=batch,
predictions=predictions
)
accuracy = float(correct)/float(total)
tqdm.write("test_acc: %f" % accuracy)
return accuracy
def log(self, file, batch, tag_predictions, segment_length_predictions):
unfold_predictions, unfold_ground_truth = self._process_predictions(
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions,
ground_truth=batch.ground_truth,
ground_truth_segmentation_length=batch.ground_truth_segmentation_length,
ground_truth_segment_length=batch.ground_truth_segment_length,
question_length=batch.questions_length
)
with open(file, "a") as f:
string = ""
for tt, ts, pt, ps, qid, cv, table_id, unfold_p, unfold_t in zip(
batch.ground_truth,
batch.ground_truth_segment_length,
tag_predictions,
segment_length_predictions,
batch.questions_ids,
batch.cell_value_length,
batch.table_map_ids,
unfold_predictions,
unfold_ground_truth
):
result = np.sum(np.abs(np.array(unfold_p) - np.array(unfold_t)), axis=-1)
string += "=======================\n"
string += ("id: " + str(qid) + "\n")
string += ("tid: " + str(table_id) + "\n")
string += ("max_column: " + str(len(cv)) + "\n")
string += ("max_cell_value_per_col: " + str(len(cv[0])) + "\n")
string += ("unfold_t: " + (', '.join([str(i) for i in unfold_t])) + "\n")
string += ("unfold_p: " + (', '.join([str(i) for i in unfold_p])) + "\n")
string += ("ts: " + (', '.join([str(i) for i in ts])) + "\n")
string += ("tt: " + (', '.join([str(i) for i in tt])) + "\n")
string += ("pt: " + (', '.join([str(i) for i in pt])) + "\n")
string += ("ps: " + (', '.join([str(i) for i in ps])) + "\n")
string += ("Result: " + str(result == 0) + "\n")
# string += ("s: " + str(scores) + "\n")
f.write(string)
def _epoch_log(self, file, num_epoch, train_accuracy, dev_accuracy, average_loss):
"""
Log epoch
:param file:
:param num_epoch:
:param train_accuracy:
:param dev_accuracy:
:param average_loss:
:return:
"""
with open(file, "a") as f:
f.write("epoch: %d, train_accuracy: %f, dev_accuracy: %f, average_loss: %f\n" % (
num_epoch, train_accuracy, dev_accuracy, average_loss))
def test(self, data_iterator, is_log=False):
tqdm.write("Testing...")
total = 0
correct = 0
file = os.path.join(self._result_log_base_path, "test_" + self._curr_time + ".log")
for i in tqdm(range(data_iterator.batch_per_epoch)):
batch = data_iterator.get_batch()
tag_predictions, segment_length_predictions, feed_dict = self._test_model.predict(batch)
tag_predictions, segment_length_predictions = self._session.run(
(tag_predictions, segment_length_predictions,),
feed_dict=feed_dict
)
correct += self._check_predictions(
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions,
ground_truth=batch.ground_truth,
ground_truth_segment_length=batch.ground_truth_segment_length,
ground_truth_segmentation_length=batch.ground_truth_segmentation_length,
question_length=batch.questions_length
)
total += batch.size
if is_log:
self.log(
file=file,
batch=batch,
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions
)
accuracy = float(correct) / float(total)
tqdm.write("test_acc: %f" % accuracy)
return accuracy
def log(self, file, batch, predictions, is_detail=False):
with open(file, "a") as f:
string = ""
for t, p, qid, cv, table_id in zip(batch.ground_truth, predictions, batch.questions_ids, batch.cell_value_length, batch.table_map_ids):
result = np.sum(np.abs(np.array(p) - np.array(t)), axis=-1)
string += "=======================\n"
string += ("id: " + str(qid) + "\n")
string += ("tid: " + str(table_id) + "\n")
string += ("max_column: " + str(len(cv)) + "\n")
string += ("max_cell_value_per_col: " + str(len(cv[0])) + "\n")
string += ("t: " + (', '.join([str(i) for i in t])) + "\n")
string += ("p: " + (', '.join([str(i) for i in p])) + "\n")
string += ("Result: " + str(result == 0) + "\n")
# string += ("s: " + str(scores) + "\n")
f.write(string)
def _epoch_log(self, file, num_epoch, train_accuracy, dev_accuracy, average_loss):
"""
Log epoch
:param file:
:param num_epoch:
:param train_accuracy:
:param dev_accuracy:
:param average_loss:
:return:
"""
with open(file, "a") as f:
f.write("epoch: %d, train_accuracy: %f, dev_accuracy: %f, average_loss: %f\n" % (num_epoch, train_accuracy, dev_accuracy, average_loss))
def test(self, data_iterator, is_log=False):
tqdm.write("Testing...")
total = 0
correct = 0
file = os.path.join(self._result_log_base_path, "test_" + self._curr_time + ".log")
for i in tqdm(range(data_iterator.batch_per_epoch)):
batch = data_iterator.get_batch()
predictions, feed_dict = self._test_model.predict(batch)
predictions = self._session.run(predictions, feed_dict=feed_dict)
correct += self._check_predictions(
predictions=predictions,
ground_truth=batch.ground_truth
)
total += batch.size
if is_log:
self.log(
file=file,
batch=batch,
predictions=predictions
)
accuracy = float(correct)/float(total)
tqdm.write("test_acc: %f" % accuracy)
return accuracy
def log(self, file, batch, tag_predictions, segment_length_predictions):
unfold_predictions, unfold_ground_truth = self._process_predictions(
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions,
ground_truth=batch.ground_truth,
ground_truth_segmentation_length=batch.ground_truth_segmentation_length,
ground_truth_segment_length=batch.ground_truth_segment_length,
question_length=batch.questions_length
)
with open(file, "a") as f:
string = ""
for tt, ts, pt, ps, qid, cv, table_id, unfold_p, unfold_t in zip(
batch.ground_truth,
batch.ground_truth_segment_length,
tag_predictions,
segment_length_predictions,
batch.questions_ids,
batch.cell_value_length,
batch.table_map_ids,
unfold_predictions,
unfold_ground_truth
):
result = np.sum(np.abs(np.array(unfold_p) - np.array(unfold_t)), axis=-1)
string += "=======================\n"
string += ("id: " + str(qid) + "\n")
string += ("tid: " + str(table_id) + "\n")
string += ("max_column: " + str(len(cv)) + "\n")
string += ("max_cell_value_per_col: " + str(len(cv[0])) + "\n")
string += ("unfold_t: " + (', '.join([str(i) for i in unfold_t])) + "\n")
string += ("unfold_p: " + (', '.join([str(i) for i in unfold_p])) + "\n")
string += ("ts: " + (', '.join([str(i) for i in ts])) + "\n")
string += ("tt: " + (', '.join([str(i) for i in tt])) + "\n")
string += ("pt: " + (', '.join([str(i) for i in pt])) + "\n")
string += ("ps: " + (', '.join([str(i) for i in ps])) + "\n")
string += ("Result: " + str(result == 0) + "\n")
# string += ("s: " + str(scores) + "\n")
f.write(string)
def test(self, data_iterator, is_log=False):
tqdm.write("Testing...")
total = 0
correct = 0
file = os.path.join(self._result_log_base_path, "test_" + self._curr_time + ".log")
for i in tqdm(range(data_iterator.batch_per_epoch)):
batch = data_iterator.get_batch()
tag_predictions, segment_length_predictions, feed_dict = self._test_model.predict(batch)
tag_predictions, segment_length_predictions = self._session.run(
(tag_predictions, segment_length_predictions,),
feed_dict=feed_dict
)
correct += self._check_predictions(
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions,
ground_truth=batch.ground_truth,
ground_truth_segment_length=batch.ground_truth_segment_length,
ground_truth_segmentation_length=batch.ground_truth_segmentation_length,
question_length=batch.questions_length
)
total += batch.size
if is_log:
self.log(
file=file,
batch=batch,
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions
)
accuracy = float(correct) / float(total)
tqdm.write("test_acc: %f" % accuracy)
return accuracy
def log(self, file, batch, tag_predictions, segment_length_predictions):
unfold_predictions, unfold_ground_truth = self._process_predictions(
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions,
ground_truth=batch.ground_truth,
ground_truth_segmentation_length=batch.ground_truth_segmentation_length,
ground_truth_segment_length=batch.ground_truth_segment_length,
question_length=batch.questions_length
)
with open(file, "a") as f:
string = ""
for tt, ts, pt, ps, qid, cv, table_id, unfold_p, unfold_t in zip(
batch.ground_truth,
batch.ground_truth_segment_length,
tag_predictions,
segment_length_predictions,
batch.questions_ids,
batch.cell_value_length,
batch.table_map_ids,
unfold_predictions,
unfold_ground_truth
):
result = np.sum(np.abs(np.array(unfold_p) - np.array(unfold_t)), axis=-1)
string += "=======================\n"
string += ("id: " + str(qid) + "\n")
string += ("tid: " + str(table_id) + "\n")
string += ("max_column: " + str(len(cv)) + "\n")
string += ("max_cell_value_per_col: " + str(len(cv[0])) + "\n")
string += ("unfold_t: " + (', '.join([str(i) for i in unfold_t])) + "\n")
string += ("unfold_p: " + (', '.join([str(i) for i in unfold_p])) + "\n")
string += ("ts: " + (', '.join([str(i) for i in ts])) + "\n")
string += ("tt: " + (', '.join([str(i) for i in tt])) + "\n")
string += ("pt: " + (', '.join([str(i) for i in pt])) + "\n")
string += ("ps: " + (', '.join([str(i) for i in ps])) + "\n")
string += ("Result: " + str(result == 0) + "\n")
# string += ("s: " + str(scores) + "\n")
f.write(string)
def _epoch_log(self, file, num_epoch, train_accuracy, dev_accuracy, average_loss):
"""
Log epoch
:param file:
:param num_epoch:
:param train_accuracy:
:param dev_accuracy:
:param average_loss:
:return:
"""
with open(file, "a") as f:
f.write("epoch: %d, train_accuracy: %f, dev_accuracy: %f, average_loss: %f\n" % (
num_epoch, train_accuracy, dev_accuracy, average_loss))
def test(self, data_iterator, is_log=False):
tqdm.write("Testing...")
total = 0
correct = 0
file = os.path.join(self._result_log_base_path, "test_" + self._curr_time + ".log")
for i in tqdm(range(data_iterator.batch_per_epoch)):
batch = data_iterator.get_batch()
tag_predictions, segment_length_predictions, feed_dict = self._test_model.predict(batch)
tag_predictions, segment_length_predictions = self._session.run(
(tag_predictions, segment_length_predictions,),
feed_dict=feed_dict
)
correct += self._check_predictions(
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions,
ground_truth=batch.ground_truth,
ground_truth_segment_length=batch.ground_truth_segment_length,
ground_truth_segmentation_length=batch.ground_truth_segmentation_length,
question_length=batch.questions_length
)
total += batch.size
if is_log:
self.log(
file=file,
batch=batch,
tag_predictions=tag_predictions,
segment_length_predictions=segment_length_predictions
)
accuracy = float(correct) / float(total)
tqdm.write("test_acc: %f" % accuracy)
return accuracy
def _epoch_log(self, file, num_epoch, train_accuracy, dev_accuracy, average_loss):
"""
Log epoch
:param file:
:param num_epoch:
:param train_accuracy:
:param dev_accuracy:
:param average_loss:
:return:
"""
with open(file, "a") as f:
f.write("epoch: %d, train_accuracy: %f, dev_accuracy: %f, average_loss: %f\n" % (num_epoch, train_accuracy, dev_accuracy, average_loss))
def test(self, data_iterator, is_log=False):
tqdm.write("Testing...")
total = 0
correct = 0
file = os.path.join(self._result_log_base_path, "test_" + self._curr_time + ".log")
for i in tqdm(range(data_iterator.batch_per_epoch)):
batch = data_iterator.get_batch()
predictions, feed_dict = self._test_model.predict(batch)
predictions = self._session.run(predictions, feed_dict=feed_dict)
correct += self._check_predictions(
predictions=predictions,
ground_truth=batch.ground_truth
)
total += batch.size
if is_log:
self.log(
file=file,
batch=batch,
predictions=predictions
)
accuracy = float(correct)/float(total)
tqdm.write("test_acc: %f" % accuracy)
return accuracy
def write(self, log):
""" Write log. """
tqdm.write(log)
tqdm.write(log, file=self.file)
self.file.flush()
self.logs.append(log)
def load_state_dict(self, state_dict):
""" Loads the logger state. """
self.logs = state_dict['logs']
# write logs.
tqdm.write(self.logs[-1])
for log in self.logs:
tqdm.write(log, file=self.file)
def _train(self, model, optimizer, train_iter, log_interval, logger, start_time):
model.train()
for iteration, batch in enumerate(tqdm(train_iter, desc='this epoch'), 1):
image, pose, visibility = Variable(batch[0]), Variable(batch[1]), Variable(batch[2])
if self.gpu:
image, pose, visibility = image.cuda(), pose.cuda(), visibility.cuda()
optimizer.zero_grad()
output = model(image)
loss = mean_squared_error(output, pose, visibility, self.use_visibility)
loss.backward()
optimizer.step()
if iteration % log_interval == 0:
log = 'elapsed_time: {0}, loss: {1}'.format(time.time() - start_time, loss.data[0])
logger.write(log)
