def train(args):
time_start = timer()
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
cuda.check_cuda_available()
if args.path_vocab == '':
vocab = create_from_dir(args.path_corpus)
else:
vocab = Vocabulary()
vocab.load(args.path_vocab)
logger.info("loaded vocabulary")
if args.context_representation != 'word': # for deps or ner context representation, we need a new context vocab for NS or HSM loss function.
vocab_context = create_from_annotated_dir(args.path_corpus, representation=args.context_representation)
else :
vocab_context = vocab
loss_func = get_loss_func(args, vocab_context)
model = get_model(args, loss_func, vocab)
if args.gpu >= 0:
model.to_gpu()
logger.debug("model sent to gpu")
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
if os.path.isfile(args.path_corpus):
train, val = get_data(args.path_corpus, vocab)
if args.test:
train = train[:100]
val = val[:100]
train_iter = WindowIterator(train, args.window, args.batchsize)
val_iter = WindowIterator(val, args.window, args.batchsize, repeat=False)
else:
train_iter = DirWindowIterator(path=args.path_corpus, vocab=vocab, window_size=args.window, batch_size=args.batchsize)
updater = training.StandardUpdater(train_iter, optimizer, converter=convert, device=args.gpu)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.path_out)
if os.path.isfile(args.path_corpus):
trainer.extend(extensions.Evaluator(val_iter, model, converter=convert, device=args.gpu))
trainer.extend(extensions.LogReport())
if os.path.isfile(args.path_corpus):
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'validation/main/loss', 'elapsed_time']))
else:
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'elapsed_time']))
# trainer.extend(extensions.ProgressBar())
trainer.run()
model = create_model(args, model, vocab)
time_end = timer()
model.metadata["execution_time"] = time_end - time_start
return model
python类LogReport()的实例源码
def main(options):
#load the config params
gpu = options['gpu']
data_path = options['path_dataset']
embeddings_path = options['path_vectors']
n_epoch = options['epochs']
batch_size = options['batchsize']
test = options['test']
embed_dim = options['embed_dim']
freeze = options['freeze_embeddings']
distance_embed_dim = options['distance_embed_dim']
#load the data
data_processor = DataProcessor(data_path)
data_processor.prepare_dataset()
train_data = data_processor.train_data
test_data = data_processor.test_data
vocab = data_processor.vocab
cnn = CNN(n_vocab=len(vocab), input_channel=1,
output_channel=100,
n_label=19,
embed_dim=embed_dim, position_dims=distance_embed_dim, freeze=freeze)
cnn.load_embeddings(embeddings_path, data_processor.vocab)
model = L.Classifier(cnn)
#use GPU if flag is set
if gpu >= 0:
model.to_gpu()
#setup the optimizer
optimizer = O.Adam()
optimizer.setup(model)
train_iter = chainer.iterators.SerialIterator(train_data, batch_size)
test_iter = chainer.iterators.SerialIterator(test_data, batch_size,repeat=False, shuffle=False)
updater = training.StandardUpdater(train_iter, optimizer, converter=convert.concat_examples, device=gpu)
trainer = training.Trainer(updater, (n_epoch, 'epoch'))
# Evaluation
test_model = model.copy()
test_model.predictor.train = False
trainer.extend(extensions.Evaluator(test_iter, test_model, device=gpu, converter=convert.concat_examples))
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy']))
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def main(options):
#load the config params
gpu = options['gpu']
data_path = options['path_dataset']
embeddings_path = options['path_vectors']
n_epoch = options['epochs']
batchsize = options['batchsize']
test = options['test']
embed_dim = options['embed_dim']
freeze = options['freeze_embeddings']
#load the data
data_processor = DataProcessor(data_path, test)
data_processor.prepare_dataset()
train_data = data_processor.train_data
dev_data = data_processor.dev_data
test_data = data_processor.test_data
vocab = data_processor.vocab
cnn = CNN(n_vocab=len(vocab), input_channel=1,
output_channel=10, n_label=2, embed_dim=embed_dim, freeze=freeze)
cnn.load_embeddings(embeddings_path, data_processor.vocab)
model = L.Classifier(cnn)
if gpu >= 0:
model.to_gpu()
#setup the optimizer
optimizer = O.Adam()
optimizer.setup(model)
train_iter = chainer.iterators.SerialIterator(train_data, batchsize)
dev_iter = chainer.iterators.SerialIterator(dev_data, batchsize,repeat=False, shuffle=False)
test_iter = chainer.iterators.SerialIterator(test_data, batchsize,repeat=False, shuffle=False)
batch1 = train_iter.next()
batch2 = dev_iter.next()
updater = training.StandardUpdater(train_iter, optimizer, converter=util.concat_examples, device=gpu)
trainer = training.Trainer(updater, (n_epoch, 'epoch'))
# Evaluation
eval_model = model.copy()
eval_model.predictor.train = False
trainer.extend(extensions.Evaluator(dev_iter, eval_model, device=gpu, converter=util.concat_examples))
test_model = model.copy()
test_model.predictor.train = False
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy']))
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def __init__(self, folder, chain, train, test, batchsize=500, resume=True, gpu=0, nepoch=1, reports=[]):
self.reports = reports
self.nepoch = nepoch
self.folder = folder
self.chain = chain
self.gpu = gpu
if self.gpu >= 0:
chainer.cuda.get_device(gpu).use()
chain.to_gpu(gpu)
self.eval_chain = eval_chain = chain.copy()
self.chain.test = False
self.eval_chain.test = True
self.testset = test
if not os.path.exists(folder):
os.makedirs(folder)
train_iter = chainer.iterators.SerialIterator(train, batchsize, shuffle=True)
test_iter = chainer.iterators.SerialIterator(test, batchsize,
repeat=False, shuffle=False)
updater = training.StandardUpdater(train_iter, chain.optimizer, device=gpu)
trainer = training.Trainer(updater, (nepoch, 'epoch'), out=folder)
# trainer.extend(TrainingModeSwitch(chain))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.Evaluator(test_iter, eval_chain, device=gpu), trigger=(1,'epoch'))
trainer.extend(extensions.snapshot_object(
chain, 'chain_snapshot_epoch_{.updater.epoch:06}'), trigger=(1,'epoch'))
trainer.extend(extensions.snapshot(
filename='snapshot_epoch_{.updater.epoch:06}'), trigger=(1,'epoch'))
trainer.extend(extensions.LogReport(trigger=(1,'epoch')), trigger=(1,'iteration'))
trainer.extend(extensions.PrintReport(
['epoch']+reports), trigger=IntervalTrigger(1,'epoch'))
self.trainer = trainer
if resume:
#if resumeFrom is not None:
# trainerFile = os.path.join(resumeFrom[0],'snapshot_epoch_{:06}'.format(resumeFrom[1]))
# S.load_npz(trainerFile, trainer)
i = 1
trainerFile = os.path.join(folder,'snapshot_epoch_{:06}'.format(i))
while i <= nepoch and os.path.isfile(trainerFile):
i = i + 1
trainerFile = os.path.join(folder,'snapshot_epoch_{:06}'.format(i))
i = i - 1
trainerFile = os.path.join(folder,'snapshot_epoch_{:06}'.format(i))
if i >= 0 and os.path.isfile(trainerFile):
S.load_npz(trainerFile, trainer)
def create_trainer(
config: TrainConfig,
project_path: str,
updater,
model: typing.Dict,
eval_func,
iterator_test,
iterator_train_eval,
loss_names,
converter=chainer.dataset.convert.concat_examples,
log_name='log.txt',
):
trainer = chainer.training.Trainer(updater, out=project_path)
log_trigger = (config.log_iteration, 'iteration')
save_trigger = (config.save_iteration, 'iteration')
eval_test_name = 'eval/test'
eval_train_name = 'eval/train'
snapshot = extensions.snapshot_object(model['encoder'], 'encoder{.updater.iteration}.model')
trainer.extend(snapshot, trigger=save_trigger)
snapshot = extensions.snapshot_object(model['generator'], 'generator{.updater.iteration}.model')
trainer.extend(snapshot, trigger=save_trigger)
snapshot = extensions.snapshot_object(model['mismatch_discriminator'], 'mismatch_discriminator{.updater.iteration}.model')
trainer.extend(snapshot, trigger=save_trigger)
trainer.extend(utility.chainer.dump_graph([
'encoder/' + loss_names[0],
'generator/' + loss_names[0],
'mismatch_discriminator/' + loss_names[0],
], out_name='main.dot'))
def _make_evaluator(iterator):
return utility.chainer.NoVariableEvaluator(
iterator,
target=model,
converter=converter,
eval_func=eval_func,
device=config.gpu,
)
trainer.extend(_make_evaluator(iterator_test), name=eval_test_name, trigger=log_trigger)
trainer.extend(_make_evaluator(iterator_train_eval), name=eval_train_name, trigger=log_trigger)
report_target = []
for evaluator_name in ['', eval_test_name + '/', eval_train_name + '/']:
for model_name in [s + '/' for s in model.keys()]:
for loss_name in set(loss_names):
report_target.append(evaluator_name + model_name + loss_name)
trainer.extend(extensions.LogReport(trigger=log_trigger, log_name=log_name))
trainer.extend(extensions.PrintReport(report_target))
return trainer
