def main(config_file):
with open(config_file) as fp:
conf = json.load(fp)['feature_extractor']
model_class = getattr(cnn_feature_extractors, conf['model'])
model = model_class(conf['n_classes'], conf['n_base_units'])
resume_file = conf['out_file'] + '.to_resume'
if os.path.exists(resume_file):
chainer.serializers.load_npz(resume_file, model)
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
device = conf.get('device', -1)
train_dataset = create_dataset(os.path.join(conf['dataset_path'], 'train'))
train_iter = chainer.iterators.SerialIterator(train_dataset, conf.get('batch_size', 10))
updater = chainer.training.StandardUpdater(train_iter, optimizer, device=device)
trainer = chainer.training.Trainer(updater, (conf['epoch'], 'epoch'), out='out')
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.LogReport())
trainer.extend(extensions.ProgressBar(update_interval=10))
test_dataset_path = os.path.join(conf['dataset_path'], 'test')
if os.path.exists(test_dataset_path):
test_dataset = create_dataset(test_dataset_path)
test_iter = chainer.iterators.SerialIterator(test_dataset, 20, repeat=False, shuffle=False)
trainer.extend(extensions.Evaluator(test_iter, model, device=device))
trainer.extend(extensions.PrintReport([
'epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy'
]))
else:
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'main/accuracy']))
trainer.run()
model = model.to_cpu()
chainer.serializers.save_npz(conf['out_file'], model)
python类training()的实例源码
def evaluation(gen_g, gen_f, test_image_folder, image_size=256, side=2):
@chainer.training.make_extension()
def _eval(trainer, it):
xp = gen_g.xp
batch = it.next()
batchsize = len(batch)
#x = []
x = xp.zeros((batchsize, 3, image_size, image_size)).astype("f")
t = xp.zeros((batchsize, 3, image_size, image_size)).astype("f")
for i in range(batchsize):
x[i, :] = xp.asarray(batch[i][0])
t[i, :] = xp.asarray(batch[i][1])
x = Variable(x)
result = gen_g(x, test=True)
img = result.data.get()
img_c = img.reshape((side, side, 3, image_size, image_size))
img_c = img_c.transpose(0,1,3,4,2)
img_c = (img + 1) *127.5
img_c = np.clip(img_c, 0, 255)
img_c = img_c.astype(np.uint8)
img_c = img_c.reshape((side, side, image_size, image_size, 3)).transpose(0,2,1,3,4).reshape((side*image_size, side*image_size, 3))[:,:,::-1]
Image.fromarray(img_c).save(test_image_folder+"/iter_"+str(trainer.updater.iteration)+"_G.jpg")
t = Variable(t)
result = gen_f(t, test=True)
img_t = result.data.get()
img_t = img_t.reshape( (side, side, 3, image_size, image_size))
img_t = img_t.transpose(0,1,3,4,2)
img_t = (img + 1) *127.5
img_t = np.clip(img_t, 0, 255)
img_t = img_t.astype(np.uint8)
img_t = img_t.reshape((side, side, image_size, image_size, 3)).transpose(0,2,1,3,4).reshape((side*image_size, side*image_size, 3))[:,:,::-1]
#print(img_t)
Image.fromarray(img_t).save(test_image_folder+"/iter_"+str(trainer.updater.iteration)+"_F.jpg")
def evaluation(trainer):
it = trainer.updater.get_iterator('test')
_eval(trainer, it)
return evaluation
def train_main(args):
"""
trains model specfied in args.
main method for train subcommand.
"""
# load text
with open(args.text_path) as f:
text = f.read()
logger.info("corpus length: %s.", len(text))
# data iterator
data_iter = DataIterator(text, args.batch_size, args.seq_len)
# load or build model
if args.restore:
logger.info("restoring model.")
load_path = args.checkpoint_path if args.restore is True else args.restore
model = load_model(load_path)
else:
net = Network(vocab_size=VOCAB_SIZE,
embedding_size=args.embedding_size,
rnn_size=args.rnn_size,
num_layers=args.num_layers,
drop_rate=args.drop_rate)
model = L.Classifier(net)
# make checkpoint directory
log_dir = make_dirs(args.checkpoint_path)
with open("{}.json".format(args.checkpoint_path), "w") as f:
json.dump(model.predictor.args, f, indent=2)
chainer.serializers.save_npz(args.checkpoint_path, model)
logger.info("model saved: %s.", args.checkpoint_path)
# optimizer
optimizer = chainer.optimizers.Adam(alpha=args.learning_rate)
optimizer.setup(model)
# clip gradient norm
optimizer.add_hook(chainer.optimizer.GradientClipping(args.clip_norm))
# trainer
updater = BpttUpdater(data_iter, optimizer)
trainer = chainer.training.Trainer(updater, (args.num_epochs, 'epoch'), out=log_dir)
trainer.extend(extensions.snapshot_object(model, filename=os.path.basename(args.checkpoint_path)))
trainer.extend(extensions.ProgressBar(update_interval=1))
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PlotReport(y_keys=["main/loss"]))
trainer.extend(LoggerExtension(text))
# training start
model.predictor.reset_state()
logger.info("start of training.")
time_train = time.time()
trainer.run()
# training end
duration_train = time.time() - time_train
logger.info("end of training, duration: %ds.", duration_train)
# generate text
seed = generate_seed(text)
generate_text(model, seed, 1024, 3)
return model
def create_trainer(
config,
project_path,
updater,
model,
eval_func,
iterator_test,
iterator_train_varidation,
loss_names,
converter=chainer.dataset.convert.concat_examples,
):
# type: (TrainConfig, str, any, typing.Dict, any, any, any, any, any) -> any
def _make_evaluator(iterator):
return utility.chainer_utility.NoVariableEvaluator(
iterator,
target=model,
converter=converter,
eval_func=eval_func,
device=config.gpu,
)
trainer = chainer.training.Trainer(updater, out=project_path)
log_trigger = (config.log_iteration, 'iteration')
save_trigger = (config.save_iteration, 'iteration')
eval_test_name = 'eval/test'
eval_train_name = 'eval/train'
snapshot = extensions.snapshot_object(model['main'], '{.updater.iteration}.model')
trainer.extend(snapshot, trigger=save_trigger)
trainer.extend(extensions.dump_graph('main/' + loss_names[0], out_name='main.dot'))
trainer.extend(_make_evaluator(iterator_test), name=eval_test_name, trigger=log_trigger)
trainer.extend(_make_evaluator(iterator_train_varidation), name=eval_train_name, trigger=log_trigger)
report_target = []
for evaluator_name in ['', eval_test_name + '/', eval_train_name + '/']:
for model_name in ['main/']:
for loss_name in loss_names:
report_target.append(evaluator_name + model_name + loss_name)
trainer.extend(extensions.LogReport(trigger=log_trigger, log_name="log.txt"))
trainer.extend(extensions.PrintReport(report_target))
return trainer
def train(args):
model = LSTMParser(args.model, args.word_emb_size, args.afix_emb_size, args.nlayers,
args.hidden_dim, args.elu_dim, args.dep_dim, args.dropout_ratio)
with open(args.model + "/params", "w") as f: log(args, f)
if args.initmodel:
print 'Load model from', args.initmodel
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print 'Load pretrained word embeddings from', args.pretrained
model.load_pretrained_embeddings(args.pretrained)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
train = LSTMParserDataset(args.model, args.train)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
val = LSTMParserDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.Adam(beta2=0.9)
# optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(1e-6))
# optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer,
device=args.gpu, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 1000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(val_iter, eval_model,
converter, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration',
'main/tagging_accuracy', 'main/tagging_loss',
'main/parsing_accuracy', 'main/parsing_loss',
'validation/main/tagging_accuracy',
'validation/main/parsing_accuracy'
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def train(args):
model = LSTMTagger(args.model, args.word_emb_size, args.afix_emb_size,
args.nlayers, args.hidden_dim, args.relu_dim, args.dropout_ratio)
with open(args.model + "/params", "w") as f:
log(args, f)
if args.initmodel:
print('Load model from', args.initmodel)
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print('Load pretrained word embeddings from', args.pretrained)
model.load_pretrained_embeddings(args.pretrained)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
train = LSTMTaggerDataset(args.model, args.train)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
val = LSTMTaggerDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(1e-6))
optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer,
device=args.gpu, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 2000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(
val_iter, eval_model, converter, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy',
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def train(args):
model = BiaffineJaLSTMParser(args.model, args.word_emb_size, args.char_emb_size,
args.nlayers, args.hidden_dim, args.dep_dim, args.dropout_ratio)
with open(args.model + "/params", "w") as f: log(args, f)
if args.initmodel:
print('Load model from', args.initmodel)
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print('Load pretrained word embeddings from', args.pretrained)
model.load_pretrained_embeddings(args.pretrained)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
train = LSTMParserDataset(args.model, args.train)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
val = LSTMParserDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.Adam(beta2=0.9)
# optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(2e-6))
# optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer,
device=args.gpu, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 1000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.ExponentialShift(
"eps", .75, 2e-3), trigger=(2500, 'iteration'))
trainer.extend(extensions.Evaluator(
val_iter, eval_model, converter, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration',
'main/tagging_accuracy', 'main/tagging_loss',
'main/parsing_accuracy', 'main/parsing_loss',
'validation/main/tagging_accuracy',
'validation/main/parsing_accuracy'
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def __init__(self, model_path, word_dim=None, char_dim=None, nlayers=2,
hidden_dim=128, relu_dim=64, dep_dim=100, dropout_ratio=0.5):
self.model_path = model_path
defs_file = model_path + "/tagger_defs.txt"
if word_dim is None:
# use as supertagger
with open(defs_file) as f:
defs = json.load(f)
self.dep_dim = defs["dep_dim"]
self.word_dim = defs["word_dim"]
self.char_dim = defs["char_dim"]
self.hidden_dim = defs["hidden_dim"]
self.relu_dim = defs["relu_dim"]
self.nlayers = defs["nlayers"]
self.train = False
self.extractor = FeatureExtractor(model_path)
else:
# training
self.dep_dim = dep_dim
self.word_dim = word_dim
self.char_dim = char_dim
self.hidden_dim = hidden_dim
self.relu_dim = relu_dim
self.nlayers = nlayers
self.train = True
with open(defs_file, "w") as f:
json.dump({"model": self.__class__.__name__,
"word_dim": self.word_dim, "char_dim": self.char_dim,
"hidden_dim": hidden_dim, "relu_dim": relu_dim,
"nlayers": nlayers, "dep_dim": dep_dim}, f)
self.targets = read_model_defs(model_path + "/target.txt")
self.words = read_model_defs(model_path + "/words.txt")
self.chars = read_model_defs(model_path + "/chars.txt")
self.in_dim = self.word_dim + self.char_dim
self.dropout_ratio = dropout_ratio
super(JaLSTMParser, self).__init__(
emb_word=L.EmbedID(len(self.words), self.word_dim),
emb_char=L.EmbedID(len(self.chars), 50, ignore_label=IGNORE),
conv_char=L.Convolution2D(1, self.char_dim,
(3, 50), stride=1, pad=(1, 0)),
lstm_f=L.NStepLSTM(nlayers, self.in_dim,
self.hidden_dim, self.dropout_ratio),
lstm_b=L.NStepLSTM(nlayers, self.in_dim,
self.hidden_dim, self.dropout_ratio),
linear_cat1=L.Linear(2 * self.hidden_dim, self.relu_dim),
linear_cat2=L.Linear(self.relu_dim, len(self.targets)),
linear_dep=L.Linear(2 * self.hidden_dim, self.dep_dim),
linear_head=L.Linear(2 * self.hidden_dim, self.dep_dim),
biaffine=Biaffine(self.dep_dim)
)
def __init__(self, model_path, word_dim=None, char_dim=None,
nlayers=2, hidden_dim=128, relu_dim=64, dropout_ratio=0.5):
self.model_path = model_path
defs_file = model_path + "/tagger_defs.txt"
if word_dim is None:
# use as supertagger
with open(defs_file) as f:
defs = json.load(f)
self.word_dim = defs["word_dim"]
self.char_dim = defs["char_dim"]
self.hidden_dim = defs["hidden_dim"]
self.relu_dim = defs["relu_dim"]
self.nlayers = defs["nlayers"]
self.train = False
self.extractor = FeatureExtractor(model_path)
else:
# training
self.word_dim = word_dim
self.char_dim = char_dim
self.hidden_dim = hidden_dim
self.relu_dim = relu_dim
self.nlayers = nlayers
self.train = True
with open(defs_file, "w") as f:
json.dump({"model": self.__class__.__name__,
"word_dim": self.word_dim, "char_dim": self.char_dim,
"hidden_dim": hidden_dim, "relu_dim": relu_dim,
"nlayers": nlayers}, f)
self.targets = read_model_defs(model_path + "/target.txt")
self.words = read_model_defs(model_path + "/words.txt")
self.chars = read_model_defs(model_path + "/chars.txt")
self.in_dim = self.word_dim + self.char_dim
self.dropout_ratio = dropout_ratio
super(JaLSTMTagger, self).__init__(
emb_word=L.EmbedID(len(self.words), self.word_dim),
emb_char=L.EmbedID(len(self.chars), 50, ignore_label=IGNORE),
conv_char=L.Convolution2D(1, self.char_dim,
(3, 50), stride=1, pad=(1, 0)),
lstm_f=L.NStepLSTM(nlayers, self.in_dim,
self.hidden_dim, 0.),
lstm_b=L.NStepLSTM(nlayers, self.in_dim,
self.hidden_dim, 0.),
conv1=L.Convolution2D(1, 2 * self.hidden_dim,
(7, 2 * self.hidden_dim), stride=1, pad=(3, 0)),
linear1=L.Linear(2 * self.hidden_dim, self.relu_dim),
linear2=L.Linear(self.relu_dim, len(self.targets)),
)
def train(args):
model = LSTMTagger(args.model, args.word_emb_size, args.char_emb_size,
args.nlayers, args.hidden_dim, args.relu_dim, args.dropout_ratio)
with open(args.model + "/params", "w") as f:
log(args, f)
if args.initmodel:
print('Load model from', args.initmodel)
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print('Load pretrained word embeddings from', args.pretrained)
model.load_pretrained_embeddings(args.pretrained)
train = LSTMTaggerDataset(args.model, args.train)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
val = LSTMTaggerDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(1e-6))
# optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 1000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(
val_iter, eval_model, converter), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy',
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def train(args):
model = PeepHoleJaLSTMParser(args.model, args.word_emb_size, args.char_emb_size,
args.nlayers, args.hidden_dim, args.relu_dim, args.dep_dim, args.dropout_ratio)
with open(args.model + "/params", "w") as f: log(args, f)
if args.initmodel:
print('Load model from', args.initmodel)
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print('Load pretrained word embeddings from', args.pretrained)
model.load_pretrained_embeddings(args.pretrained)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
converter = lambda x, device: \
concat_examples(x, device=device, padding=-1)
train = LSTMParserDataset(args.model, args.train)
train_iter = SerialIterator(train, args.batchsize)
val = LSTMParserDataset(args.model, args.val)
val_iter = SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.Adam(beta2=0.9)
# optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(1e-6))
# optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 2000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(val_iter, eval_model,
converter, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration', 'main/tagging_loss',
'main/tagging_accuracy', 'main/tagging_loss',
'main/parsing_accuracy', 'main/parsing_loss',
'validation/main/tagging_accuracy',
'validation/main/parsing_accuracy'
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def train(args):
model = LSTMParser(args.model, args.word_emb_size, args.afix_emb_size, args.nlayers,
args.hidden_dim, args.elu_dim, args.dep_dim, args.dropout_ratio)
with open(args.model + "/params", "w") as f: log(args, f)
if args.initmodel:
print('Load model from', args.initmodel)
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print('Load pretrained word embeddings from', args.pretrained)
model.load_pretrained_embeddings(args.pretrained)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
train = LSTMParserDataset(args.model, args.train)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
val = LSTMParserDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.Adam(beta2=0.9)
# optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(1e-6))
# optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer,
device=args.gpu, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 1000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(val_iter, eval_model,
converter, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration',
'main/tagging_accuracy', 'main/tagging_loss',
'main/parsing_accuracy', 'main/parsing_loss',
'validation/main/tagging_accuracy',
'validation/main/parsing_accuracy'
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def compress_traindata(args):
words = OrderedDict()
print("reading embedding vocabulary")
for word in open(args.vocab):
words[word.strip()] = 1
suffixes = defaultdict(int)
suffixes["UNK"] = 1
caps = defaultdict(int)
target = defaultdict(int)
traindata = open(args.path)
len_traindata = 0
print("reading training file")
for line in traindata:
len_traindata += 1
items = line.strip().split(" ")
target[items[-1]] += 1
for item in items[:-1]:
word, suffix, cap = item.split("|")
if words.has_key(word):
words[word] += 1
else:
words[word] = 1
suffixes[suffix] += 1
caps[cap] += 1
def out_dict(d, outfile, freq_cut=-1):
print("writing to {}".format(outfile))
res = {}
with open(outfile, "w") as out:
i = 0
for item, n in d.items():
if freq_cut <= n:
out.write("{} {}\n".format(item, n))
res[item] = i
i += 1
return res
word2id = out_dict(words, os.path.join(args.out, "words.txt"))
suffix2id = out_dict(suffixes, os.path.join(args.out, "suffixes.txt"))
cap2id = out_dict(caps, os.path.join(args.out, "caps.txt"))
target2id = out_dict(target, os.path.join(args.out, "target.txt"), freq_cut=10)
traindata.seek(0)
new_traindata = os.path.join(args.out, "traindata.txt")
print("writing to {}".format(new_traindata))
with open(new_traindata, "w") as out:
for i, line in enumerate(traindata):
items = line.strip().split(" ")
if not target2id.has_key(items[-1]):
continue
target =items[-1]
new_line = ""
for j, item in enumerate(items[:-1]):
word, suffix, cap = item.split("|")
if not word2id.has_key(word):
word = "*UNKNOWN*"
if not suffix2id.has_key(suffix):
suffix = "UNK"
new_line += "|".join([word, suffix, cap]) + " "
out.write(new_line + target + "\n")
def train(args):
model = JaCCGEmbeddingTagger(args.model,
args.word_emb_size, args.char_emb_size)
if args.initmodel:
print('Load model from', args.initmodel)
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print('Load pretrained word embeddings from', args.pretrained)
model.load_pretrained_embeddings(args.pretrained)
train = JaCCGTaggerDataset(args.model, args.train)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
val = JaCCGTaggerDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.AdaGrad()
optimizer.setup(model)
# optimizer.add_hook(WeightDecay(1e-8))
my_converter = lambda x, dev: convert.concat_examples(x, dev, (None,-1,None,None))
updater = training.StandardUpdater(train_iter, optimizer, converter=my_converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 1000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(
val_iter, eval_model, my_converter), trigger=val_interval)
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.snapshot(), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy',
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def train(args):
model = PeepHoleLSTMTagger(args.model, args.word_emb_size, args.afix_emb_size,
args.nlayers, args.hidden_dim, args.relu_dim, args.dropout_ratio)
with open(args.model + "/params", "w") as f:
log(args, f)
if args.initmodel:
print('Load model from', args.initmodel)
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print('Load pretrained word embeddings from', args.pretrained)
model.load_pretrained_embeddings(args.pretrained)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
converter = lambda x, device: \
concat_examples(x, device=device, padding=-1)
train = LSTMTaggerDataset(args.model, args.train)
train_iter = SerialIterator(train, args.batchsize)
val = LSTMTaggerDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(1e-6))
optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer,
device=args.gpu, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 1000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(
val_iter, eval_model, converter, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy',
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def train(args):
model = PeepHoleLSTMParser(args.model, args.word_emb_size, args.afix_emb_size, args.nlayers,
args.hidden_dim, args.elu_dim, args.dep_dim, args.dropout_ratio, args.gpu >= 0)
with open(args.model + "/params", "w") as f: log(args, f)
if args.initmodel:
print 'Load model from', args.initmodel
chainer.serializers.load_npz(args.initmodel, model)
if args.pretrained:
print 'Load pretrained word embeddings from', args.pretrained
model.load_pretrained_embeddings(args.pretrained)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
converter = lambda x, device: \
concat_examples(x, device=device, padding=-1)
train = LSTMParserDataset(args.model, args.train)
train_iter = SerialIterator(train, args.batchsize)
val = LSTMParserDataset(args.model, args.val)
val_iter = chainer.iterators.SerialIterator(
val, args.batchsize, repeat=False, shuffle=False)
optimizer = chainer.optimizers.Adam(beta2=0.9)
# optimizer = chainer.optimizers.MomentumSGD(momentum=0.7)
optimizer.setup(model)
optimizer.add_hook(WeightDecay(1e-6))
# optimizer.add_hook(GradientClipping(5.))
updater = training.StandardUpdater(train_iter, optimizer,
device=args.gpu, converter=converter)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.model)
val_interval = 1000, 'iteration'
log_interval = 200, 'iteration'
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(val_iter, eval_model,
converter, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
model, 'model_iter_{.updater.iteration}'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.PrintReport([
'epoch', 'iteration', 'main/tagging_loss',
'main/tagging_accuracy', 'main/tagging_loss',
'main/parsing_accuracy', 'main/parsing_loss',
'validation/main/tagging_loss', 'validation/main/tagging_accuracy',
'validation/main/parsing_loss', 'validation/main/parsing_accuracy'
]), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def start(self):
""" Train pose net. """
# set random seed.
if self.seed is not None:
random.seed(self.seed)
np.random.seed(self.seed)
if self.gpu >= 0:
chainer.cuda.cupy.random.seed(self.seed)
# initialize model to train.
model = AlexNet(self.Nj, self.use_visibility)
if self.resume_model:
serializers.load_npz(self.resume_model, model)
# prepare gpu.
if self.gpu >= 0:
chainer.cuda.get_device(self.gpu).use()
model.to_gpu()
# load the datasets.
train = PoseDataset(self.train, data_augmentation=self.data_augmentation)
val = PoseDataset(self.val, data_augmentation=False)
# training/validation iterators.
train_iter = chainer.iterators.MultiprocessIterator(
train, self.batchsize)
val_iter = chainer.iterators.MultiprocessIterator(
val, self.batchsize, repeat=False, shuffle=False)
# set up an optimizer.
optimizer = self._get_optimizer()
optimizer.setup(model)
if self.resume_opt:
chainer.serializers.load_npz(self.resume_opt, optimizer)
# set up a trainer.
updater = training.StandardUpdater(train_iter, optimizer, device=self.gpu)
trainer = training.Trainer(
updater, (self.epoch, 'epoch'), os.path.join(self.out, 'chainer'))
# standard trainer settings
trainer.extend(extensions.dump_graph('main/loss'))
val_interval = (10, 'epoch')
trainer.extend(TestModeEvaluator(val_iter, model, device=self.gpu), trigger=val_interval)
# save parameters and optimization state per validation step
resume_interval = (self.epoch/10, 'epoch')
trainer.extend(extensions.snapshot_object(
model, "epoch-{.updater.epoch}.model"), trigger=resume_interval)
trainer.extend(extensions.snapshot_object(
optimizer, "epoch-{.updater.epoch}.state"), trigger=resume_interval)
trainer.extend(extensions.snapshot(
filename="epoch-{.updater.epoch}.iter"), trigger=resume_interval)
# show log
log_interval = (10, "iteration")
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.observe_lr(), trigger=log_interval)
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss', 'lr']), trigger=log_interval)
trainer.extend(extensions.ProgressBar(update_interval=10))
# start training
if self.resume:
chainer.serializers.load_npz(self.resume, trainer)
trainer.run()
def train(args):
time_start = timer()
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
cuda.check_cuda_available()
if args.path_vocab == '':
vocab = create_from_dir(args.path_corpus)
else:
vocab = Vocabulary()
vocab.load(args.path_vocab)
logger.info("loaded vocabulary")
if args.context_representation != 'word': # for deps or ner context representation, we need a new context vocab for NS or HSM loss function.
vocab_context = create_from_annotated_dir(args.path_corpus, representation=args.context_representation)
else :
vocab_context = vocab
loss_func = get_loss_func(args, vocab_context)
model = get_model(args, loss_func, vocab)
if args.gpu >= 0:
model.to_gpu()
logger.debug("model sent to gpu")
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
if os.path.isfile(args.path_corpus):
train, val = get_data(args.path_corpus, vocab)
if args.test:
train = train[:100]
val = val[:100]
train_iter = WindowIterator(train, args.window, args.batchsize)
val_iter = WindowIterator(val, args.window, args.batchsize, repeat=False)
else:
train_iter = DirWindowIterator(path=args.path_corpus, vocab=vocab, window_size=args.window, batch_size=args.batchsize)
updater = training.StandardUpdater(train_iter, optimizer, converter=convert, device=args.gpu)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.path_out)
if os.path.isfile(args.path_corpus):
trainer.extend(extensions.Evaluator(val_iter, model, converter=convert, device=args.gpu))
trainer.extend(extensions.LogReport())
if os.path.isfile(args.path_corpus):
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'validation/main/loss', 'elapsed_time']))
else:
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'elapsed_time']))
# trainer.extend(extensions.ProgressBar())
trainer.run()
model = create_model(args, model, vocab)
time_end = timer()
model.metadata["execution_time"] = time_end - time_start
return model
def main(options):
#load the config params
gpu = options['gpu']
data_path = options['path_dataset']
embeddings_path = options['path_vectors']
n_epoch = options['epochs']
batch_size = options['batchsize']
test = options['test']
embed_dim = options['embed_dim']
freeze = options['freeze_embeddings']
distance_embed_dim = options['distance_embed_dim']
#load the data
data_processor = DataProcessor(data_path)
data_processor.prepare_dataset()
train_data = data_processor.train_data
test_data = data_processor.test_data
vocab = data_processor.vocab
cnn = CNN(n_vocab=len(vocab), input_channel=1,
output_channel=100,
n_label=19,
embed_dim=embed_dim, position_dims=distance_embed_dim, freeze=freeze)
cnn.load_embeddings(embeddings_path, data_processor.vocab)
model = L.Classifier(cnn)
#use GPU if flag is set
if gpu >= 0:
model.to_gpu()
#setup the optimizer
optimizer = O.Adam()
optimizer.setup(model)
train_iter = chainer.iterators.SerialIterator(train_data, batch_size)
test_iter = chainer.iterators.SerialIterator(test_data, batch_size,repeat=False, shuffle=False)
updater = training.StandardUpdater(train_iter, optimizer, converter=convert.concat_examples, device=gpu)
trainer = training.Trainer(updater, (n_epoch, 'epoch'))
# Evaluation
test_model = model.copy()
test_model.predictor.train = False
trainer.extend(extensions.Evaluator(test_iter, test_model, device=gpu, converter=convert.concat_examples))
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy']))
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def main(options):
#load the config params
gpu = options['gpu']
data_path = options['path_dataset']
embeddings_path = options['path_vectors']
n_epoch = options['epochs']
batchsize = options['batchsize']
test = options['test']
embed_dim = options['embed_dim']
freeze = options['freeze_embeddings']
#load the data
data_processor = DataProcessor(data_path, test)
data_processor.prepare_dataset()
train_data = data_processor.train_data
dev_data = data_processor.dev_data
test_data = data_processor.test_data
vocab = data_processor.vocab
cnn = CNN(n_vocab=len(vocab), input_channel=1,
output_channel=10, n_label=2, embed_dim=embed_dim, freeze=freeze)
cnn.load_embeddings(embeddings_path, data_processor.vocab)
model = L.Classifier(cnn)
if gpu >= 0:
model.to_gpu()
#setup the optimizer
optimizer = O.Adam()
optimizer.setup(model)
train_iter = chainer.iterators.SerialIterator(train_data, batchsize)
dev_iter = chainer.iterators.SerialIterator(dev_data, batchsize,repeat=False, shuffle=False)
test_iter = chainer.iterators.SerialIterator(test_data, batchsize,repeat=False, shuffle=False)
batch1 = train_iter.next()
batch2 = dev_iter.next()
updater = training.StandardUpdater(train_iter, optimizer, converter=util.concat_examples, device=gpu)
trainer = training.Trainer(updater, (n_epoch, 'epoch'))
# Evaluation
eval_model = model.copy()
eval_model.predictor.train = False
trainer.extend(extensions.Evaluator(dev_iter, eval_model, device=gpu, converter=util.concat_examples))
test_model = model.copy()
test_model.predictor.train = False
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy']))
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
