gen_nltk_dataset.py 文件源码

def train_brown_lemmatizer(output_folder):
    obs_set = np.load(os.path.join(output_folder, 'X_train.npy'))
    out_set = np.load(os.path.join(output_folder, 'y_train.npy'))
    count_set = np.load(os.path.join(output_folder, 'K_train.npy'))
    nn_param_set = train_lemmatizer(
        obs_set,
        out_set,
        count_set,
        window_size=[2,2],
        positive_samples_only=True,
        batch_size=128,
        param_scale=0.01,
        num_epochs=4000,
        step_size=0.001,
        l2_lambda=0.1)

    if output_folder:
        with open(os.path.join(output_folder, 'nn_param_dict.pkl'), 'w') as f:
            dill.dump(nn_param_set, f)