data_utils.py 文件源码

def write_features_hdf5(train, valid, test):
    f = h5py.File(args.hdf5, "w")

    train_grp = f.create_group("train")
    train_x = train_grp.create_dataset("train_x", train[0].shape, dtype='f', compression="gzip", compression_opts=9)
    train_y = train_grp.create_dataset("train_y", train[1].shape, dtype='i', compression="gzip", compression_opts=9)

    valid_grp = f.create_group("valid")
    valid_x = valid_grp.create_dataset("valid_x", valid[0].shape, dtype='f', compression="gzip", compression_opts=9)
    valid_y = valid_grp.create_dataset("valid_y", valid[1].shape, dtype='i', compression="gzip", compression_opts=9)

    test_grp = f.create_group("test")
    test_x = test_grp.create_dataset("test_x", test[0].shape, dtype='f', compression="gzip", compression_opts=9)
    test_y = test_grp.create_dataset("test_y", test[1].shape, dtype='i', compression="gzip", compression_opts=9)

    train_x.write_direct(np.ascontiguousarray(train[0], dtype=train[0].dtype))
    train_y.write_direct(np.ascontiguousarray(train[1], dtype=train[1].dtype))
    valid_x.write_direct(np.ascontiguousarray(valid[0], dtype=valid[0].dtype))
    valid_y.write_direct(np.ascontiguousarray(valid[1], dtype=valid[1].dtype))
    test_x.write_direct(np.ascontiguousarray(test[0], dtype=test[0].dtype))
    test_y.write_direct(np.ascontiguousarray(test[1], dtype=test[1].dtype))

    f.close()