dataset.py 文件源码

def encode_and_pack(vocab, fin, fout, input_sepline=sepline):

    packer = msgpack.Packer()
    for line in fin:
        words = input_sepline(line)
        encoded = [vocab.sos_id]
        encoded.extend([vocab.get_id(word) for word in words])
        encoded.append(vocab.eos_id)
        fout.write(packer.pack(encoded))