book_parser.py 文件源码

def read_all_games(fn_in, fn_out):
    g = h5py.File(fn_out, 'w')
    X = g.create_dataset('x', (0, 28 * 8), dtype='b', maxshape=(None, 28 * 8), chunks=True) # dtype='b'
    M = g.create_dataset('m', (0, 1), dtype='float32', maxshape=(None, 1), chunks=True)
    size = 0
    line = 0
    for game in read_games(fn_in):
        game = parse_game(game)
        if game is None:
            continue

        for x, m in game :
            if line + 1 >= size:
                g.flush()
                size = 2 * size + 1
                print 'resizing to', size
                [d.resize(size=size, axis=0) for d in (X, M)]

            X[line] = numpy.packbits(x)
            M[line] = m

            line += 1

    print 'shrink to', line
    [d.resize(size=line, axis=0) for d in (X, M)]   # shrink to fit
    g.close()