load_files.py 文件源码

def load_vectors_mmaped(VECTORS_FILE, NOFTYPES, D, UPDATE_VECTORS):
    if (not os.path.exists('vectors.mymemmap')) or UPDATE_VECTORS == 1:
        v = np.memmap('vectors.mymemmap', dtype='float', mode='w+', shape=(NOFTYPES, D))
        i = 0

        # Showing percentage to user
        limit = 100000
        with open(VECTORS_FILE, 'r') as f:
            for line in f:
                if i >= limit:
                    print(limit/float(NOFTYPES)*100)
                    limit += 100000
                line = line.strip()
                if len(line) > 0:
                    v[i][:] = map(float, line.split(' '))
                i += 1
    else:
        v = np.memmap('vectors.mymemmap', dtype='float', mode='r', shape = (NOFTYPES, D))
    return v