def __init__(self, lmdb_dir, shuffle=True):
self._lmdb = lmdb.open(lmdb_dir, readonly=True, lock=False,
map_size=1099511627776 * 2, max_readers=100)
self._txn = self._lmdb.begin()
self._shuffle = shuffle
self._size = self._txn.stat()['entries']
if shuffle:
self.keys = self._txn.get('__keys__')
if not self.keys:
self.keys = []
with timed_operation("Loading LMDB keys ...", log_start=True), \
tqdm(total=self._size, ascii=True) as pbar:
for k in self._txn.cursor():
if k != '__keys__':
self.keys.append(k)
pbar.update()
评论列表
文章目录