def load_lmdb_dataset(db_name):
X = []
Y = []
env = lmdb.open(db_name, readonly=True)
with env.begin() as txn:
cursor = txn.cursor()
for key, value in cursor:
datum = Datum()
datum.ParseFromString(value)
flat_x = np.fromstring(datum.data, dtype=np.uint8)
X.append(flat_x.reshape(datum.channels, datum.height, datum.width))
Y.append(datum.label)
env.close()
assert len(X) == len(Y)
X = np.asarray(X)
Y = np.asarray(Y)
return X, Y
python类open()的实例源码
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
#plt.imshow(data)
#plt.show()
return X, np.array(y)
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
def read_db(db_name):
lmdb_env = lmdb.open(db_name)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()
X = []
y = []
cnts = {}
for key, value in lmdb_cursor:
datum.ParseFromString(value)
label = datum.label
data = caffe.io.datum_to_array(datum)
#data = data.swapaxes(0, 2).swapaxes(0, 1)
X.append(data)
y.append(label)
if label not in cnts:
cnts[label] = 0
cnts[label] += 1
#plt.imshow(data)
#plt.show()
return X, np.array(y), cnts
frames_to_caffe_datum_proto_lmdb.py 文件源码
项目：video-tools
作者: achalddave
项目源码
文件源码
阅读 17
收藏 0
点赞 0
评论 0
def load_image_datum(image_path, resize_height=None, resize_width=None):
"""Load an image in a Caffe datum in BGR order.
Args:
image_path (str): Path to an image.
resize_height (int): Height to resize an image to. If 0 or None, the
image is not resized.
resize_width (int): Width to resize an image to. If 0 or None, the
image is not resized.
Returns:
image_datum (caffe Datum): Contains the image in BGR order after
resizing.
"""
image = Image.open(image_path)
if resize_height and resize_width:
image = image.resize((resize_width, resize_height))
# Image has shape (height, width, num_channels), where the
# channels are in RGB order.
image = np.array(image)
# Convert image from RGB to BGR.
image = image[:, :, ::-1]
# Convert image to (num_channels, height, width) shape.
image = image.transpose((2, 0, 1))
return caffe.io.array_to_datum(image).SerializeToString()
def dump_one_lmdb(path, offset):
with lmdb.open(path, map_size=map_size) as env, \
env.begin().cursor() as lmdb_cursor:
num_entries = env.stat()['entries']
# Unfortunately, it seems the only way to set the cursor to an
# arbitrary key index (without knowing the key) is to literally call
# next() repeatedly.
lmdb_cursor.next()
for i in tqdm(range(offset)):
lmdb_cursor.next()
video_frame = video_frames_pb2.LabeledVideoFrame()
video_frame.ParseFromString(lmdb_cursor.value())
image_proto = video_frame.frame.image
image = np.fromstring(image_proto.data,
dtype=np.uint8).reshape(
image_proto.channels, image_proto.height,
image_proto.width).transpose((1, 2, 0))
image = Image.fromarray(image, 'RGB')
image.save('tmp.png')
print(lmdb_cursor.key())
print(', '.join([label.name for label in video_frame.label]))
def main():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'labels_hdf5',
help=('Maps video names to a binary matrix of shape (num_frames, '
'num_labels).'))
parser.add_argument('output_lmdb')
args = parser.parse_args()
map_size = 2e9
lmdb_environment = lmdb.open(args.output_lmdb, map_size=int(map_size))
with lmdb_environment.begin(write=True) as lmdb_transaction, h5py.File(
args.labels_hdf5) as labels:
for video_name, file_labels in tqdm(labels.items()):
file_labels = np.asarray(file_labels)
for frame_number, frame_labels in enumerate(file_labels):
key = '{}-{}'.format(video_name, frame_number + 1)
lmdb_transaction.put(key, frame_labels.tobytes())
frames_to_labeled_video_frames_lmdb.py 文件源码
项目：video-tools
作者: achalddave
项目源码
文件源码
阅读 19
收藏 0
点赞 0
评论 0
def load_image(image_path, resize_height=None, resize_width=None):
"""Load an image in video_frames.Image format.
Args:
image_path (str): Path to an image.
resize_height (int): Height to resize an image to. If 0 or None, the
image is not resized.
resize_width (int): Width to resize an image to. If 0 or None, the
image is not resized.
Returns:
image_datum (numpy array): Contains the image in BGR order after
resizing.
"""
image = Image.open(image_path)
if resize_height and resize_width:
image = image.resize((resize_width, resize_height))
# Image has shape (height, width, num_channels), where the
# channels are in RGB order.
image = np.array(image)
# Convert image from RGB to BGR.
image = image[:, :, ::-1]
# Convert image to (num_channels, height, width) shape.
image = image.transpose((2, 0, 1))
return image
def write_prototxt(output_file,prototxt):
''' Write prototxt to file.
Usage: write_prototxt(output_file,prototxt_dictionary) '''
with open(output_file,'w') as f:
if 'name' in prototxt.keys():
f.write('name: \"'+prototxt['name']+'\" \n')
if 'layer' in prototxt.keys():
f.write(get_prototxt_string(prototxt['layer'],0,'layer').replace('\n\n','\n'))
def __getitem__(self, index):
assert index <= len(self), 'index range error'
index += 1
with self.env.begin(write=False) as txn:
img_key = 'image-%09d' % index
imgbuf = txn.get(img_key)
buf = six.BytesIO()
buf.write(imgbuf)
buf.seek(0)
try:
img = Image.open(buf).convert('L')
except IOError:
print('Corrupted image for %d' % index)
return self[index + 1]
if self.transform is not None:
img = self.transform(img)
label_key = 'label-%09d' % index
label = str(txn.get(label_key))
if self.target_transform is not None:
label = self.target_transform(label)
return (img, label)
def __init__(self, dirpath):
# Open LMDB environment in read-only mode
self._lmdb_env = lmdb.open(dirpath, readonly=True, max_dbs=NB_DBS)
# Open the default database(s) associated with the environment
self.data_db = self._lmdb_env.open_db(DATA_DB)
self.meta_db = self._lmdb_env.open_db(META_DB)
# Read the metadata
self.nb_samples = int(self.get_meta_str(NB_SAMPLES))
def close(self):
"""Close the environment.
Invalidates any open iterators, cursors, and transactions.
"""
self._lmdb_env.close()
def close(self):
"""Close the environment.
Before closing, the number of samples is written to `meta_db`.
Invalidates any open iterators, cursors, and transactions.
"""
self.set_meta_str(NB_SAMPLES, self.nb_samples)
self._lmdb_env.close()
