def __init__(self, feat_stride, scales, ratios, output_score,
rpn_pre_nms_top_n, rpn_post_nms_top_n, threshold, rpn_min_size):
super(ProposalOperator, self).__init__()
self._feat_stride = feat_stride
self._scales = np.fromstring(scales[1:-1], dtype=float, sep=',')
self._ratios = np.fromstring(ratios[1:-1], dtype=float, sep=',')
self._anchors = generate_anchors(base_size=self._feat_stride, scales=self._scales, ratios=self._ratios)
self._num_anchors = self._anchors.shape[0]
self._output_score = output_score
self._rpn_pre_nms_top_n = rpn_pre_nms_top_n
self._rpn_post_nms_top_n = rpn_post_nms_top_n
self._threshold = threshold
self._rpn_min_size = rpn_min_size
if DEBUG:
print('feat_stride: {}'.format(self._feat_stride))
print('anchors:')
print(self._anchors)
python类fromstring()的实例源码
def vec2bin(input_path, output_path):
input_fd = open(input_path, "rb")
output_fd = open(output_path, "wb")
header = input_fd.readline()
output_fd.write(header)
vocab_size, vector_size = map(int, header.split())
for line in tqdm(range(vocab_size)):
word = []
while True:
ch = input_fd.read(1)
output_fd.write(ch)
if ch == b' ':
word = b''.join(word).decode('utf-8')
break
if ch != b'\n':
word.append(ch)
vector = np.fromstring(input_fd.readline(), sep=' ', dtype='float32')
output_fd.write(vector.tostring())
input_fd.close()
output_fd.close()
def enwik8_raw_data(data_path=None, num_test_symbols=5000000):
"""Load raw data from data directory "data_path".
The raw Hutter prize data is at:
http://mattmahoney.net/dc/enwik8.zip
Args:
data_path: string path to the directory where simple-examples.tgz has
been extracted.
num_test_symbols: number of symbols at the end that make up the test set
Returns:
tuple (train_data, valid_data, test_data, unique)
where each of the data objects can be passed to hutter_iterator.
"""
data_path = os.path.join(data_path, "enwik8")
raw_data = _read_symbols(data_path)
raw_data = np.fromstring(raw_data, dtype=np.uint8)
unique, data = np.unique(raw_data, return_inverse=True)
train_data = data[: -2 * num_test_symbols]
valid_data = data[-2 * num_test_symbols: -num_test_symbols]
test_data = data[-num_test_symbols:]
return train_data, valid_data, test_data, unique
def text8_raw_data(data_path=None, num_test_symbols=5000000):
"""Load raw data from data directory "data_path".
The raw text8 data is at:
http://mattmahoney.net/dc/text8.zip
Args:
data_path: string path to the directory where simple-examples.tgz has
been extracted.
num_test_symbols: number of symbols at the end that make up the test set
Returns:
tuple (train_data, valid_data, test_data, unique)
where each of the data objects can be passed to text8_iterator.
"""
data_path = os.path.join(data_path, "text8")
raw_data = _read_symbols(data_path)
raw_data = np.fromstring(raw_data, dtype=np.uint8)
unique, data = np.unique(raw_data, return_inverse=True)
train_data = data[: -2 * num_test_symbols]
valid_data = data[-2 * num_test_symbols: -num_test_symbols]
test_data = data[-num_test_symbols:]
return train_data, valid_data, test_data, unique
def load_bin_vec(fname, vocab):
"""
Loads word vecs from word2vec bin file
"""
word_vecs = OrderedDict()
with open(fname, "rb") as f:
header = f.readline()
vocab_size, layer1_size = map(int, header.split())
binary_len = np.dtype('float32').itemsize * layer1_size
for line in xrange(vocab_size):
word = []
while True:
ch = f.read(1)
if ch == ' ':
word = ''.join(word)
break
if ch != '\n':
word.append(ch)
if word in vocab:
idx = vocab[word]
word_vecs[idx] = np.fromstring(f.read(binary_len), dtype='float32')
else:
f.read(binary_len)
return word_vecs
def test_if_items_patch_updates_stock_filter(self, init_db, headers, redis, session, client, api):
body = [{
'name': 'test',
'stores': [{'id': 1}],
'schema': {'properties': {'id': {'type': 'string'}}, 'type': 'object', 'id_names': ['id']}
}]
client = await client
await client.post('/item_types/', headers=headers, data=ujson.dumps(body))
body = [{'id': 'test'}]
resp = await client.post('/item_types/1/items?store_id=1', headers=headers, data=ujson.dumps(body))
assert resp.status == 201
test_model = _all_models['store_items_test_1']
await ItemsIndicesMap(test_model).update(session)
body = [{'id': 'test', '_operation': 'delete'}]
resp = await client.patch('/item_types/1/items?store_id=1', headers=headers, data=ujson.dumps(body))
stock_filter = np.fromstring(await redis.get('store_items_test_1_stock_filter'), dtype=np.bool).tolist()
assert stock_filter == [False]
def predict(self, input_file):
# img = base64.b64decode(input_base64)
# img_array = np.fromstring(img, np.uint8)
# input_file = cv2.imdecode(img_array, 1)
# ip_converted = preprocessing.resizing(input_base64)
segmented_image = preprocessing.image_segmentation(
preprocessing.resizing(input_file)
)
# processed_image = preprocessing.removebg(segmented_image)
detect = pycolor.detect_color(
segmented_image,
self._mapping_file
)
return (detect)
def load_poses(self):
"""Load ground truth poses from file."""
print('Loading poses for sequence ' + self.sequence + '...')
pose_file = os.path.join(self.pose_path, self.sequence + '.txt')
# Read and parse the poses
try:
self.T_w_cam0 = []
with open(pose_file, 'r') as f:
for line in f.readlines():
T = np.fromstring(line, dtype=float, sep=' ')
T = T.reshape(3, 4)
T = np.vstack((T, [0, 0, 0, 1]))
self.T_w_cam0.append(T)
print('done.')
except FileNotFoundError:
print('Ground truth poses are not avaialble for sequence ' +
self.sequence + '.')
def enwik8_raw_data(data_path=None, num_test_symbols=5000000):
"""Load raw data from data directory "data_path".
The raw Hutter prize data is at:
http://mattmahoney.net/dc/enwik8.zip
Args:
data_path: string path to the directory where simple-examples.tgz has
been extracted.
num_test_symbols: number of symbols at the end that make up the test set
Returns:
tuple (train_data, valid_data, test_data, unique)
where each of the data objects can be passed to hutter_iterator.
"""
data_path = os.path.join(data_path, "enwik8")
raw_data = _read_symbols(data_path)
raw_data = np.fromstring(raw_data, dtype=np.uint8)
unique, data = np.unique(raw_data, return_inverse=True)
train_data = data[: -2 * num_test_symbols]
valid_data = data[-2 * num_test_symbols: -num_test_symbols]
test_data = data[-num_test_symbols:]
return train_data, valid_data, test_data, unique
def text8_raw_data(data_path=None, num_test_symbols=5000000):
"""Load raw data from data directory "data_path".
The raw text8 data is at:
http://mattmahoney.net/dc/text8.zip
Args:
data_path: string path to the directory where simple-examples.tgz has
been extracted.
num_test_symbols: number of symbols at the end that make up the test set
Returns:
tuple (train_data, valid_data, test_data, unique)
where each of the data objects can be passed to text8_iterator.
"""
data_path = os.path.join(data_path, "text8")
raw_data = _read_symbols(data_path)
raw_data = np.fromstring(raw_data, dtype=np.uint8)
unique, data = np.unique(raw_data, return_inverse=True)
train_data = data[: -2 * num_test_symbols]
valid_data = data[-2 * num_test_symbols: -num_test_symbols]
test_data = data[-num_test_symbols:]
return train_data, valid_data, test_data, unique
def load_word_vectors(file_destination):
"""
This method loads the word vectors from the supplied file destination.
It loads the dictionary of word vectors and prints its size and the vector dimensionality.
"""
print "Loading pretrained word vectors from", file_destination
word_dictionary = {}
try:
f = codecs.open(file_destination, 'r', 'utf-8')
for line in f:
line = line.split(" ", 1)
key = unicode(line[0].lower())
word_dictionary[key] = numpy.fromstring(line[1], dtype="float32", sep=" ")
except:
print "Word vectors could not be loaded from:", file_destination
return {}
print len(word_dictionary), "vectors loaded from", file_destination
return word_dictionary
def checkImageIsValid(imageBin):
if imageBin is None:
return False
try:
imageBuf = np.fromstring(imageBin, dtype=np.uint8)
img = cv2.imdecode(imageBuf, cv2.IMREAD_GRAYSCALE)
imgH, imgW = img.shape[0], img.shape[1]
except:
return False
else:
if imgH * imgW == 0:
return False
return True
def get_frame_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.SequenceExample()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.context.feature['video_id'].bytes_list.value[0]
label = example.context.feature['labels'].int64_list.value[:]
rgbs = []
audios = []
rgb_feature = example.feature_lists.feature_list['rgb'].feature
for i in range(len(rgb_feature)):
rgb = np.fromstring(rgb_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
rgb = utils.Dequantize(rgb, 2, -2)
rgbs.append(rgb)
audio_feature = example.feature_lists.feature_list['audio'].feature
for i in range(len(audio_feature)):
audio = np.fromstring(audio_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
audio = utils.Dequantize(audio, 2, -2)
audios.append(audio)
rgbs = np.array(rgbs)
audios = np.array(audios)
features.append((video_id, label, rgbs, audios))
return features
def get_frame_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.SequenceExample()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.context.feature['video_id'].bytes_list.value[0]
label = example.context.feature['labels'].int64_list.value[:]
rgbs = []
audios = []
rgb_feature = example.feature_lists.feature_list['rgb'].feature
for i in range(len(rgb_feature)):
rgb = np.fromstring(rgb_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
rgb = utils.Dequantize(rgb, 2, -2)
rgbs.append(rgb)
audio_feature = example.feature_lists.feature_list['audio'].feature
for i in range(len(audio_feature)):
audio = np.fromstring(audio_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
audio = utils.Dequantize(audio, 2, -2)
audios.append(audio)
rgbs = np.array(rgbs)
audios = np.array(audios)
features.append((video_id, label, rgbs, audios))
return features
def get_frame_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.SequenceExample()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.context.feature['video_id'].bytes_list.value[0]
label = example.context.feature['labels'].int64_list.value[:]
rgbs = []
audios = []
rgb_feature = example.feature_lists.feature_list['rgb'].feature
for i in range(len(rgb_feature)):
rgb = np.fromstring(rgb_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
rgb = utils.Dequantize(rgb, 2, -2)
rgbs.append(rgb)
audio_feature = example.feature_lists.feature_list['audio'].feature
for i in range(len(audio_feature)):
audio = np.fromstring(audio_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
audio = utils.Dequantize(audio, 2, -2)
audios.append(audio)
rgbs = np.array(rgbs)
audios = np.array(audios)
features.append((video_id, label, rgbs, audios))
return features
def _unpack_data_block(f, blocksize, packing):
"""
Private method to read a block from a file into a NumPy array.
"""
return numpy.fromstring(f.read(blocksize), packing)
def get_full_alignment_base_quality_scores(read):
"""
Returns base quality scores for the full read alignment, inserting zeroes for deletions and removing
inserted and soft-clipped bases. Therefore, only returns quality for truly aligned sequenced bases.
Args:
read (pysam.AlignedSegment): read to get quality scores for
Returns:
np.array: numpy array of quality scores
"""
quality_scores = np.fromstring(read.qual, dtype=np.byte) - tk_constants.ILLUMINA_QUAL_OFFSET
start_pos = 0
for operation,length in read.cigar:
operation = cr_constants.cigar_numeric_to_category_map[operation]
if operation == 'D':
quality_scores = np.insert(quality_scores, start_pos, [0] * length)
elif operation == 'I' or operation == 'S':
quality_scores = np.delete(quality_scores, np.s_[start_pos:start_pos + length])
if not operation == 'I' and not operation == 'S':
start_pos += length
return start_pos, quality_scores
def get_qvs(qual):
if qual is None:
return None
return numpy.fromstring(qual, dtype=numpy.byte) - ILLUMINA_QUAL_OFFSET
def get_bases_qual(qual, cutoff):
if qual is None:
return None
qvs = numpy.fromstring(qual, dtype=numpy.byte) - ILLUMINA_QUAL_OFFSET
return numpy.count_nonzero(qvs[qvs > cutoff])
def get_min_qual(qual):
if qual is None or len(qual) == 0:
return None
return (numpy.fromstring(qual, dtype=numpy.byte) - ILLUMINA_QUAL_OFFSET).min()
