def prepare_inception_data(o_dir, i_dir):
if not os.path.exists(o_dir):
os.makedirs(o_dir)
cnt = 0
bar = progressbar.ProgressBar(redirect_stdout=True,
max_value=progressbar.UnknownLength)
for root, subFolders, files in os.walk(i_dir):
if files:
for f in files:
if 'jpg' in f:
f_name = str(cnt) + '_ins.' + f.split('.')[-1]
cnt += 1
file_dir = os.path.join(root, f)
dest_path = os.path.join(o_dir, f)
dest_new_name = os.path.join(o_dir, f_name)
copy(file_dir, o_dir)
os.rename(dest_path, dest_new_name)
bar.update(cnt)
bar.finish()
print('Total number of files: {}'.format(cnt))
python类UnknownLength()的实例源码
def load_images(o_dir, i_dir, n_images=3000, size=128):
prepare_inception_data(o_dir, i_dir)
image_list = []
done = False
cnt = 0
bar = progressbar.ProgressBar(redirect_stdout=True,
max_value=progressbar.UnknownLength)
for root, dirs, files in os.walk(o_dir):
if files:
for f in files:
cnt += 1
file_dir = os.path.join(root, f)
image_list.append(ip.load_image_inception(file_dir, 0))
bar.update(cnt)
if len(image_list) == n_images:
done = True
break
if done:
break
bar.finish()
print('Finished Loading Files')
return image_list
def __call__(self, progress, data, width):
if progress.end_time:
return self.finish_msg
if progress.max_value is progressbar.UnknownLength:
bar = progressbar.BouncingBar
else:
bar = progressbar.Bar
line = bar.__call__(self, progress, data, width)
if data["percentage"] is None:
msg = self.msg
else:
msg = "%s (%d%%)" % (self.msg, data["percentage"])
offset = width // 2 - len(msg) // 2
return line[:offset] + msg + line[offset + len(msg):]
def _parse(self, xml_file):
self.xml_file = xml_file
self.key_count = 0
if not self.suppliers:
self.suppliers = IceCatSupplierMapping(log=self.log, auth=self.auth, data_dir=self.data_dir)
if not self.categories:
self.categories = IceCatCategoryMapping(log=self.log, data_dir=self.data_dir, auth=self.auth)
print("Parsing products from index file:", xml_file)
with progressbar.ProgressBar(max_value=progressbar.UnknownLength) as self.bar:
with open(self.xml_file, 'rb') as f:
self.o = xmltodict.parse(f, attr_prefix='', postprocessor=self._postprocessor,
namespace_separator='', process_namespaces=True, namespaces=self._namespaces)
f.closed
# peel down to file key
self.o = self.o['icecat-interface']['files.index']['file']
self.log.info("Parsed {} products from IceCat catalog".format(str(len(self.o))))
return len(self.o)
def on_finish(self):
if self.progress:
try:
self.progress.update(self.finished)
except Exception:
self.progress.update(progressbar.UnknownLength)
self.finished += 1
def _search_for_type(self, type_attribute: str) -> list:
"""The actual request to misp. Skip attributes which are proposed for deletion.
:param type_attribute: MISP event attribute type to search for
:returns: List of values per event matching type_attribute"""
results = self.misp.search(type_attribute=type_attribute, deleted=False)
attribute_values = []
self._debug('Processing MISP results...')
bar = progressbar.ProgressBar(max_value=progressbar.UnknownLength)
for idx, events in enumerate(results.get('response', None)):
bar.update(idx)
event_info = events.get('Event').get('info')
event_id = events.get('Event').get('id')
if self.ignore and event_id in self.ignore:
continue
attribute_values.append({'info': event_info,
'id': event_id,
'values': []})
for values in events.get('Event').get('Attribute'):
# Skip attributed which are proposed to delete
shadow_attribute = values.get('ShadowAttribute', None)
if len(shadow_attribute) > 0 and shadow_attribute[0].get('proposal_to_delete', False):
continue
# Skip attributes which are not marked for ids export
if not values.get('to_ids', None):
continue
if type_attribute in values.get('type'):
attribute_values[idx]['values'].append(values.get('value'))
return attribute_values
def on_finish(self):
if self.progress:
try:
self.progress.update(self.finished)
except Exception:
self.progress.update(progressbar.UnknownLength)
self.finished += 1
def __init__(self, present, past=None, max_value=1, vars=None,
**kwargs):
self.present = present
self.sub_bar = None
self.finished = None
if past is None:
past = present
self.msg_bar = MessageBar(
msg=present, finish_msg="%s finished in" % past)
widgets = [self.msg_bar, " "]
if max_value is None:
widgets.append(progressbar.Timer(format="%(elapsed)s"))
else:
widgets.append(progressbar.ETA(
format="ETA: %(eta)s",
format_finished="%(elapsed)s"))
if vars is not None:
self.var_vals = progressbar.FormatCustomText(
" (" + ", ".join("%s: %%(%s)s" % (v, v) for v in vars) + ")",
{v: "---" for v in vars})
widgets.append(self.var_vals)
else:
self.var_vals = None
def update_thread():
while not self.finished:
if self.sub_bar is None or self.sub_bar.finished:
self.update()
time.sleep(0.001)
self.thread = threading.Thread(target=update_thread)
self.thread.daemon = True
if max_value is None:
max_value = progressbar.UnknownLength
super(ProgressBar, self).__init__(
poll_interval=0.1, widgets=widgets, fd=sys.stdout,
max_value=max_value, **kwargs)
