def knn_masked_data(trX,trY,missing_data_dir, input_shape, k):
raw_im_data = np.loadtxt(join(script_dir,missing_data_dir,'index.txt'),delimiter=' ',dtype=str)
raw_mask_data = np.loadtxt(join(script_dir,missing_data_dir,'index_mask.txt'),delimiter=' ',dtype=str)
# Using 'brute' method since we only want to do one query per classifier
# so this will be quicker as it avoids overhead of creating a search tree
knn_m = KNeighborsClassifier(algorithm='brute',n_neighbors=k)
prob_Y_hat = np.zeros((raw_im_data.shape[0],int(np.max(trY)+1)))
total_images = raw_im_data.shape[0]
pbar = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('\rProcessed %(value)d of %(max)d Images '), progressbar.Bar()], maxval=total_images, term_width=50).start()
for i in range(total_images):
mask_im=load_image(join(script_dir,missing_data_dir,raw_mask_data[i][0]), input_shape,1).reshape(np.prod(input_shape))
mask = np.logical_not(mask_im > eps) # since mask is 1 at missing locations
v_im=load_image(join(script_dir,missing_data_dir,raw_im_data[i][0]), input_shape, 255).reshape(np.prod(input_shape))
rep_mask = np.tile(mask,(trX.shape[0],1))
# Corrupt whole training set according to the current mask
corr_trX = np.multiply(trX, rep_mask)
knn_m.fit(corr_trX, trY)
prob_Y_hat[i,:] = knn_m.predict_proba(v_im.reshape(1,-1))
pbar.update(i)
pbar.finish()
return prob_Y_hat
python类FormatLabel()的实例源码
def __init__(self, name, max_value=100, history_len=5, display=True,
display_data={'train':['loss', 'accuracy'], 'test':['loss', 'accuracy']},
level=logging.INFO, train_log_mode='TRAIN_PROGRESS', test_log_mode='TEST_PROGRESS'):
super(ProgressbarLogger, self).__init__(
name, level=level, display=display, logfile=None,
train_log_mode=train_log_mode, test_log_mode=test_log_mode)
self.train_log_data = {}
self.test_log_data = {}
self.max_value = max_value
self.history_len = history_len
self.display_data = display_data
self.mode['TRAIN_PROGRESS'] = self.log_train_progress
self.mode['TEST_PROGRESS'] = self.log_test_progress
# create logging format
self.widgets = [progressbar.FormatLabel('(%(value)d of %(max)s)'),
' ', progressbar.Percentage(),
' ', progressbar.Bar()]
self.dynamic_data = {k+'_'+kk: 0.0 for k in display_data.keys() for kk in display_data[k]}
diff_data = {'diff_'+k+'_'+kk: 0.0 for k in display_data.keys() for kk in display_data[k]}
self.dynamic_data.update(diff_data)
for t in display_data.keys():
ddstr = ' [' + t + ']'
for s in display_data[t]:
value_name = t + '_' + s
ddstr = ddstr + ' ' + s + ':' + '%(' + value_name + ').3f (%(diff_' + value_name + ').3f)'
self.widgets.append(progressbar.FormatLabel(ddstr))
self.widgets.extend(['|', progressbar.FormatLabel('Time: %(elapsed)s'), '|', progressbar.AdaptiveETA()])
def update(self, n_total, n_lc_updated, n_scaling_out, n_scaled_out, n_services_installed, n_scaling_in, n_complete, *args):
msg = '[Patching {0} ASGs]: '.format(n_total)
stages = []
if n_lc_updated is not 0:
stages.append('{0} Launch Configs Updated'.format(n_lc_updated))
if n_scaling_out is not 0:
stages.append('{0} Scaling Out'.format(n_scaling_out))
if n_scaled_out is not 0:
stages.append('{0} Installing Services'.format(n_scaled_out))
if n_scaling_in is not 0:
stages.append('{0} Scaling In'.format(n_scaling_in))
if n_complete is not 0:
stages.append('{0} Complete'.format(n_complete))
msg += ', '.join(stages)
self.widgets[4] = FormatLabel(msg)
t1 = (5 / n_total) * n_lc_updated
t2 = (10 / n_total) * n_scaling_out
t3 = (30 / n_total) * n_scaled_out
t4 = (70 / n_total) * n_services_installed
t5 = (75 / n_total) * n_scaling_in
t6 = (100 / n_total) * n_complete
self.total_progress = t1 + t2 + t3 + t4 + t5 + t6
def print_status_stream(title, stream):
widgets = [title, FormatLabel(''), ' ', Percentage(), ' ', Bar(), ' ', RotatingMarker()]
bar = None
if sys.stderr.isatty():
bar = progressbar.ProgressBar(widgets=widgets, max_value=255)
def print_error(status):
print(status['error'])
def print_status(status):
progress = status.get('progressDetail')
if progress:
widgets[1] = FormatLabel("%12s" % (status['status']))
prog = int(round(255 * ((progress['current'] / progress['total']))))
if bar is not None:
bar.update(prog)
def print_unknown(status):
print(status)
for line in stream:
status = json.loads(line.decode('utf8'))
if 'error' in status:
print_error(status)
elif 'status' in status:
print_status(status)
else:
print_unknown(status)
def __init__(self):
self.stop_running = threading.Event()
self.progress_thread = threading.Thread(target=self.init_progress)
self.progress_thread.daemon = True
spinner = RotatingMarker()
spinner.INTERVAL = datetime.timedelta(milliseconds=100)
self.widgets = [spinner, ' ', Percentage(), ' ', FormatLabel('Calculating patch requirements'), ' ', Bar(), ' ', FormatLabel('')]
self.progress = ProgressBar(redirect_stdout=True, widgets=self.widgets, max_value=100)
self.progress.update(0)
def finish(self, total):
msg = '[Patching {0} ASGs]: {0} Complete'.format(total)
self.widgets[4] = FormatLabel(msg)
self.progress.finish()
def init_progress(self):
while not self.stop_running.is_set():
p = self.total_progress
if math.isnan(p) or p is None or p == 0:
p = 1
t = datetime.datetime.utcnow()
s = (t - self.start_time).total_seconds()
elapsed = progressbar.utils.format_time(s)
self.widgets[8] = FormatLabel('Elapsed Time: {0}'.format(elapsed))
self.progress.update(p)
time.sleep(0.2)
def create_widgets(self):
label = self.label
bar = Bar(left='[', right=']')
fmtlabel = FormatLabel(" %(value)d/%(max)d ")
(w, _) = get_terminal_size()
lw = int(w * 0.5)
if len(label)>lw-1:
s = lw-4
label = label[:s]
label += "... "
else:
label = label.ljust(lw, ' ')
return [ label, bar, fmtlabel ]
def convert_dataset(args):
try:
if args.min_rects > args.max_rects:
raise ValueError('min_rect must be less than or equal to max_rect.')
if args.min_width > args.max_width:
raise ValueError('min_width must be less than or equal to max_width.')
try:
os.makedirs(args.output_dir)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(args.output_dir):
pass
else:
raise ValueError('output_dir argument is not a valid path.')
total_images = len(args.filenames)
params = zip(args.filenames, [args] * total_images)
pool = Pool(initializer=init_worker)
pbar = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('\rProcessed %(value)d of %(max)d Images '), progressbar.Bar()], maxval=total_images, term_width=50).start()
try:
results = pool.imap_unordered(corrupt_source_image, params, chunksize=max(int(math.sqrt(len(args.filenames)))/2, 10))
for i in range(len(args.filenames)):
next(results)
pbar.update(i+1)
pool.close()
pool.join()
pbar.finish()
except KeyboardInterrupt:
pool.terminate()
pool.join()
pbar.finish()
raise
except ValueError as e:
print
print 'Bad parameters:', e
raise e
except KeyboardInterrupt:
print
if __name__ == '__main__':
print 'User stopped generation!'
raise
except:
print
print "Unexpected error:", sys.exc_info()[0]
raise
# Main routine
def convert_dataset(args):
try:
if args.min_rects > args.max_rects:
raise ValueError('min_rect must be less than or equal to max_rect.')
if args.min_width > args.max_width:
raise ValueError('min_width must be less than or equal to max_width.')
try:
os.makedirs(args.output_dir)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(args.output_dir):
pass
else:
raise ValueError('output_dir argument is not a valid path.')
total_images = len(args.filenames)
params = zip(args.filenames, [args] * total_images)
pool = Pool(initializer=init_worker)
pbar = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('\rProcessed %(value)d of %(max)d Images '), progressbar.Bar()], maxval=total_images, term_width=50).start()
try:
results = pool.imap_unordered(corrupt_source_image, params, chunksize=max(int(math.sqrt(len(args.filenames)))/2, 10))
for i in range(len(args.filenames)):
next(results)
pbar.update(i+1)
pool.close()
pool.join()
pbar.finish()
except KeyboardInterrupt:
pool.terminate()
pool.join()
pbar.finish()
raise
except ValueError as e:
print
print 'Bad parameters:', e
raise e
except KeyboardInterrupt:
print
if __name__ == '__main__':
print 'User stopped generation!'
raise
except:
print
print "Unexpected error:", sys.exc_info()[0]
raise
# Main routine
