def render(self, length=None, progress=False):
"""
Render this signal into an numpy array of floats. Return the array.
:param length: The length to render, in seconds. Optional.
:param progress: Whether to show a progress bar for rendering
"""
if progress and not progressbar:
print('Install the progressbar module to see a progress bar for rendering')
progress = False
duration = self.duration if length is None else length * SAMPLE_RATE
if duration == float('inf'):
duration = 3*SAMPLE_RATE
else:
duration = int(duration)
out = numpy.empty((duration, 1))
pbar = progressbar.ProgressBar(widgets=['Rendering: ', progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA()], maxval=duration-1).start() if progress else None
for i in range(duration):
out[i] = self.amplitude(i)
if pbar: pbar.update(i)
if pbar: pbar.finish()
return out
python类Percentage()的实例源码
def main():
uri, outfile, dataset = get_arguments()
fd = tempfile.NamedTemporaryFile()
progress = ProgressBar(widgets=[Percentage(), ' ', Bar(), ' ', ETA(), ' ', FileTransferSpeed()])
def update(count, blockSize, totalSize):
if progress.maxval is None:
progress.maxval = totalSize
progress.start()
progress.update(min(count * blockSize, totalSize))
urllib.urlretrieve(uri, fd.name, reporthook = update)
if dataset == 'zinc12':
df = pandas.read_csv(fd.name, delimiter = '\t')
df = df.rename(columns={'SMILES':'structure'})
df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
elif dataset == 'chembl22':
df = pandas.read_table(fd.name,compression='gzip')
df = df.rename(columns={'canonical_smiles':'structure'})
df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
pass
else:
df = pandas.read_csv(fd.name, delimiter = '\t')
df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
def collect_mailids(server):
folders = server.list_folders()
#construct progressbar
progressbar_widgets = [
'[Searching for mails on server] ',
progressbar.Percentage(),
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ']
progressbar_instance = progressbar.ProgressBar(widgets=progressbar_widgets, maxval=len(folders)).start()
#collect all mailids for all folders
folder_contents = {}
folder_progress = 0
for flags, delimiter, folder in folders:
#read all mailids for the folder
server.select_folder(folder, readonly=True)
folder_contents[folder] = server.search()
#update progrssbar
folder_progress += 1
progressbar_instance.update(folder_progress)
progressbar_instance.finish()
return folder_contents
def download(download_list, total_download_size):
progressbar_widgets = [
'[Downloading mails ] ',
progressbar.Percentage(),
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA(), ' ',
bitmath.integrations.BitmathFileTransferSpeed()]
progressbar_instance = progressbar.ProgressBar(widgets=progressbar_widgets, maxval=int(total_download_size)).start()
downloaded_size = bitmath.Byte(0)
for folder, mails in download_list.items():
server.select_folder(folder, readonly=True)
for mailid, mailfilename, mailsize in mails:
#make parent directory
if not os.path.isdir(os.path.dirname(mailfilename)):
os.makedirs(os.path.dirname(mailfilename))
#download mail
with open(mailfilename, 'wb') as mailfile:
mailfile.write(server.fetch([mailid], ['RFC822'])[mailid][b'RFC822'])
#update progressbar
downloaded_size += mailsize
progressbar_instance.update(int(downloaded_size))
progressbar_instance.finish()
def progressbarize(iterable, progress=False):
"""Construct progressbar for loops if progressbar requested, otherwise return directly iterable.
:param iterable: iterable to use
:param progress: True if print progressbar
"""
if progress:
# The casting to list is due to possibly yielded value that prevents
# ProgressBar to compute overall ETA
return progressbar.ProgressBar(widgets=[
progressbar.Timer(), ', ',
progressbar.Percentage(), ', ',
progressbar.SimpleProgress(), ', ',
progressbar.ETA()
])(list(iterable))
return iterable
def bruteforce():
import progressbar
from time import sleep
bar = progressbar.ProgressBar(maxval=60, \
widgets=[progressbar.Bar('==', '[', ']'), ' ', progressbar.Percentage()])
bar.start()
for i in xrange(10):
bar.update(i+1)
sleep(0.05)
wordlist = "/root/2fassassin/crack/wordlist/2fa-wordlist.txt"
target = "/root/2fassassin/loot/*.pfx"
sign = ""
sign += "crackpkcs12 -v -b"
sign += " "
sign += target
sign += "| tee crack.log"
os.system(sign)
bar.finish()
sys.exit()
def bruteforce():
import progressbar
from time import sleep
bar = progressbar.ProgressBar(maxval=60, \
widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
bar.start()
for i in xrange(10):
bar.update(i+1)
sleep(0.05)
wordlist = "/root/2fassassin/crack/wordlist/2fa-wordlist.txt"
target = "/root/2fassassin/loot/*.pfx"
sign = ""
sign += "crackpkcs12 -v -b"
sign += " "
sign += target
sign += "| tee crack.log"
os.system(sign)
bar.finish()
sys.exit()
def Steg_brute(ifile, dicc):
i = 0
ofile = ifile.split('.')[0] + "_flag.txt"
nlines = len(open(dicc).readlines())
with open(dicc, 'r') as passFile:
pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=nlines).start()
for line in passFile.readlines():
password = line.strip('\n')
r = commands.getoutput("steghide extract -sf %s -p '%s' -xf %s" % (ifile, password, ofile))
if not "no pude extraer" in r and not "could not extract" in r:
print(color.GREEN + "\n\n " + r + color.ENDC)
print("\n\n [+] " + color.INFO + "Information obtained with password:" + color.GREEN + " %s\n" % password + color.ENDC)
if check_file(ofile):
with open(ofile, 'r') as outfile:
for line in outfile.readlines():
print(line)
break
pbar.update(i + 1)
i += 1
def __init__(self, *args, **kwargs):
self.dld = FileDownloader()
self.dld.stage(self.cmd_name)
load_continents()
load_oceans()
load_currencies()
load_languages()
self.widgets = [
MemoryUsage(),
progressbar.ETA(),
' |Processed: ',
progressbar.Counter(),
' |Done: ',
progressbar.Percentage(),
progressbar.Bar(),
]
return super().__init__(*args, **kwargs)
def __init__(self, name, max_value=100, history_len=5, display=True,
display_data={'train':['loss', 'accuracy'], 'test':['loss', 'accuracy']},
level=logging.INFO, train_log_mode='TRAIN_PROGRESS', test_log_mode='TEST_PROGRESS'):
super(ProgressbarLogger, self).__init__(
name, level=level, display=display, logfile=None,
train_log_mode=train_log_mode, test_log_mode=test_log_mode)
self.train_log_data = {}
self.test_log_data = {}
self.max_value = max_value
self.history_len = history_len
self.display_data = display_data
self.mode['TRAIN_PROGRESS'] = self.log_train_progress
self.mode['TEST_PROGRESS'] = self.log_test_progress
# create logging format
self.widgets = [progressbar.FormatLabel('(%(value)d of %(max)s)'),
' ', progressbar.Percentage(),
' ', progressbar.Bar()]
self.dynamic_data = {k+'_'+kk: 0.0 for k in display_data.keys() for kk in display_data[k]}
diff_data = {'diff_'+k+'_'+kk: 0.0 for k in display_data.keys() for kk in display_data[k]}
self.dynamic_data.update(diff_data)
for t in display_data.keys():
ddstr = ' [' + t + ']'
for s in display_data[t]:
value_name = t + '_' + s
ddstr = ddstr + ' ' + s + ':' + '%(' + value_name + ').3f (%(diff_' + value_name + ').3f)'
self.widgets.append(progressbar.FormatLabel(ddstr))
self.widgets.extend(['|', progressbar.FormatLabel('Time: %(elapsed)s'), '|', progressbar.AdaptiveETA()])
def deleteHostsByHostgroup(groupname):
hostgroup = zapi.hostgroup.get(output=['groupid'],filter={'name': groupname})
if hostgroup.__len__() != 1:
logger.error('Hostgroup not found: %s\n\tFound this: %s' % (groupname,hostgroup))
groupid = int(hostgroup[0]['groupid'])
hosts = zapi.host.get(output=['name','hostid'],groupids=groupid)
total = len(hosts)
logger.info('Hosts found: %d' % (total))
if ( args.run ):
x = 0
bar = ProgressBar(maxval=total,widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
logger.echo = False
for host in hosts:
x = x + 1
bar.update(x)
logger.debug('(%d/%d) >> Removing >> %s' % (x, total, host))
out = zapi.globo.deleteMonitors(host['name'])
bar.finish()
logger.echo = True
else:
logger.info('No host removed due to --no-run arg. Full list of hosts:')
for host in hosts:
logger.info('%s' % host['name'])
return
def hosts_disable_all():
"""
status de host 0 = enabled
status de host 1 = disabled
"""
logger.info('Disabling all hosts, in blocks of 1000')
hosts = zapi.host.get(output=[ 'hostid' ], search={ 'status': 0 })
maxval = int(ceil(hosts.__len__())/1000+1)
bar = ProgressBar(maxval=maxval,widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
i = 0
for i in xrange(maxval):
block = hosts[:1000]
del hosts[:1000]
result = zapi.host.massupdate(hosts=[ x for x in block ], status=1)
i += 1
bar.update(i)
bar.finish()
logger.info('Done')
return
def proxy_passive_to_active():
"""
status de prxy 5 = active
status de prxy 6 = passive
"""
logger.info('Change all proxys to active')
proxys = zapi.proxy.get(output=[ 'shorten', 'host' ],
filter={ 'status': 6 })
if ( proxys.__len__() == 0 ):
logger.info('Done')
return
bar = ProgressBar(maxval=proxys.__len__(),widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
i = 0
for x in proxys:
i += 1
proxyid = x['proxyid']
result = zapi.proxy.update(proxyid=proxyid, status=5)
logger.echo = False
logger.debug('Changed from passive to active proxy: %s' % (x['host']))
bar.update(i)
bar.finish()
logger.echo = True
logger.info('Done')
return
def log_stats(self, generation, population, selected, n_mutated, scores):
"""
Stats to be logged:
Generation : Represant the iteration you're on
mutated : Represent the mutated population
mutation_percent : Percentage of mutation
fitness_ind : Fitness of all individue
group_fitness : Fitness of all groups
population : Population at this time.
score : One score for the Puzzle
record: fitness_ind and fitness_group compiled by stats. (min max avg)
:param generation: The current iteration you're on
:param population: The current population you're using
:param n_mutated: The number of mutated element.
:return:
"""
record = self.stats.compile(population)
self.populations.append(population)
self.logbook.record(generations=generation, mutated=n_mutated, mutation_percent=config.mutate_inpd,
nb_full_connected=len([x for x in population if x.fitness_ind.values[0] == 4]),
connections_completions=scores[0], score=scores[1], selected=selected, **record)
# I case we need to keep famous big scores.
# self.famous.update(self.pop)
def train(self, epochs, batch_size, learning_rate, save_to=None):
self.train_step = pt.apply_optimizer(tf.train.AdamOptimizer(learning_rate, epsilon=1), losses = [self.error_function])
init = tf.initialize_all_variables()
self.sess.run(init)
pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=epochs).start()
while self.get_epoch() < epochs:
input_data = self.hdf5reader.next()
_, loss_value = self.sess.run(
[self.train_step, self.error_function],
{
self.encoder.input_data: input_data
}
)
pbar.update(self.get_epoch())
pbar.finish()
def images_to_hdf5(dir_path, output_hdf5, size = (112,112), channels = 3, resize_to = None):
files = sorted(os.listdir(dir_path))
nr_of_images = len(files)
if resize_to:
size = resize_to
i = 0
pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=nr_of_images).start()
data = np.empty(shape=(nr_of_images, size[0], size[1], channels), dtype=np.uint8)
for f in files:
datum = imread(dir_path + '/' + f)
if resize_to:
datum = np.asarray(Image.fromarray((datum), 'RGB').resize((size[0],size[1]), PIL.Image.ANTIALIAS))
data[i,:,:,:] = datum
i = i + 1
pbar.update(i)
pbar.finish()
with h5py.File(output_hdf5, 'w') as hf:
hf.create_dataset('data', data=data)
def load_corpus(self, corenlpserver, process=True):
"""
Use the PubMed web services to retrieve the title and abstract of each PMID
:param corenlpserver:
:param process:
:return:
"""
time_per_abs = []
widgets = [pb.Percentage(), ' ', pb.Bar(), ' ', pb.AdaptiveETA(), ' ', pb.Timer()]
pbar = pb.ProgressBar(widgets=widgets, maxval=len(self.pmids), redirect_stdout=True).start()
for i, pmid in enumerate(self.pmids):
t = time.time()
newdoc = PubmedDocument(pmid)
if newdoc.abstract == "":
logging.info("ignored {} due to the fact that no abstract was found".format(pmid))
continue
newdoc.process_document(corenlpserver, "biomedical")
self.documents["PMID" + pmid] = newdoc
abs_time = time.time() - t
time_per_abs.append(abs_time)
pbar.update(i+1)
pbar.finish()
abs_avg = sum(time_per_abs)*1.0/len(time_per_abs)
logging.info("average time per abstract: %ss" % abs_avg)
def getProgress(self, url, fileSize):
status = json.loads(urllib.urlopen(url).read())
if len(status["data"]) ==0 :
logger.info(url + " upload done ")
return True
widgets = ['Progress: ', Percentage(), ' ', Bar(
marker=RotatingMarker('>-=')), ' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets, maxval=fileSize).start()
upload_size = 0
while upload_size < fileSize:
_response = self.doGet(url)
_data = json.loads(_response)
upload_size = long(_data["data"]["upload_size"])
total_size = long(_data["data"]["total_size"])
if upload_size == 0 and total_size == 0:
break
pbar.update(upload_size)
time.sleep(1)
pbar.finish()
logger.info(url + " upload done")
return True
"""
??????
"""
def scrape_mlb_odds_range(min_date=None, max_date=None):
min_date = min_date or datetime.datetime.today() - datetime.timedelta(days=1)
max_date = max_date or datetime.datetime.today()
if isinstance(min_date, basestring):
min_date = parser.parse(min_date)
if isinstance(max_date, basestring):
max_date = parser.parse(max_date)
date = min_date
pbar = progressbar.ProgressBar(widgets=[progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA()],
maxval=int((max_date-min_date).total_seconds() / (60*60*24)) + 1)
pbar.start()
saved = 0
hit = 0
while date <= max_date:
day_odds = load_odds_for_day(date)
if day_odds is not None and len(day_odds) > 0:
save_sbr_odds_info('mlb', date, day_odds)
saved += 1
hit += 1
date += datetime.timedelta(days=1)
pbar.update(value=hit)
pbar.finish()
return saved
def scrape_nba_odds_range(min_date=None, max_date=None):
min_date = min_date or datetime.datetime.today() - datetime.timedelta(days=1)
max_date = max_date or datetime.datetime.today()
if isinstance(min_date, basestring):
min_date = parser.parse(min_date)
if isinstance(max_date, basestring):
max_date = parser.parse(max_date)
date = min_date
pbar = progressbar.ProgressBar(widgets=[progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA()],
maxval=int((max_date-min_date).total_seconds() / (60*60*24)) + 1)
pbar.start()
saved = 0
hit = 0
while date <= max_date:
day_odds = load_odds_for_day(date)
if day_odds is not None and len(day_odds) > 0:
save_sbr_odds_info('nba', date, day_odds)
saved += 1
hit += 1
date += datetime.timedelta(days=1)
pbar.update(value=hit)
pbar.finish()
return saved
def __iter__(self):
if self.count != 0:
widgets = [
'%s: ' % (self.caption,),
progressbar.Percentage(),
' ',
progressbar.Bar(),
' ',
progressbar.ETA(),
]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=self.count)
pbar.start()
for idx, item in enumerate(self.iterator):
yield item
pbar.update(idx)
pbar.finish()
def _setup_progress(self, options):
if options.progress:
if self.beanstalk:
# With Beanstalk C&C we don't know how many...
self.progress = progressbar.ProgressBar(
redirect_stdout=True,
redirect_stderr=True,
widgets=[
'Total: ',
progressbar.Counter(),
', ',
progressbar.Timer()
])
else:
self.progress = progressbar.ProgressBar(
redirect_stdout=True,
redirect_stderr=True,
widgets=[
progressbar.Percentage(),
progressbar.Bar(),
' (', progressbar.ETA(), ') ',
])
else:
self.progress = None
def __init__(self, options):
self.wildcards = []
self.options = options
self.domains = []
if options.domains:
self.domains += filter(None, options.domains.read().split("\n"))
self.domains += options.domain
self.domains = list(set(self.domains))
random.shuffle(self.domains)
self.resolvers = map(str.strip, filter(None, options.resolvers.read().split("\n")))
random.shuffle(self.resolvers)
self.names = [X for X in self._load_names(options.names)]
if options.progress:
self.progress = progressbar.ProgressBar(
redirect_stdout=True,
redirect_stderr=True,
widgets=[
progressbar.Percentage(),
progressbar.Bar(),
' (', progressbar.ETA(), ') ',
])
else:
self.progress = None
self.finished = 0
LOG.info("%d names, %d resolvers, %d domains",
len(self.names), len(self.resolvers), len(self.domains))
def compute_embeddings(images):
"""Runs inference on an image.
Args:
image: Image file names.
Returns:
Dict mapping image file name to embedding.
"""
# Creates graph from saved GraphDef.
create_graph()
filename_to_emb = {}
config = tf.ConfigProto(device_count = {'GPU': 0})
bar = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
with tf.Session(config=config) as sess:
i = 0
for image in bar(images):
if not tf.gfile.Exists(image):
tf.logging.fatal('File does not exist %s', image)
image_data = tf.gfile.FastGFile(image, 'rb').read()
# Some useful tensors:
# 'softmax:0': A tensor containing the normalized prediction across
# 1000 labels.
# 'pool_3:0': A tensor containing the next-to-last layer containing 2048
# float description of the image.
# 'DecodeJpeg/contents:0': A tensor containing a string providing JPEG
# encoding of the image.
# Runs the softmax tensor by feeding the image_data as input to the graph.
softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')
embedding_tensor = sess.graph.get_tensor_by_name('pool_3:0')
embedding = sess.run(embedding_tensor,
{'DecodeJpeg/contents:0': image_data})
filename_to_emb[image] = embedding.reshape(2048)
i += 1
# print(image, i, len(images))
return filename_to_emb
# temp_dir is a subdir of temp
def main(project_id, video_basename, sampling_rate=3):
# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' # or any {'0', '1', '2'}
video_name = video_basename[:video_basename.index('.')]
# extract video frames
extracted_frame_dir = os.path.join('temp', project_id, video_name, 'frames')
mkdir_p(extracted_frame_dir)
if not os.path.isdir(extracted_frame_dir):
os.mkdir(extracted_frame_dir)
video_path = os.path.join('videos', project_id, video_basename)
vidcap = cv2.VideoCapture(video_path)
print('Extracting video frames...')
bar = progressbar.ProgressBar(maxval=101, widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
bar.start()
fps = vidcap.get(CV_CAP_PROP_FPS)# TODO
fps = fps if fps != float('nan') else 25
print 'actual fps', fps, 'sampling rate', sampling_rate
success, image = vidcap.read()
frames_to_extract = range(0, int(vidcap.get(CV_CAP_PROP_FRAME_COUNT)), int(round(fps / sampling_rate)))
frame_count = len(frames_to_extract)
for frame_pos in bar(frames_to_extract):
vidcap.set(CV_CAP_PROP_POS_FRAMES, frame_pos)
success, image = vidcap.read()
# print('Read a new frame: %f ms'% vidcap.get(CV_CAP_PROP_POS_MSEC), success)
cv2.imwrite(os.path.join(extracted_frame_dir, "%09d.jpg" % vidcap.get(CV_CAP_PROP_POS_MSEC)), image) # TODO (might still work)
bar.finish()
def download(number, save_dir='./'):
"""Download pre-trained word vector
:param number: integer, default ``None``
:param save_dir: str, default './'
:return: file path for downloaded file
"""
df = load_datasets()
row = df.iloc[[number]]
url = ''.join(row.URL)
if not url:
print('The word vector you specified was not found. Please specify correct name.')
widgets = ['Test: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets)
def dlProgress(count, blockSize, totalSize):
if pbar.max_value is None:
pbar.max_value = totalSize
pbar.start()
pbar.update(min(count * blockSize, totalSize))
file_name = url.split('/')[-1]
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path = os.path.join(save_dir, file_name)
path, _ = urlretrieve(url, save_path, reporthook=dlProgress)
pbar.finish()
return path
def __enter__(self):
self.bar = progressbar.ProgressBar(
widgets=[
progressbar.Percentage(),
' ',
progressbar.Bar(),
progressbar.FileTransferSpeed(),
' ',
progressbar.ETA(),
],
max_value=self.max_value,
)
self.fd = open(self.output_path, 'wb')
return self
def train(self):
data = Data(self.train_dat, self.train_lab)
batch_num = self.length/self.batch_size if self.length%self.batch_size == 0 else self.length/self.batch_size + 1
model = self.add_model()
with self.sess as sess:
tf.initialize_all_variables().run()
for ite in range(self.iterations):
print "Iteration {}".format(ite)
cost = 0.
pbar = pb.ProgressBar(widgets=[pb.Percentage(), pb.Bar(), pb.ETA()], maxval=batch_num).start()
for i in range(batch_num):
batch_x, batch_y = data.next_batch(self.batch_size)
c, _ = self.sess.run([model['loss'], model['optimizer']], feed_dict={model['train_x']:batch_x, model['train_y']:batch_y, model['p_keep_dens']:0.75})
cost += c / batch_num
pbar.update(i+1)
pbar.finish()
print ">>cost: {}".format(cost)
t_acc, d_acc = self.eval(model, 3000)
# early stop
if t_acc >= 0.995 and d_acc >= 0.995:
break
self.predict(model)
def get_pbar(num, prefix=""):
assert isinstance(prefix, str)
pbar = pb.ProgressBar(widgets=[prefix, pb.Percentage(), pb.Bar(), pb.ETA()], maxval=num)
return pbar
def __init__(self, nbytes, nfiles):
self._total_bytes = nbytes
self._pending_files = nfiles
self._transferring_files = 0
self._complete_files = 0
self._lock = Lock()
self._data = {}
widgets = ['Progress: ', Percentage(), ' ', Bar(left='[',right=']'),
' ', Timer(format='Time: %s'), ' ', FileTransferSpeed()]
if self._total_bytes > 0:
self.pbar = ProgressBar(widgets=widgets, maxval=self._total_bytes).start()
else:
self.pbar = ProgressBar(widgets=widgets, maxval=nfiles).start()
