def main():
uri, outfile, dataset = get_arguments()
fd = tempfile.NamedTemporaryFile()
progress = ProgressBar(widgets=[Percentage(), ' ', Bar(), ' ', ETA(), ' ', FileTransferSpeed()])
def update(count, blockSize, totalSize):
if progress.maxval is None:
progress.maxval = totalSize
progress.start()
progress.update(min(count * blockSize, totalSize))
urllib.urlretrieve(uri, fd.name, reporthook = update)
if dataset == 'zinc12':
df = pandas.read_csv(fd.name, delimiter = '\t')
df = df.rename(columns={'SMILES':'structure'})
df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
elif dataset == 'chembl22':
df = pandas.read_table(fd.name,compression='gzip')
df = df.rename(columns={'canonical_smiles':'structure'})
df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
pass
else:
df = pandas.read_csv(fd.name, delimiter = '\t')
df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
python类FileTransferSpeed()的实例源码
def getProgress(self, url, fileSize):
status = json.loads(urllib.urlopen(url).read())
if len(status["data"]) ==0 :
logger.info(url + " upload done ")
return True
widgets = ['Progress: ', Percentage(), ' ', Bar(
marker=RotatingMarker('>-=')), ' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets, maxval=fileSize).start()
upload_size = 0
while upload_size < fileSize:
_response = self.doGet(url)
_data = json.loads(_response)
upload_size = long(_data["data"]["upload_size"])
total_size = long(_data["data"]["total_size"])
if upload_size == 0 and total_size == 0:
break
pbar.update(upload_size)
time.sleep(1)
pbar.finish()
logger.info(url + " upload done")
return True
"""
??????
"""
def download(number, save_dir='./'):
"""Download pre-trained word vector
:param number: integer, default ``None``
:param save_dir: str, default './'
:return: file path for downloaded file
"""
df = load_datasets()
row = df.iloc[[number]]
url = ''.join(row.URL)
if not url:
print('The word vector you specified was not found. Please specify correct name.')
widgets = ['Test: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets)
def dlProgress(count, blockSize, totalSize):
if pbar.max_value is None:
pbar.max_value = totalSize
pbar.start()
pbar.update(min(count * blockSize, totalSize))
file_name = url.split('/')[-1]
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path = os.path.join(save_dir, file_name)
path, _ = urlretrieve(url, save_path, reporthook=dlProgress)
pbar.finish()
return path
def __enter__(self):
self.bar = progressbar.ProgressBar(
widgets=[
progressbar.Percentage(),
' ',
progressbar.Bar(),
progressbar.FileTransferSpeed(),
' ',
progressbar.ETA(),
],
max_value=self.max_value,
)
self.fd = open(self.output_path, 'wb')
return self
def __init__(self, nbytes, nfiles):
self._total_bytes = nbytes
self._pending_files = nfiles
self._transferring_files = 0
self._complete_files = 0
self._lock = Lock()
self._data = {}
widgets = ['Progress: ', Percentage(), ' ', Bar(left='[',right=']'),
' ', Timer(format='Time: %s'), ' ', FileTransferSpeed()]
if self._total_bytes > 0:
self.pbar = ProgressBar(widgets=widgets, maxval=self._total_bytes).start()
else:
self.pbar = ProgressBar(widgets=widgets, maxval=nfiles).start()
def setup_progressbar(self):
from progressbar import ProgressBar, FileTransferSpeed, Bar, Percentage, ETA
return ProgressBar(widgets=[FileTransferSpeed(), ' <<<', Bar(), '>>> ', Percentage(), ' ', ETA()])
def doUploadFileProgress(self,filePath, url):
startTime = getNow()
result = False
try:
widgets = ['Progress: ', Percentage(), ' ', Bar(
marker=RotatingMarker('>-=')), ' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets, maxval=os.path.getsize(filePath)).start()
progress = Progress()
fileSizeStr = formatSize(os.path.getsize(filePath))
logger.info("??????{0} ?? {1}".format(filePath,fileSizeStr))
stream = file_with_callback(filePath, 'rb', progress.update,os.path.basename(filePath),pbar)
params = {"filedata": stream}
datagen, headers = multipart_encode(params)
upload_request =urllib2.Request(url, datagen, headers)
response = urllib2.urlopen(upload_request).read()
endTime = getNow()
totlaTime = caltime(startTime, endTime)
logger.info("??????{0} ????{1} ????{2} ????{3} ??{4} ????{5}"
.format(filePath,startTime, endTime, fileSizeStr, totlaTime,response))
#???????????'4b ? 0'???json ??errmsg("Extra data", s, end, len(s) ??????,??????????
if "code\":0" in response.replace(' ', ''):
result = True
else:
result = json.loads(response)["code"] == 0
except Exception as e:
logger.error("??????{0} exception: {1}".format(filePath,e))
return result
def _addresses_to_check_with_caching(self, show_progress=True):
num_addrs = len(list(self._addresses_to_check()))
widgets = ['ROP: ', progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()),
' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
progress = progressbar.ProgressBar(widgets=widgets, maxval=num_addrs)
if show_progress:
progress.start()
self._cache = dict()
seen = dict()
for i, a in enumerate(self._addresses_to_check()):
if show_progress:
progress.update(i)
try:
bl = self.project.factory.block(a)
if bl.size > self._max_block_size:
continue
block_data = bl.bytes
except (SimEngineError, SimMemoryError):
continue
if block_data in seen:
self._cache[seen[block_data]].add(a)
continue
else:
if len(bl.vex.constant_jump_targets) == 0 and not self._block_has_ip_relative(a, bl):
seen[block_data] = a
self._cache[a] = set()
yield a
if show_progress:
progress.finish()
def upload(self, data):
"""
???????? ?????? ? ????????? (???????? ?? send)
"""
if options.progress:
written = 0
start = int(time.time())
bar = None
try:
total = int(self._content_length)
if pclProgressBar:
try:
widgets = ["--> Upload: ", pclProgressBar.Percentage(), " ", pclProgressBar.Bar(left = "[", marker = "=", right = "]"), " ", pclProgressBar.ETA(), " ", pclProgressBar.FileTransferSpeed()]
bar = pclProgressBar.ProgressBar(widgets = widgets, maxval = total).start()
except:
total = pcl_human(total)
else:
total = pcl_human(total)
except:
total = "-"
datablock = data.read(self._options.chunk)
while datablock:
self.sock.sendall(datablock)
if self._options.progress:
written += len(datablock)
if bar:
bar.update(written)
else:
delta = int(time.time()) - start
if delta > 0:
sys.stderr.write("--> Upload: {0}/{1} ({2}/s){3}\r".format(pcl_human(written), total, pcl_human(written / delta), " " * 12))
datablock = data.read(self._options.chunk)
if self._options.progress:
if bar:
bar.finish()
else:
sys.stderr.write("{0}\r".format(" " * 33))
def pcl_query_download(options, response, filename):
"""
???????? ????? ?? ?????????
?????????:
options (pclOptions) -- ????? ??????????
response (HTTPResponse) -- HTTP ?????
filename (str) -- ??? ?????????? ????? ??? ??????
"""
if options.progress:
read = 0
start = int(time.time())
bar = None
try:
total = int(response.info().get("Content-Length"))
if pclProgressBar:
try:
widgets = ["--> Download: ", pclProgressBar.Percentage(), " ", pclProgressBar.Bar(left = "[", marker = "=", right = "]"), " ", pclProgressBar.ETA(), " ", pclProgressBar.FileTransferSpeed()]
bar = pclProgressBar.ProgressBar(widgets = widgets, maxval = total).start()
except:
total = pcl_human(total)
else:
total = pcl_human(total)
except:
total = "-"
with open(filename, "wb") as fd:
while True:
part = response.read(options.chunk)
if not part:
break
fd.write(part)
if options.progress:
read += len(part)
if bar:
bar.update(read)
else:
delta = int(time.time()) - start
if delta > 0:
sys.stderr.write("--> Download: {0}/{1} ({2}/s){3}\r".format(pcl_human(read), total, pcl_human(read / delta), " " * 12))
if options.progress:
if bar:
bar.finish()
else:
sys.stderr.write("{0}\r".format(" " * 35))
def run_collapse(self, hosts, cmd):
progress = None
if self.progressbar:
from progressbar import ProgressBar, Percentage, Bar, ETA, FileTransferSpeed
progress = ProgressBar(
widgets=["Running: ", Percentage(), ' ', Bar(marker='.'), ' ', ETA(), ' ', FileTransferSpeed()],
maxval=len(hosts))
codes = {"total": 0, "error": 0, "success": 0}
outputs = defaultdict(list)
def worker(host, cmd):
p = Popen(self.get_parallel_ssh_options(host, cmd), stdout=PIPE, stderr=PIPE)
o = ""
while True:
outs, _, _ = select([p.stdout, p.stderr], [], [])
outline = errline = ""
if p.stdout in outs:
outline = p.stdout.readline()
if p.stderr in outs:
errline = p.stderr.readline()
o += outline + errline
if outline == "" and errline == "" and p.poll() is not None:
break
if o == "":
o = colored("[ No Output ]\n", "yellow")
outputs[o].append(host)
if p.poll() == 0:
codes["success"] += 1
else:
codes["error"] += 1
codes["total"] += 1
if self.progressbar:
progress.update(codes["total"])
pool = Pool(self.ssh_threads)
if self.progressbar:
progress.start()
for host in hosts:
pool.start(Greenlet(worker, host, cmd))
try:
pool.join()
except KeyboardInterrupt:
pass
if self.progressbar:
progress.finish()
self.print_exec_results(codes)
print()
for output, hosts in outputs.items():
msg = " %s " % ','.join(hosts)
table_width = min([len(msg) + 2, terminal_size()[0]])
cprint("=" * table_width, "blue", attrs=["bold"])
cprint(msg, "blue", attrs=["bold"])
cprint("=" * table_width, "blue", attrs=["bold"])
print(output)
