def run(argv):
gencode_filepath = get_generated_path('sites/genes/gencode-{}.gtf.gz'.format(genes_version))
genes_filepath = common_filepaths['genes']
if not os.path.exists(genes_filepath):
print('genes-{}.bed will be stored at {!r}'.format(genes_version, genes_filepath))
if not os.path.exists(gencode_filepath):
make_basedir(gencode_filepath)
wget.download(
url="ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_25/GRCh37_mapping/gencode.v25lift37.annotation.gtf.gz",
out=gencode_filepath
)
print('')
genes = get_all_genes(gencode_filepath)
genes = dedup_ensg(genes)
genes = dedup_symbol(genes)
make_basedir(genes_filepath)
with open(genes_filepath, 'w') as f:
writer = csv.DictWriter(f, delimiter='\t', fieldnames='chrom start end symbol ensg'.split(), lineterminator='\n')
writer.writerows(genes)
else:
print("gencode is at {!r}".format(genes_filepath))
python类download()的实例源码
def download_content_file(self,dir_name,url):
working_directory = os.getcwd()
filename = None
#if directory exists just catch error
try:
os.mkdir(dir_name)
except:
pass
try:
os.chdir(dir_name)
filename = wget.download(url)
except Exception as inst:
print("had exception on wget: ", inst)
pass
#reset directory
os.chdir(working_directory)
return filename
def get_data():
"""
Download the image,depth and segmentation data:
Returns, the h5 database.
"""
if not osp.exists(DB_FNAME):
try:
colorprint(Color.BLUE,'\tdownloading data (56 M) from: '+DATA_URL,bold=True)
print
sys.stdout.flush()
out_fname = 'data.tar.gz'
wget.download(DATA_URL,out=out_fname)
tar = tarfile.open(out_fname)
tar.extractall()
tar.close()
os.remove(out_fname)
colorprint(Color.BLUE,'\n\tdata saved at:'+DB_FNAME,bold=True)
sys.stdout.flush()
except:
print colorize(Color.RED,'Data not found and have problems downloading.',bold=True)
sys.stdout.flush()
sys.exit(-1)
# open the h5 file and return:
return h5py.File(DB_FNAME,'r')
def configure(self, app_id, trigger, resources, dependencies):
"""
This method is meant to download the resources, resolve the dependencies, and prepare the app environment
:param app_id:
:param trigger:
:param resources:
:param dependencies:
:return:
"""
instance = self.get_instance()
instance.set_id(app_id)
instance.set_command(trigger)
self.create_env(instance)
self.get_resources(instance, resources)
self.resolve_dependencies(dependencies)
return instance
#def start(self, instance):
# self.spawn(instance.get_command())
# return instance
def download_file(self, url, path):
"""
logic that does all the work when downloading the resources
:param url:
:param path:
:return:
"""
file_name = url.split("/")[-1]
file_location = path + "/" + file_name
try:
os.remove(file_location)
except OSError:
pass
wget.download(url, out=file_location, bar=None)
return file_location
def Downloadfiles(urls_metadata,output,target):
path = None
try:
filename = None
print "\nDo you like downloading these files to analyze metadata(Y/N)?"
#Convert to lower the input
resp = raw_input().lower()
if (resp == 'n'):
print "Exiting"
exit(1)
if ((resp != 'y') and (resp != 'n')):
print "The option is not valided. Please, try again it"
if (resp =='y'):
path = str(target) + '/temp'
for url in urls_metadata:
filename = wget.download(url,str(path))
Analyze_Metadata(filename)
time.sleep(3)
#Delete temp folder
os.system('rm -r ' + str(path))
Display_Export_Metadata(metadata_files,output,target)
except Exception as e:
print str(e)
def Downloadfiles(urls_metadata):
print "\nDo you like downloading these files to analyze metadata(Y/N)?"
#try:
resp = raw_input()
if (resp == 'N'):
print "Exiting"
exit(1)
if ((resp != 'Y') and (resp != 'N')):
print "The option is not valided. Please, try again it"
if (resp =='Y'):
print "Indicate the location where keep the files downloaded"
path = raw_input()
try:
for url in urls_metadata:
try:
filename = wget.download(url,path)
Analyze_Metadata(filename)
except:
pass
Display_Export_Metadata(metadata_files)
except:
pass
#********************************************************#
#Definition and treatment of the parameters
def Downloadfiles(urls_metadata):
print "\nDo you like downloading these files to analyze metadata(Y/N)?"
#try:
resp = raw_input()
if (resp == 'N'):
print "Exiting"
exit(1)
if ((resp != 'Y') and (resp != 'N')):
print "The option is not valided. Please, try again it"
if (resp =='Y'):
try:
for url in urls_metadata:
try:
filename= wget.download(url,"/opt/")
Analyze_Metadata(filename)
except Exception, e:
print e
except:
pass
#********************************************************#
#Definition and treatment of the parameters
def Downloadfiles(urls_metadata):
print "\nDo you like downloading these files to analyze metadata(Y/N)?"
#try:
resp = raw_input()
if (resp == 'N'):
print "Exiting"
exit(1)
if ((resp != 'Y') and (resp != 'N')):
print "The option is not valided. Please, try again it"
if (resp =='Y'):
print "Indicate the location where keep the files downloaded"
path = raw_input()
try:
for url in urls_metadata:
try:
filename = wget.download(url,path)
Analyze_Metadata(filename)
except:
pass
Display_Export_Metadata(metadata_files)
except:
pass
#********************************************************#
#Definition and treatment of the parameters
def Downloadfiles(urls_metadata):
print "\nDo you like downloading these files to analyze metadata(Y/N)?"
#try:
resp = raw_input()
if (resp == 'N'):
print "Exiting"
exit(1)
if ((resp != 'Y') and (resp != 'N')):
print "The option is not valided. Please, try again it"
if (resp =='Y'):
try:
for url in urls_metadata:
try:
filename= wget.download(url,"/opt/")
Analyze_Metadata(filename)
except Exception, e:
print e
except:
pass
#********************************************************#
#Definition and treatment of the parameters
def Downloadfiles(urls_metadata):
print "\nDo you like downloading these files to analyze metadata(Y/N)?"
#try:
resp = raw_input().lower()
if (resp == 'n'):
print "Exiting"
exit(1)
if ((resp != 'y') and (resp != 'n')):
print "The option is not valided. Please, try again it"
if (resp =='y'):
print "Indicate the location where you want to keep the files downloaded"
path = raw_input()
try:
for url in urls_metadata:
try:
filename= wget.download(url,path)
Analyze_Metadata(filename)
except Exception, e:
print e
except:
pass
#********************************************************#
#Definition and treatment of the parameters
def main():
root_path = 'an4/'
name = 'an4'
wget.download('http://www.speech.cs.cmu.edu/databases/an4/an4_raw.bigendian.tar.gz')
tar = tarfile.open('an4_raw.bigendian.tar.gz')
tar.extractall()
os.makedirs(args.target_dir)
_format_data(root_path, 'train', name, 'an4_clstk')
_format_data(root_path, 'test', name, 'an4test_clstk')
shutil.rmtree(root_path)
os.remove('an4_raw.bigendian.tar.gz')
train_path = args.target_dir + '/train/'
test_path = args.target_dir + '/test/'
print ('\n', 'Creating manifests...')
create_manifest(train_path, 'an4_train')
create_manifest(test_path, 'an4_val')
def download_figshare(file_name, file_ext, dir_path='./', change_name = None):
prepare_data_dir(dir_path)
url = 'https://ndownloader.figshare.com/files/' + file_name
wget.download(url, out=dir_path)
file_path = os.path.join(dir_path, file_name)
if file_ext == '.zip':
zip_ref = zipfile.ZipFile(file_path,'r')
if change_name is not None:
dir_path = os.path.join(dir_path, change_name)
zip_ref.extractall(dir_path)
zip_ref.close()
os.remove(file_path)
elif file_ext == '.tar.bz2':
tar_ref = tarfile.open(file_path,'r:bz2')
if change_name is not None:
dir_path = os.path.join(dir_path, change_name)
tar_ref.extractall(dir_path)
tar_ref.close()
os.remove(file_path)
elif change_name is not None:
os.rename(file_path, os.path.join(dir_path, change_name))
# Download QM9 dataset
def run(argv):
if not os.path.exists(clean_filepath):
print('dbsnp will be stored at {!r}'.format(clean_filepath))
if not os.path.exists(raw_filepath):
# dbSNP downloads are described at <https://www.ncbi.nlm.nih.gov/variation/docs/human_variation_vcf/>
# This file includes chr-pos-ref-alt-rsid and 4X a bunch of useless columns:
dbsnp_url = 'ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606_b150_GRCh37p13/VCF/00-All.vcf.gz'
print('Downloading dbsnp!')
make_basedir(raw_filepath)
raw_tmp_filepath = get_tmp_path(raw_filepath)
wget.download(url=dbsnp_url, out=raw_tmp_filepath)
print('')
os.rename(raw_tmp_filepath, raw_filepath)
print('Done downloading.')
print('Converting {} -> {}'.format(raw_filepath, clean_filepath))
make_basedir(clean_filepath)
clean_tmp_filepath = get_tmp_path(clean_filepath)
run_script(r'''
gzip -cd '{raw_filepath}' |
grep -v '^#' |
perl -F'\t' -nale 'print "$F[0]\t$F[1]\t$F[2]\t$F[3]\t$F[4]"' | # Gotta declare that it's tab-delimited, else it's '\s+'-delimited I think.
gzip > '{clean_tmp_filepath}'
'''.format(raw_filepath=raw_filepath, clean_tmp_filepath=clean_tmp_filepath))
os.rename(clean_tmp_filepath, clean_filepath)
print("dbsnp is at '{clean_filepath}'".format(clean_filepath=clean_filepath))
def download_ref(build, chrom):
'''Download a chromosome reference file, and remove the header and newlines so we can seek to positions.'''
dest_filepath = ref_filepath(build, chrom, download=False)
if os.path.exists(dest_filepath):
return
dl_filepath = get_tmp_path('dl-chrom-{}-{}'.format(build['hg'], chrom))
if not os.path.exists(dl_filepath):
dl_tmp_filepath = get_tmp_path(dl_filepath)
url = 'ftp://hgdownload.cse.ucsc.edu/goldenPath/{}/chromosomes/chr{}.fa.gz'.format(build['hg'], chrom)
wget.download(url=url, out=dl_tmp_filepath)
print('')
os.rename(dl_tmp_filepath, dl_filepath)
tmp_filepath = get_tmp_path(dest_filepath)
run_script(r'''
gzip -cd '{dl_filepath}' |
tail -n +2 |
tr -d "\n" > '{tmp_filepath}'
'''.format(dl_filepath=dl_filepath, tmp_filepath=tmp_filepath))
os.rename(tmp_filepath, dest_filepath)
print("ref is at", dest_filepath)
def ref_filepath(build, chrom, download=True):
filepath = get_cacheable_file_location('ref', 'reference-{}-chrom-{}.fa'.format(build['hg'], chrom))
if download: download_ref(build, chrom)
return filepath
def download_ckpt(url):
fname = wget.download(url)
with tarfile.open(fname) as tar:
tar.extractall()
ckpt_name = tar.getnames()[0]
if not os.path.exists(FLAGS.checkpoints_dir):
os.makedirs(FLAGS.checkpoints_dir)
shutil.move(ckpt_name, FLAGS.checkpoints_dir)
os.remove(fname)
def download_file(url, directory='./', local_fname=None, force_write=False):
"""Download the data from source url, unless it's already here.
Args:
filename: string, name of the file in the directory.
work_directory: string, path to working directory.
source_url: url to download from if file doesn't exist.
Returns:
Path to resulting file.
"""
if not os.path.isdir(directory):
os.mkdir(directory)
if local_fname is None:
local_fname = url.split('/')[-1]
local_fname = os.path.join(directory, local_fname)
if os.path.exists(local_fname) and not force_write:
print ("File [{}] existed!".format(local_fname))
return local_fname
print ("Downloading file [{}] from [{}]".format(local_fname, url))
try:
import wget
return wget.download(url, local_fname)
except:
return _single_thread_download(url, local_fname)
file_download_operations.py 文件源码
项目：netsocadmin2
作者: UCCNetworkingSociety
项目源码
文件源码
阅读 23
收藏 0
点赞 0
评论 0
def download_to(url, path_to_dir):
"""
Downloads a file from a given to a target directory.
Returns the file name if the downloaded file.
"""
logger.debug("downloading file from %s to %s" % (url, path_to_dir))
filename = wget.download(url, out=path_to_dir, bar=None)
return filename
def run(self):
"""Runs install.
"""
self.initOptions()
#Try to download gmsh
if self.dGmsh:
self.downloadGmsh()
else:
self.gmshDownloaded=False
#Try to download fiji
if self.dFiji:
self.downloadFiji()
else:
self.fijiDownloaded=False
#Run setuptools install
install.run(self)
#Print log info
log.info("Overriding setuptools mode of scripts ...")
#Add Data and edit file permissions
self.addData()
def downloadFileIfNotExist(self,url):
"""Downloads URL if file does not already exist.
Args:
url (str): URL to download.
Returns:
tuple: Tuple containing:
* fnDL (str): Donwload filename
* folderFn (str): Filename of extracted download files
"""
import wget
cwd=os.getcwd()+"/"
if not os.path.exists(cwd+os.path.basename(url)):
print cwd+os.path.basename(url) +" does not exist, will download it."
folderFn=wget.download(url)
else:
print cwd+os.path.basename(url) +" alreay exists, will not download."
folderFn=os.path.basename(url)
fnDL=str(folderFn)
print
return folderFn, fnDL
def downloadGmshWin(self,arch,gmshVersion):
"""Downloads Gmsh from Gmsh website for Windows
Args:
arch (str): System architecture, e.g. 64/32.
gmshVersion (str): gmshVersion String, e.g. 2.12.0 .
Returns:
tuple: Tuple containing:
* fnDL (str): Donwload filename
* folderFn (str): Filename of extracted download files
"""
#Download Gmsh
url='http://gmsh.info/bin/Windows/gmsh-'+gmshVersion+'-Windows'+arch+'.zip'
folderFn, fnDL=self.downloadFileIfNotExist(url)
#Decompress
import zipfile
with zipfile.ZipFile(folderFn) as zf:
zf.extractall()
folderFn='gmsh-'+gmshVersion+'-Windows'
self.gmshPath='executables/gmsh/bin/gmsh.exe'
return fnDL,folderFn
def downloadFijiLinux(self,arch):
"""Downloads Fiji from Fiji website for Linux.
Args:
arch (str): System architecture, e.g. 64/32.
Returns:
tuple: Tuple containing:
* fnDL (str): Donwload filename
* folderFn (str): Filename of extracted download files
"""
import wget
#Download Fiji
url='http://downloads.imagej.net/fiji/latest/fiji-linux'+arch+'.zip'
folderFn=wget.download(url)
fnDL=str(folderFn)
print
#Decompress
import zipfile
with zipfile.ZipFile(folderFn) as zf:
zf.extractall()
folderFn='Fiji.app'
self.fijiPath='executables/Fiji.app/./ImageJ-linux64'
return fnDL,folderFn
def downloadFijiWin(self,arch):
"""Downloads Fiji from Fiji website for Windows.
Args:
arch (str): System architecture, e.g. 64/32.
Returns:
tuple: Tuple containing:
* fnDL (str): Donwload filename
* folderFn (str): Filename of extracted download files
"""
import wget
#Download fiji
url='http://downloads.imagej.net/fiji/latest/fiji-win'+arch+'.zip'
folderFn=wget.download(url)
fnDL=str(folderFn)
print
#Decompress
import zipfile
with zipfile.ZipFile(folderFn) as zf:
zf.extractall()
folderFn='Fiji.app'
self.fijiPath='executables/Fiji.app/ImageJ-linux64.exe'
return fnDL,folderFn
def downloadFijiOSX(self,arch):
"""Downloads Fiji from Fiji website for OSX.
Returns:
tuple: Tuple containing:
* fnDL (str): Donwload filename
* folderFn (str): Filename of extracted download files
"""
import wget
#Download fiji
url='http://downloads.imagej.net/fiji/latest/fiji-macosx.dmg'
folderFn=wget.download(url)
fnDL=str(folderFn)
print
#Mount dmg file
os.system('hdiutil attach '+folderFn)
cwd=os.getcwd()
#Copy fiji executable to cwd
os.system('cp -rv /Volumes/Fiji/Fiji.app '+ cwd)
#Unmount gmsh
os.system('hdiutil detach /Volumes/Fiji')
folderFn='Fiji.app'
self.fijiPath='executables/Fiji.app/Contents/MacOS/./ImageJ-macosx'
return fnDL,folderFn
def add_arguments(self, parser):
parser.add_argument("--reset", dest="reset", action="store_true",
help="If true, clear the list of images and download all of them again")
def ShowResults(newlist,num_files,target):
print "Files in the target "+target+" are:\n"
print "Files indexed:", len (urls_final)
for i in urls_final:
if i not in newlist:
newlist.append(i)
print i
#verify if the user wants to export results
if output == 'Y':
#Only it can enter if -j is put in the execution
ExportResults(newlist)
#Call to function to download the files
Downloadfiles(newlist)
#MAIN
def ShowResults(newlist,num_files,target):
print "Files in the target "+target+" are:\n"
print "Files indexed:", len (urls_final)
for i in urls_final:
if i not in newlist:
newlist.append(i)
print i
#verify if the user wants to export results
if output == 'Y':
#Only it can enter if -j is put in the execution
ExportResults(newlist)
#Call to function to download the files
Downloadfiles(newlist)
#INICIO MAIN
def ShowResults(newlist,num_files,target):
print "Files in the target "+target+" are:\n"
print "Files indexed:", len (urls_final)
for i in urls_final:
if i not in newlist:
newlist.append(i)
print i
#verify if the user wants to export results
if output == 'Y':
#Only it can enter if -j is put in the execution
ExportResults(newlist)
#Call to function to download the files
Downloadfiles(newlist)
#INICIO MAIN
def Downloadfiles(urls_metadata,output):
try:
print "\nDo you like downloading these files to analyze metadata(Y/N)?"
#Convert to lower the input
resp = raw_input().lower()
if (resp == 'n'):
print "Exiting"
exit(1)
if ((resp != 'y') and (resp != 'n')):
print "The option is not valided. Please, try again it"
if (resp =='y'):
print "Indicate the location where you want to keep the files downloaded",
path = raw_input()
try:
for url in urls_metadata:
try:
filename = wget.download(url,path)
Analyze_Metadata(filename)
except:
pass
Display_Export_Metadata(metadata_files,output)
except:
pass
except Exception as e:
print str(e)
#********************************************************#
#Definition and treatment of the parameters
