def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
python类iglob()的实例源码
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def iter_paths(prefix=None, from_dir=None, only_dirs=False):
if prefix:
start_at = os.path.expandvars(os.path.expanduser(prefix))
# TODO: implement env var completion.
if not prefix.startswith(('%', '$', '~')):
start_at = os.path.join(from_dir, prefix)
start_at = os.path.expandvars(os.path.expanduser(start_at))
prefix_split = os.path.split(prefix)
prefix_len = len(prefix_split[1])
if ('/' in prefix and not prefix_split[0]):
prefix_len = 0
for path in glob.iglob(start_at + '*'):
if not only_dirs or os.path.isdir(path):
suffix = ('/' if os.path.isdir(path) else '')
item = os.path.split(path)[1]
yield prefix + (item + suffix)[prefix_len:]
else:
prefix = from_dir
start_at = os.path.expandvars(os.path.expanduser(prefix))
for path in glob.iglob(start_at + '*'):
if not only_dirs or os.path.isdir(path):
yield path[len(start_at):] + ('' if not os.path.isdir(path) else '/')
def iload(files, load_func, **kwargs):
"""
Create a stream of arrays from files, which are loaded lazily.
Parameters
----------
pattern : iterable of str or str
Either an iterable of filenames or a glob-like pattern str.
load_func : callable, optional
Function taking a filename as its first arguments
kwargs
Keyword arguments are passed to ``load_func``.
Yields
------
arr: `~numpy.ndarray`
Loaded data.
"""
if isinstance(files, str):
files = iglob(files)
files = iter(files)
yield from map(partial(load_func, **kwargs), files)
# pmap does not support local functions
def _read_config(self):
"""
This initializes `_key_map` and _children.
"""
self._key_map = {}
self._children = []
root_dct = self.root.get_dict()
base_directory = os.path.dirname(self.root.file)
for section, contents in root_dct.iteritems():
# find all !includedir lines, add configuration to self._children and self._sectionmap
if section.startswith('!includedir'):
relative_directory = section.split(' ', 1)[1]
directory = os.path.abspath(os.path.join(base_directory, relative_directory))
# include all files in the directory
for filename in iglob(os.path.join(directory, '*.cnf')):
# order is not guaranteed, according to mysql docs
# parse every file, return parsing result
self._read_child_config(filename)
elif section.startswith('!'):
raise NotImplementedError()
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def create_seed_and_test_random(factor, start_id):
# Only use 1/factor of the crop images
# for example there are 10000 crops and a factor of 100
#then only 100 of them would be the random seed and test images.
# A factor of 0 would be 100%
# This should be changed to percent!
crops = []
image_ids = []
for filename in glob.iglob(crop_dir + '*.png'):
crops.append(filename)
for filename in crops:
renamed = filename.replace("_", "")
image_id = int(renamed.replace('.png', '').replace('/home/pkrush/cents/', ''))
if image_id < start_id:
continue
renamed = crop_dir + str(image_id) + '.png'
os.rename(filename, renamed)
rand_int = random.randint(0, factor)
if rand_int == 0:
image_ids.append(image_id)
pickle.dump(image_ids, open(data_dir + 'seed_image_ids.pickle', "wb"))
pickle.dump(image_ids, open(data_dir + 'test_image_ids.pickle', "wb"))
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert_midi2mp3():
""" Convert all midi files of the given directory to mp3
"""
input_dir = 'docs/midi/'
output_dir = 'docs/mp3/'
assert os.path.exists(input_dir)
os.makedirs(output_dir, exist_ok=True)
print('Converting:')
i = 0
for filename in glob.iglob(os.path.join(input_dir, '**/*.mid'), recursive=True):
print(filename)
in_name = filename
out_name = os.path.join(output_dir, os.path.splitext(os.path.basename(filename))[0] + '.mp3')
command = 'timidity {} -Ow -o - | ffmpeg -i - -acodec libmp3lame -ab 64k {}'.format(in_name, out_name) # TODO: Redirect stdout to avoid polluting the screen (have cleaner printing)
subprocess.call(command, shell=True)
i += 1
print('{} files converted.'.format(i))
def clean(path):
for f_xml in glob.iglob(os.path.join(path, '*.xml')):
print(f_xml)
f_txt = os.path.splitext(f_xml)[0]
with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:
root = ET.parse(f_xml).getroot()[0]
for doc in root.findall('doc'):
for e in doc.findall('seg'):
fd_txt.write(e.text.strip() + '\n')
xml_tags = ['<url', '<keywords', '<talkid', '<description',
'<reviewer', '<translator', '<title', '<speaker']
for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
print(f_orig)
f_txt = f_orig.replace('.tags', '')
with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \
io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:
for l in fd_orig:
if not any(tag in l for tag in xml_tags):
fd_txt.write(l.strip() + '\n')
def __init__(self, path, text_field, label_field, **kwargs):
"""Create an IMDB dataset instance given a path and fields.
Arguments:
path: Path to the dataset's highest level directory
text_field: The field that will be used for text data.
label_field: The field that will be used for label data.
Remaining keyword arguments: Passed to the constructor of
data.Dataset.
"""
fields = [('text', text_field), ('label', label_field)]
examples = []
for label in ['pos', 'neg']:
for fname in glob.iglob(os.path.join(path, label, '*.txt')):
with open(fname, 'r') as f:
text = f.readline()
examples.append(data.Example.fromlist([text, label], fields))
super(IMDB, self).__init__(examples, fields, **kwargs)
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def iglob(*args, include_hidden=False, **kwargs):
"""
A :py:func:`glob.iglob` that **optionally** but **truly** excludes hidden
files (i.e. even on *Windows*).
:py:func:`glob._ishidden`, which is implicitly used by :py:func:`glob.glob`
and :py:func:`glob.iglob` always filters out *dot* files but does not mind
about file's *HIDDEN* attribute on Windows.
**CAUTION:** this function **is not** thread-safe as it installs a trap at
runtime (i.e. for :py:func:`glob._ishidden`). The ``glob`` module must not
be used concurrently to this function.
"""
orig_ishidden = _glob._ishidden
if include_hidden:
_glob._ishidden = lambda x: False
else:
# original glob._ishidden() only removes "dot" files
# on windows, files have a "hidden" attribute
_glob._ishidden = _ishidden
try:
yield from _glob.iglob(*args, **kwargs)
finally:
_glob._ishidden = orig_ishidden
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def __init__(self, pattern="**/summary.json", output_filename=None,
verbose=True, **kargs):
super().__init__()
from sequana import sequana_debug_level
sequana_debug_level(level="INFO")
if verbose is False:
sequana_debug_level(level="WARNING")
logger.info("Sequana Summary is still a tool in progress and have been " +
" tested with the quality_control pipeline only for now.")
self.title = "Sequana multiple summary"
self.devtools = DevTools()
self.filenames = list(glob.iglob(pattern, recursive=True))
self.summaries = [ReadSummary(filename) for filename in self.filenames]
self.projects = [ReadSummary(filename).data['project'] for filename in self.filenames]
self.create_report_content()
self.create_html(output_filename)
def combine_cell_files(folder, globber, verbose=False):
dfs = []
for filename in glob.iglob(os.path.join(folder, globber)):
if verbose:
print(f'Reading {filename} ...')
channel = os.path.basename(os.path.dirname(filename))
df = pd.read_csv(filename, index_col=0)
df.index = pd.MultiIndex.from_product(([channel], df.index),
names=['channel', 'cell_id'])
dfs.append(df)
combined = pd.concat(dfs)
return combined
def read_tissue_smushed(self, folder, verbose=False, tissue=None):
smusheds = {}
if tissue is None:
globber = glob.iglob(os.path.join(folder, 'smushed-*'))
else:
globber = glob.iglob(os.path.join(folder, f'smushed-{tissue}*'))
for filename in globber:
if verbose:
print(f'Reading {filename} ...')
tissue = filename.split('smushed-')[-1].split('.')[0]
tissue = tissue.split('-')[0]
df = pd.read_csv(filename, index_col=0)
df.rename(columns={'0': 0, '1': 1}, inplace=True)
smusheds[tissue] = df
assert len(df.columns.difference([0, 1, 'cluster'])) == 0
return smusheds
def get_vf_num_by_pci_address(pci_addr):
"""Get the VF number based on a VF's pci address
A VF is associated with an VF number, which ip link command uses to
configure it. This number can be obtained from the PCI device filesystem.
"""
VIRTFN_RE = re.compile("virtfn(\d+)")
virtfns_path = "/sys/bus/pci/devices/%s/physfn/virtfn*" % (pci_addr)
vf_num = None
try:
for vf_path in glob.iglob(virtfns_path):
if re.search(pci_addr, os.readlink(vf_path)):
t = VIRTFN_RE.search(vf_path)
vf_num = t.group(1)
break
except Exception:
pass
if vf_num is None:
raise exception.PciDeviceNotFoundById(id=pci_addr)
return vf_num
def _index(cls, path, types):
if sys.version_info >= (3, 5):
# Python version >=3.5 supports glob
import glob
for img_type in types:
for filename in glob.iglob(
(path + '/**/' + img_type), recursive=True
):
f_base = os.path.basename(filename)
cls._names.update({f_base: filename})
else:
# Python version <=3.4
import fnmatch
for root, dirnames, filenames in os.walk(path):
for img_type in types:
for f_base in fnmatch.filter(filenames, img_type):
filename = os.path.join(root, f_base)
cls._names.update({f_base: filename})
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def get_files(directory, pattern, recursive=True):
""" Return the full path to all files in directory matching the specified
pattern.
Arguments:
directory (str): Directory path in which to look
pattern (str): A glob pattern for filenames
recursive (bool): Searches recursively if True
Returns:
A list of matching file paths
"""
# This yields an iterator which really speeds up looking through large, flat directories
if recursive is False:
it = glob.iglob(os.path.join(directory, pattern))
return it
# If we want to recurse, use os.walk instead
matches = list()
for root, dirnames, filenames in os.walk(directory):
matches.extend([os.path.join(root, ss) for ss in
fnmatch.filter(filenames, pattern)])
return matches
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
def convert(installers, dest_dir, verbose):
require_pkgresources('wheel convert')
# Only support wheel convert if pkg_resources is present
from ..wininst2wheel import bdist_wininst2wheel
from ..egg2wheel import egg2wheel
for pat in installers:
for installer in iglob(pat):
if os.path.splitext(installer)[1] == '.egg':
conv = egg2wheel
else:
conv = bdist_wininst2wheel
if verbose:
sys.stdout.write("{0}... ".format(installer))
sys.stdout.flush()
conv(installer, dest_dir)
if verbose:
sys.stdout.write("OK\n")
