def assign_brandenburg(self, *args, **options):
brandenburg_state = State.objects.get(name='Brandenburg')
excel_file = pd.ExcelFile(options['filename'])
df = excel_file.parse('Brandenburg')
assigned_auths = defaultdict(list)
locations = {}
for _, row in df.iterrows():
auth = SupervisionAuthority.objects.get(state=brandenburg_state, name=row['name'])
locations[auth] = GEOSGeometry('POINT(%f %f)' % (row['lng'], row['lat']), srid=4326)
assigned_districts = row[u'Landkreis-Zuständigkeit'].splitlines()
for district_name in assigned_districts:
districts = District.objects.filter(part_of=brandenburg_state, name=district_name)
if len(districts) != 1:
print(district_name)
print(districts)
else:
assigned_auths[districts[0]].append(auth)
for nursinghome in NursingHome.objects.filter(supervision_authority__isnull=True,
state=brandenburg_state):
district = District.objects.get(geom__covers=nursinghome.geo)
auths = assigned_auths[district]
if len(auths) == 1:
nursinghome.supervision_authority = auths[0]
nursinghome.save()
else:
min_distance = None
best_auth = None
for auth, point in locations.items():
if auth not in auths:
continue
dist = NursingHome.objects.filter(pk=nursinghome.pk
).annotate(distance=Distance('geo', point))
dist = dist[0].distance.m
if min_distance is None or dist < min_distance:
min_distance = dist
best_auth = auth
nursinghome.supervision_authority = best_auth
nursinghome.save()
python类filter()的实例源码
def assign_rheinlandpfalz(self, *args, **options):
rp_state = State.objects.get(name='Rheinland-Pfalz')
excel_file = pd.ExcelFile(options['filename'])
df = excel_file.parse('Rheinland-Pfalz')
assigned = defaultdict(list)
for _, row in df.iterrows():
auth = SupervisionAuthority.objects.get(state=rp_state, name=row['name'])
district_names = row[u'Landkreis-Zuständigkeit'].splitlines()
for district_name in district_names:
only = None
if '|' in district_name:
district_name, only = district_name.split('|')
only = only.split(',')
districts = District.objects.filter(part_of=rp_state, name=district_name)
if len(districts) == 0:
districts = District.objects.filter(part_of=rp_state, name__contains=district_name)
if len(districts) == 0:
districts = District.objects.filter(part_of=rp_state, name__contains=district_name.split()[0])
if len(districts) == 0:
districts = District.objects.filter(part_of=rp_state, name__istartswith=re.sub('\W', '', district_name))
if len(districts) > 1:
if 'Kreis' in district_name:
districts = districts.filter(kind_detail__contains='Landkreis')
if 'Stadt' in district_name:
districts = districts.filter(kind_detail__contains='Stadt')
if len(districts) != 1:
print(districts)
print(u'District not one: %s' % district_name)
continue
assigned[auth].append((districts[0], only))
for auth, district_list in assigned.items():
for district, only in district_list:
if only is None:
NursingHome.objects.filter(state=rp_state, district=district, supervision_authority__isnull=True).update(supervision_authority=auth)
continue
for muni_name in only:
muni_name = muni_name.strip()
munis = Municipality.objects.filter(part_of=district, name__contains=muni_name)
if len(munis) > 1:
munis = Municipality.objects.filter(part_of=district, name=muni_name)
if len(munis) != 1:
print('Did not find %s' % muni_name)
continue
muni = munis[0]
NursingHome.objects.filter(state=rp_state, district=district, supervision_authority__isnull=True, geo__coveredby=muni.geom).update(supervision_authority=auth)
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
globs = (self.exclude_package_data.get('', [])
+ self.exclude_package_data.get(package, []))
bad = []
for pattern in globs:
bad.extend(
fnmatch.filter(
files, os.path.join(src_dir, convert_path(pattern))
)
)
bad = dict.fromkeys(bad)
seen = {}
return [
f for f in files if f not in bad
and f not in seen and seen.setdefault(f,1) # ditch dupes
]
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
globs = (self.exclude_package_data.get('', [])
+ self.exclude_package_data.get(package, []))
bad = []
for pattern in globs:
bad.extend(
fnmatch.filter(
files, os.path.join(src_dir, convert_path(pattern))
)
)
bad = dict.fromkeys(bad)
seen = {}
return [
f for f in files if f not in bad
and f not in seen and seen.setdefault(f, 1) # ditch dupes
]
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def test_diff_roles():
from ldap2pg.manager import SyncManager, Role, RoleSet
m = SyncManager()
pgroles = RoleSet([
Role('drop-me'),
Role('alter-me'),
Role('nothing'),
])
ldaproles = RoleSet([
Role('alter-me', options=dict(LOGIN=True)),
Role('nothing'),
Role('create-me')
])
queries = [q.args[0] for q in m.diff(pgroles, set(), ldaproles, set())]
assert fnfilter(queries, 'ALTER ROLE "alter-me" WITH* LOGIN*;')
assert fnfilter(queries, 'CREATE ROLE "create-me" *;')
assert fnfilter(queries, '*DROP ROLE "drop-me";*')
assert not fnfilter(queries, '*nothing*')
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
globs = (
self.exclude_package_data.get('', [])
+ self.exclude_package_data.get(package, [])
)
bad = set(
item
for pattern in globs
for item in fnmatch.filter(
files,
os.path.join(src_dir, convert_path(pattern)),
)
)
seen = collections.defaultdict(itertools.count)
return [
fn
for fn in files
if fn not in bad
# ditch dupes
and not next(seen[fn])
]
def _build(inputs):
paths = []
for f in inputs:
if os.path.isdir(f):
# Walk through the directory and get all PDF files
# Credit: https://stackoverflow.com/a/2186565/4856091
for root, dirnames, filenames in os.walk(f):
for filename in fnmatch.filter(filenames, "*.pdf"):
paths.append(os.path.join(root, filename))
elif f.endswith(".pdf"):
paths.append(f)
else:
# Get the contents as list of files
_files = [line.strip() for line in open(f).readlines()]
_files = [_f for _f in _files
if (not _f.startswith("#")) and (_f != "")]
paths += _files
return paths
def __search_folders(self, key):
result = SearchResult('FOLDERS')
folders = []
for rootfolder in self.__settings['folders']:
dirnames = next(os.walk(rootfolder))[1]
for dirname in fnmatch.filter(dirnames, '*{}*'.format(key)):
folders.append(os.path.join(rootfolder, dirname))
if len(folders) == 0:
result.add_message('No folder to remove.', None)
else:
for folder in folders:
result.add_message(
'Folder \'{}\' will be removed.'
.format(folder)
)
result.add_action(
lambda f=folder: self.__remove_folder(f)
)
return result
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def _process_paths(self):
for path in self.path_list:
if os.path.isfile(path):
self._process(path)
elif os.path.isdir(path):
for root, dirs, files in os.walk(path):
for filename in fnmatch.filter(files, self.extension):
self._process(os.path.join(root, filename))
else:
raise TypeError('%s should be either dir or file' % path)
if self.pretend:
logging.info('No actual changes was done, run without \'-p\' to overwrite source files')
logging.info('%d files processed' % self.files_processed)
def get_mosfit_hash(salt=u''):
"""Return a unique hash for the MOSFiT code."""
import fnmatch
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
matches = []
for root, dirnames, filenames in os.walk(dir_path):
for filename in fnmatch.filter(filenames, '*.py'):
matches.append(os.path.join(root, filename))
matches = list(sorted(list(matches)))
code_str = salt
for match in matches:
with codecs.open(match, 'r', 'utf-8') as f:
code_str += f.read()
return hashlib.sha512(hash_bytes(code_str)).hexdigest()[:16]
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
patterns = self._get_platform_patterns(
self.package_data,
package,
src_dir,
)
globs_expanded = map(glob, patterns)
# flatten the expanded globs into an iterable of matches
globs_matches = itertools.chain.from_iterable(globs_expanded)
glob_files = filter(os.path.isfile, globs_matches)
files = itertools.chain(
self.manifest_files.get(package, []),
glob_files,
)
return self.exclude_data_files(package, src_dir, files)
def exclude_data_files(self, package, src_dir, files):
"""Filter filenames for package's data files in 'src_dir'"""
files = list(files)
patterns = self._get_platform_patterns(
self.exclude_package_data,
package,
src_dir,
)
match_groups = (
fnmatch.filter(files, pattern)
for pattern in patterns
)
# flatten the groups of matches into an iterable of matches
matches = itertools.chain.from_iterable(match_groups)
bad = set(matches)
keepers = (
fn
for fn in files
if fn not in bad
)
# ditch dupes
return list(_unique_everseen(keepers))
def test(self):
"""
Runs the binutils `make check`
"""
ret = build.make(self.src_dir, extra_args='check', ignore_status=True)
errors = 0
for root, _, filenames in os.walk(self.src_dir):
for filename in fnmatch.filter(filenames, '*.log'):
filename = os.path.join(root, filename)
logfile = filename[:-4] + ".log"
os.system('cp ' + logfile + ' ' + self.logdir)
with open(logfile) as result:
for line in result.readlines():
if line.startswith('FAIL'):
errors += 1
self.log.error(line)
if errors:
self.fail("%s test(s) failed, check the log for details." % errors)
elif ret:
self.fail("'make check' finished with %s, but no FAIL lines were "
"found." % ret)
def test(self):
"""
Run the `make check` on ltrace
"""
ret = build.make(self.src_lt, extra_args='check', ignore_status=True)
errors = 0
for root, _, filenames in os.walk('.'):
for filename in fnmatch.filter(filenames, '*.log'):
filename = os.path.join(root, filename)
shutil.copy(filename, self.logdir)
with open(filename) as result:
for line in result.readlines():
if line.startswith('FAIL'):
errors += 1
self.log.error(line)
if errors:
self.fail("%s test(s) failed, check the log for details." % errors)
elif ret:
self.fail("'make check' finished with %s, but no FAIL lines were "
"found." % ret)
def _GetFilesWithExt(root_dir, ext):
"""Gets all files with a given extension.
Args:
root_dir: Directory in which to search for files.
ext: Extension to look for (including dot)
Returns:
A list of absolute paths to files that match.
"""
files = []
for root, _, filenames in os.walk(root_dir):
basenames = fnmatch.filter(filenames, '*.' + ext)
files.extend([os.path.join(root, basename)
for basename in basenames])
return files
def _index(cls, path, types):
if sys.version_info >= (3, 5):
# Python version >=3.5 supports glob
import glob
for img_type in types:
for filename in glob.iglob(
(path + '/**/' + img_type), recursive=True
):
f_base = os.path.basename(filename)
cls._names.update({f_base: filename})
else:
# Python version <=3.4
import fnmatch
for root, dirnames, filenames in os.walk(path):
for img_type in types:
for f_base in fnmatch.filter(filenames, img_type):
filename = os.path.join(root, f_base)
cls._names.update({f_base: filename})
