def _load_experiments(self, data_folder, name_or_patterns):
if not isinstance(name_or_patterns, (list, tuple)):
name_or_patterns = [name_or_patterns]
files = []
for name_or_pattern in name_or_patterns:
matched_files = glob(
osp.join(data_folder, name_or_pattern)) # golb gives a list of all files satisfying pattern
files += matched_files # this will include twice the same file if it satisfies 2 patterns
experiments = []
progress_f = None
params_f = None
pkl_data = None
for f in files:
if os.path.isdir(f):
try:
progress = self._read_data(osp.join(f, "progress.csv"))
params = self._read_params(osp.join(f, "params.json"))
params["exp_name"] = osp.basename(f)
if os.path.isfile(osp.join(f, "params.pkl")):
pkl_data = joblib.load(osp.join(f, "params.pkl"))
experiments.append(Experiment(progress, params, pkl_data))
else:
experiments.append(Experiment(progress, params))
except Exception as e:
print(e)
elif 'progress.csv' in f: # in case you're giving as datafolder the dir that contains the files!
progress_f = self._read_data(f)
elif 'params.json' in f:
params_f = self._read_params(f)
elif 'params.pkl' in f:
print('about to load', f)
pkl_data = joblib.load(f)
if params_f and progress_f:
if pkl_data:
experiments.append(Experiment(progress_f, params_f, pkl_data))
else:
experiments.append(Experiment(progress_f, params_f))
self._experiments = experiments
评论列表
文章目录
