evaluate_new.py 文件源码

def evaluate_hyperparameters(dataset, iterator, args):
    # Select features
    if args.features is not None and args.features != dataset.feature_names:
        print('selecting features ...')
        features = _explode_features(args.features)
        start = timeit.default_timer()
        dataset = dataset.dataset_from_feature_names(features)
        print('done, took %fs' % (timeit.default_timer() - start))
    print('')

    states = range(3, 22 + 1)  # = [3,...,22]
    topologies = ['full', 'left-to-right-full', 'left-to-right-1', 'left-to-right-2']
    n_combinations = len(states) * len(topologies)

    curr_step = 0
    combinations = []
    measures = []
    for state in states:
        for topology in topologies:
            curr_step += 1
            prefix = '%.3d_%d_%s' % (curr_step, state, topology)
            print('(%.3d/%.3d) evaluating state=%d and topology=%s ...' % (curr_step, n_combinations, state, topology))
            start = timeit.default_timer()
            try:
                # Configure args from which the HMMs are created
                args.n_states = state
                args.topology = topology

                ll_stats = _compute_averaged_pos_and_neg_lls(dataset, iterator, prefix, args)
                measure = _compute_measure(ll_stats, dataset, args)
            except:
                measure = np.nan
            if measure is np.isnan(measure):
                print('measure: not computable')
            else:
                print('measure: %f' % measure)
            combinations.append((str(state), topology))
            measures.append(measure)
            print('done, took %fs' % (timeit.default_timer() - start))
            print('')

    best_idx = np.nanargmax(np.array(measures))  # get the argmax ignoring NaNs
    print('best combination with score %f: %s' % (measures[best_idx], ', '.join(combinations[best_idx])))
    print('detailed reports have been saved')

    # Save results
    assert len(combinations) == len(measures)
    if args.output_dir is not None:
        filename = '_results.csv'
        with open(os.path.join(args.output_dir, filename), 'wb') as f:
            writer = csv.writer(f, delimiter=';')
            writer.writerow(['', 'idx', 'measure', 'combination'])
            for idx, (measure, combination) in enumerate(zip(measures, combinations)):
                selected = '*' if best_idx == idx else ''
                writer.writerow([selected, '%d' % idx, '%f' % measure, ', '.join(combination)])