evaluate.py 文件源码

def validate(classifier, train, test, args, report_base_name):
    print('\nTraining classifier on %d samples ...' % len(train.X))
    start = timeit.default_timer()
    classifier.fit(train.X, train.y)
    stop = timeit.default_timer()
    print('Classifier trained, took %f seconds' % (stop - start))

    for method in args.loglikelihood_methods:
        report_name = report_base_name + '_' + method

        if args.calculate_distances:
            print('\nCalculating distances ...')
            start = timeit.default_timer()
            distances = classifier.distances(loglikelihood_method=method, n_samples=500)
            print('Distances calculated, took %f seconds' % (timeit.default_timer() - start))

            report = _generate_distance_reports(distances, target_names=train.target_names)
            _handle_report(report, report_name + '_distances', args)

            # Calculate proto symbol space
            #mds = MDS(n_components=5, dissimilarity='precomputed')
            #coordinates = mds.fit_transform(distances)
            #_plot_proto_symbol_space(coordinates, train.target_names, report_name + '_scatter', args)

        # Get loglikelihoods for train set
        print('\nValidating classifier on training set with %d samples ...' % len(train.X))
        loglikelihoods_train = _calculate_loglikelihoods(classifier, train.X, method)
        report = _generate_loglikelihood_reports(loglikelihoods_train, train.y, target_names=train.target_names)
        _handle_report(report, report_name + '_train_loglikelihoods', args)

        # Fit decision makers
        loglikelihoods_test = None
        for idx, decision_maker in enumerate(get_decision_makers(args)):
            if decision_maker is not None:
                name = args.decision_makers[idx]
                if hasattr(decision_maker, 'fit') and callable(getattr(decision_maker, 'fit')):
                    print('\nTraining decision maker %s on %d loglikelihoods ...' % (name, len(loglikelihoods_train)))
                    decision_maker.fit(loglikelihoods_train, train.y)
                    print('Decision maker trained, took %f seconds' % (stop - start))
                else:
                    print('\nUsing decision maker %s ...' % name)
                y_pred = _calculate_predictions(decision_maker, loglikelihoods_train)
                report = _generate_classification_reports(train.y, y_pred, target_names=train.target_names)
                _handle_report(report, report_name + '_train_classification_' + name, args)

            # Validate on test set
            print('\nValidating classifier on test set with %d samples ...' % len(test.X))
            if loglikelihoods_test is None:
                loglikelihoods_test = _calculate_loglikelihoods(classifier, test.X, method)
                report = _generate_loglikelihood_reports(loglikelihoods_test, test.y, target_names=test.target_names)
                _handle_report(report, report_name + '_test_loglikelihoods', args)
            if decision_maker is not None:
                y_pred = _calculate_predictions(decision_maker, loglikelihoods_test)
                report = _generate_classification_reports(test.y, y_pred, target_names=test.target_names)
                _handle_report(report, report_name + '_test_classification_' + name, args)