task4_audio_tagging.py 文件源码

def do_system_evaluation(dataset, dataset_evaluation_mode, result_path):

    # Set warnings off, sklearn metrics will trigger warning for classes without
    # predicted samples in F1-scoring. This is just to keep printing clean.
    #warnings.simplefilter("ignore")

    fold_wise_class_eer = numpy.zeros((len(dataset.folds(mode=dataset_evaluation_mode)), dataset.audio_tag_count))

    for fold in dataset.folds(mode=dataset_evaluation_mode):
        class_wise_eer       = numpy.zeros((dataset.audio_tag_count))
        results = []
        result_filename = get_result_filename(fold=fold, path=result_path)
        if os.path.isfile(result_filename):
            with open(result_filename, 'rt') as f:
                for row in csv.reader(f, delimiter=','):
                    results.append(row)
        else:
            raise IOError("Result file not found [%s]" % result_filename)

        for tag_id,tag in enumerate(dataset.audio_tags):

            y_true_binary = []
            y_true_file = []
            y_score = []
            for result in results:
                if tag == result[1]:
                    relative_path = dataset.package_list[0]['local_audio_path'].replace(dataset.local_path,'')[1:] + os.path.sep + result[0]
                    y_true_file.append(result[0])
                    if tag in dataset.file_meta(relative_path)[0]['tags']:
                        y_true_binary.append(1)
                    else:
                        y_true_binary.append(0)

                    y_score.append(float(result[2]))

            if numpy.any(y_true_binary):
                class_wise_eer[tag_id] = compute_eer(result_filename, tag, dict(zip(y_true_file, y_true_binary)))
            else:
                class_wise_eer[tag_id] = None

        fold_wise_class_eer[fold - 1 if fold > 0 else fold, :] = class_wise_eer

    print "  File-wise evaluation, over %d folds" % (dataset.fold_count)

    print "     {:20s} | {:8s}".format('Tag', 'EER')
    print "     ==============================================="
    labels = numpy.array([dataset.tagcode_to_taglabel(t) for t in dataset.audio_tags])
    for i in numpy.argsort(labels):
        print "     {:20s} | {:3.3f} ".format(labels[i],
                                                                    numpy.nanmean(fold_wise_class_eer[:,i])
                                                                    )
    print "     ==============================================="
    print "     {:20s} | {:3.3f} ".format('Mean error',
                                                      numpy.mean(numpy.nanmean(fold_wise_class_eer))
                                                      )
    # Restore warnings to default settings
    warnings.simplefilter("default")