train_tuner.py 文件源码

def arrau_precision_n(test_scores, num_true, all_candidates, punctuation_ids, ns):
    """
    Precision at n measure is the number of instances where the any crowd's answer occur within ranker's firs n choices
    For more details take a look at: http://www.aclweb.org/anthology/D13-1030
    The first num_true_antec[i] dev_scores are predicted scores for true antecedents of the i-th sentence w/ PA

    :param test_scores: \in [batch_size, num of candidates], for every sent w\ PA predicted scores for its candidates
    :param num_true_antec: \in [batch_size], for every sent w\ PA number of true antecedents
    :return: list of size 10
    """
    precisions = []
    for i in range(ns):
        precision = 0
        for k, item in enumerate(test_scores):
            ranks = len(item) - rankdata(item, method='ordinal').astype(int)
            #precision += min(1, len(set(ranks[:i+1]) & set(range(num_true))))
            counter = 0
            for pred in ranks[:i+1]:
                for gold in range(num_true[k]):
                    sym_difference = list(set(all_candidates[k][pred]) ^ set(all_candidates[k][gold]))
                    intersection_strip = [s for s in sym_difference if s not in punctuation_ids]
                    if len(intersection_strip) <= 1:
                        counter += 1
            precision += min(1, counter)
        precision /= float(len(test_scores))
        precision *= 100
        precisions.append(precision)
    return precisions