python类precision_recall_fscore_support()的实例源码

def _update_tsg_metrics(self, y_true, y_pred, prob):
        self.tsg_gene_pred = pd.Series(y_pred, self.y.index)
        self.tsg_gene_score = pd.Series(prob, self.y.index)

        # compute metrics for classification
        self.tsg_gene_count[self.num_pred] = sum(y_pred)
        prec, recall, fscore, support = metrics.precision_recall_fscore_support(y_true, y_pred)
        tsg_col = 1  # column for metrics relate to tsg
        self.tsg_precision[self.num_pred] = prec[tsg_col]
        self.tsg_recall[self.num_pred] = recall[tsg_col]
        self.tsg_f1_score[self.num_pred] = fscore[tsg_col]
        self.logger.debug('Tsg Iter %d: Precission=%s, Recall=%s, f1_score=%s' % (
                          self.num_pred + 1, str(prec), str(recall), str(fscore)))

        # compute ROC curve metrics
        fpr, tpr, thresholds = metrics.roc_curve(y_true, prob)
        self.tsg_tpr_array[self.num_pred, :] = interp(self.tsg_fpr_array, fpr, tpr)
        #self.tsg_tpr_array[0] = 0.0

        # compute Precision-Recall curve metrics
        p, r, thresh = metrics.precision_recall_curve(y_true, prob)
        p, r, thresh = p[::-1], r[::-1], thresh[::-1]  # reverse order of results
        self.tsg_precision_array[self.num_pred, :] = interp(self.tsg_recall_array, r, p)

def addProbabilistFold(self, fold_id, true_labels, predicted_proba, threshold = None):
        if threshold is None:
            for threshold in self.thresholds:
                self.addProbabilistFold(fold_id, true_labels, predicted_proba, threshold = threshold)
        else:
            predicted_labels = np.array(predicted_proba) > threshold / 100
            precision, recall, f_score, _ = precision_recall_fscore_support(true_labels, predicted_labels,
                                                                            average = 'binary')
            if len(predicted_labels) == 0:
                fp = 0
                tn = 0
            else:
                conf_matrix = confusion_matrix(true_labels, predicted_labels, [True, False])
                fp = conf_matrix[1][0]
                tn = conf_matrix[1][1]
            fp_tn = fp + tn
            if fp_tn == 0:
                false_alarm_rate = 0
            else:
                false_alarm_rate = fp / (fp + tn)
            self.fold_perf[threshold][fold_id, :] = [precision, recall, false_alarm_rate, f_score]

def addNonProbabilistFold(self, fold_id, true_labels, predicted_labels):
        precision, recall, f_score, _ = precision_recall_fscore_support(true_labels, predicted_labels,
                                                                        average = 'binary')
        accuracy = accuracy_score(true_labels, predicted_labels)
        if len(predicted_labels) == 0:
                fp = 0
                tn = 0
        else:
            conf_matrix = confusion_matrix(true_labels, predicted_labels, [True, False])
            fp = conf_matrix[1][0]
            tn = conf_matrix[1][1]
        fp_tn = fp + tn
        if fp_tn == 0:
            false_alarm_rate = 0
        else:
            false_alarm_rate = fp / (fp + tn)
        self.fold_perf[fold_id, :] = [precision, recall, false_alarm_rate, f_score, accuracy]

def test_svm_estimator(estimator, notes, encodings_train, labels_train,
                       encodings_test, labels_test):
    t0 = time()
    estimator.fit(encodings_train, labels_train)
    print("Time cons: %.2fs, type: %s" % (time() - t0, notes))
    predicted = estimator.predict(encodings_test)
    accuracy = metrics.accuracy_score(labels_test, predicted)
    print("Accuracy: %.5f" % accuracy)
    report = metrics.classification_report(labels_test, predicted)
    print(report)
    prec_recall_f_score = metrics.precision_recall_fscore_support(
        labels_test, predicted)
    print('-' * 10)
    prec_recall_f_score_dict = {
        'prec': np.mean(prec_recall_f_score[0]),
        'recall': np.mean(prec_recall_f_score[1]),
        'f_score': np.mean(prec_recall_f_score[2])
    }
    return accuracy, prec_recall_f_score_dict

def score(self, X, y=None, **kwargs):
        """
        Generates the Scikit-Learn classification_report

        Parameters
        ----------

        X : ndarray or DataFrame of shape n x m
            A matrix of n instances with m features

        y : ndarray or Series of length n
            An array or series of target or class values

        """
        y_pred = self.predict(X)
        keys   = ('precision', 'recall', 'f1')
        self.scores = precision_recall_fscore_support(y, y_pred)
        self.scores = map(lambda s: dict(zip(self.classes_, s)), self.scores[0:3])
        self.scores = dict(zip(keys, self.scores))

        return self.draw(y, y_pred)

def score(self, X, y=None, **kwargs):
        """
        Generates the Scikit-Learn precision_recall_fscore_support

        Parameters
        ----------

        X : ndarray or DataFrame of shape n x m
            A matrix of n instances with m features

        y : ndarray or Series of length n
            An array or series of target or class values

        Returns
        -------

        ax : the axis with the plotted figure
        """
        y_pred = self.predict(X)
        self.scores  = precision_recall_fscore_support(y, y_pred)
        self.support = dict(zip(self.classes_, self.scores[-1]))
        return self.draw()

def precision_recall_at_x_proportion(test_labels, test_predictions, x_proportion=0.01,
                                     return_cutoff=False):
    """Compute precision, recall, F1 for a specified fraction of the test set.

    :params list test_labels: true labels on test set
    :params list test_predicted: predicted labels on test set
    :params float x_proportion: proportion of the test set to flag
    :params bool return_cutoff: if True return the cutoff probablility
    :returns float precision: fraction correctly flagged
    :returns float recall: fraction of the positive class recovered
    :returns float f1: 
    """

    cutoff_index = int(len(test_predictions) * x_proportion)
    cutoff_index = min(cutoff_index, len(test_predictions) - 1)

    sorted_by_probability = np.sort(test_predictions)[::-1]
    cutoff_probability = sorted_by_probability[cutoff_index]

    test_predictions_binary = [1 if x > cutoff_probability else 0 for x in test_predictions]

    precision, recall, f1, _ = metrics.precision_recall_fscore_support(
        test_labels, test_predictions_binary)

    # Only interested in metrics for label 1
    precision, recall, f1 = precision[1], recall[1], f1[1]

    if return_cutoff:
        return precision, recall, f1, cutoff_probability
    else:
        return precision, recall, f1

def calc(gr_truth, predicted):
    # precision, recall, fscore, _ = score(gr_truth, predicted, average='micro')
    # print('precision: {}'.format(precision))
    # print('recall: {}'.format(recall))
    # print('fscore: {}'.format(fscore))
    # print('jaccard: {}'.format(jaccard_similarity_score(gr_truth, predicted, normalize=True)))
    # print('mutual: {}'.format(mutual_info_score(gr_truth, predicted)))
    # print('mutual adj: {}'.format(adjusted_mutual_info_score(gr_truth, predicted)))
    # print('mutual norm: {}'.format(normalized_mutual_info_score(gr_truth, predicted)))
    return normalized_mutual_info_score(gr_truth, predicted)

def compute_f1(predictions, labels):
    """
    Compute the F1 for FAVOR and AGAINST classes, as well as the average of the two.
    """
    _, _, f1, _ = precision_recall_fscore_support(labels, predictions,
                                                  warn_for=("f1"))
    f1_against = f1[0]
    f1_favor = f1[2]
    f1_overall = (f1_against + f1_favor) / 2
    return f1_against, f1_favor, f1_overall

def f1_score_wrapper(y_true, y_pred):
    # y_pred_2 = np.asarray([1 if b > a else 0 for [a, b] in y_pred])
    y_pred_2 = np.argmax(y_pred, axis=1)
    # print("F1 score inputs:")
    # print(y_true)
    # print(y_pred_2)
    # print("---")
    p, r, f1, s = precision_recall_fscore_support(y_true, y_pred_2)
    return accuracy_score(y_true, y_pred_2) if 0 in f1 else np.mean(f1)

def _update_metrics(self, y_true, y_pred,
                        onco_prob, tsg_prob):
        # record which genes were predicted what
        self.driver_gene_pred = pd.Series(y_pred, self.y.index)
        self.driver_gene_score = pd.Series(onco_prob+tsg_prob, self.y.index)

        # evaluate performance
        prec, recall, fscore, support = metrics.precision_recall_fscore_support(y_true, y_pred,
                                                                                average='macro')
        cancer_gene_pred = ((onco_prob + tsg_prob)>.5).astype(int)
        self.cancer_gene_count[self.num_pred] = np.sum(cancer_gene_pred)
        self.precision[self.num_pred] = prec
        self.recall[self.num_pred] = recall
        self.f1_score[self.num_pred] = fscore

        # compute Precision-Recall curve metrics
        driver_prob = onco_prob + tsg_prob
        driver_true = (y_true > 0).astype(int)
        p, r, thresh = metrics.precision_recall_curve(driver_true, driver_prob)
        p, r, thresh = p[::-1], r[::-1], thresh[::-1]  # reverse order of results
        thresh = np.insert(thresh, 0, 1.0)
        self.driver_precision_array[self.num_pred, :] = interp(self.driver_recall_array, r, p)
        self.driver_threshold_array[self.num_pred, :] = interp(self.driver_recall_array, r, thresh)

        # calculate prediction summary statistics
        prec, recall, fscore, support = metrics.precision_recall_fscore_support(driver_true, cancer_gene_pred)
        self.driver_precision[self.num_pred] = prec[1]
        self.driver_recall[self.num_pred] = recall[1]

        # save driver metrics
        fpr, tpr, thresholds = metrics.roc_curve(driver_true, driver_prob)
        self.driver_tpr_array[self.num_pred, :] = interp(self.driver_fpr_array, fpr, tpr)

def _update_onco_metrics(self, y_true, y_pred, prob):
        self.onco_gene_pred = pd.Series(y_pred, self.y.index)
        self.onco_gene_score = pd.Series(prob, self.y.index)

        # compute metrics for classification
        self.onco_gene_count[self.num_pred] = sum(y_pred)
        prec, recall, fscore, support = metrics.precision_recall_fscore_support(y_true, y_pred)
        self.onco_precision[self.num_pred] = prec[self.onco_num]
        self.onco_recall[self.num_pred] = recall[self.onco_num]
        self.onco_f1_score[self.num_pred] = fscore[self.onco_num]
        self.logger.debug('Onco Iter %d: Precission=%s, Recall=%s, f1_score=%s' % (
                          self.num_pred + 1, str(prec), str(recall), str(fscore)))

        # compute ROC curve metrics
        fpr, tpr, thresholds = metrics.roc_curve(y_true, prob)
        self.onco_tpr_array[self.num_pred, :] = interp(self.onco_fpr_array, fpr, tpr)
        #self.onco_mean_tpr[0] = 0.0

        # compute Precision-Recall curve metrics
        p, r, thresh = metrics.precision_recall_curve(y_true, prob)
        p, r, thresh = p[::-1], r[::-1], thresh[::-1]  # reverse order of results
        thresh = np.insert(thresh, 0, 1.0)
        self.onco_precision_array[self.num_pred, :] = interp(self.onco_recall_array, r, p)
        self.onco_threshold_array[self.num_pred, :] = interp(self.onco_recall_array, r, thresh)

def arg_p_r_f(Y_true, Y_pred, labels, **kwargs):

    macro_p = []
    macro_r = []
    macro_f = []

    micro_true = []
    micro_pred = []

    for y_true, y_pred in zip(Y_true, Y_pred):
        p, r, f, _ = precision_recall_fscore_support(y_true, y_pred,
                                                     **kwargs)
        macro_p.append(p)
        macro_r.append(r)
        macro_f.append(f)

        micro_true.extend(y_true)
        micro_pred.extend(y_pred)

    micro_p, micro_r, micro_f, _ = precision_recall_fscore_support(
        micro_true, micro_pred, **kwargs
    )
    kwargs.pop('average')
    per_class_fs = f1_score(micro_true, micro_pred, average=None, **kwargs)

    res = {
        'p_macro': np.mean(macro_p),
        'r_macro': np.mean(macro_r),
        'f_macro': np.mean(macro_f),
        'p_micro': micro_p,
        'r_micro': micro_r,
        'f_micro': micro_f
    }

    for label, per_class_f in zip(sorted(labels), per_class_fs):
        res['f_class_{}'.format(label)] = per_class_f

    return res

def save_results(y_test, y_pred, labels, fold_number=0):
    pickle.dump(y_test, open("y_test_fold{number}.plk".format(number=fold_number), "w"))
    pickle.dump(y_pred, open("y_pred_fold{number}.plk".format(number=fold_number), "w"))
    print classification_report(y_test, y_pred)
    print confusion_matrix(y_test, y_pred)
    print "Micro stats:"
    print precision_recall_fscore_support(y_test, y_pred, average='micro')
    print "Macro stats:"
    print precision_recall_fscore_support(y_test, y_pred, average='macro')
    try:
        visualization.plot_confusion_matrix(confusion_matrix(y_test, y_pred),
                                            title="Test CM fold{number}".format(number=fold_number),
                                            labels=labels)
    except:
        pass

def prediction(clf, X, y):
    y_pred = clf.predict(X)
    y_test = y
    print classification_report(y_test, y_pred)
    # print confusion_matrix(y_test, y_pred)
    print "Micro stats:"
    print precision_recall_fscore_support(y_test, y_pred, average='micro')
    print "Macro stats:"
    print precision_recall_fscore_support(y_test, y_pred, average='macro')

def write_score(name, gold_labels, pred_scores, classes, average_classes):
    classes, average_classes = np.array(classes), np.array(average_classes)
    gold_scores = LabelBinarizer().fit(classes).transform(gold_labels)
    pred_labels = classes[np.argmax(pred_scores, axis=1)]

    with closing(Tee('{}.txt'.format(name), 'w')):
        precision, recall, fscore, _ = precision_recall_fscore_support(gold_labels, pred_labels, labels=classes)
        for t in zip(classes, precision, recall, fscore):
            print('{}: P={:.2f}, R={:.2f}, F1={:.2f}'.format(*t))
        print('Accuracy: {:.4f}'.format(accuracy_score(gold_labels, pred_labels)))
        print('F1 average: {:.4f}'.format(np.mean(fscore[LabelEncoder().fit(classes).transform(average_classes)])))

    with PdfPages('{}.pdf'.format(name)) as pdf:
        fpr = {}
        tpr = {}
        roc_auc = {}
        for i in range(len(classes)):
            fpr[i], tpr[i], _ = roc_curve(gold_scores[:, i], pred_scores[:, i])
            roc_auc[i] = auc(fpr[i], tpr[i])
        fpr['micro'], tpr['micro'], _ = roc_curve(gold_scores.ravel(), pred_scores.ravel())
        roc_auc['micro'] = auc(fpr['micro'], tpr['micro'])
        plt.figure()
        plt.plot(fpr['micro'], tpr['micro'], label='micro-average (area = {:.2f})'.format(roc_auc['micro']))
        for i in range(len(classes)):
            plt.plot(fpr[i], tpr[i], label='{0} (area = {1:.2f})'.format(i, roc_auc[i]))
        plt.plot([0, 1], [0, 1], 'k--')
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.xlabel('False Positive Rate')
        plt.ylabel('True Positive Rate')
        plt.title('ROC Curves')
        plt.legend(loc='lower right')
        pdf.savefig()

def f1(average_classes):
    # noinspection PyShadowingNames
    def f1(y_true, y_pred, theano=False):
        if theano:
            raise NotImplementedError
        else:
            return np.mean(precision_recall_fscore_support(y_true, np.argmax(y_pred, axis=-1))[2][average_classes])

    return f1

def GBDT_classify(train_dataSet_path, test_dataSet_path, train_one_and_two_result_as_proba_path):

    train_data = pd.read_csv(train_dataSet_path)
    train_data = train_data.as_matrix()
    X_train = train_data[:, 2:-1]  # select columns 0 through end-1
    y_train = train_data[:, -1]  # select column end

    test_data = pd.read_csv(test_dataSet_path)
    test_data = test_data.as_matrix()
    X_test = test_data[:, 2:-1]  # select columns 0 through end-1
    y_test = test_data[:, -1]  # select column end 

    clf = GradientBoostingClassifier(n_estimators=200)
    clf.fit(X_train, y_train)
    pre_y_test = clf.predict_proba(X_test)
    print pre_y_test
    print("GBDT Metrics : {0}".format(precision_recall_fscore_support(y_test, pre_y_test)))

    print u'????.....'
    f_result = open(test_dataSet_prob_path, 'w')
    for i in range(0, len(pre_y_test)):
        if i==0:
            print str(pre_y_test[i][0])
        if i==len(pre_y_test)-1:
            print str(pre_y_test[i][0])
        f_result.write(str(pre_y_test[i][0]) + '\n')   

    return clf

def _f1_score(self, y_pred, y_true):
        y_pred_2 = np.argmax(y_pred, axis=1)
        p, r, f1, s = precision_recall_fscore_support(y_true, y_pred_2)
        return accuracy_score(y_true, y_pred_2) if 0 in f1 else np.mean(f1)

def f1_score_wrapper(y_true, y_pred):
    # y_pred_2 = np.asarray([1 if b > a else 0 for [a, b] in y_pred])
    y_pred_2 = np.argmax(y_pred, axis=1)
    # print("F1 score inputs:")
    # print(y_true)
    # print(y_pred_2)
    # print("---")
    p, r, f1, s = precision_recall_fscore_support(y_true, y_pred_2)
    return accuracy_score(y_true, y_pred_2) if 0 in f1 else np.mean(f1)

def eval_performance(y_true, y_pred):
    '''
    Evaluate the performance of a multiclass classification model.
    :param y_true: the gold-standard labels
    :param y_pred: the predictions
    :return: mean F1
    '''
    pre, rec, f1, support = metrics.precision_recall_fscore_support(y_true, y_pred, average='weighted')
    print '=== Performance ==='
    print 'Mean precision:  %.03f%%' % pre # (100*sum(pre * support)/sum(support))
    print 'Mean recall:     %.03f%%' % rec # (100*sum(rec * support)/sum(support))
    print 'Mean F1:         %.03f%%' % f1 # mean_f1
    return pre, rec, f1, support

def get_f1_pre_rec(self, labels, prediction):

        pre, rec, f1, _ = precision_recall_fscore_support(
            y_true = labels, 
            y_pred = prediction,
            labels = [self.labels_dict[i] for i in self.labels_list])

        counts = np.zeros([2, 1])
        for i in labels:
            counts[i] += 1

        return np.expand_dims(pre,1), np.expand_dims(rec,1), np.expand_dims(f1,1), counts

def get_f1_pre_rec(self, labels, prediction):

        pre, rec, f1, _ = precision_recall_fscore_support(
            y_true = labels, 
            y_pred = prediction,
            labels = [self.labels_dict[i] for i in self.labels_list])

        counts = np.zeros([6, 1])
        for i in labels:
            counts[i] += 1

        return np.expand_dims(pre,1), np.expand_dims(rec,1), np.expand_dims(f1,1), counts

def get_f1_pre_rec(self, labels, prediction):

        pre, rec, f1, _ = precision_recall_fscore_support(
            y_true = labels, 
            y_pred = prediction,
            labels = [self.labels_dict[i] for i in self.labels_list])

        counts = np.zeros([len(self.labels_list), 1])
        for i in labels:
            counts[i] += 1

        return np.expand_dims(pre,1), np.expand_dims(rec,1), np.expand_dims(f1,1), counts

def multilabel_classification_report(y_true, y_pred, fmt='.3f', target_names=None):
    y_true = check_multilabel_array(y_true)
    y_pred = check_multilabel_array(y_pred)
    if y_true.shape != y_pred.shape:
        raise ValueError('y_true and y_pred must have equal shapes')
    n_labels = y_true.shape[1]
    if target_names is not None and len(target_names) != n_labels:
        raise ValueError('target_names must specify a name for all %d labels' % n_labels)

    # Collect stats
    precision, recall, f1_score, support = precision_recall_fscore_support(y_true, y_pred)
    tp, fp, tn, fn = multilabel_tp_fp_tn_fn_scores(y_true, y_pred)
    accuracy = multilabel_accuracy(y_true, y_pred)

    # Generate data for table, where each row represents a label
    headers = ['', 'precision', 'recall', 'f1-score', 'accuracy', 'support', 'TP', 'TN', 'FP', 'FN']
    data = []
    for label_idx in range(n_labels):
        target_name = str(label_idx) if target_names is None else target_names[label_idx]
        row = [target_name, precision[label_idx], recall[label_idx], f1_score[label_idx], accuracy[label_idx],
               support[label_idx], tp[label_idx], tn[label_idx], fp[label_idx], fn[label_idx]]
        data.append(row)

    # Calculate summaries for all values
    summary = ['avg / total', np.average(precision), np.average(recall), np.average(f1_score), np.average(accuracy),
               np.sum(support), np.sum(tp), np.sum(tn), np.sum(fp), np.sum(fn)]
    data.append(summary)

    return tabulate(data, headers=headers, floatfmt=fmt)

def evaluate_prediction(y_true, y_pred):
    """ evaluate prediction performance, given the ground truth

    :param y_true: correct target values
    :param y_pred: predicted values
    :return: confusion matrix, tp, tn, fp, fn precision, recall, F-score, and support
    """
    cm = confusion_matrix(y_true, y_pred)
    precision, recall, fscore, support = precision_recall_fscore_support(y_true, y_pred)
    return cm, tp_tn_fp_fn(cm), precision, recall, fscore, support

def test_metrics(self):
        Y = np.random.randint(0,2,size=(2,5,5))
        Yhat = np.random.randint(0,2,size=(2,5,5))

        C,acc,prec,recall,f1 = emlib.metrics(Y, Yhat, display=False)
        prec2, recall2, f12, supp = smetrics(np.reshape(Y, (Y.size,)), 
                np.reshape(Yhat, (Yhat.size,)))

        self.assertAlmostEqual(prec, prec2[1])
        self.assertAlmostEqual(recall, recall2[1])
        self.assertAlmostEqual(f1, f12[1])

def print_data(y,y_):
    p,r,f,s = precision_recall_fscore_support(y,y_)
    print('precision:\t{}'.format(p[1]))
    print('recall:\t\t{}'.format(r[1]))
    print('f1 score:\t{}'.format(f[1]))

def report_raw(self):
        precision, recall, f1, support = precision_recall_fscore_support(self.test_labels, self.predict_labels,
                                                                         labels=self.categories)
        prec_average, rec_average, f1_average, _ = precision_recall_fscore_support(self.test_labels,
                                                                                   self.predict_labels,
                                                                                   average='macro',
                                                                                   labels=self.categories)
        support_total = sum(support)
        matrix = [precision.tolist(), recall.tolist(), f1.tolist(), support.tolist()]
        matrix = [list(i) for i in zip(*matrix)]
        matrix.append([prec_average, rec_average, f1_average, support_total])
        return matrix

def GBDT_classify(train_dataSet_path, test_dataSet_path, train_one_and_two_result_as_proba_path):

    train_data = pd.read_csv(train_dataSet_path)
    train_data = train_data.as_matrix()
    X_train = train_data[:, 2:-1]  # select columns 0 through end-1
    y_train = train_data[:, -1]  # select column end

    test_data = pd.read_csv(test_dataSet_path)
    test_data = test_data.as_matrix()
    X_test = test_data[:, 2:-1]  # select columns 0 through end-1
    y_test = test_data[:, -1]  # select column end 

    clf = GradientBoostingClassifier(n_estimators=200)
    clf.fit(X_train, y_train)
    pre_y_test = clf.predict_proba(X_test)
    print pre_y_test
    print("GBDT Metrics : {0}".format(precision_recall_fscore_support(y_test, pre_y_test)))

    print u'????.....'
    f_result = open(test_dataSet_prob_path, 'w')
    for i in range(0, len(pre_y_test)):
        if i==0:
            print str(pre_y_test[i][0])
        if i==len(pre_y_test)-1:
            print str(pre_y_test[i][0])
        f_result.write(str(pre_y_test[i][0]) + '\n')   

    return clf