diff --git a/deepchem/metrics/__init__.py b/deepchem/metrics/__init__.py index f471e9e97a..0be35fb3a6 100644 --- a/deepchem/metrics/__init__.py +++ b/deepchem/metrics/__init__.py @@ -37,3 +37,4 @@ from deepchem.metrics.score_function import rms_score from deepchem.metrics.score_function import mae_score from deepchem.metrics.score_function import bedroc_score +from deepchem.metrics.score_function import concordance_index diff --git a/deepchem/metrics/metric.py b/deepchem/metrics/metric.py index 47160f2c42..10b5092390 100644 --- a/deepchem/metrics/metric.py +++ b/deepchem/metrics/metric.py @@ -528,20 +528,11 @@ def __init__(self, if mode is None: # These are some smart defaults if self.metric.__name__ in [ - "roc_auc_score", - "matthews_corrcoef", - "recall_score", - "accuracy_score", - "kappa_score", - "cohen_kappa_score", - "precision_score", - "balanced_accuracy_score", - "prc_auc_score", - "f1_score", - "bedroc_score", - "jaccard_score", - "jaccard_index", - "pixel_error", + "roc_auc_score", "matthews_corrcoef", "recall_score", + "accuracy_score", "kappa_score", "cohen_kappa_score", + "precision_score", "balanced_accuracy_score", "prc_auc_score", + "f1_score", "bedroc_score", "jaccard_score", "jaccard_index", + "pixel_error" ]: mode = "classification" # These are some smart defaults corresponding to sklearn's required @@ -561,7 +552,8 @@ def __init__(self, classification_handling_mode = None elif self.metric.__name__ in [ "pearson_r2_score", "r2_score", "mean_squared_error", - "mean_absolute_error", "rms_score", "mae_score", "pearsonr" + "mean_absolute_error", "rms_score", "mae_score", "pearsonr", + "concordance_index" ]: mode = "regression" else: diff --git a/deepchem/metrics/score_function.py b/deepchem/metrics/score_function.py index 8ff57ee069..ec8e31686e 100644 --- a/deepchem/metrics/score_function.py +++ b/deepchem/metrics/score_function.py @@ -162,3 +162,55 @@ def bedroc_score(y_true: np.ndarray, y_pred: np.ndarray, alpha: float = 20.0): scores = sorted(scores, key=lambda pair: pair[1], reverse=True) return CalcBEDROC(scores, 0, alpha) + + +def concordance_index(y_true: np.ndarray, y_pred: np.ndarray) -> float: + """Compute Concordance index. + + Statistical metric indicates the quality of the predicted ranking. + Please confirm details from [1]_. + + Parameters + ---------- + y_true: np.ndarray + continous value + y_pred: np.ndarray + Predicted value + + Returns + ------- + float + score between [0,1] + + References + ---------- + .. [1] Steck, Harald, et al. "On ranking in survival analysis: + Bounds on the concordance index." Advances in neural information processing systems (2008): 1209-1216. + """ + + idx = np.argsort(y_true) + y_true = y_true[idx] + y_pred = y_pred[idx] + + pairs = 0 + correct_pairs = 0.0 + + for i in range(len(y_true)): + true_a = y_true[i] + pred_a = y_pred[i] + + for j in range(i + 1, len(y_true)): + true_b = y_true[j] + pred_b = y_pred[j] + if true_a != true_b: + pairs += 1 + if pred_a == pred_b: + correct_pairs += 0.5 + elif pred_a < pred_b: + correct_pairs += true_a < true_b + else: + correct_pairs += true_a > true_b + + assert pairs > 0, 'No pairs for comparision' + + return correct_pairs / pairs diff --git a/deepchem/metrics/tests/test_metrics.py b/deepchem/metrics/tests/test_metrics.py index 343d2f82c7..5cd665174a 100644 --- a/deepchem/metrics/tests/test_metrics.py +++ b/deepchem/metrics/tests/test_metrics.py @@ -68,3 +68,23 @@ def test_bedroc_score(): np.concatenate([worst_pred_actives, worst_pred_inactives])) worst_score = dc.metrics.bedroc_score(y_true, y_pred_worst) np.testing.assert_almost_equal(worst_score, 0.0, 4) + + +def test_concordance_index(): + """Test concordance index.""" + + metric = dc.metrics.Metric(dc.metrics.concordance_index) + + y_true = np.array([1, 3, 5, 4, 2]) + y_pred = np.array([3, 1, 5, 4, 2]) + assert metric.compute_singletask_metric(y_true, y_pred) == 0.7 + + # best case + y_true = np.array([1, 3, 5, 4, 2]) + y_pred = np.array([1, 3, 5, 4, 2]) + assert metric.compute_singletask_metric(y_true, y_pred) == 1.0 + + # duplicate prediction value + y_true = np.array([1, 3, 5, 4, 2]) + y_pred = np.array([1, 3, 4, 4, 2]) + assert metric.compute_singletask_metric(y_true, y_pred) == 0.95 diff --git a/docs/metrics.rst b/docs/metrics.rst index 403bec37da..72032083b3 100644 --- a/docs/metrics.rst +++ b/docs/metrics.rst @@ -79,6 +79,8 @@ DeepChem has a variety of different metrics which are useful for measuring model .. autofunction:: deepchem.metrics.bedroc_score +.. autofunction:: deepchem.metrics.concordance_index + .. autofunction:: deepchem.metrics.genomic_metrics.get_motif_scores .. autofunction:: deepchem.metrics.genomic_metrics.get_pssm_scores