Merge d5974e7 into 6c7d02e

deepchem/metrics/__init__.py

@@ -37,3 +37,4 @@
 from deepchem.metrics.score_function import rms_score
 from deepchem.metrics.score_function import mae_score
 from deepchem.metrics.score_function import bedroc_score
+from deepchem.metrics.score_function import concordance_index

deepchem/metrics/metric.py

@@ -528,20 +528,11 @@ def __init__(self,
     if mode is None:
       # These are some smart defaults
       if self.metric.__name__ in [
-          "roc_auc_score",
-          "matthews_corrcoef",
-          "recall_score",
-          "accuracy_score",
-          "kappa_score",
-          "cohen_kappa_score",
-          "precision_score",
-          "balanced_accuracy_score",
-          "prc_auc_score",
-          "f1_score",
-          "bedroc_score",
-          "jaccard_score",
-          "jaccard_index",
-          "pixel_error",
+          "roc_auc_score", "matthews_corrcoef", "recall_score",
+          "accuracy_score", "kappa_score", "cohen_kappa_score",
+          "precision_score", "balanced_accuracy_score", "prc_auc_score",
+          "f1_score", "bedroc_score", "jaccard_score", "jaccard_index",
+          "pixel_error"
       ]:
         mode = "classification"
         # These are some smart defaults corresponding to sklearn's required
@@ -561,7 +552,8 @@ def __init__(self,
             classification_handling_mode = None
       elif self.metric.__name__ in [
           "pearson_r2_score", "r2_score", "mean_squared_error",
-          "mean_absolute_error", "rms_score", "mae_score", "pearsonr"
+          "mean_absolute_error", "rms_score", "mae_score", "pearsonr",
+          "concordance_index"
       ]:
         mode = "regression"
       else:

deepchem/metrics/score_function.py

@@ -162,3 +162,55 @@ def bedroc_score(y_true: np.ndarray, y_pred: np.ndarray, alpha: float = 20.0):
   scores = sorted(scores, key=lambda pair: pair[1], reverse=True)
 
   return CalcBEDROC(scores, 0, alpha)
+
+
+def concordance_index(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+  """Compute Concordance index.
+
+  Statistical metric indicates the quality of the predicted ranking.
+  Please confirm details from [1]_.
+
+  Parameters
+  ----------
+  y_true: np.ndarray
+    continous value
+  y_pred: np.ndarray
+    Predicted value
+
+  Returns
+  -------
+  float
+    score between [0,1]
+
+  References
+  ----------
+  .. [1] Steck, Harald, et al. "On ranking in survival analysis:
+     Bounds on the concordance index." Advances in neural information processing systems (2008): 1209-1216.
+  """
+
+  idx = np.argsort(y_true)
+  y_true = y_true[idx]
+  y_pred = y_pred[idx]
+
+  pairs = 0
+  correct_pairs = 0.0
+
+  for i in range(len(y_true)):
+    true_a = y_true[i]
+    pred_a = y_pred[i]
+
+    for j in range(i + 1, len(y_true)):
+      true_b = y_true[j]
+      pred_b = y_pred[j]
+      if true_a != true_b:
+        pairs += 1
+        if pred_a == pred_b:
+          correct_pairs += 0.5
+        elif pred_a < pred_b:
+          correct_pairs += true_a < true_b
+        else:
+          correct_pairs += true_a > true_b
+
+  assert pairs > 0, 'No pairs for comparision'
+
+  return correct_pairs / pairs

deepchem/metrics/tests/test_metrics.py

@@ -68,3 +68,23 @@ def test_bedroc_score():
       np.concatenate([worst_pred_actives, worst_pred_inactives]))
   worst_score = dc.metrics.bedroc_score(y_true, y_pred_worst)
   np.testing.assert_almost_equal(worst_score, 0.0, 4)
+
+
+def test_concordance_index():
+  """Test concordance index."""
+
+  metric = dc.metrics.Metric(dc.metrics.concordance_index)
+
+  y_true = np.array([1, 3, 5, 4, 2])
+  y_pred = np.array([3, 1, 5, 4, 2])
+  assert metric.compute_singletask_metric(y_true, y_pred) == 0.7
+
+  # best case
+  y_true = np.array([1, 3, 5, 4, 2])
+  y_pred = np.array([1, 3, 5, 4, 2])
+  assert metric.compute_singletask_metric(y_true, y_pred) == 1.0
+
+  # duplicate prediction value
+  y_true = np.array([1, 3, 5, 4, 2])
+  y_pred = np.array([1, 3, 4, 4, 2])
+  assert metric.compute_singletask_metric(y_true, y_pred) == 0.95

docs/metrics.rst

@@ -79,6 +79,8 @@ DeepChem has a variety of different metrics which are useful for measuring model
 
 .. autofunction:: deepchem.metrics.bedroc_score
 
+.. autofunction:: deepchem.metrics.concordance_index
+
 .. autofunction:: deepchem.metrics.genomic_metrics.get_motif_scores
 
 .. autofunction:: deepchem.metrics.genomic_metrics.get_pssm_scores