Merge 2b6313c into 8be0ba0

csrank/metrics.py

@@ -7,7 +7,8 @@
 
 __all__ = ['zero_one_rank_loss', 'zero_one_rank_loss_for_scores',
            'zero_one_rank_loss_for_scores_ties',
-           'make_ndcg_at_k_loss', 'kendalls_tau_for_scores', 'spearman_correlation_for_scores', "zero_one_accuracy",
+           'make_ndcg_at_k_loss', 'kendalls_tau_for_scores',
+           'spearman_correlation_for_scores', "zero_one_accuracy",
            "zero_one_accuracy_for_scores"]
 
 
@@ -16,12 +17,15 @@ def zero_one_rank_loss(y_true, y_pred):
     mask = K.greater(y_true[:, None] - y_true[:, :, None], 0)
     # Count the number of mistakes (here position difference less than 0)
     mask2 = K.less(y_pred[:, None] - y_pred[:, :, None], 0)
+    mask3 = K.equal(y_pred[:, None] - y_pred[:, :, None], 0)
 
     # Calculate Transpositions
     transpositions = tf.logical_and(mask, mask2)
     transpositions = K.sum(K.cast(transpositions, dtype='float32'), axis=[1, 2])
 
     n_objects = K.max(y_true) + 1
+    transpositions += (K.sum(K.cast(mask3, dtype='float32'), axis=[1, 2])
+                       - n_objects) / 4.
     denominator = K.cast((n_objects * (n_objects - 1.)) / 2., dtype='float32')
     result = transpositions / denominator
     return K.mean(result)
@@ -37,19 +41,7 @@ def zero_one_accuracy(y_true, y_pred):
 
 
 def zero_one_rank_loss_for_scores(y_true, s_pred):
-    y_true, s_pred = tensorify(y_true), tensorify(s_pred)
-    mask = K.greater(y_true[:, None] - y_true[:, :, None], 0)
-    # Count the number of mistakes (score difference is larger than 0)
-    mask2 = K.greater(s_pred[:, None] - s_pred[:, :, None], 0)
-
-    # Calculate Transpositions
-    transpositions = tf.logical_and(mask, mask2)
-    transpositions = K.sum(K.cast(transpositions, dtype='float32'), axis=[1, 2])
-
-    n_objects = K.max(y_true) + 1
-    denominator = K.cast((n_objects * (n_objects - 1.)) / 2., dtype='float32')
-    result = transpositions / denominator
-    return K.mean(result)
+    return zero_one_rank_loss_for_scores_ties(y_true, s_pred)
 
 
 def zero_one_rank_loss_for_scores_ties(y_true, s_pred):
@@ -74,21 +66,22 @@ def make_ndcg_at_k_loss(k=5):
     def ndcg(y_true, y_pred):
         y_true, y_pred = tensorify(y_true), tensorify(y_pred)
         n_objects = K.cast(K.int_shape(y_pred)[1], 'float32')
-        y_true_f = K.cast(y_true, 'float32')
-        relevance = K.pow(2., n_objects - y_pred - 1.) - 1.
+        relevance = K.pow(2., n_objects - y_true - 1.) - 1.
+        relevance_pred = K.pow(2., n_objects - y_pred - 1.) - 1.
 
         # Calculate ideal dcg:
         toprel, toprel_ind = tf.nn.top_k(relevance, k)
         log_term = K.log(K.arange(k, dtype='float32') + 2.) / K.log(2.)
         idcg = K.sum(toprel / log_term, axis=-1, keepdims=True)
 
         # Calculate actual dcg:
-        toppred, toppred_ind = tf.nn.top_k(y_pred, k)
+        toppred, toppred_ind = tf.nn.top_k(relevance_pred, k)
         row_ind = K.cumsum(K.ones_like(toppred_ind), axis=0) - 1
         ind = K.stack([row_ind, toppred_ind], axis=-1)
-        pred_rel = K.sum(tf.gather_nd(relevance, ind) / log_term, axis=-1, keepdims=True)
-        loss = 1 - pred_rel / idcg
-        return loss
+        pred_rel = K.sum(tf.gather_nd(relevance, ind) / log_term, axis=-1,
+                         keepdims=True)
+        gain = pred_rel / idcg
+        return gain
 
     return ndcg
 

csrank/tests/test_metrics.py

@@ -0,0 +1,86 @@
+import pytest
+import numpy as np
+from keras import backend as K
+from numpy.testing import assert_almost_equal
+
+from csrank.metrics import zero_one_rank_loss, zero_one_rank_loss_for_scores, \
+    zero_one_accuracy, make_ndcg_at_k_loss
+
+
+@pytest.fixture(scope="module",
+                params=[(True, False), (False, True)],
+                ids=['NoTies', 'Ties'])
+def problem_for_pred(request):
+    is_numpy, ties = request.param
+    y_true = np.arange(5)[None, :]
+    # We test the error by swapping one adjacent pair:
+    if ties:
+        y_pred = np.array([[0, 2, 1, 2, 3]])
+    else:
+        y_pred = np.array([[0, 2, 1, 3, 4]])
+    if is_numpy:
+        return y_true, y_pred, ties
+    y_true_tensor = K.constant(y_true)
+    y_pred_tensor = K.constant(y_pred)
+    return y_true_tensor, y_pred_tensor, ties
+
+
+@pytest.fixture(scope="module",
+                params=[(True, False), (False, True)],
+                ids=['NoTies', 'Ties'])
+def problem_for_scores(request):
+    is_numpy, ties = request.param
+    y_true = np.arange(5)[None, :]
+    # We test the error by swapping one adjacent pair:
+    if ties:
+        y_scores = np.array([[1., 0.8, 0.9, 0.8, 0.6]])
+    else:
+        y_scores = np.array([[1., 0.8, 0.9, 0.7, 0.6]])
+    if is_numpy:
+        return y_true, y_scores, ties
+    y_true_tensor = K.constant(y_true)
+    y_scores_tensor = K.constant(y_scores)
+    return y_true_tensor, y_scores_tensor, ties
+
+
+def test_zero_one_rank_loss(problem_for_pred):
+    y_true_tensor, y_pred_tensor, ties = problem_for_pred
+    score = zero_one_rank_loss(y_true_tensor, y_pred_tensor)
+    real_score = K.eval(score)
+    if ties:
+        assert_almost_equal(actual=real_score, desired=np.array([0.15]))
+    else:
+        assert_almost_equal(actual=real_score, desired=np.array([0.1]))
+
+
+def test_zero_one_rank_loss_for_scores(problem_for_scores):
+    y_true_tensor, y_scores_tensor, ties = problem_for_scores
+
+    score = zero_one_rank_loss_for_scores(y_true_tensor, y_scores_tensor)
+    real_score = K.eval(score)
+    if ties:
+        assert_almost_equal(actual=real_score, desired=np.array([0.15]))
+    else:
+        assert_almost_equal(actual=real_score, desired=np.array([0.1]))
+
+
+def test_zero_one_accuracy(problem_for_pred):
+    y_true_tensor, y_pred_tensor, ties = problem_for_pred
+
+    score = zero_one_accuracy(y_true_tensor, y_pred_tensor)
+    real_score = K.eval(score)
+    assert_almost_equal(actual=real_score, desired=np.array([0.]))
+
+
+def test_ndcg(problem_for_pred):
+    y_true_tensor, y_pred_tensor, ties = problem_for_pred
+
+    ndcg = make_ndcg_at_k_loss(k=2)
+    gain = ndcg(y_true_tensor, y_pred_tensor)
+    real_gain = K.eval(gain)
+
+    expected_dcg = 15. + 3. / np.log2(3.)
+    expected_idcg = 15. + 7. / np.log2(3.)
+    assert_almost_equal(actual=real_gain,
+                        desired=np.array([[expected_dcg/expected_idcg]]),
+                        decimal=5)