supress value error when label contains one label

chainer_chemistry/training/extensions/roc_auc_evaluator.py

@@ -1,4 +1,5 @@
 import copy
+from logging import getLogger
 
 import numpy
 
@@ -73,6 +74,10 @@ class ROCAUCEvaluator(Evaluator):
             are considered as negative.
         ignore_labels (int or list or None): labels to be ignored.
             `None` is used to not ignore all labels.
+        raise_value_error (bool): If False, `ValueError` caused by
+            `roc_auc_score` calculation is suppressed and ignored with warning
+            message.
+        logger:
 
     Attributes:
         converter: Converter function.
@@ -86,13 +91,16 @@ class ROCAUCEvaluator(Evaluator):
 
     def __init__(self, iterator, target, converter=convert.concat_examples,
                  device=None, eval_hook=None, eval_func=None, name=None,
-                 pos_labels=1, ignore_labels=None):
+                 pos_labels=1, ignore_labels=None, raise_value_error=True,
+                 logger=None):
         super(ROCAUCEvaluator, self).__init__(
             iterator, target, converter=converter, device=device,
             eval_hook=eval_hook, eval_func=eval_func)
         self.name = name
         self.pos_labels = _to_list(pos_labels)
         self.ignore_labels = _to_list(ignore_labels)
+        self.raise_value_error = raise_value_error
+        self.logger = logger or getLogger()
 
     def evaluate(self):
         iterator = self._iterators['main']
@@ -132,7 +140,19 @@ def evaluate(self):
         # --- set positive labels to 1, negative labels to 0 ---
         pos_indices = numpy.in1d(t_total, self.pos_labels)
         t_total = numpy.where(pos_indices, 1, 0)
-        roc_auc = metrics.roc_auc_score(t_total, y_total)
+        try:
+            roc_auc = metrics.roc_auc_score(t_total, y_total)
+        except ValueError as e:
+            if self.raise_value_error:
+                raise e
+            else:
+                # This is usually caused by the following
+                # Only one class present in y_true.
+                # ROC AUC score is not defined in that case
+                self.logger.warning(
+                    'ValueError detected during roc_auc_score calculation. {}'
+                    .format(e.args))
+                roc_auc = numpy.nan
 
         observation = {}
         with reporter.report_scope(observation):

tests/training_tests/extensions_tests/test_roc_auc_evaluator.py

@@ -30,6 +30,15 @@ def data1():
     return y, t
 
 
+@pytest.fixture
+def data2():
+    # Example of bad example case
+    # `t` only contains correct label, `y` is dummy predict value by predictor
+    t = numpy.array([0, 0, 0, 0], dtype=numpy.int32)[:, None]
+    y = numpy.array([0.1, 0.4, 0.35, 0.8], dtype=numpy.float32)[:, None]
+    return y, t
+
+
 class DummyPredictor(chainer.Chain):
 
     def __call__(self, y):
@@ -95,5 +104,32 @@ def _test_roc_auc_evaluator_with_labels(data1):
     assert result['val/main/roc_auc'] == expected_roc_auc
 
 
+def test_roc_auc_evaluator_raise_value_error(data2):
+    with pytest.raises(ValueError):
+        _test_roc_auc_evaluator_raise_error(data2, raise_value_error=True)
+
+    res = _test_roc_auc_evaluator_raise_error(data2, raise_value_error=False)
+    assert numpy.isnan(res)
+
+
+def _test_roc_auc_evaluator_raise_error(data, raise_value_error=True):
+
+    predictor = DummyPredictor()
+    dataset = NumpyTupleDataset(*data)
+
+    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
+    evaluator = ROCAUCEvaluator(
+        iterator, predictor, name='train',
+        pos_labels=1, ignore_labels=None,
+        raise_value_error=raise_value_error
+    )
+    repo = chainer.Reporter()
+    repo.add_observer('target', predictor)
+    with repo:
+        observation = evaluator.evaluate()
+
+    return observation['target/roc_auc']
+
+
 if __name__ == '__main__':
     pytest.main([__file__, '-v', '-s'])