Metrics for multi-label classification for using with tf.keras

[Abhijit-2592]

Top-K Metrics are widely used in assessing the quality of Multi-Label classification. tf.metrics.recall_at_k and tf.metrics.precision_at_k cannot be directly used with tf.keras! Even if we wrap it accordingly for tf.keras, In most cases it will raise NaNs because of numerical instability. Since we don't have out of the box metrics that can be used for monitoring multi-label classification training using tf.keras. I came up with the following plugin for Tensorflow 1.X version. This can also be easily ported to Tensorflow 2.0.

import tensorflow as tf
K = tf.keras.backend

class MetricsAtTopK:
    def __init__(self, k):
        self.k = k

    def _get_prediction_tensor(self, y_pred):
        """Takes y_pred and creates a tensor of same shape with 1 in indices where, the values are in top_k
        """
        topk_values, topk_indices = tf.nn.top_k(y_pred, k=self.k, sorted=False, name="topk")
        # the topk_indices are along last axis (1). Add indices for axis=0
        ii, _ = tf.meshgrid(tf.range(tf.shape(y_pred)[0]), tf.range(self.k), indexing='ij')
        index_tensor = tf.reshape(tf.stack([ii, topk_indices], axis=-1), shape=(-1, 2))
        prediction_tensor = tf.sparse_to_dense(sparse_indices=index_tensor,
                                               output_shape=tf.shape(y_pred),
                                               default_value=0,
                                               sparse_values=1.0,
                                               validate_indices=False
                                               )
        prediction_tensor = tf.cast(prediction_tensor, K.floatx())
        return prediction_tensor

    def true_positives_at_k(self, y_true, y_pred):
        prediction_tensor = self._get_prediction_tensor(y_pred=y_pred)
        true_positive = K.sum(tf.multiply(prediction_tensor, y_true))
        return true_positive

    def false_positives_at_k(self, y_true, y_pred):
        prediction_tensor = self._get_prediction_tensor(y_pred=y_pred)
        true_positive = K.sum(tf.multiply(prediction_tensor, y_true))
        c2 = K.sum(prediction_tensor)  # TP + FP
        false_positive = c2 - true_positive
        return false_positive

    def false_negatives_at_k(self, y_true, y_pred):
        prediction_tensor = self._get_prediction_tensor(y_pred=y_pred)
        true_positive = K.sum(tf.multiply(prediction_tensor, y_true))
        c3 = K.sum(y_true)  # TP + FN
        false_negative = c3 - true_positive
        return false_negative

    def precision_at_k(self, y_true, y_pred):
        prediction_tensor = self._get_prediction_tensor(y_pred=y_pred)
        true_positive = K.sum(tf.multiply(prediction_tensor, y_true))
        c2 = K.sum(prediction_tensor)  # TP + FP
        return true_positive/(c2+K.epsilon())

    def recall_at_k(self, y_true, y_pred):
        prediction_tensor = self._get_prediction_tensor(y_pred=y_pred)
        true_positive = K.sum(tf.multiply(prediction_tensor, y_true))
        c3 = K.sum(y_true)  # TP + FN
        return true_positive/(c3+K.epsilon())

    def f1_at_k(self, y_true, y_pred):
        precision = self.precision_at_k(y_true=y_true, y_pred=y_pred)
        recall = self.recall_at_k(y_true=y_true, y_pred=y_pred)
        f1 = (2*precision*recall)/(precision+recall+K.epsilon())
        return f1

### Usage:
metrics = MetricsAtTopK(k=5)
# model definition
# ...
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc', 
                                                                     metrics.true_positives_at_k, 
                                                                     metrics.false_positives_at_k,
                                                                     metrics.false_negatives_at_k,
                                                                     metrics.recall_at_k,
                                                                     metrics.precision_at_k,
                                                                     metrics.f1_at_k,
                                                                    ]
model.fit(...)  # as usual
 

Is this something which we can integrate to Tensorflow? If so I will be glad to open up a Pull Request

[shashvatshahi1998]

@Abhijit-2592 I have opened a pull request already about adding multilabel classification, but i am not getting correct location to add those lines so that they can work properly

[Abhijit-2592]

Can you link the pull request?

[shashvatshahi1998]

#28204

[pavithrasv]

@Abhijit-2592 Have you tried the new 2.0 metrics? https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/metrics.py#L1072 Please take a look and let me know if they would work for you. We have implementations of precision at K and recall at K. They are called Precision and Recall and top_k is accepted as a parameter in these metrics.

[pavithrasv]

Closing this issue for now, please feel free to re-open if this isn't resolved.

[SumNeuron]

@pavithrasv I'd like to reopen the issue if possible. My reasons are as follows:

1. the metric has an annoying control dependency:

with ops.control_dependencies([
      check_ops.assert_greater_equal(
          y_pred,
          math_ops.cast(0.0, dtype=y_pred.dtype),
          message='predictions must be >= 0'),
      check_ops.assert_less_equal(
          y_pred,
          math_ops.cast(1.0, dtype=y_pred.dtype),
          message='predictions must be <= 1')
  ]):

why is this annoying? if using a sigmoid cross entropy for the loss (as multi-labels are independent), tf offers the function:

tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits)

Since this function applies a sigmoid transformation to the logits (output of the model), then the user shouldn't have in their final layer a sigmoid activation. Otherwise it is sigmoid(sigmoid(...)). Since this is the case, values are not gaurenteed to be between [0, 1], especially when training first begins. So Precision (and other similar tf.keras.metrics) are not even able to be included as this will throw an error ending training.

2. in the precision (and similar) metrics, the thing of interest multi_label is called with the default value (False):

# inside Precision update_state
return metrics_utils.update_confusion_matrix_variables(
        {
            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives
        },
        y_true,
        y_pred,
        thresholds=self.thresholds,
        top_k=self.top_k,
        class_id=self.class_id,
        sample_weight=sample_weight) # <---- multi_label is not set to True, nor do we as users
        # have the option to do so

# function signature
def update_confusion_matrix_variables(variables_to_update,
                                      y_true,
                                      y_pred,
                                      thresholds,
                                      top_k=None,
                                      class_id=None,
                                      sample_weight=None,
                                      multi_label=False, # <---- defaults to false
                                      label_weights=None):

collectively, this means that multi_label metrics are not supported via tf.keras api.

While the update_confusion_matrix_variables function is great (and the associated structure), to an outsider who might want to contribute / track down why it doesn't work with their code, it is could be daunting.

Perhaps tf.keras.metrics could single out multilabel metrics?

e.g. based on the subclassing Metric documentation:

class MultiLabelMacroSpecificity(tf.keras.metrics.Metric):
    
    def __init__(self, name='multi_label_macro_specificity', threshold=0.5, **kwargs):        
        super(MultiLabelMacroSpecificity, self).__init__(name=name, **kwargs)
        self.specificity = self.add_weight(name='mlm_spec', initializer='zeros')        
        self.threshold       = tf.constant(threshold)

        # replace this with tf confusion_matrix utils
        self.true_negatives  = self.add_weight(name='fn', initializer='zeros')
        self.false_positives = self.add_weight(name='fp', initializer='zeros')
    
    def update_state(self, y_true, y_pred):
        
        # Compare predictions and threshold.        
        pred_is_pos  = tf.greater(tf.cast(y_pred, tf.float32), self.threshold)            
        # |-- in case of soft labeling        
        label_is_pos = tf.greater(tf.cast(y_true, tf.float32), self.threshold)                
        label_is_neg = tf.logical_not(tf.cast(label_is_pos, tf.bool))
        
        self.true_negatives.assign_add(tf.reduce_sum(tf.cast(label_is_neg, tf.float32)))
        self.false_positives.assign_add(
            tf.reduce_sum(tf.cast(tf.logical_and(pred_is_pos, label_is_neg), tf.float32))
        )
        
        tn = self.true_negatives
        fp = self.false_positives
        specificity = tf.div_no_nan(tn, tf.add(tn, fp))
        self.specificity.assign(specificity)
        return specificity
    
    def result(self):
        return self.specificity        

[pavithrasv]

Thank you, i see you have opened another issue for the pending features, will reply on that.