Metrics.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Import useful packages
import tensorflow as tf

def evaluation(y, prediction):
    '''

    This is the evaluation metrics.
    Here, we provide a four-class classification evaluation codes
    The files will be updated to adapt multiply classes soon.

    Notice: All the evaluations during training will be saved in the TensorBoard.

    Use
        tensorboard --logdir="Your_Checkpoint_File_Address" --host=127.0.0.1
    to view and download the evaluation files.

    BTW, you can definitely edit the following codes to adapt your own classes,
    e.g., if Two classes, u can just delete the three and four class.

    Args:
        y: The True Labels (with One-hot representation)
        prediction: The predicted output probability

    Returns:
        Single class evaluation: T1_accuracy, T1_Precision, T1_Recall, T1_F_Score,
                                 T2_accuracy, T2_Precision, T2_Recall, T2_F_Score,
                                 T3_accuracy, T3_Precision, T3_Recall, T3_F_Score,
                                 T4_accuracy, T4_Precision, T4_Recall, T4_F_Score,

        Global model evaluation: Global_Average_Accuracy,
                                 Kappa_Metric,
                                 Macro_Global_Precision,
                                 Macro_Global_Recall,
                                 Macro_Global_F1_Score

    '''

    # Calculate Accuracy
    # Add metrics to TensorBoard.
    with tf.name_scope('Evalution'):
        # Calculate Each Task Accuracy
        with tf.name_scope('Each_Class_accuracy'):
            # Task 1 Accuracy
            with tf.name_scope('T1_accuracy'):
                # Number of Classified Correctly
                y_T1 = tf.equal(tf.argmax(y, 1), 0)
                prediction_T1 = tf.equal(tf.argmax(prediction, 1), 0)
                T1_Corrected_Num = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T1, prediction_T1), tf.float32))

                # Number of All the Test Samples
                T1_all_Num = tf.reduce_sum(tf.cast(y_T1, tf.float32))

                # Task 1 Accuracy
                T1_accuracy = tf.divide(T1_Corrected_Num, T1_all_Num)
                tf.summary.scalar('T1_accuracy', T1_accuracy)

                T1_TP = T1_Corrected_Num
                T1_TN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T1), tf.math.logical_not(prediction_T1)),
                            tf.float32))
                T1_FP = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T1), prediction_T1), tf.float32))
                T1_FN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(y_T1, tf.math.logical_not(prediction_T1)), tf.float32))

                with tf.name_scope("T1_Precision"):
                    T1_Precision = T1_TP / (T1_TP + T1_FP)
                    tf.summary.scalar('T1_Precision', T1_Precision)

                with tf.name_scope("T1_Recall"):
                    T1_Recall = T1_TP / (T1_TP + T1_FN)
                    tf.summary.scalar('T1_Recall', T1_Recall)

                with tf.name_scope("T1_F_Score"):
                    T1_F_Score = (2 * T1_Precision * T1_Recall) / (T1_Precision + T1_Recall)
                    tf.summary.scalar('T1_F_Score', T1_F_Score)

            # Task 2 Accuracy
            with tf.name_scope('T2_accuracy'):
                # Number of Classified Correctly
                y_T2 = tf.equal(tf.argmax(y, 1), 1)
                prediction_T2 = tf.equal(tf.argmax(prediction, 1), 1)
                T2_Corrected_Num = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T2, prediction_T2), tf.float32))

                # Number of All the Test Samples
                T2_all_Num = tf.reduce_sum(tf.cast(y_T2, tf.float32))

                # Task 2 Accuracy
                T2_accuracy = tf.divide(T2_Corrected_Num, T2_all_Num)
                tf.summary.scalar('T2_accuracy', T2_accuracy)

                T2_TP = T2_Corrected_Num
                T2_TN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T2), tf.math.logical_not(prediction_T2)),
                            tf.float32))
                T2_FP = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T2), prediction_T2), tf.float32))
                T2_FN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(y_T2, tf.math.logical_not(prediction_T2)), tf.float32))

                with tf.name_scope("T2_Precision"):
                    T2_Precision = T2_TP / (T2_TP + T2_FP)
                    tf.summary.scalar('T2_Precision', T2_Precision)

                with tf.name_scope("T2_Recall"):
                    T2_Recall = T2_TP / (T2_TP + T2_FN)
                    tf.summary.scalar('T2_Recall', T2_Recall)

                with tf.name_scope("T2_F_Score"):
                    T2_F_Score = (2 * T2_Precision * T2_Recall) / (T2_Precision + T2_Recall)
                    tf.summary.scalar('T2_F_Score', T2_F_Score)

            # Task 3 Accuracy
            with tf.name_scope('T3_accuracy'):
                # Number of Classified Correctly
                y_T3 = tf.equal(tf.argmax(y, 1), 2)
                prediction_T3 = tf.equal(tf.argmax(prediction, 1), 2)
                T3_Corrected_Num = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T3, prediction_T3), tf.float32))

                # Number of All the Test Samples
                T3_all_Num = tf.reduce_sum(tf.cast(y_T3, tf.float32))

                # Task 3 Accuracy
                T3_accuracy = tf.divide(T3_Corrected_Num, T3_all_Num)
                tf.summary.scalar('T3_accuracy', T3_accuracy)

                T3_TP = T3_Corrected_Num
                T3_TN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T3), tf.math.logical_not(prediction_T3)),
                            tf.float32))
                T3_FP = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T3), prediction_T3), tf.float32))
                T3_FN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(y_T3, tf.math.logical_not(prediction_T3)), tf.float32))

                with tf.name_scope("T3_Precision"):
                    T3_Precision = T3_TP / (T3_TP + T3_FP)
                    tf.summary.scalar('T3_Precision', T3_Precision)

                with tf.name_scope("T3_Recall"):
                    T3_Recall = T3_TP / (T3_TP + T3_FN)
                    tf.summary.scalar('T3_Recall', T3_Recall)

                with tf.name_scope("T3_F_Score"):
                    T3_F_Score = (2 * T3_Precision * T3_Recall) / (T3_Precision + T3_Recall)
                    tf.summary.scalar('T3_F_Score', T3_F_Score)

            # Task 4 Accuracy
            with tf.name_scope('T4_accuracy'):
                # Number of Classified Correctly
                y_T4 = tf.equal(tf.argmax(y, 1), 3)
                prediction_T4 = tf.equal(tf.argmax(prediction, 1), 3)
                T4_Corrected_Num = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T4, prediction_T4), tf.float32))

                # Number of All the Test Samples
                T4_all_Num = tf.reduce_sum(tf.cast(y_T4, tf.float32))

                # Task 4 Accuracy
                T4_accuracy = tf.divide(T4_Corrected_Num, T4_all_Num)
                tf.summary.scalar('T4_accuracy', T4_accuracy)

                T4_TP = T4_Corrected_Num
                T4_TN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T4), tf.math.logical_not(prediction_T4)),
                            tf.float32))
                T4_FP = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(tf.math.logical_not(y_T4), prediction_T4), tf.float32))
                T4_FN = tf.reduce_sum(
                    tf.cast(tf.math.logical_and(y_T4, tf.math.logical_not(prediction_T4)), tf.float32))

                with tf.name_scope("T4_Precision"):
                    T4_Precision = T4_TP / (T4_TP + T4_FP)
                    tf.summary.scalar('T4_Precision', T4_Precision)

                with tf.name_scope("T4_Recall"):
                    T4_Recall = T4_TP / (T4_TP + T4_FN)
                    tf.summary.scalar('T4_Recall', T4_Recall)

                with tf.name_scope("T4_F_Score"):
                    T4_F_Score = (2 * T4_Precision * T4_Recall) / (T4_Precision + T4_Recall)
                    tf.summary.scalar('T4_F_Score', T4_F_Score)

        # Calculate the Confusion Matrix
        with tf.name_scope("Confusion_Matrix"):
            with tf.name_scope("T1_Label"):
                T1_T1 = T1_Corrected_Num
                T1_T2 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T1, prediction_T2), tf.float32))
                T1_T3 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T1, prediction_T3), tf.float32))
                T1_T4 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T1, prediction_T4), tf.float32))

                T1_T1_percent = tf.divide(T1_T1, T1_all_Num)
                T1_T2_percent = tf.divide(T1_T2, T1_all_Num)
                T1_T3_percent = tf.divide(T1_T3, T1_all_Num)
                T1_T4_percent = tf.divide(T1_T4, T1_all_Num)

                tf.summary.scalar('T1_T1_percent', T1_T1_percent)
                tf.summary.scalar('T1_T2_percent', T1_T2_percent)
                tf.summary.scalar('T1_T3_percent', T1_T3_percent)
                tf.summary.scalar('T1_T4_percent', T1_T4_percent)

            with tf.name_scope("T2_Label"):
                T2_T1 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T2, prediction_T1), tf.float32))
                T2_T2 = T2_Corrected_Num
                T2_T3 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T2, prediction_T3), tf.float32))
                T2_T4 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T2, prediction_T4), tf.float32))

                T2_T1_percent = tf.divide(T2_T1, T2_all_Num)
                T2_T2_percent = tf.divide(T2_T2, T2_all_Num)
                T2_T3_percent = tf.divide(T2_T3, T2_all_Num)
                T2_T4_percent = tf.divide(T2_T4, T2_all_Num)

                tf.summary.scalar('T2_T1_percent', T2_T1_percent)
                tf.summary.scalar('T2_T2_percent', T2_T2_percent)
                tf.summary.scalar('T2_T3_percent', T2_T3_percent)
                tf.summary.scalar('T2_T4_percent', T2_T4_percent)

            with tf.name_scope("T3_Label"):
                T3_T1 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T3, prediction_T1), tf.float32))
                T3_T2 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T3, prediction_T2), tf.float32))
                T3_T3 = T3_Corrected_Num
                T3_T4 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T3, prediction_T4), tf.float32))

                T3_T1_percent = tf.divide(T3_T1, T3_all_Num)
                T3_T2_percent = tf.divide(T3_T2, T3_all_Num)
                T3_T3_percent = tf.divide(T3_T3, T3_all_Num)
                T3_T4_percent = tf.divide(T3_T4, T3_all_Num)

                tf.summary.scalar('T3_T1_percent', T3_T1_percent)
                tf.summary.scalar('T3_T2_percent', T3_T2_percent)
                tf.summary.scalar('T3_T3_percent', T3_T3_percent)
                tf.summary.scalar('T3_T4_percent', T3_T4_percent)

            with tf.name_scope("T4_Label"):
                T4_T1 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T4, prediction_T1), tf.float32))
                T4_T2 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T4, prediction_T2), tf.float32))
                T4_T3 = tf.reduce_sum(tf.cast(tf.math.logical_and(y_T4, prediction_T3), tf.float32))
                T4_T4 = T4_Corrected_Num

                T4_T1_percent = tf.divide(T4_T1, T4_all_Num)
                T4_T2_percent = tf.divide(T4_T2, T4_all_Num)
                T4_T3_percent = tf.divide(T4_T3, T4_all_Num)
                T4_T4_percent = tf.divide(T4_T4, T4_all_Num)

                tf.summary.scalar('T4_T1_percent', T4_T1_percent)
                tf.summary.scalar('T4_T2_percent', T4_T2_percent)
                tf.summary.scalar('T4_T3_percent', T4_T3_percent)
                tf.summary.scalar('T4_T4_percent', T4_T4_percent)

        with tf.name_scope('Global_Evalution_Metrics'):
            # Global Average Accuracy - Simple Algorithm
            with tf.name_scope('Global_Average_Accuracy'):
                correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
                Global_Average_Accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
                tf.summary.scalar('Global_Average_Accuracy', Global_Average_Accuracy)

            with tf.name_scope('Kappa_Metric'):
                Test_Set_Num = T1_all_Num + T2_all_Num + T3_all_Num + T4_all_Num

                Actual_T1 = T1_all_Num
                Actual_T2 = T2_all_Num
                Actual_T3 = T3_all_Num
                Actual_T4 = T4_all_Num

                Prediction_T1 = T1_T1 + T2_T1 + T3_T1 + T4_T1
                Prediction_T2 = T1_T2 + T2_T2 + T3_T2 + T4_T2
                Prediction_T3 = T1_T3 + T2_T3 + T3_T3 + T4_T3
                Prediction_T4 = T1_T4 + T2_T4 + T3_T4 + T4_T4

                p0 = (T1_T1 + T2_T2 + T3_T3 + T4_T4) / Test_Set_Num
                pe = (Actual_T1 * Prediction_T1 + Actual_T2 * Prediction_T2 + Actual_T3 * Prediction_T3 + Actual_T4 * Prediction_T4) / \
                     (Test_Set_Num * Test_Set_Num)

                Kappa_Metric = (p0 - pe) / (1 - pe)
                tf.summary.scalar('Kappa_Metric', Kappa_Metric)

            with tf.name_scope('Micro_Averaged_Evalution'):
                with tf.name_scope("Micro_Averaged_Confusion_Matrix"):
                    TP_all = T1_TP + T2_TP + T3_TP + T4_TP
                    TN_all = T1_TN + T2_TN + T3_TN + T4_TN
                    FP_all = T1_FP + T2_FP + T3_FP + T4_FP
                    FN_all = T1_FN + T2_FN + T3_FN + T4_FN

                with tf.name_scope("Micro_Global_Precision"):
                    Micro_Global_Precision = TP_all / (TP_all + FP_all)
                    tf.summary.scalar('Micro_Global_Precision', Micro_Global_Precision)

                with tf.name_scope("Micro_Global_Recall"):
                    Micro_Global_Recall = TP_all / (TP_all + FN_all)
                    tf.summary.scalar('Micro_Global_Recall', Micro_Global_Recall)

                with tf.name_scope("Micro_Global_F1_Score"):
                    Micro_Global_F1_Score = (2 * Micro_Global_Precision * Micro_Global_Recall) / (
                            Micro_Global_Precision + Micro_Global_Recall)
                    tf.summary.scalar('Micro_Global_F1_Score', Micro_Global_F1_Score)

            with tf.name_scope('Macro_Averaged_Evalution'):
                with tf.name_scope("Macro_Global_Precision"):
                    Macro_Global_Precision = (T1_Precision + T2_Precision + T3_Precision + T4_Precision) / 4
                    tf.summary.scalar('Macro_Global_Precision', Macro_Global_Precision)

                with tf.name_scope("Macro_Global_Recall"):
                    Macro_Global_Recall = (T1_Recall + T2_Recall + T3_Recall + T4_Recall) / 4
                    tf.summary.scalar('Macro_Global_Recall', Macro_Global_Recall)

                with tf.name_scope("Macro_Global_F1_Score"):
                    Macro_Global_F1_Score = (T1_F_Score + T2_F_Score + T3_F_Score + T4_F_Score) / 4
                    tf.summary.scalar('Macro_Global_F1_Score', Macro_Global_F1_Score)

    # You DON'T have to return these because all the criterias have been saved in the TensorBoard
    # return T1_accuracy, T1_Precision, T1_Recall, T1_F_Score, \
    #        T2_accuracy, T2_Precision, T2_Recall, T2_F_Score, \
    #        T3_accuracy, T3_Precision, T3_Recall, T3_F_Score, \
    #        T4_accuracy, T4_Precision, T4_Recall, T4_F_Score, \
    #        Global_Average_Accuracy, Kappa_Metric, \
    #        Macro_Global_Precision, Macro_Global_Recall, Macro_Global_F1_Score

    # Instead, you can only return Accuracy Criteria to compare the capacity of your Model
    return Global_Average_Accuracy