metric.proto

// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =============================================================================

syntax = "proto3";

package tensorflow.metadata.v0;

import "google/protobuf/any.proto";
import "google/protobuf/wrappers.proto";
import "google/protobuf/descriptor.proto";
import "tensorflow_metadata/proto/v0/path.proto";

option cc_enable_arenas = true;
option java_package = "org.tensorflow.metadata.v0";
option java_multiple_files = true;

// Metric type indicates which direction of a real-valued metric is "better".
// For most message types, this is invariant. For custom message types,
// is_maximized == true is like MAXIMIZE, and otherwise MINIMIZE.
enum MetricType {
  UNKNOWN = 0;
  // Maximize the metric (i.e. a utility).
  MAXIMIZE = 1;
  // Minimize the metric (i.e. a loss).
  MINIMIZE = 2;
  // Look for a field is_maximized.
  CUSTOM = 3;
}

extend google.protobuf.MessageOptions {
  MetricType metric_type = 227673489;
}

// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/binary_accuracy
message BinaryAccuracy {
  option (metric_type) = MAXIMIZE;
}

// categorical_accuracy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/categorical_accuracy
message CategoricalAccuracy {
  option (metric_type) = MAXIMIZE;
}

// categorical_crossentropy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/categorical_crossentropy
message CategoricalCrossEntropy {
  option (metric_type) = MINIMIZE;
}

// cosine(...)
// cosine_proximity(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/cosine_proximity
// DEPRECATED
message Cosine {
  option (metric_type) = MINIMIZE;
}

// Linear Hinge Loss
// hinge(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/hinge
// DEPRECATED
message Hinge {
  option (metric_type) = MINIMIZE;
}

// kld(...)
// kullback_leibler_divergence(...)
// KLD(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/kullback_leibler_divergence
// DEPRECATED
message KullbackLeiblerDivergence {
  option (metric_type) = MINIMIZE;
}

// MAE(...)
// mae(...)
// mean_absolute_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_absolute_error
message MeanAbsoluteError {
  option (metric_type) = MINIMIZE;
}

// MAPE(...)
// mape(...)
// mean_absolute_percentage_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_absolute_percentage_error
message MeanAbsolutePercentageError {
  option (metric_type) = MINIMIZE;
}

// MSE(...)
// mse(...)
// mean_squared_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_squared_error
message MeanSquaredError {
  option (metric_type) = MINIMIZE;
}

// msle(...)
// MSLE(...)
// mean_squared_logarithmic_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_squared_logarithmic_error
message MeanSquaredLogarithmicError {
  option (metric_type) = MINIMIZE;
}

// poisson(...)
// DEPRECATED
message Poisson {
  option (metric_type) = MINIMIZE;
}

// squared_hinge(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/squared_hinge
// DEPRECATED
message SquaredHinge {
  option (metric_type) = MINIMIZE;
}

// top_k_categorical_accuracy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/top_k_categorical_accuracy
message TopKCategoricalAccuracy {
  option (metric_type) = MAXIMIZE;
}

// sparse_top_k_categorical_accuracy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/sparse_top_k_categorical_accuracy
// DEPRECATED
message SparseTopKCategoricalAccuracy {
  option (metric_type) = MAXIMIZE;
}

// Binary cross entropy as a metric is equal to the negative log likelihood
// (see logistic regression).
// In addition, when used to solve a binary classification task, binary cross
// entropy implies that the binary label will maximize binary accuracy.
// binary_crossentropy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/binary_crossentropy
message BinaryCrossEntropy {
  option (metric_type) = MINIMIZE;
}

// AKA the negative log likelihood or log loss.
// Given a label y\in {0,1} and a predicted probability p in [0,1]:
// -yln(p)-(1-y)ln(1-p)
// TODO(martinz): if this is interpreted the same as binary_cross_entropy,
// we may need to revisit the semantics.
// DEPRECATED
message LogisticRegression {
  option (metric_type) = MINIMIZE;
}

// Area under curve for the ROC-curve.
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/AUC
message AUC {
  option (metric_type) = MAXIMIZE;
}

// Area under curve for the precision-recall-curve.
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/AUC
message AUCPrecisionRecall {
  option (metric_type) = MAXIMIZE;
}

// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/SensitivityAtSpecificity
message SensitivityAtSpecificity {
  option (metric_type) = MAXIMIZE;

  // Minimal required specificity, (0.0, 1.0).
  google.protobuf.DoubleValue specificity = 1;
}

// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/SpecificityAtSensitivity
message SpecificityAtSensitivity {
  option (metric_type) = MAXIMIZE;

  // Minimal required sensitivity, (0.0, 1.0).
  google.protobuf.DoubleValue sensitivity = 1;
}

// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/PrecisionAtRecall
message PrecisionAtRecall {
  option (metric_type) = MAXIMIZE;

  // Minimal required recall, (0.0, 1.0).
  google.protobuf.DoubleValue recall = 1;
}

// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/RecallAtPrecision
message RecallAtPrecision {
  option (metric_type) = MAXIMIZE;

  // Minimal required precision, (0.0, 1.0).
  google.protobuf.DoubleValue precision = 1;
}

message FalseNegativeRateAtThreshold {
  option (metric_type) = MAXIMIZE;

  // Threshold to apply to a prediction to determine positive vs negative.
  // Note: if the model is calibrated, the threshold can be thought of as a
  // probability so the threshold has a stable, intuitive semantic.
  // However, not all solutions may be calibrated, and not all computations of
  // the metric may operate on a calibrated score. In AutoTFX, the final model
  // metrics are computed on a calibrated score, but the metrics computed within
  // the model selection process are uncalibrated. Be aware of this possible
  // skew in the metrics between model selection and final model evaluation.
  google.protobuf.DoubleValue threshold = 1;
}

message FalsePositiveRateAtThreshold {
  option (metric_type) = MAXIMIZE;

  // Threshold to apply to a prediction to determine positive vs negative.
  // Note: if the model is calibrated, the threshold can be thought of as a
  // probability so the threshold has a stable, intuitive semantic.
  // However, not all solutions may be calibrated, and not all computations of
  // the metric may operate on a calibrated score. In AutoTFX, the final model
  // metrics are computed on a calibrated score, but the metrics computed within
  // the model selection process are uncalibrated. Be aware of this possible
  // skew in the metrics between model selection and final model evaluation.
  google.protobuf.DoubleValue threshold = 1;
}

message PrecisionAtK {
  option (metric_type) = MAXIMIZE;
}

message MeanReciprocalRank {}

// https://www.tensorflow.org/responsible_ai/model_remediation/api_docs/python/model_remediation/min_diff/losses/MMDLoss
message MaximumMeanDiscrepancy {
  option (metric_type) = MINIMIZE;

  // Kernel to apply to the predictions. Currently supported values are
  // 'gaussian' and 'laplace'. Defaults to 'gaussian'.
  string kernel = 1;
}

// The mean of the prediction across the dataset.
message PredictionMean {}

// Area under ROC-curve calculated globally for MultiClassClassification (model
// predicts a single label) or MultiLabelClassification (model predicts class
// probabilities). The area is calculated by treating the entire set of data as
// an aggregate result, and computing a single metric rather than k metrics
// (one for each target label) that get averaged together. For example, the FPR
// and TPR at a given point on the AUC curve for k targer labels are:
//   FPR = (FP1 + FP2 + ... + FPk) / ((FP1 + FP2 + ... + FPk) +
//                                    (TN1 + TN2 + ... + TNk))
//   TPR = (TP1 + TP2 + ... +TPk) / ((TP1 + TP2 + ... + TPk) +
//                                   (FN1 + FN2 + ... + FNk))
message MicroAUC {
  option (metric_type) = MAXIMIZE;
}

// Cross entropy for MultiLabelClassification where each target and
// prediction is the probabily of belonging to that class independent of other
// classes.
message MultilabelCrossEntropy {
  option (metric_type) = MINIMIZE;
}

// DEPRECATED
message BlockUtility {
  option (metric_type) = MAXIMIZE;

  repeated double weight = 1;
}

// A custom metric.
// Prefer using or adding an explicit metric message
// and only use this generic message as a last resort.
// NEXT_TAG: 4
message CustomMetric {
  option (metric_type) = CUSTOM;

  // The display name of a metric computed by the model. The name should match
  // ^[a-zA-Z0-9\s]{1,25}$ and must be unique across all performance metrics.
  // Trailing and leading spaces will be truncated before matching.
  string name = 1;

  // True if the metric is maximized: false if it is minimized.
  // Must be specified if the CustomMetric is used as an objective.
  bool is_maximized = 2;

  // RegistrySpec is a full specification of the custom metric and its
  // construction based on the binary’s metric registry. New custom metrics must
  // be linked to the binary and registered in its metric registry to be
  // identifiable via this specification.
  message RegistrySpec {
    // Identifier of the metric class in the metric registry of the binary.
    string key = 1;

    // Generic proto describing the configuration for the metric to be computed.
    // It's upto the implementer of the metric to parse this configuration.
    google.protobuf.Any config = 2;
  }

  // Specification of the metric in the binary’s metric registry.
  RegistrySpec registry_spec = 3;
}

// Performance metrics measure the quality of a model. They need not be
// differentiable.
message PerformanceMetric {
  oneof performance_metric {
    AUC auc = 1;
    AUCPrecisionRecall auc_precision_recall = 26;
    BinaryAccuracy binary_accuracy = 2;
    BinaryCrossEntropy binary_cross_entropy = 3;
    BlockUtility block_utility = 4 [deprecated = true];
    CategoricalAccuracy categorical_accuracy = 5;
    CategoricalCrossEntropy categorical_cross_entropy = 6;
    Cosine cosine = 7 [deprecated = true];
    Hinge hinge = 8 [deprecated = true];
    KullbackLeiblerDivergence kullback_leibler_divergence = 9
        [deprecated = true];
    LogisticRegression logistic_regression = 10 [deprecated = true];
    MeanAbsoluteError mean_absolute_error = 11;
    MeanAbsolutePercentageError mean_absolute_percentage_error = 12;
    MeanSquaredError squared_error = 13;
    MeanSquaredLogarithmicError mean_squared_logarithmic_error = 14;
    MeanReciprocalRank mean_reciprocal_rank = 15;
    MicroAUC micro_auc = 27;
    MultilabelCrossEntropy multi_label_cross_entropy = 28;
    Poisson poisson = 16 [deprecated = true];
    PrecisionAtK precision_at_k = 17;
    SquaredHinge squared_hinge = 18 [deprecated = true];
    SparseTopKCategoricalAccuracy sparse_top_k_categorical_accuracy = 19
        [deprecated = true];
    TopKCategoricalAccuracy top_k_categorical_accuracy = 20;
    CustomMetric custom_metric = 21;
    SensitivityAtSpecificity sensitivity_at_specificity = 22;
    SpecificityAtSensitivity specificity_at_sensitivity = 23;
    PrecisionAtRecall precision_at_recall = 24;
    RecallAtPrecision recall_at_precision = 25;
  }
  // NEXT_TAG: 37;
}