[Model Overview] Object Detection Metrics support

apps/widget/src/app/ModelAssessment.tsx

@@ -56,16 +56,17 @@ export class ModelAssessment extends React.Component<IModelAssessmentProps> {
         return callFlaskService(this.props.config, data, "/get_exp");
       };
       callBack.requestObjectDetectionMetrics = async (
-        trueY: number[][][],
-        predictedY: number[][][],
+        selectionIndexes: number[][],
         aggregateMethod: string,
         className: string,
-        iouThresh: number
+        iouThresh: number,
+        abortSignal: AbortSignal
       ): Promise<any[]> => {
         return callFlaskService(
           this.props.config,
-          [trueY, predictedY, aggregateMethod, className, iouThresh],
-          "/get_object_detection_metrics"
+          [selectionIndexes, aggregateMethod, className, iouThresh],
+          "/get_object_detection_metrics",
+          abortSignal
         );
       };
       callBack.requestPredictions = async (data: any[]): Promise<any[]> => {

libs/core-ui/src/lib/Context/ModelAssessmentContext.tsx

@@ -138,11 +138,11 @@ export interface IModelAssessmentContext {
     | undefined;
   requestObjectDetectionMetrics?:
     | ((
-        trueY: number[][][],
-        predictedY: number[][][],
+        selectionIndexes: number[][],
         aggregateMethod: string,
         className: string,
-        iouThresh: number
+        iouThresh: number,
+        abortSignal: AbortSignal
       ) => Promise<any[]>)
     | undefined;
   requestSplinePlotDistribution?: (
@@ -173,6 +173,7 @@ export const defaultModelAssessmentContext: IModelAssessmentContext = {
   modelMetadata: {} as IExplanationModelMetadata,
   modelType: undefined,
   requestExp: undefined,
+  requestObjectDetectionMetrics: undefined,
   requestLocalFeatureExplanations: undefined,
   requestPredictions: undefined,
   selectedErrorCohort: {} as ErrorCohort,

libs/core-ui/src/lib/util/JointDataset.ts

@@ -42,6 +42,8 @@ export class JointDataset {
   public static readonly PredictedYLabel = "PredictedY";
   public static readonly ProbabilityYRoot = "ProbabilityClass";
   public static readonly TrueYLabel = "TrueY";
+  public static readonly ObjectDetectionPredictedYLabel = "ObjectDetectionPredictedY";
+  public static readonly ObjectDetectionTrueYLabel = "ObjectDetectionTrueY";
   public static readonly DitherLabel = "Dither";
   public static readonly DitherLabel2 = "Dither2";
   public static readonly ClassificationError = "ClassificationError";

libs/core-ui/src/lib/util/JointDatasetUtils.ts

@@ -23,6 +23,8 @@ export interface IJointDatasetArgs {
   metadata: IExplanationModelMetadata;
   featureMetaData?: IFeatureMetaData;
   targetColumn?: string | string[];
+  objectDetectionTrueY?: number[][][],
+  objectDetectionPredictedY?: number[][][]
 }
 
 export enum ColumnCategories {

libs/core-ui/src/lib/util/ObjectDetectionStatisticsUtils.ts

@@ -8,50 +8,41 @@ import {
   TotalCohortSamples
 } from "../Interfaces/IStatistic";
 
-import { JointDataset } from "./JointDataset";
-
 export enum ObjectDetectionMetrics {
   MeanAveragePrecision = "meanAveragePrecision",
   AveragePrecision = "averagePrecision",
   AverageRecall = "averageRecall"
 }
 
 export const generateObjectDetectionStats: (
-  jointDataset: JointDataset,
   selectionIndexes: number[][]
 ) => ILabeledStatistic[][] = (
-  jointDataset: JointDataset,
   selectionIndexes: number[][]
 ): ILabeledStatistic[][] => {
-  const numLabels = jointDataset.numLabels;
+
   return selectionIndexes.map((selectionArray) => {
     const count = selectionArray.length;
 
-    // TODO: replace placeholder values with flask endpoint calls to python backend.
-    const meanAveragePrecision = 42;
-    const averagePrecision = 42;
-    const averageRecall = 42;
-
     return [
       {
         key: TotalCohortSamples,
         label: localization.Interpret.Statistics.samples,
-        stat: count / numLabels // TODO: remove numLabels from here when using jointDataset elsewhere.
+        stat: count
       },
       {
         key: ObjectDetectionMetrics.MeanAveragePrecision,
         label: localization.Interpret.Statistics.meanAveragePrecision,
-        stat: meanAveragePrecision
+        stat: 0
       },
       {
         key: ObjectDetectionMetrics.AveragePrecision,
         label: localization.Interpret.Statistics.averagePrecision,
-        stat: averagePrecision
+        stat: 0
       },
       {
         key: ObjectDetectionMetrics.AverageRecall,
         label: localization.Interpret.Statistics.averageRecall,
-        stat: averageRecall
+        stat: 0
       }
     ];
   });

libs/core-ui/src/lib/util/StatisticsUtils.ts

@@ -279,7 +279,7 @@ export const generateMetrics: (
     });
   }
   if (modelType === ModelTypes.ObjectDetection) {
-    return generateObjectDetectionStats(jointDataset, selectionIndexes);
+    return generateObjectDetectionStats(selectionIndexes);
   }
   const outcomes = jointDataset.unwrap(JointDataset.ClassificationError);
   return selectionIndexes.map((selectionArray) => {

libs/interpret-vision/src/lib/VisionExplanationDashboard/Interfaces/IVisionExplanationDashboardProps.ts

@@ -17,11 +17,11 @@ export interface IVisionExplanationDashboardProps {
     abortSignal: AbortSignal
   ) => Promise<any[]>;
   requestObjectDetectionMetrics?: (
-    trueY: number[][][],
-    predictedY: number[][][],
+    selectionIndexes: number[][],
     aggregateMethod: string,
     className: string,
-    iouThresh: number
+    iouThresh: number,
+    abortSignal: AbortSignal
   ) => Promise<any[]>;
   selectedCohort: ErrorCohort;
   setSelectedCohort: (cohort: ErrorCohort) => void;

libs/model-assessment/src/lib/ModelAssessmentDashboard/Controls/ModelOverview/ModelOverview.tsx

@@ -33,7 +33,8 @@ import {
   TelemetryLevels,
   TelemetryEventName,
   DatasetTaskType,
-  ImageClassificationMetrics
+  ImageClassificationMetrics,
+  TotalCohortSamples
 } from "@responsible-ai/core-ui";
 import { localization } from "@responsible-ai/localization";
 import React from "react";
@@ -52,6 +53,12 @@ import { getSelectableMetrics } from "./StatsTableUtils";
 
 interface IModelOverviewProps {
   telemetryHook?: (message: ITelemetryEvent) => void;
+  requestObjectDetectionMetrics?: (
+    selectionIndexes: number[][],
+    aggregateMethod: string,
+    className: string,
+    iouThresh: number
+  ) => Promise<any[]>;
 }
 
 interface IModelOverviewState {
@@ -70,6 +77,9 @@ interface IModelOverviewState {
   datasetBasedCohorts: ErrorCohort[];
   featureBasedCohortLabeledStatistics: ILabeledStatistic[][];
   featureBasedCohorts: ErrorCohort[];
+  aggregateMethod: string,
+  className: string,
+  iouThresh: number
 }
 
 const datasetCohortViewPivotKey = "datasetCohortView";
@@ -86,6 +96,7 @@ export class ModelOverview extends React.Component<
 
   public constructor(props: IModelOverviewProps) {
     super(props);
+
     this.state = {
       chartConfigurationIsVisible: false,
       datasetBasedCohorts: [],
@@ -99,7 +110,10 @@ export class ModelOverview extends React.Component<
       selectedFeatures: [],
       selectedFeaturesContinuousFeatureBins: {},
       selectedMetrics: [],
-      showHeatmapColors: true
+      showHeatmapColors: true,
+      aggregateMethod: localization.ModelAssessment.ModelOverview.metricTypes.macro,
+      className: "",
+      iouThresh: 70
     };
   }
 
@@ -333,6 +347,7 @@ export class ModelOverview extends React.Component<
               <ObjectDetectionWidgets
                 classNames={classNames}
                 dataset={this.context.dataset}
+                modelOverview={this}
               />
             )}
           </Stack>
@@ -513,18 +528,96 @@ export class ModelOverview extends React.Component<
   }
 
   private updateDatasetCohortStats(): void {
+    let selectionIndexes : number[][] = this.context.errorCohorts.map((errorCohort) =>
+      errorCohort.cohort.unwrap(JointDataset.IndexLabel)
+    );
     const datasetCohortMetricStats = generateMetrics(
       this.context.jointDataset,
-      this.context.errorCohorts.map((errorCohort) =>
-        errorCohort.cohort.unwrap(JointDataset.IndexLabel)
-      ),
+      selectionIndexes,
       this.context.modelMetadata.modelType
     );
 
     this.setState({
       datasetBasedCohorts: this.context.errorCohorts,
       datasetCohortLabeledStatistics: datasetCohortMetricStats
     });
+
+    this.updateObjectDetectionMetrics(selectionIndexes, true);
+  }
+
+  private updateObjectDetectionMetrics(
+    selectionIndexes: number[][],
+    isDatasetCohort: boolean): void {
+    if (this.context.requestObjectDetectionMetrics &&
+        selectionIndexes.length > 0 &&
+        this.state.aggregateMethod.length > 0 &&
+        this.state.className.length > 0 &&
+        this.state.iouThresh) {
+      console.log('entered endpoint')
+
+      this.context.requestObjectDetectionMetrics(
+        selectionIndexes,
+        this.state.aggregateMethod,
+        this.state.className,
+        this.state.iouThresh,
+        new AbortController().signal
+      ).then(
+        (result) => {
+          // TODO: assert length of result and selectionIndexes are the same.
+          let updatedMetricStats : ILabeledStatistic[][] = [];
+
+          for (let cohortIndex = 0; cohortIndex < result.length; cohortIndex++) {
+            let count = selectionIndexes[cohortIndex].length;
+
+            let [meanAveragePrecision, averagePrecision, averageRecall] = result[cohortIndex];
+
+            let updatedCohortMetricStats = [
+              {
+                key: TotalCohortSamples,
+                label: localization.Interpret.Statistics.samples,
+                stat: count
+              },
+              {
+                key: ObjectDetectionMetrics.MeanAveragePrecision,
+                label: localization.Interpret.Statistics.meanAveragePrecision,
+                stat: meanAveragePrecision
+              },
+              {
+                key: ObjectDetectionMetrics.AveragePrecision,
+                label: localization.Interpret.Statistics.averagePrecision,
+                stat: averagePrecision
+              },
+              {
+                key: ObjectDetectionMetrics.AverageRecall,
+                label: localization.Interpret.Statistics.averageRecall,
+                stat: averageRecall
+              }
+            ];
+
+            updatedMetricStats.push(updatedCohortMetricStats);
+
+          }
+
+          isDatasetCohort
+            ? this.updateDatasetCohortState(updatedMetricStats)
+            : this.updateFeatureCohortState(updatedMetricStats);
+
+        }
+      )
+    }
+    else {
+      console.log(this.context.requestObjectDetectionMetrics);
+      console.log(selectionIndexes.length)
+      console.log(this.state.aggregateMethod);
+      console.log(this.state.className);
+      console.log(this.state.iouThresh);
+    }
+  }
+
+  private updateDatasetCohortState(cohortMetricStats: ILabeledStatistic[][]): void {
+    this.setState({
+      datasetCohortLabeledStatistics: cohortMetricStats
+    });
   }
 
   private async updateFeatureCohortStats(): Promise<void> {
@@ -537,18 +630,28 @@ export class ModelOverview extends React.Component<
       this.state.selectedFeaturesContinuousFeatureBins
     );
 
+    let selectionIndexes: number[][] = featureBasedCohorts.map((errorCohort) =>
+      errorCohort.cohort.unwrap(JointDataset.IndexLabel)
+    );
+
     const featureCohortMetricStats = generateMetrics(
       this.context.jointDataset,
-      featureBasedCohorts.map((errorCohort) =>
-        errorCohort.cohort.unwrap(JointDataset.IndexLabel)
-      ),
+      selectionIndexes,
       this.context.modelMetadata.modelType
     );
 
     this.setState({
       featureBasedCohortLabeledStatistics: featureCohortMetricStats,
       featureBasedCohorts
     });
+
+    this.updateObjectDetectionMetrics(selectionIndexes, false);
+  }
+
+  private updateFeatureCohortState(cohortMetricStats: ILabeledStatistic[][]): void {
+    this.setState({
+      featureBasedCohortLabeledStatistics: cohortMetricStats
+    });
   }
 
   private ifCohortIndexesEquals(a: number[], b: number[]): boolean {