Add telemetry hook (#282)

* publish * telemetry hook * address comment
interpretml · Jul 9, 2020 · e82fc09 · e82fc09
1 parent a6b72e5
commit e82fc09
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 0 deletions.
diff --git a/visualization/dashboard/src/MLIDashboard/ExplanationDashboard.tsx b/visualization/dashboard/src/MLIDashboard/ExplanationDashboard.tsx
@@ -58,6 +58,7 @@ import { ModelExplanationUtils } from './ModelExplanationUtils';
 import { IBarChartConfig } from './SharedComponents/IBarChartConfig';
 import { EbmExplanation } from './Controls/EbmExplanation';
 import { JointDataset } from './JointDataset';
+import { TelemetryLevels } from './Interfaces/ITelemetryMessage';
 
 const s = require('./ExplanationDashboard.css');
 const RowIndex = 'rowIndex';
@@ -130,6 +131,13 @@ export class ExplanationDashboard extends React.Component<IExplanationDashboardP
         const modelMetadata = ExplanationDashboard.buildModelMetadata(props);
         const errorMessage = ExplanationDashboard.validateInputs(props, modelMetadata);
         if (errorMessage !== undefined) {
+            if (props.telemetryHook !== undefined) {
+                props.telemetryHook({
+                    message: "Invalid inputs",
+                    level: TelemetryLevels.error,
+                    context: errorMessage
+                });
+            }
             return {
                 modelMetadata,
                 explanationGenerators,

diff --git a/visualization/dashboard/src/MLIDashboard/Interfaces/IExplanationDashboardProps.ts b/visualization/dashboard/src/MLIDashboard/Interfaces/IExplanationDashboardProps.ts
@@ -1,4 +1,5 @@
 import IStringsParam from './IStringsParam';
+import { ITelemetryMessage } from './ITelemetryMessage';
 
 // This is the interface of the data to be provided by any glue code, be it the ModelExplanationController, the Jupyter widget,
 // or some future extension. The Explanation Dashboard opperates on this data object, and an optional chart config that specifies
@@ -29,6 +30,7 @@ export interface IExplanationDashboardProps {
     shouldInitializeIcons?: boolean;
     iconUrl?: string;
     explanationMethod?: string;
+    telemetryHook?: (message: ITelemetryMessage) => void;
     requestPredictions?: (request: any[], abortSignal: AbortSignal) => Promise<any[]>;
     requestLocalFeatureExplanations?: (
         request: any[],

diff --git a/visualization/dashboard/src/MLIDashboard/Interfaces/ITelemetryMessage.ts b/visualization/dashboard/src/MLIDashboard/Interfaces/ITelemetryMessage.ts
@@ -0,0 +1,18 @@
+export enum TelemetryLevels {
+    error = 'Error',
+    warning = 'Warning',
+    info = 'Info',
+    trace = 'Trace'
+}
+
+export type TelemetryLevel = 
+    TelemetryLevels.error | 
+    TelemetryLevels.warning |
+    TelemetryLevels.info |
+    TelemetryLevels.trace;
+
+export interface ITelemetryMessage {
+    level: TelemetryLevel;
+    message?: string;
+    context?: any;
+}
diff --git a/visualization/dashboard/src/MLIDashboard/Interfaces/index.ts b/visualization/dashboard/src/MLIDashboard/Interfaces/index.ts
@@ -1,2 +1,3 @@
 export * from './IExplanationDashboardProps';
 export * from './IStringsParam';
+export * from './ITelemetryMessage'
diff --git a/visualization/dashboard/src/MLIDashboard/NewExplanationDashboard.tsx b/visualization/dashboard/src/MLIDashboard/NewExplanationDashboard.tsx
@@ -3,6 +3,7 @@ import {
     IExplanationDashboardProps,
     IMultiClassLocalFeatureImportance,
     ISingleClassLocalFeatureImportance,
+    TelemetryLevels,
 } from './Interfaces';
 import { JointDataset } from './JointDataset';
 import { ModelMetadata } from 'mlchartlib';
@@ -287,6 +288,13 @@ export class NewExplanationDashboard extends React.PureComponent<
         const globalProps = NewExplanationDashboard.buildGlobalProperties(props);
         // consider taking filters in as param arg for programatic users
         const cohorts = [new Cohort(localization.Cohort.defaultLabel, jointDataset, [])];
+        if (validationCheck.errorStrings.length !== 0 && props.telemetryHook !== undefined) {
+            props.telemetryHook({
+                message: "Invalid inputs",
+                level: TelemetryLevels.error,
+                context: validationCheck.errorStrings.length
+            });
+        }
         return {
             cohorts,
             validationWarnings: validationCheck.errorStrings,

diff --git a/visualization/test/test-page/App.jsx b/visualization/test/test-page/App.jsx
@@ -251,6 +251,7 @@ import { createTheme } from "@uifabric/styling";
                           this.generateRandomScore :
                           this.generateRandomProbs.bind(this, classDimension)}
                         stringParams={{contextualHelp: this.messages}}
+                        telemetryHook={(er) => {console.error(er.message)}}
                         theme={theme}
                         explanationMethod="mimic"
                         locale={this.state.language}
@@ -270,6 +271,7 @@ import { createTheme } from "@uifabric/styling";
                         }}
                         requestPredictions={this.generateRandomProbs.bind(this, classDimension)}
                         stringParams={{contextualHelp: this.messages}}
+                        telemetryHook={(er) => {console.error(er.message)}}
                         theme={theme}
                         locale={this.state.language}
                         key={new Date()}