Implemented Performance Monitoring in Azure DevOps with Olive

.azure_pipelines/performance_check/best_metrics.json

@@ -0,0 +1,62 @@
+{
+    "bert": {
+        "cpu": {
+            "accuracy-accuracy": 0.88,
+            "latency-avg": {
+                "8272CL": 15.5,
+                "E5-2673": 44.5,
+                "8171M": 20.89,
+                "8370C": 18
+            }
+        },
+        "gpu": {
+            "accuracy-accuracy": 0.9,
+            "latency-avg": 1.61
+        }
+    },
+    "deberta": {
+        "cpu": {
+            "accuracy-accuracy": 0.84,
+            "latency-avg": {
+                "8272CL": 57.74,
+                "E5-2673": 117.00,
+                "8171M": 93.37,
+                "8370C": 0
+            }
+        },
+        "gpu": {
+            "accuracy-accuracy": 0.88,
+            "latency-avg": 8.011
+        }
+    },
+    "distilbert": {
+        "cpu": {
+            "accuracy-accuracy": 0.94,
+            "latency-avg": {
+                "8272CL": 4.5,
+                "E5-2673": 11.0,
+                "8171M": 5.9,
+                "8370C": 4.57
+            }
+        },
+        "gpu": {
+            "accuracy-accuracy": 0.94,
+            "latency-avg": 0.91
+        }
+    },
+    "roberta_large": {
+        "cpu": {
+            "accuracy-accuracy": 0.88,
+            "latency-avg": {
+                "8272CL": 52.38,
+                "E5-2673": 140.34,
+                "8171M": 69.33,
+                "8370C": 0
+            }
+        },
+        "gpu": {
+            "accuracy-accuracy": 0.89,
+            "latency-avg": 6.164
+        }
+    }
+}

.azure_pipelines/performance_check/configs/bert.json

@@ -25,15 +25,13 @@
                     "type": "accuracy",
                     "backend": "huggingface_metrics",
                     "sub_types": [
-                        {"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}}
-                    ]
+                        {"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.05}}                    ]
                 },
                 {
                     "name": "latency",
                     "type": "latency",
                     "sub_types": [
-                        {"name": "avg", "priority": 2, "goal": {"type": "percent-min-improvement", "value": 20}}
-                    ]
+                        {"name": "avg", "priority": 2, "goal": {"type": "percent-min-improvement", "value": 5}}                    ]
                 }
             ]
         }
@@ -81,7 +79,7 @@
         "clean_cache": true,
         "evaluator": "common_evaluator",
         "execution_providers": ["CPUExecutionProvider"],
-        "cache_dir": "cache",
-        "output_dir" : "models/bert_ptq"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/bert_ptq"
     }
 }

.azure_pipelines/performance_check/configs/bert_gpu.json

@@ -49,9 +49,6 @@
             "type": "OrtTransformersOptimization",
             "disable_search": true,
             "config": {
-                "model_type": "bert",
-                "num_heads": 12,
-                "hidden_size": 768,
                 "float16": true
             }
         },
@@ -79,7 +76,7 @@
         "evaluator": "common_evaluator",
         "execution_providers": ["CUDAExecutionProvider"],
         "clean_cache": true,
-        "cache_dir": "cache",
-        "output_dir" : "models/bert_gpu"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/bert_gpu"
     }
 }

.azure_pipelines/performance_check/configs/deberta.json

@@ -30,7 +30,7 @@
                     "type": "accuracy",
                     "backend": "huggingface_metrics",
                     "sub_types": [
-                        {"name": "accuracy", "priority": 1}
+                        {"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.05}}
 
                     ]
                 },
@@ -53,17 +53,12 @@
         },
         "transformers_optimization": {
             "type": "OrtTransformersOptimization",
-            "disable_search": true,
-            "config": {
-                "model_type": "bert",
-                "num_heads": 12,
-                "hidden_size": 768,
-                "float16": false
-            }
+            "disable_search": true
         },
         "quantization": {
             "type": "OnnxQuantization",
             "config": {
+                "quant_mode": "dynamic",
                 "quant_preprocess": true,
                 "data_config": "__input_model_data_config__"
             }
@@ -87,7 +82,7 @@
         "clean_cache": true,
         "evaluator": "common_evaluator",
         "execution_providers": ["CPUExecutionProvider"],
-        "cache_dir": "cache",
-        "output_dir" : "models/microsoft-deberta"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/microsoft-deberta"
     }
 }

.azure_pipelines/performance_check/configs/deberta_gpu.json

@@ -30,15 +30,15 @@
                     "type": "accuracy",
                     "backend": "huggingface_metrics",
                     "sub_types": [
-                        {"name": "accuracy", "priority": 1}
+                        {"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}}
 
                     ]
                 },
                 {
                     "name": "latency",
                     "type": "latency",
                     "sub_types": [
-                        {"name": "avg", "priority": 2}
+                        {"name": "avg", "priority": 2, "goal": {"type": "percent-min-improvement", "value": 20}}
                     ]
                 }
             ]
@@ -55,9 +55,6 @@
             "type": "OrtTransformersOptimization",
             "disable_search": true,
             "config": {
-                "model_type": "bert",
-                "num_heads": 12,
-                "hidden_size": 768,
                 "float16": true
             }
         },
@@ -85,7 +82,7 @@
         "clean_cache": true,
         "evaluator": "common_evaluator",
         "execution_providers": ["CUDAExecutionProvider"],
-        "cache_dir": "cache",
-        "output_dir" : "models/microsoft-deberta_cuda"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/microsoft-deberta_cuda"
     }
 }

.azure_pipelines/performance_check/configs/distilbert.json

@@ -26,14 +26,14 @@
                     "type": "accuracy",
                     "backend": "huggingface_metrics",
                     "sub_types": [
-                        {"name": "accuracy", "priority": 1}
+                        {"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}}
                     ]
                 },
                 {
                     "name": "latency",
                     "type": "latency",
                     "sub_types": [
-                        {"name": "avg", "priority": 2}
+                        {"name": "avg", "priority": 2, "goal": {"type": "percent-min-improvement", "value": 20}}
                     ]
                 }
             ]
@@ -48,13 +48,7 @@
         },
         "transformers_optimization": {
             "type": "OrtTransformersOptimization",
-            "disable_search": true,
-            "config": {
-                "model_type": "bert",
-                "num_heads": 12,
-                "hidden_size": 768,
-                "float16": false
-            }
+            "disable_search": true
         },
         "quantization": {
             "type": "OnnxQuantization",
@@ -79,10 +73,11 @@
                 "seed": 0
             }
         },
+        "log_severity_level": 0,
         "clean_cache": true,
         "evaluator": "common_evaluator",
         "execution_providers": ["CPUExecutionProvider"],
-        "cache_dir": "cache",
-        "output_dir" : "models/distilbert"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/distilbert"
     }
 }

.azure_pipelines/performance_check/configs/distilbert_gpu.json

@@ -49,9 +49,6 @@
             "type": "OrtTransformersOptimization",
             "disable_search": true,
             "config": {
-                "model_type": "bert",
-                "num_heads": 12,
-                "hidden_size": 768,
                 "float16": true
             }
         },
@@ -79,7 +76,7 @@
         "evaluator": "common_evaluator",
         "execution_providers": ["CUDAExecutionProvider"],
         "clean_cache": true,
-        "cache_dir": "cache",
-        "output_dir" : "models/distilbert_cuda"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/distilbert_cuda"
     }
 }

.azure_pipelines/performance_check/configs/roberta_large.json

@@ -52,13 +52,7 @@
         },
         "transformers_optimization": {
             "type": "OrtTransformersOptimization",
-            "disable_search": true,
-            "config": {
-                "model_type": "bert",
-                "num_heads": 12,
-                "hidden_size": 768,
-                "float16": false
-            }
+            "disable_search": true
         },
         "quantization": {
             "type": "OnnxQuantization",
@@ -86,7 +80,7 @@
         "clean_cache": true,
         "evaluator": "common_evaluator",
         "execution_providers": ["CPUExecutionProvider"],
-        "cache_dir": "cache",
-        "output_dir" : "models/roberta_large"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/roberta_large"
     }
 }

.azure_pipelines/performance_check/configs/roberta_large_gpu.json

@@ -54,9 +54,6 @@
             "type": "OrtTransformersOptimization",
             "disable_search": true,
             "config": {
-                "model_type": "bert",
-                "num_heads": 12,
-                "hidden_size": 768,
                 "float16": true
             }
         },
@@ -84,7 +81,7 @@
         "evaluator": "common_evaluator",
         "execution_providers": ["CUDAExecutionProvider"],
         "clean_cache": true,
-        "cache_dir": "cache",
-        "output_dir" : "models/roberta_large"
+        "cache_dir": "run_cache/olive/cache",
+        "output_dir" : "run_cache/olive/roberta_large"
     }
 }