Update torchbench model expected accuracy values after pinning numpy (#…

…129213) After pinning numpy on torchbench, we need to move torchbench inductor benchmark jobs out of unstable state asap, so that more failures don't sneak it. I'm updating the expected values here to make trunk green. Pull Request resolved: #129213 Approved by: https://github.com/xuzhao9, https://github.com/malfet, https://github.com/desertfire
pytorch · Jun 22, 2024 · b72ef9d · b72ef9d
1 parent f42d5b6
commit b72ef9d
Show file tree

Hide file tree

Showing 13 changed files with 13 additions and 65 deletions.
diff --git a/benchmarks/dynamo/ci_expected_accuracy/aot_eager_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/aot_eager_torchbench_inference.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,pass,6
-
-
-
 LearningToPaint,pass,0
 
 
@@ -150,7 +146,7 @@ hf_Bert_large,pass,0
 
 
 
-hf_BigBird,pass,13
+hf_BigBird,pass,0
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/aot_eager_torchbench_training.csv b/benchmarks/dynamo/ci_expected_accuracy/aot_eager_torchbench_training.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,eager_fail_to_run,0
-
-
-
 LearningToPaint,pass,6
 
 
@@ -98,7 +94,7 @@ hf_Bert_large,pass,6
 
 
 
-hf_BigBird,pass,19
+hf_BigBird,pass,6
 
 
 
@@ -286,4 +282,4 @@ vision_maskrcnn,pass,33
 
 
 
-yolov3,pass,8
+yolov3,fail_accuracy,8
diff --git a/benchmarks/dynamo/ci_expected_accuracy/aot_inductor_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/aot_inductor_torchbench_inference.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,fail_to_run,0
-
-
-
 LearningToPaint,pass,0
 
 
@@ -138,7 +134,7 @@ hf_Bert_large,pass,0
 
 
 
-hf_BigBird,fail_to_run,0
+hf_BigBird,fail_accuracy,0
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/cpu_inductor_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/cpu_inductor_torchbench_inference.csv
@@ -10,10 +10,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,model_fail_to_load,0
-
-
-
 LearningToPaint,pass,0
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/dynamic_aot_eager_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/dynamic_aot_eager_torchbench_inference.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,pass,6
-
-
-
 LearningToPaint,pass,0
 
 
@@ -150,7 +146,7 @@ hf_Bert_large,pass,0
 
 
 
-hf_BigBird,pass,13
+hf_BigBird,fail_to_run,0
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/dynamic_aot_eager_torchbench_training.csv b/benchmarks/dynamo/ci_expected_accuracy/dynamic_aot_eager_torchbench_training.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,eager_fail_to_run,0
-
-
-
 LearningToPaint,pass,6
 
 
@@ -98,7 +94,7 @@ hf_Bert_large,pass,6
 
 
 
-hf_BigBird,pass,19
+hf_BigBird,fail_to_run,3
 
 
 
@@ -282,4 +278,4 @@ vision_maskrcnn,pass,33
 
 
 
-yolov3,pass,8
+yolov3,fail_accuracy,8
diff --git a/benchmarks/dynamo/ci_expected_accuracy/dynamic_cpu_inductor_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/dynamic_cpu_inductor_torchbench_inference.csv
@@ -10,10 +10,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,model_fail_to_load,0
-
-
-
 LearningToPaint,pass,0
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/dynamic_inductor_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/dynamic_inductor_torchbench_inference.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,pass,6
-
-
-
 LearningToPaint,pass,0
 
 
@@ -150,7 +146,7 @@ hf_Bert_large,pass,0
 
 
 
-hf_BigBird,fail_accuracy,13
+hf_BigBird,fail_to_run,0
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/dynamic_inductor_torchbench_training.csv b/benchmarks/dynamo/ci_expected_accuracy/dynamic_inductor_torchbench_training.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,eager_fail_to_run,0
-
-
-
 LearningToPaint,pass,6
 
 
@@ -98,7 +94,7 @@ hf_Bert_large,pass,6
 
 
 
-hf_BigBird,pass,19
+hf_BigBird,fail_to_run,3
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/dynamo_eager_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/dynamo_eager_torchbench_inference.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,pass,6
-
-
-
 LearningToPaint,pass,0
 
 
@@ -150,7 +146,7 @@ hf_Bert_large,pass,0
 
 
 
-hf_BigBird,pass,13
+hf_BigBird,pass,0
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/dynamo_eager_torchbench_training.csv b/benchmarks/dynamo/ci_expected_accuracy/dynamo_eager_torchbench_training.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,eager_fail_to_run,0
-
-
-
 LearningToPaint,pass,6
 
 
@@ -98,7 +94,7 @@ hf_Bert_large,pass,6
 
 
 
-hf_BigBird,pass,19
+hf_BigBird,pass,6
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/inductor_torchbench_inference.csv b/benchmarks/dynamo/ci_expected_accuracy/inductor_torchbench_inference.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,pass,6
-
-
-
 LearningToPaint,pass,0
 
 
@@ -150,7 +146,7 @@ hf_Bert_large,pass,0
 
 
 
-hf_BigBird,fail_accuracy,13
+hf_BigBird,fail_accuracy,0
 
 
 

diff --git a/benchmarks/dynamo/ci_expected_accuracy/inductor_torchbench_training.csv b/benchmarks/dynamo/ci_expected_accuracy/inductor_torchbench_training.csv
@@ -14,10 +14,6 @@ Background_Matting,pass_due_to_skip,0
 
 
 
-DALLE2_pytorch,eager_fail_to_run,0
-
-
-
 LearningToPaint,pass,6
 
 
@@ -98,7 +94,7 @@ hf_Bert_large,pass,6
 
 
 
-hf_BigBird,pass,19
+hf_BigBird,fail_to_run,3