#8373: Update perf thresholds for perf governor

tenstorrent · May 15, 2024 · 70cbb38 · 70cbb38
1 parent 83eb4d4
commit 70cbb38
Show file tree

Hide file tree

Showing 7 changed files with 19 additions and 22 deletions.
diff --git a/models/demos/metal_BERT_large_11/tests/test_perf_bert11.py b/models/demos/metal_BERT_large_11/tests/test_perf_bert11.py
@@ -150,9 +150,9 @@ def run_perf_bert11(
 @pytest.mark.parametrize(
     "batch_size, model_config_str, expected_inference_time, expected_compile_time, inference_iterations",
     (
-        [7, "BFLOAT8_B-SHARDED", 0.0364, 10, 10],
-        [8, "BFLOAT8_B-SHARDED", 0.0364, 10, 10],
-        [12, "BFLOAT8_B-SHARDED", 0.0364, 10, 10],
+        [7, "BFLOAT8_B-SHARDED", 0.0358, 3, 10],
+        [8, "BFLOAT8_B-SHARDED", 0.0358, 3, 10],
+        [12, "BFLOAT8_B-SHARDED", 0.0358, 3, 10],
     ),
 )
 def test_perf_bare_metal(

diff --git a/models/demos/resnet/tests/test_perf_accuracy_resnet.py b/models/demos/resnet/tests/test_perf_accuracy_resnet.py
@@ -166,7 +166,7 @@ def run_perf_resnet(
 @pytest.mark.models_performance_bare_metal
 @pytest.mark.parametrize(
     "batch_size, expected_inference_time, expected_compile_time, iterations",
-    ((16, 0.015, 33, 160), (20, 0.0185, 33, 160)),
+    ((16, 0.015, 14, 160), (20, 0.014, 14, 160)),
 )
 def test_perf_bare_metal(
     device,

diff --git a/models/demos/resnet/tests/test_perf_resnet.py b/models/demos/resnet/tests/test_perf_resnet.py
@@ -121,20 +121,17 @@ def run_perf_resnet(
     logger.info(f"resnet50 {comments} inference time (avg): {inference_time_avg}")
     logger.info(f"resnet50 compile time: {compile_time}")
 
-    assert inference_time_avg < expected_inference_time, f"resnet50 {comments} inference is too slow"
-    assert compile_time < expected_compile_time, f"resnet50 {comments} compilation is too slow"
-
 
 @skip_for_wormhole_b0(reason_str="Not tested on single WH")
 @pytest.mark.parametrize("device_l1_small_size", [32768], indirect=True)
 @pytest.mark.models_performance_bare_metal
 @pytest.mark.parametrize(
     "batch_size, expected_inference_time, expected_compile_time",
     (
-        (1, 0.015, 25),
-        (2, 0.015, 25),
-        (16, 0.0135, 25),  # Issue 7816 Inference time
-        (20, 0.0135, 25),  # Issue 7816 Inference time
+        (1, 0.001, 1),
+        (2, 0.001, 1),
+        (16, 0.0085, 1),  # Issue 7816 Inference time
+        (20, 0.0095, 1),  # Issue 7816 Inference time
     ),
 )
 def test_perf_bare_metal(

diff --git a/models/demos/ttnn_falcon7b/tests/test_perf_falcon.py b/models/demos/ttnn_falcon7b/tests/test_perf_falcon.py
@@ -359,11 +359,11 @@ def convert_to_ttnn(model, name):
 @pytest.mark.parametrize(
     "llm_mode, batch, seq_len, kv_cache_len, expected_inference_time",
     (
-        ("prefill", 1, 128, 0, 0.30),
-        ("prefill", 1, 256, 0, 0.44),
-        ("decode", 32, 1, 128, 0.27),
-        ("decode", 32, 1, 1024, 0.35),
-        ("decode", 32, 1, 2047, 0.48),
+        ("prefill", 1, 128, 0, 0.28),
+        ("prefill", 1, 256, 0, 0.385),
+        ("decode", 32, 1, 128, 0.25),
+        ("decode", 32, 1, 1024, 0.307),
+        ("decode", 32, 1, 2047, 0.34),
     ),
     ids=[
         "prefill_seq128",

diff --git a/tests/ttnn/integration_tests/bert/test_performance.py b/tests/ttnn/integration_tests/bert/test_performance.py
@@ -57,9 +57,9 @@ def preprocess_inputs(
 
 def get_expected_times(bert):
     return {
-        ttnn_bert: (15, 32),
-        ttnn_optimized_bert: (12, 0.08),
-        ttnn_optimized_sharded_bert: (12, 0.08),
+        ttnn_bert: (0.1, 0.1),
+        ttnn_optimized_bert: (5.3, 0.07),
+        ttnn_optimized_sharded_bert: (5, 0.07),
     }[bert]
 
 

diff --git a/tests/ttnn/integration_tests/resnet/test_performance.py b/tests/ttnn/integration_tests/resnet/test_performance.py
@@ -55,7 +55,7 @@ def test_perf_device_bare_metal(batch_size, test, expected_perf):
 @pytest.mark.parametrize("device_l1_small_size", [24576], indirect=True)
 @pytest.mark.parametrize(
     "model_name,batch_size,act_dtype,weight_dtype,math_fidelity,expected_compile_time,expected_inference_time",
-    [("ResNet50", 20, ttnn.bfloat8_b, ttnn.bfloat8_b, ttnn.MathFidelity.LoFi, 23, 0.04)],  ## pass
+    [("ResNet50", 20, ttnn.bfloat8_b, ttnn.bfloat8_b, ttnn.MathFidelity.LoFi, 14, 0.015)],  ## pass
 )
 def test_performance(
     device,

diff --git a/tests/ttnn/integration_tests/whisper/test_performance.py b/tests/ttnn/integration_tests/whisper/test_performance.py
@@ -17,8 +17,8 @@
 
 def get_expected_times(functional_whisper):
     return {
-        ttnn_functional_whisper: (30.0, 7.0),
-        ttnn_optimized_functional_whisper: (30.0, 7.0),
+        ttnn_functional_whisper: (10, 4.11),
+        ttnn_optimized_functional_whisper: (1.2, 1.35),
     }[functional_whisper]