From 8090144a2d663d353f8ca850b079f909a89a1ae8 Mon Sep 17 00:00:00 2001 From: Artem Yerofieiev Date: Thu, 23 May 2024 23:20:22 +0000 Subject: [PATCH] #8364: Split tests requiring fast runtime mode off --- .github/workflows/ttnn-post-commit.yaml | 17 +++++++++++------ pytest.ini | 1 + tests/ttnn/conftest.py | 14 ++++++++++++++ .../unit_tests/operations/test_experimental.py | 2 ++ .../unit_tests/operations/test_transformer.py | 9 +++++---- tests/ttnn/unit_tests/test_deallocate.py | 1 + .../test_pre_and_post_operation_hook.py | 1 + tests/ttnn/unit_tests/test_reports.py | 5 +++++ tests/ttnn/unit_tests/test_tracer.py | 2 ++ tests/ttnn/unit_tests/test_tutorials.py | 1 + .../ttnn/unit_tests/test_validate_decorator.py | 1 + 11 files changed, 44 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ttnn-post-commit.yaml b/.github/workflows/ttnn-post-commit.yaml index e9626cac2a5..177c8dcbee1 100644 --- a/.github/workflows/ttnn-post-commit.yaml +++ b/.github/workflows/ttnn-post-commit.yaml @@ -18,12 +18,17 @@ jobs: # # N300 {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"], name: N300}, ] - test-group: [ - {name: ttnn group 1, cmd: pytest $TT_METAL_HOME/tests/ttnn/unit_tests -v --splits 2 --group 1}, - {name: ttnn group 2, cmd: pytest $TT_METAL_HOME/tests/ttnn/unit_tests -v --splits 2 --group 2}, - {name: ttnn cpp tests, cmd: ./build/test/ttnn/unit_tests_ttnn}, - - ] + test-group: + - name: ttnn group 1 + cmd: pytest $TT_METAL_HOME/tests/ttnn/unit_tests -v --splits 2 --group 1 -m "not disable_fast_runtime_mode" + - name: ttnn group 2 + cmd: pytest $TT_METAL_HOME/tests/ttnn/unit_tests -v --splits 2 --group 2 -m "not disable_fast_runtime_mode" + - name: ttnn group 3 + cmd: pytest $TT_METAL_HOME/tests/ttnn/unit_tests -m requires_fast_runtime_mode_off + env: + TTNN_CONFIG_OVERRIDES: '{"enable_fast_runtime_mode": false}' + - name: ttnn cpp tests + cmd: ./build/test/ttnn/unit_tests_ttnn name: ${{ matrix.test-group.name }} ${{ matrix.runner-info.arch }} ${{ matrix.runner-info.name }} env: TT_METAL_ENV: ${{ vars.TT_METAL_ENV }} diff --git a/pytest.ini b/pytest.ini index b1f2bdbc22c..f0388b12ae3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,3 +14,4 @@ markers = models_performance_virtual_machine: mark model silicon tests for performance on virtual_machine models_device_performance_bare_metal: mark model silicon tests for device performance on bare metal model_perf_t3000: mark model silicon tests for performance on t3000 bare metal + requires_fast_runtime_mode_off diff --git a/tests/ttnn/conftest.py b/tests/ttnn/conftest.py index 23c2ac7e76b..b9858751630 100644 --- a/tests/ttnn/conftest.py +++ b/tests/ttnn/conftest.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 +import os import copy import datetime import json @@ -24,6 +25,19 @@ def pytest_make_parametrize_id(config, val, argname): return f"{argname}={val}" +def pytest_collection_modifyitems(config, items): + if not ttnn.CONFIG.enable_fast_runtime_mode: + return + + logger.warning("Fast Runtime Mode is ON. Skipping tests tagged with @pytest.mark.requires_fast_runtime_mode_off") + skip_unmarked = pytest.mark.skip(reason="Skipping test with requires_fast_runtime_mode_off") + for item in items: + logger.warning(item.keywords) + if "requires_fast_runtime_mode_off" in item.keywords: + logger.warning(f"Skipping {item}") + item.add_marker(skip_unmarked) + + @pytest.fixture(autouse=True) def pre_and_post(request): original_config = copy.copy(ttnn.CONFIG) diff --git a/tests/ttnn/unit_tests/operations/test_experimental.py b/tests/ttnn/unit_tests/operations/test_experimental.py index c7e7296f4ff..f43f95edb77 100644 --- a/tests/ttnn/unit_tests/operations/test_experimental.py +++ b/tests/ttnn/unit_tests/operations/test_experimental.py @@ -12,6 +12,7 @@ @skip_for_wormhole_b0() +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("height", [32]) @pytest.mark.parametrize("width", [32]) def test_ttnn_experimental_tensor_exp(device, height, width): @@ -69,6 +70,7 @@ def test_ttnn_experimental_operations_primary_moreh_matmul(device, m_size, k_siz assert_with_pcc(torch_output_tensor, output_tensor) +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("input_a_is_sharded", [True, False]) @pytest.mark.parametrize("output_is_sharded", [True, False]) @pytest.mark.parametrize("m_size, num_cores", [[25088, 98]]) diff --git a/tests/ttnn/unit_tests/operations/test_transformer.py b/tests/ttnn/unit_tests/operations/test_transformer.py index f9b2a5457b7..ba807cdafbd 100644 --- a/tests/ttnn/unit_tests/operations/test_transformer.py +++ b/tests/ttnn/unit_tests/operations/test_transformer.py @@ -496,10 +496,11 @@ def test_concatenate_heads_when_head_size_is_not_a_multiple_of_32(device): with pytest.raises(RuntimeError) as e: output_tensor = ttnn.transformer.concatenate_heads(input_tensor) - assert ( - "Head size must be a multiple of 32! Update matmul that uses the output of this operation to have the padding in the weights!" - in str(e.value) - ) + + assert ( + "Head size must be a multiple of 32! Update matmul that uses the output of this operation to have the padding in the weights!" + in str(e.value) + ) input_tensor = torch.nn.functional.pad(torch_input_tensor, (0, padded_head_size - head_size), "constant", 0) input_tensor = ttnn.from_torch( diff --git a/tests/ttnn/unit_tests/test_deallocate.py b/tests/ttnn/unit_tests/test_deallocate.py index 03f519b7404..b1647ba689f 100644 --- a/tests/ttnn/unit_tests/test_deallocate.py +++ b/tests/ttnn/unit_tests/test_deallocate.py @@ -11,6 +11,7 @@ @pytest.mark.parametrize("h", [32]) @pytest.mark.parametrize("w", [2 * 32]) +@pytest.mark.requires_fast_runtime_mode_off def test_deallocate(device, h, w): torch_input_tensor = torch.rand((h, w), dtype=torch.bfloat16) diff --git a/tests/ttnn/unit_tests/test_pre_and_post_operation_hook.py b/tests/ttnn/unit_tests/test_pre_and_post_operation_hook.py index 29c3339c4fa..4ab073e80f5 100644 --- a/tests/ttnn/unit_tests/test_pre_and_post_operation_hook.py +++ b/tests/ttnn/unit_tests/test_pre_and_post_operation_hook.py @@ -36,6 +36,7 @@ def post_hook_to_print_output(operation, args, kwargs, output): @skip_for_wormhole_b0() +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("batch_size", [1]) @pytest.mark.parametrize("h", [32]) @pytest.mark.parametrize("w", [32]) diff --git a/tests/ttnn/unit_tests/test_reports.py b/tests/ttnn/unit_tests/test_reports.py index 4ea914be9d8..9d71554cd6c 100644 --- a/tests/ttnn/unit_tests/test_reports.py +++ b/tests/ttnn/unit_tests/test_reports.py @@ -12,6 +12,7 @@ import ttnn.database +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("height", [1024]) @pytest.mark.parametrize("width", [1024]) def test_enable_logging(device, height, width): @@ -45,6 +46,7 @@ def test_enable_logging(device, height, width): assert len(operations) == 5 +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("height", [1024]) @pytest.mark.parametrize("width", [1024]) def test_enable_logging_and_enable_graph_report(device, height, width): @@ -67,6 +69,7 @@ def test_enable_logging_and_enable_graph_report(device, height, width): ttnn.to_torch(output_tensor) +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("height", [1024]) @pytest.mark.parametrize("width", [1024]) def test_enable_logging_and_enable_detailed_buffer_report(device, height, width): @@ -107,6 +110,7 @@ def test_enable_logging_and_enable_detailed_buffer_report(device, height, width) assert len(buffer_pages) > 0 +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("height", [1024]) @pytest.mark.parametrize("width", [1024]) def test_enable_logging_and_enable_comparison_mode(device, height, width): @@ -139,6 +143,7 @@ def test_enable_logging_and_enable_comparison_mode(device, height, width): assert len(operations) > 0 +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("height", [1024]) @pytest.mark.parametrize("width", [1024]) def test_enable_logging_and_enable_detailed_tensor_report(device, height, width): diff --git a/tests/ttnn/unit_tests/test_tracer.py b/tests/ttnn/unit_tests/test_tracer.py index d5ae0595e1e..536f81ac6df 100644 --- a/tests/ttnn/unit_tests/test_tracer.py +++ b/tests/ttnn/unit_tests/test_tracer.py @@ -28,6 +28,7 @@ def test_exp(): @skip_for_wormhole_b0() +@pytest.mark.requires_fast_runtime_mode_off def test_reshape(): with trace(): tensor = torch.randint(0, 100, (4, 64)) @@ -40,6 +41,7 @@ def test_reshape(): @skip_for_wormhole_b0() +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("show_modules", [True, False]) def test_torch_bert(show_modules): model_name = "google/bert_uncased_L-4_H-256_A-4" diff --git a/tests/ttnn/unit_tests/test_tutorials.py b/tests/ttnn/unit_tests/test_tutorials.py index 4dd3c458f41..4d23607a834 100644 --- a/tests/ttnn/unit_tests/test_tutorials.py +++ b/tests/ttnn/unit_tests/test_tutorials.py @@ -26,6 +26,7 @@ def collect_tutorials(): @skip_for_wormhole_b0() +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("notebook_path", collect_tutorials()) def test_tutorials(notebook_path): with open(notebook_path) as f: diff --git a/tests/ttnn/unit_tests/test_validate_decorator.py b/tests/ttnn/unit_tests/test_validate_decorator.py index b2d8628bfb8..dbaffd58a46 100644 --- a/tests/ttnn/unit_tests/test_validate_decorator.py +++ b/tests/ttnn/unit_tests/test_validate_decorator.py @@ -12,6 +12,7 @@ @skip_for_wormhole_b0() +@pytest.mark.requires_fast_runtime_mode_off @pytest.mark.parametrize("batch_size", [1]) @pytest.mark.parametrize("h", [32]) @pytest.mark.parametrize("w", [32])