From 8f51d3ba29a8ddc8b8a311c7f5a33a39e2c08e90 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 9 Sep 2025 19:05:55 -0400 Subject: [PATCH 1/3] Add CUMULATIVE_TEST_TOKENS_PER_SEQUENCE env var Signed-off-by: Abhishek --- tests/models/test_decoders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/test_decoders.py b/tests/models/test_decoders.py index 45628a24..422e85a0 100644 --- a/tests/models/test_decoders.py +++ b/tests/models/test_decoders.py @@ -69,7 +69,7 @@ USE_MICRO_MODELS = os.environ.get("FMS_TEST_SHAPES_USE_MICRO_MODELS", "1") == "1" USE_DISTRIBUTED = os.environ.get("FMS_TEST_SHAPES_DISTRIBUTED", "0") == "1" TIMING = os.environ.get("TIMING", "") - +CUMULATIVE_TEST_TOKENS_PER_SEQUENCE = os.environ.get("FMS_TEST_SHAPES_CUMULATIVE_TEST_TOKENS_PER_SEQUENCE", "1024") ATTN_TYPE = os.environ.get("FMS_TEST_SHAPES_ATTN_TYPE", "sdpa") attention_map = { "sdpa": "sdpa_causal", @@ -608,7 +608,7 @@ def _metric_calculator(r: torch.Tensor, t: torch.Tensor): ) return (cross_entropy, diff) - iters = 1024 // max_new_tokens + iters = CUMULATIVE_TEST_TOKENS_PER_SEQUENCE // max_new_tokens ce_fail_responses_list = [] diff_fail_responses_list = [] total_tokens = 0 From 1374fd20b532a3919f0a3b917aa14b3b23f53eb4 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 9 Sep 2025 19:11:53 -0400 Subject: [PATCH 2/3] Lint check Signed-off-by: Abhishek --- tests/models/test_decoders.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/models/test_decoders.py b/tests/models/test_decoders.py index 422e85a0..78585be7 100644 --- a/tests/models/test_decoders.py +++ b/tests/models/test_decoders.py @@ -69,7 +69,9 @@ USE_MICRO_MODELS = os.environ.get("FMS_TEST_SHAPES_USE_MICRO_MODELS", "1") == "1" USE_DISTRIBUTED = os.environ.get("FMS_TEST_SHAPES_DISTRIBUTED", "0") == "1" TIMING = os.environ.get("TIMING", "") -CUMULATIVE_TEST_TOKENS_PER_SEQUENCE = os.environ.get("FMS_TEST_SHAPES_CUMULATIVE_TEST_TOKENS_PER_SEQUENCE", "1024") +CUMULATIVE_TEST_TOKENS_PER_SEQUENCE = os.environ.get( + "FMS_TEST_SHAPES_CUMULATIVE_TEST_TOKENS_PER_SEQUENCE", "1024" +) ATTN_TYPE = os.environ.get("FMS_TEST_SHAPES_ATTN_TYPE", "sdpa") attention_map = { "sdpa": "sdpa_causal", From 823b56171aabae74a76f295e8da38ebaa4261a91 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Wed, 10 Sep 2025 10:37:12 -0400 Subject: [PATCH 3/3] int wrap of CUMULATIVE_TEST_TOKENS_PER_SEQUENCE Signed-off-by: Abhishek --- tests/models/test_decoders.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/models/test_decoders.py b/tests/models/test_decoders.py index 78585be7..45959169 100644 --- a/tests/models/test_decoders.py +++ b/tests/models/test_decoders.py @@ -69,8 +69,8 @@ USE_MICRO_MODELS = os.environ.get("FMS_TEST_SHAPES_USE_MICRO_MODELS", "1") == "1" USE_DISTRIBUTED = os.environ.get("FMS_TEST_SHAPES_DISTRIBUTED", "0") == "1" TIMING = os.environ.get("TIMING", "") -CUMULATIVE_TEST_TOKENS_PER_SEQUENCE = os.environ.get( - "FMS_TEST_SHAPES_CUMULATIVE_TEST_TOKENS_PER_SEQUENCE", "1024" +CUMULATIVE_TEST_TOKENS_PER_SEQUENCE = int( + os.environ.get("FMS_TEST_SHAPES_CUMULATIVE_TEST_TOKENS_PER_SEQUENCE", "1024") ) ATTN_TYPE = os.environ.get("FMS_TEST_SHAPES_ATTN_TYPE", "sdpa") attention_map = { @@ -610,7 +610,7 @@ def _metric_calculator(r: torch.Tensor, t: torch.Tensor): ) return (cross_entropy, diff) - iters = CUMULATIVE_TEST_TOKENS_PER_SEQUENCE // max_new_tokens + iters = int(CUMULATIVE_TEST_TOKENS_PER_SEQUENCE) // max_new_tokens ce_fail_responses_list = [] diff_fail_responses_list = [] total_tokens = 0