From 2c817852b9d7b0dc95058ea265f2858806418fc0 Mon Sep 17 00:00:00 2001 From: Tomasz Janiszewski Date: Tue, 17 Mar 2026 13:29:29 +0100 Subject: [PATCH 1/2] Update maxToolCalls and minToolCalls in eval.yaml --- e2e-tests/mcpchecker/eval.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e-tests/mcpchecker/eval.yaml b/e2e-tests/mcpchecker/eval.yaml index 3b6e73e..f28d41e 100644 --- a/e2e-tests/mcpchecker/eval.yaml +++ b/e2e-tests/mcpchecker/eval.yaml @@ -50,7 +50,7 @@ config: argumentsMatch: cveName: "CVE-2016-1000031" minToolCalls: 1 - maxToolCalls: 3 + maxToolCalls: 4 # Test 4: Non-existent CVE # Expects 3 calls because "Is CVE detected in my clusters?" triggers comprehensive check @@ -118,7 +118,7 @@ config: argumentsMatch: cveName: "CVE-2021-44228" minToolCalls: 1 - maxToolCalls: 3 + maxToolCalls: 4 # Test 10: Multiple CVEs in one prompt - path: tasks/cve-multiple.yaml @@ -135,8 +135,8 @@ config: minToolCalls: 2 maxToolCalls: 6 - # Test 11: RHSA detection (should fail gracefully) + # Test 11: RHSA detection should work - path: tasks/rhsa-not-supported.yaml assertions: - minToolCalls: 0 - maxToolCalls: 1 + minToolCalls: 1 + maxToolCalls: 4 From 0c1322af2ddb826b9a01e43c1d972f5407a6c96c Mon Sep 17 00:00:00 2001 From: Tomasz Janiszewski Date: Tue, 17 Mar 2026 13:46:35 +0100 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Tomasz Janiszewski --- e2e-tests/mcpchecker/eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/mcpchecker/eval.yaml b/e2e-tests/mcpchecker/eval.yaml index f28d41e..f95d03d 100644 --- a/e2e-tests/mcpchecker/eval.yaml +++ b/e2e-tests/mcpchecker/eval.yaml @@ -63,7 +63,7 @@ config: argumentsMatch: cveName: "CVE-2099-00001" minToolCalls: 1 - maxToolCalls: 3 + maxToolCalls: 5 # Test 5: CVE with specific cluster filter (does exist) # Claude does comprehensive checking even for single cluster (orchestrator, deployments, nodes)