From f782d24b87dc97b302281fb755f188b575c750e1 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 6 Aug 2026 22:44:23 +0200 Subject: [PATCH 1/2] fix(gooddata-eval): stop the metric-skill simulated user from dropping MAQL clauses agentic_metric_skill's simulated-user reply (generate_simulated_response) is what keeps a multi-turn metric-creation conversation going after the agent asks a clarifying question -- it prompts an LLM to answer as the user, using the fixture's expected_output.maql as its only source of truth. The prompt told it to "reply briefly" with no instruction to preserve the MAQL's structure. In practice it would silently drop a WHERE/filter clause, or paraphrase a label id, whenever the agent's question didn't happen to ask about that part directly -- so a well-behaved agent, faithfully following the (already-wrong) simulated answer, still failed the eval. Reproduced live twice against a real gdc-mic-ai-evaluation fixture ("Create a metric for total ecommerce spend", expects SELECT {metric/spend_amount_-_cutcgco} WHERE {label/ecommerce_indicator_code} = "1"): 1. Simulated reply dropped "_code" off ecommerce_indicator_code, anchoring the agent on a sibling attribute that doesn't have that filter. 2. Simulated reply picked one of 3 metric options the agent offered and said "please proceed with that" -- never mentioning the WHERE clause that expected_output required, even though it had it in hand. Confirmed via a 5x-repeated A/B test that this is a prompt problem, not a model-capability one: swapping gpt-4o-mini for gpt-4o under the OLD prompt did not fix it (still dropped the clause); the NEW prompt fixes it on the ORIGINAL gpt-4o-mini (1/5 -> 5/5 runs preserving the exact filter). Fix: instruct the simulating LLM to (a) ensure every clause of the expected MAQL is eventually satisfied even if the agent's question didn't ask about it, (b) quote field/label identifiers verbatim rather than paraphrase them, and (c) proactively add a filter the agent's own offered options omitted. Also drop "reply briefly" and raise max_tokens 150->300, since brevity was part of what squeezed the filter clause out. This brings metric_skill's simulated-user prompt in line with alert_skill's generate_simulated_alert_response, which already passes structured facts + explicit "proactively tell the agent X" instructions rather than one freely-paraphrased string -- not a new pattern for this codebase. Added a regression test asserting the sent prompt preserves clause-fidelity language and the raised max_tokens. Full gooddata-eval suite: 272 passed (9 pre-existing unrelated failures, confirmed identical on clean master before this change -- missing openai extra in test env, and two unrelated test files). Co-Authored-By: Claude Sonnet 5 --- .../core/agentic/metric_skill.py | 9 +++-- .../tests/test_agentic_metric_skill.py | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 2e2b5b9b1..35bee3975 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -88,13 +88,16 @@ def generate_simulated_response(agent_message: str, expected_output: dict) -> st prompt = ( f"You are simulating a user in a conversation with a BI assistant that creates metrics. " f"The assistant said: '{agent_message}'. " - f"The user originally asked to create a metric with MAQL: {expected_maql}. " - f"Reply briefly as the user, providing any clarification the assistant needs." + f"The user's ground-truth intended metric is exactly this MAQL: {expected_maql}. " + f"Reply as the user. You MUST ensure every clause of that MAQL (including any WHERE/filter " + f"conditions) is eventually satisfied, and quote field/label identifiers verbatim from it -- " + f"never paraphrase or drop a clause, even if the assistant's question doesn't explicitly ask " + f"about it. If the assistant's offered options omit a required filter, add it yourself." ) response = client.chat.completions.create( model="gpt-4o-mini", messages=[{"role": "user", "content": prompt}], - max_tokens=150, + max_tokens=300, ) return response.choices[0].message.content or "Please proceed." diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 67a163e92..47c74aa20 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -1,5 +1,7 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import sys +import types from unittest.mock import MagicMock, patch import pytest @@ -8,6 +10,7 @@ MetricRunResult, _delete_metric, _normalize_maql, + generate_simulated_response, run_agentic_metric_skill, ) from gooddata_eval.core.models import ChatResult @@ -21,6 +24,38 @@ def test_normalize_maql_removes_select_wrapper(): assert _normalize_maql("(SELECT {metric/abc})") == "{metric/abc}" +def test_generate_simulated_response_prompt_preserves_maql_fidelity(monkeypatch): + """Regression test for a live-reproduced bug: the old prompt ("reply briefly", + no instruction to cover clauses the assistant didn't ask about) let the + simulating LLM silently drop a MAQL's WHERE clause or paraphrase a label id -- + confirmed via a 5x-repeated A/B test (1/5 vs 5/5 fidelity) that this was the + prompt, not the model (gpt-4o did not fix it under the old prompt either). + """ + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.choices = [MagicMock(message=MagicMock(content="ok"))] + mock_client.chat.completions.create.return_value = mock_response + + # `openai` is an optional [llm-judge] extra, not installed in this test env -- + # inject a fake module rather than patching a real one (mirrors how the source + # itself does `from openai import OpenAI` as a local, guarded import). + fake_openai_module = types.SimpleNamespace(OpenAI=MagicMock(return_value=mock_client)) + monkeypatch.setitem(sys.modules, "openai", fake_openai_module) + + expected_output = {"maql": 'SELECT {metric/spend_amount_-_cutcgco} WHERE {label/ecommerce_indicator_code} = "1"'} + generate_simulated_response("Which base metric should I use?", expected_output) + + call_kwargs = mock_client.chat.completions.create.call_args.kwargs + sent_prompt = call_kwargs["messages"][0]["content"] + + assert "verbatim" in sent_prompt + assert "every clause" in sent_prompt + assert "WHERE" in sent_prompt or "filter" in sent_prompt.lower() + assert "reply briefly" not in sent_prompt.lower() + assert call_kwargs["max_tokens"] >= 300 + + def test_metric_run_result_fields(): r = MetricRunResult( conversation_id="c1", From 006ccb520e81e11819211d89720932df89314553 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 6 Aug 2026 22:49:41 +0200 Subject: [PATCH 2/2] test(gooddata-eval): assert exact MAQL string appears in simulated-user prompt Addresses CodeRabbit review comment on #1718: the regression test only checked for generic instruction words ("verbatim", "every clause"), not that expected_output["maql"] itself made it into the prompt -- a regression that stripped the metric/label reference or filter value entirely could still pass. Assert the exact MAQL string is present. Co-Authored-By: Claude Sonnet 5 --- packages/gooddata-eval/tests/test_agentic_metric_skill.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 47c74aa20..3de90cdf5 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -49,6 +49,7 @@ def test_generate_simulated_response_prompt_preserves_maql_fidelity(monkeypatch) call_kwargs = mock_client.chat.completions.create.call_args.kwargs sent_prompt = call_kwargs["messages"][0]["content"] + assert expected_output["maql"] in sent_prompt assert "verbatim" in sent_prompt assert "every clause" in sent_prompt assert "WHERE" in sent_prompt or "filter" in sent_prompt.lower()