fix: build judge input as string; strip legacy judge config messages

packages/ai-providers/server-ai-langchain/src/ldai_langchain/langchain_agent_runner.py

@@ -33,7 +33,7 @@ def __init__(self, agent: Any):
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
@@ -42,7 +42,7 @@ async def run(
         Delegates to the compiled LangChain agent, which handles
         the tool-calling loop internally.
 
-        :param input: The user prompt or input to the agent
+        :param input: The user prompt string to the agent
         :param output_type: Reserved for future structured output support;
             currently ignored.
         :return: :class:`RunnerResult` with ``content``, ``raw`` response, and

packages/ai-providers/server-ai-langchain/src/ldai_langchain/langchain_model_runner.py

@@ -24,8 +24,9 @@ class LangChainModelRunner(Runner):
     :meth:`run`.
     """
 
-    def __init__(self, llm: BaseChatModel):
+    def __init__(self, llm: BaseChatModel, config_messages: Optional[List[LDMessage]] = None):
         self._llm = llm
+        self._config_messages: List[LDMessage] = list(config_messages or [])
 
     def get_llm(self) -> BaseChatModel:
         """
@@ -37,37 +38,28 @@ def get_llm(self) -> BaseChatModel:
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
         Run the LangChain model with the given input.
 
-        :param input: A string prompt or a list of :class:`LDMessage` objects
+        Prepends any config messages (system prompt, instructions, etc.) stored
+        at construction time before the user message.
+
+        :param input: A string prompt
         :param output_type: Optional JSON schema dict requesting structured output.
             When provided, ``parsed`` on the returned :class:`RunnerResult` is
             populated with the parsed JSON document.
         :return: :class:`RunnerResult` containing ``content``, ``metrics``,
             ``raw`` and (when ``output_type`` is set) ``parsed``.
         """
-        messages = self._coerce_input(input)
+        messages = self._config_messages + [LDMessage(role='user', content=input)]
 
         if output_type is not None:
             return await self._run_structured(messages, output_type)
         return await self._run_completion(messages)
 
-    # convert_messages_to_langchain only accepts List[LDMessage]; _coerce_input
-    # normalizes a bare string to [LDMessage(role='user', ...)] before that step.
-    @staticmethod
-    def _coerce_input(input: Any) -> List[LDMessage]:
-        if isinstance(input, str):
-            return [LDMessage(role='user', content=input)]
-        if isinstance(input, list):
-            return input
-        raise TypeError(
-            f"Unsupported input type for LangChainModelRunner.run: {type(input).__name__}"
-        )
-
     async def _run_completion(self, messages: List[LDMessage]) -> RunnerResult:
         try:
             langchain_messages = convert_messages_to_langchain(messages)

packages/ai-providers/server-ai-langchain/src/ldai_langchain/langchain_runner_factory.py

@@ -69,4 +69,5 @@ def create_model(self, config: AIConfigKind) -> LangChainModelRunner:
         :return: LangChainModelRunner ready to invoke the model
         """
         llm = create_langchain_model(config)
-        return LangChainModelRunner(llm)
+        config_messages = list(getattr(config, 'messages', None) or [])
+        return LangChainModelRunner(llm, config_messages)

packages/ai-providers/server-ai-langchain/tests/test_langchain_provider.py

@@ -233,8 +233,7 @@ async def test_returns_success_true_for_string_content(self, mock_llm):
         mock_llm.ainvoke = AsyncMock(return_value=mock_response)
         provider = LangChainModelRunner(mock_llm)
 
-        messages = [LDMessage(role='user', content='Hello')]
-        result = await provider.run(messages)
+        result = await provider.run('Hello')
 
         assert result.metrics.success is True
         assert result.content == 'Test response'
@@ -246,8 +245,7 @@ async def test_returns_success_false_for_non_string_content_and_logs_warning(sel
         mock_llm.ainvoke = AsyncMock(return_value=mock_response)
         provider = LangChainModelRunner(mock_llm)
 
-        messages = [LDMessage(role='user', content='Hello')]
-        result = await provider.run(messages)
+        result = await provider.run('Hello')
 
         assert result.metrics.success is False
         assert result.content == ''
@@ -259,8 +257,7 @@ async def test_returns_success_false_when_model_invocation_throws_error(self, mo
         mock_llm.ainvoke = AsyncMock(side_effect=error)
         provider = LangChainModelRunner(mock_llm)
 
-        messages = [LDMessage(role='user', content='Hello')]
-        result = await provider.run(messages)
+        result = await provider.run('Hello')
 
         assert result.metrics.success is False
         assert result.content == ''
@@ -284,9 +281,8 @@ async def test_returns_success_true_for_successful_invocation(self, mock_llm):
         mock_llm.with_structured_output = MagicMock(return_value=mock_structured_llm)
         provider = LangChainModelRunner(mock_llm)
 
-        messages = [LDMessage(role='user', content='Hello')]
         response_structure = {'type': 'object', 'properties': {}}
-        result = await provider.run(messages, output_type=response_structure)
+        result = await provider.run('Hello', output_type=response_structure)
 
         assert result.metrics.success is True
         assert result.parsed == parsed_data
@@ -300,9 +296,8 @@ async def test_returns_success_false_when_structured_model_invocation_throws_err
         mock_llm.with_structured_output = MagicMock(return_value=mock_structured_llm)
         provider = LangChainModelRunner(mock_llm)
 
-        messages = [LDMessage(role='user', content='Hello')]
         response_structure = {'type': 'object', 'properties': {}}
-        result = await provider.run(messages, output_type=response_structure)
+        result = await provider.run('Hello', output_type=response_structure)
 
         assert result.metrics.success is False
         assert result.parsed is None

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_agent_runner.py

@@ -47,7 +47,7 @@ def __init__(
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
@@ -56,7 +56,7 @@ async def run(
         Delegates to the OpenAI Agents SDK ``Runner.run``, which handles the
         tool-calling loop internally.
 
-        :param input: The user prompt or input to the agent
+        :param input: The user prompt string to the agent
         :param output_type: Reserved for future structured output support;
             currently ignored.
         :return: :class:`RunnerResult` with ``content``, ``raw`` response, and

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_model_runner.py

@@ -28,46 +28,37 @@ def __init__(
         client: AsyncOpenAI,
         model_name: str,
         parameters: Dict[str, Any],
+        config_messages: Optional[List[LDMessage]] = None,
     ):
         self._client = client
         self._model_name = model_name
         self._parameters = parameters
+        self._config_messages: List[LDMessage] = list(config_messages or [])
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
         Run the OpenAI model with the given input.
 
-        :param input: A string prompt or a list of :class:`LDMessage` objects
+        Prepends any config messages (system prompt, instructions, etc.) stored
+        at construction time before the user message.
+
+        :param input: A string prompt
         :param output_type: Optional JSON schema dict requesting structured output.
             When provided, ``parsed`` on the returned :class:`RunnerResult` is
             populated with the parsed JSON document.
         :return: :class:`RunnerResult` containing ``content``, ``metrics``,
             ``raw`` and (when ``output_type`` is set) ``parsed``.
         """
-        try:
-            messages = self._coerce_input(input)
-        except TypeError as error:
-            log.warning(f'OpenAI model runner received unsupported input type: {error}')
-            return RunnerResult(content='', metrics=LDAIMetrics(success=False, usage=None))
+        messages = self._config_messages + [LDMessage(role='user', content=input)]
 
         if output_type is not None:
             return await self._run_structured(messages, output_type)
         return await self._run_completion(messages)
 
-    @staticmethod
-    def _coerce_input(input: Any) -> List[LDMessage]:
-        if isinstance(input, str):
-            return [LDMessage(role='user', content=input)]
-        if isinstance(input, list):
-            return input
-        raise TypeError(
-            f"Unsupported input type for OpenAIModelRunner.run: {type(input).__name__}"
-        )
-
     async def _run_completion(self, messages: List[LDMessage]) -> RunnerResult:
         try:
             response = await self._client.chat.completions.create(

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_runner_factory.py

@@ -100,7 +100,8 @@ def create_model(self, config: AIConfigKind) -> OpenAIModelRunner:
         tool_defs = parameters.pop('tools', None) or []
         if tool_defs:
             parameters['tools'] = normalize_tool_types(tool_defs)
-        return OpenAIModelRunner(self._client, model_name, parameters)
+        config_messages = list(getattr(config, 'messages', None) or [])
+        return OpenAIModelRunner(self._client, model_name, parameters, config_messages)
 
     def get_client(self) -> AsyncOpenAI:
         """

packages/ai-providers/server-ai-openai/tests/test_openai_provider.py

@@ -4,8 +4,6 @@
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
-from ldai import LDMessage
-
 from ldai_openai import OpenAIModelRunner, OpenAIRunnerFactory, get_ai_metrics_from_response, get_ai_usage_from_response
 
 
@@ -143,8 +141,7 @@ async def test_invokes_openai_chat_completions_and_returns_response(self, mock_c
         mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Hello!')]
-        result = await provider.run(messages)
+        result = await provider.run('Hello!')
 
         mock_client.chat.completions.create.assert_called_once_with(
             model='gpt-3.5-turbo',
@@ -172,8 +169,7 @@ async def test_returns_unsuccessful_response_when_no_content(self, mock_client):
         mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Hello!')]
-        result = await provider.run(messages)
+        result = await provider.run('Hello!')
 
         assert result.content == ''
         assert result.metrics.success is False
@@ -190,8 +186,7 @@ async def test_returns_unsuccessful_response_when_choices_empty(self, mock_clien
         mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Hello!')]
-        result = await provider.run(messages)
+        result = await provider.run('Hello!')
 
         assert result.content == ''
         assert result.metrics.success is False
@@ -204,8 +199,7 @@ async def test_returns_unsuccessful_response_when_exception_thrown(self, mock_cl
         mock_client.chat.completions.create = AsyncMock(side_effect=Exception('API Error'))
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Hello!')]
-        result = await provider.run(messages)
+        result = await provider.run('Hello!')
 
         assert result.content == ''
         assert result.metrics.success is False
@@ -234,7 +228,6 @@ async def test_invokes_openai_with_structured_output(self, mock_client):
         mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Tell me about a person')]
         response_structure = {
             'type': 'object',
             'properties': {
@@ -245,7 +238,7 @@ async def test_invokes_openai_with_structured_output(self, mock_client):
             'required': ['name', 'age', 'city'],
         }
 
-        result = await provider.run(messages, output_type=response_structure)
+        result = await provider.run('Tell me about a person', output_type=response_structure)
 
         assert result.parsed == {'name': 'John', 'age': 30, 'city': 'New York'}
         assert result.content == '{"name": "John", "age": 30, "city": "New York"}'
@@ -269,10 +262,9 @@ async def test_returns_unsuccessful_when_no_content_in_structured_response(self,
         mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Tell me about a person')]
         response_structure = {'type': 'object'}
 
-        result = await provider.run(messages, output_type=response_structure)
+        result = await provider.run('Tell me about a person', output_type=response_structure)
 
         assert result.parsed is None
         assert result.content == ''
@@ -293,10 +285,9 @@ async def test_handles_json_parsing_errors(self, mock_client):
         mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Tell me about a person')]
         response_structure = {'type': 'object'}
 
-        result = await provider.run(messages, output_type=response_structure)
+        result = await provider.run('Tell me about a person', output_type=response_structure)
 
         assert result.parsed is None
         assert result.content == 'invalid json content'
@@ -312,10 +303,9 @@ async def test_returns_unsuccessful_response_when_exception_thrown(self, mock_cl
         mock_client.chat.completions.create = AsyncMock(side_effect=Exception('API Error'))
 
         provider = OpenAIModelRunner(mock_client, 'gpt-3.5-turbo', {})
-        messages = [LDMessage(role='user', content='Tell me about a person')]
         response_structure = {'type': 'object'}
 
-        result = await provider.run(messages, output_type=response_structure)
+        result = await provider.run('Tell me about a person', output_type=response_structure)
 
         assert result.parsed is None
         assert result.content == ''

packages/sdk/server-ai/src/ldai/client.py

@@ -9,7 +9,7 @@
 from ldai import log
 from ldai.agent_graph import AgentGraphDefinition
 from ldai.evaluator import Evaluator
-from ldai.judge import Judge
+from ldai.judge import Judge, _strip_legacy_judge_messages
 from ldai.managed_agent import ManagedAgent
 from ldai.managed_agent_graph import ManagedAgentGraph
 from ldai.managed_model import ManagedModel
@@ -237,6 +237,10 @@ def _extract_evaluation_metric_key(variation: Dict[str, Any]) -> Optional[str]:
 
         evaluation_metric_key = _extract_evaluation_metric_key(variation)
 
+        # strip legacy judge template messages before creating config
+        if messages:
+            messages = _strip_legacy_judge_messages(messages)
+
         config = AIJudgeConfig(
             key=key,
             enabled=bool(enabled),