diff --git a/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsUtilsTest.java b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsUtilsTest.java new file mode 100644 index 000000000..9f260366d --- /dev/null +++ b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsUtilsTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.integrations.chatmodels.openai; + +import com.openai.models.chat.completions.ChatCompletionMessage; +import org.apache.flink.agents.api.chat.messages.ChatMessage; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for how {@link OpenAIChatCompletionsUtils} carries a provider refusal from a Chat + * Completions message onto the returned {@link ChatMessage}. The converter is shared by every Chat + * Completions connection in this module. + */ +class OpenAIChatCompletionsUtilsTest { + + @Test + @DisplayName("A refusal reason is carried into extraArgs") + void testRefusalPreservedInExtraArgs() { + // A refused response has no content, so the reason is the only thing separating it + // from a genuinely empty completion. + ChatCompletionMessage message = + ChatCompletionMessage.builder() + .content(Optional.empty()) + .refusal("I cannot help with that") + .build(); + + ChatMessage result = OpenAIChatCompletionsUtils.convertFromOpenAIMessage(message); + + assertThat(result.getExtraArgs()).containsEntry("refusal", "I cannot help with that"); + } + + @Test + @DisplayName("No refusal key is added when the provider did not refuse") + void testNoRefusalKeyWhenAbsent() { + ChatCompletionMessage message = + ChatCompletionMessage.builder().content("hello").refusal(Optional.empty()).build(); + + ChatMessage result = OpenAIChatCompletionsUtils.convertFromOpenAIMessage(message); + + assertThat(result.getExtraArgs()).doesNotContainKey("refusal"); + } +} diff --git a/python/flink_agents/integrations/chat_models/openai/openai_utils.py b/python/flink_agents/integrations/chat_models/openai/openai_utils.py index 16f49e841..9601d9ebc 100644 --- a/python/flink_agents/integrations/chat_models/openai/openai_utils.py +++ b/python/flink_agents/integrations/chat_models/openai/openai_utils.py @@ -196,7 +196,11 @@ def convert_to_openai_message(message: ChatMessage) -> ChatCompletionMessagePara def convert_from_openai_message( message: ChatCompletionMessage, extra_args: Dict[str, Any] ) -> ChatMessage: - """Convert an OpenAI message to a chat message.""" + """Convert an OpenAI message to a chat message. + + A provider refusal is surfaced under extra_args["refusal"], including when + the refusal reason is an empty string. + """ tool_calls = [] if message.tool_calls: # Generate internal UUID for each tool call while preserving @@ -214,6 +218,8 @@ def convert_from_openai_message( } for tool_call in message.tool_calls ] + if message.refusal is not None: + extra_args = {**extra_args, "refusal": message.refusal} return ChatMessage( role=MessageRole(message.role), content=message.content or "", diff --git a/python/flink_agents/integrations/chat_models/openai/tests/test_openai_native_structured_output.py b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_native_structured_output.py index f31492494..ce6909f78 100644 --- a/python/flink_agents/integrations/chat_models/openai/tests/test_openai_native_structured_output.py +++ b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_native_structured_output.py @@ -47,6 +47,7 @@ def _connection() -> OpenAIChatModelConnection: mock_message.role = "assistant" mock_message.content = "ok" mock_message.tool_calls = None + mock_message.refusal = None mock_client.chat.completions.create.return_value.choices = [ MagicMock(message=mock_message) ] diff --git a/python/flink_agents/integrations/chat_models/openai/tests/test_openai_response_parsing.py b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_response_parsing.py new file mode 100644 index 000000000..07c5e3d91 --- /dev/null +++ b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_response_parsing.py @@ -0,0 +1,52 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +import pytest +from openai.types.chat import ChatCompletionMessage + +from flink_agents.integrations.chat_models.openai.openai_utils import ( + convert_from_openai_message, +) + + +@pytest.mark.parametrize("refusal", ["I cannot help with that", ""]) +def test_refusal_is_preserved_in_extra_args(refusal: str) -> None: + """A provider refusal reaches the caller through extra_args.""" + # A refused response carries no content, so the reason is the only thing that + # distinguishes it from a genuinely empty completion. An empty reason is still + # a refusal, which a truthiness guard would silently drop. Callers hand in an + # extra_args already holding token metrics, so recording the reason must add to + # that dict rather than replace it. The reason belongs in extra_args alone: + # folding it into content would make a refusal read as an ordinary answer. + message = ChatCompletionMessage(role="assistant", content=None, refusal=refusal) + + result = convert_from_openai_message(message, {"promptTokens": 3}) + + assert result.extra_args["refusal"] == refusal + assert result.extra_args["promptTokens"] == 3 + assert result.content == "" + + +def test_no_refusal_key_when_refusal_absent() -> None: + """A response that was not refused leaves no refusal key behind.""" + # extra_args is merged back into the outbound assistant message, so a null + # refusal key here would be echoed to the provider on every later request. + message = ChatCompletionMessage(role="assistant", content="ok", refusal=None) + + result = convert_from_openai_message(message, {}) + + assert "refusal" not in result.extra_args