From 0c5eb92100482ca1047edfb9795f251cb710fffd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 00:47:03 +0000 Subject: [PATCH 1/2] CAMEL-23928: Add tool-calling behavior tests and documentation Add behavioral unit and integration tests covering hallucinated tool name recovery, max tool round trips, tool execution error handling, and the AiServices customizer hook. Document AgentConfiguration tool-calling options and the customizer escape hatch in the component guide. Co-authored-by: Cursor --- ...tConfigurationToolCallingBehaviorTest.java | 207 ++++++++++++++++++ .../docs/langchain4j-agent-component.adoc | 42 ++++ ...gChain4jAgentAiServicesCustomizerTest.java | 149 +++++++++++++ 3 files changed, 398 insertions(+) create mode 100644 components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java create mode 100644 components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentAiServicesCustomizerTest.java diff --git a/components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java b/components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java new file mode 100644 index 0000000000000..9fd01535e17c7 --- /dev/null +++ b/components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.langchain4j.agent.api; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import dev.langchain4j.agent.tool.Tool; +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ToolExecutionResultMessage; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.service.Result; +import dev.langchain4j.service.tool.ToolErrorHandlerResult; +import dev.langchain4j.service.tool.ToolExecutionErrorHandler; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Behavioral tests for CAMEL-23928: tool-calling options and the AiServices customizer hook wired through + * {@link AbstractAgent#configureBuilder(dev.langchain4j.service.AiServices, dev.langchain4j.service.tool.ToolProvider)}. + */ +class AgentConfigurationToolCallingBehaviorTest { + + private final AtomicInteger chatRound = new AtomicInteger(); + private final AtomicBoolean beforeToolExecutionInvoked = new AtomicBoolean(); + private final AtomicReference hallucinatedToolName = new AtomicReference<>(); + private final AtomicBoolean toolExecutionErrorHandled = new AtomicBoolean(); + + @BeforeEach + void resetState() { + chatRound.set(0); + beforeToolExecutionInvoked.set(false); + hallucinatedToolName.set(null); + toolExecutionErrorHandled.set(false); + } + + @Test + void hallucinatedToolNameStrategyAllowsAgentToRecover() { + ChatModel chatModel = new ChatModel() { + @Override + public ChatResponse doChat(ChatRequest request) { + if (chatRound.getAndIncrement() == 0) { + ToolExecutionRequest hallucinated = ToolExecutionRequest.builder() + .id("h1") + .name("task_complete") + .arguments("{}") + .build(); + return ChatResponse.builder() + .aiMessage(AiMessage.builder().toolExecutionRequests(List.of(hallucinated)).build()) + .build(); + } + return ChatResponse.builder().aiMessage(AiMessage.from("recovered")).build(); + } + }; + + AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(chatModel) + .withMaxToolCallingRoundTrips(3) + .withHallucinatedToolNameStrategy(request -> { + hallucinatedToolName.set(request.name()); + return ToolExecutionResultMessage.from(request, "Tool not found: " + request.name()); + }); + + Agent agent = new AgentWithoutMemory(configuration); + Result result = agent.chat(new AiAgentBody<>("complete the task"), null); + + assertThat(result.content()).isEqualTo("recovered"); + assertThat(hallucinatedToolName.get()).isEqualTo("task_complete"); + assertThat(chatRound.get()).isEqualTo(2); + } + + @Test + void maxToolCallingRoundTripsIsEnforced() { + ChatModel alwaysRequestsTool = new ChatModel() { + @Override + public ChatResponse doChat(ChatRequest request) { + ToolExecutionRequest toolRequest = ToolExecutionRequest.builder() + .id("loop") + .name("countItems") + .arguments("{}") + .build(); + return ChatResponse.builder() + .aiMessage(AiMessage.builder().toolExecutionRequests(List.of(toolRequest)).build()) + .build(); + } + }; + + AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(alwaysRequestsTool) + .withCustomTools(List.of(new CountItemsTool())) + .withMaxToolCallingRoundTrips(1); + + Agent agent = new AgentWithoutMemory(configuration); + + assertThatThrownBy(() -> agent.chat(new AiAgentBody<>("count"), null)) + .hasMessageContaining("exceeded 1 tool calling round trips"); + } + + @Test + void toolExecutionErrorHandlerAndCompensationAllowRecovery() { + ChatModel chatModel = new ChatModel() { + @Override + public ChatResponse doChat(ChatRequest request) { + if (chatRound.getAndIncrement() == 0) { + ToolExecutionRequest toolRequest = ToolExecutionRequest.builder() + .id("f1") + .name("failOperation") + .arguments("{}") + .build(); + return ChatResponse.builder() + .aiMessage(AiMessage.builder().toolExecutionRequests(List.of(toolRequest)).build()) + .build(); + } + return ChatResponse.builder().aiMessage(AiMessage.from("handled")).build(); + } + }; + + ToolExecutionErrorHandler errorHandler = (error, context) -> { + toolExecutionErrorHandled.set(true); + return ToolErrorHandlerResult.text("tool failed safely"); + }; + + AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(chatModel) + .withCustomTools(List.of(new FailingTool())) + .withMaxToolCallingRoundTrips(3) + .withCompensateOnToolErrors(true) + .withToolExecutionErrorHandler(errorHandler); + + Agent agent = new AgentWithoutMemory(configuration); + Result result = agent.chat(new AiAgentBody<>("run failing tool"), null); + + assertThat(result.content()).isEqualTo("handled"); + assertThat(toolExecutionErrorHandled).isTrue(); + } + + @Test + void aiServicesCustomizerCanConfigureBeforeToolExecution() { + ChatModel chatModel = new ChatModel() { + @Override + public ChatResponse doChat(ChatRequest request) { + if (chatRound.getAndIncrement() == 0) { + ToolExecutionRequest toolRequest = ToolExecutionRequest.builder() + .id("c1") + .name("countItems") + .arguments("{}") + .build(); + return ChatResponse.builder() + .aiMessage(AiMessage.builder().toolExecutionRequests(List.of(toolRequest)).build()) + .build(); + } + return ChatResponse.builder().aiMessage(AiMessage.from("done")).build(); + } + }; + + AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(chatModel) + .withCustomTools(List.of(new CountItemsTool())) + .withMaxToolCallingRoundTrips(3) + .withAiServicesCustomizer(builder -> builder.beforeToolExecution( + before -> beforeToolExecutionInvoked.set(true))); + + Agent agent = new AgentWithoutMemory(configuration); + Result result = agent.chat(new AiAgentBody<>("count items"), null); + + assertThat(result.content()).isEqualTo("done"); + assertThat(beforeToolExecutionInvoked).isTrue(); + } + + static class CountItemsTool { + + @Tool(name = "countItems", value = "Returns a fixed count") + int countItems() { + return 42; + } + } + + static class FailingTool { + + @Tool(name = "failOperation", value = "Always fails") + String failOperation() { + throw new RuntimeException("Simulated tool failure"); + } + } +} diff --git a/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-component.adoc b/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-component.adoc index e706511219885..5d2fd309ffa91 100644 --- a/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-component.adoc +++ b/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-component.adoc @@ -159,6 +159,48 @@ Agents are configured using the `AgentConfiguration` class which provides a flue * Retrieval Augmentor (for RAG functionality) * Input and Output Guardrails * Concurrent tool execution (`withExecuteToolsConcurrently`) for parallel Camel route tools and MCP tools within one LLM round trip +* Tool-calling control: round-trip limits, hallucinated tool handling, and error compensation +* AiServices builder customizer for advanced LangChain4j options + +==== Tool-calling options + +`AgentConfiguration` exposes the most common LangChain4j `AiServices` tool-calling settings as first-class fluent methods. These are applied automatically when Camel creates an agent from the configuration bean (inline agent mode or a registered `AgentWithMemory` / `AgentWithoutMemory` bean): + +[cols="1,2"] +|=== +| Method | Purpose + +| `withMaxToolCallingRoundTrips(int)` | Limits how many tool-calling round trips the LLM may perform per request (`0` = unset) +| `withHallucinatedToolNameStrategy(...)` | Handles requests for tool names that do not exist (for example a hallucinated `task_complete` tool) +| `withToolExecutionErrorHandler(...)` | Converts tool execution exceptions into tool result messages for the LLM +| `withToolArgumentsErrorHandler(...)` | Handles invalid or unparsable tool arguments +| `withCompensateOnToolErrors(Boolean)` | Sends tool errors back to the LLM so it can recover instead of failing the exchange +| `withExecuteToolsConcurrently()` / `withExecuteToolsConcurrently(Executor)` | Runs multiple tool calls from one LLM turn in parallel +|=== + +._Java-only: recover from a hallucinated tool name_ +[source,java] +---- +AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(chatModel) + .withMaxToolCallingRoundTrips(5) + .withHallucinatedToolNameStrategy(request -> + ToolExecutionResultMessage.from(request, "Unknown tool: " + request.name())); +---- + +Some LangChain4j builder options (for example `beforeToolExecution`, `afterToolExecution`, or `toolSearchStrategy`) are not wrapped individually. Use the customizer escape hatch instead: + +._Java-only: configure any remaining AiServices builder option_ +[source,java] +---- +AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(chatModel) + .withAiServicesCustomizer(builder -> builder + .beforeToolExecution(ctx -> log.info("Calling tool {}", ctx.toolExecutionRequest().name())) + .afterToolExecution(exec -> log.info("Tool finished: {}", exec.result()))); +---- + +The customizer runs in `AbstractAgent.configureBuilder()` after all standard Camel wiring (tool providers, MCP, guardrails, RAG, and the first-class tool-calling options above) and before `build()` is called. ==== Concurrent tool execution diff --git a/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentAiServicesCustomizerTest.java b/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentAiServicesCustomizerTest.java new file mode 100644 index 0000000000000..31c13fa5c2282 --- /dev/null +++ b/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentAiServicesCustomizerTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.langchain4j.agent; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ToolExecutionResultMessage; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.response.ChatResponse; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.langchain4j.agent.api.AgentConfiguration; +import org.apache.camel.component.langchain4j.agent.api.AiAgentBody; +import org.apache.camel.spi.Registry; +import org.apache.camel.test.junit6.CamelTestSupport; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for CAMEL-23928: {@link AgentConfiguration} tool-calling options and AiServices customizer when + * used through the langchain4j-agent component. + */ +class LangChain4jAgentAiServicesCustomizerTest extends CamelTestSupport { + + private static final String TAG = "aiservices-customizer"; + + private final AtomicInteger chatRound = new AtomicInteger(); + private final AtomicBoolean customizerInvoked = new AtomicBoolean(); + private final AtomicReference hallucinatedToolName = new AtomicReference<>(); + + @BeforeEach + void resetState() { + chatRound.set(0); + customizerInvoked.set(false); + hallucinatedToolName.set(null); + } + + @Override + protected void bindToRegistry(Registry registry) { + registry.bind("hallucinationConfig", new AgentConfiguration() + .withChatModel(createHallucinationRecoveryModel()) + .withMaxToolCallingRoundTrips(3) + .withHallucinatedToolNameStrategy(request -> { + hallucinatedToolName.set(request.name()); + return ToolExecutionResultMessage.from(request, "unknown tool: " + request.name()); + })); + + registry.bind("customizerConfig", new AgentConfiguration() + .withChatModel(createSingleToolModel()) + .withMaxToolCallingRoundTrips(3) + .withAiServicesCustomizer(builder -> { + customizerInvoked.set(true); + builder.beforeToolExecution(before -> { + }); + })); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:hallucination") + .to("langchain4j-agent:test?agentConfiguration=#hallucinationConfig&tags=" + TAG); + + from("direct:customizer") + .to("langchain4j-agent:test?agentConfiguration=#customizerConfig&tags=" + TAG); + + from("ai-tool:routeCounter?tags=" + TAG + "&description=Route-backed counter") + .setBody(constant("counted")); + } + }; + } + + @Test + void hallucinatedToolNameStrategyWorksThroughLangchain4jAgentEndpoint() { + String response = template.requestBody("direct:hallucination", new AiAgentBody<>("finish task"), String.class); + + assertThat(response).isEqualTo("recovered via route"); + assertThat(hallucinatedToolName.get()).isEqualTo("task_complete"); + } + + @Test + void aiServicesCustomizerIsAppliedWhenUsingAgentConfigurationBean() { + String response = template.requestBody("direct:customizer", new AiAgentBody<>("count"), String.class); + + assertThat(response).isEqualTo("done"); + assertThat(customizerInvoked).isTrue(); + } + + private ChatModel createHallucinationRecoveryModel() { + return new ChatModel() { + @Override + public ChatResponse doChat(ChatRequest request) { + if (chatRound.getAndIncrement() == 0) { + ToolExecutionRequest hallucinated = ToolExecutionRequest.builder() + .id("h1") + .name("task_complete") + .arguments("{}") + .build(); + return ChatResponse.builder() + .aiMessage(AiMessage.builder().toolExecutionRequests(List.of(hallucinated)).build()) + .build(); + } + return ChatResponse.builder().aiMessage(AiMessage.from("recovered via route")).build(); + } + }; + } + + private ChatModel createSingleToolModel() { + return new ChatModel() { + @Override + public ChatResponse doChat(ChatRequest request) { + if (chatRound.getAndIncrement() == 0) { + ToolExecutionRequest toolRequest = ToolExecutionRequest.builder() + .id("r1") + .name("routeCounter") + .arguments("{}") + .build(); + return ChatResponse.builder() + .aiMessage(AiMessage.builder().toolExecutionRequests(List.of(toolRequest)).build()) + .build(); + } + return ChatResponse.builder().aiMessage(AiMessage.from("done")).build(); + } + }; + } +} From 4436100b529b9e9f979e5a8b7b96c03cf273419d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 01:06:50 +0000 Subject: [PATCH 2/2] CAMEL-23928: Regenerate catalog docs and harden round-trip test Sync langchain4j-agent component documentation into the catalog mirror and use a less brittle assertion for max tool round-trip enforcement. Co-authored-by: Cursor --- .../docs/langchain4j-agent-component.adoc | 42 +++++++++++++++++++ ...tConfigurationToolCallingBehaviorTest.java | 3 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-component.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-component.adoc index e706511219885..5d2fd309ffa91 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-component.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-component.adoc @@ -159,6 +159,48 @@ Agents are configured using the `AgentConfiguration` class which provides a flue * Retrieval Augmentor (for RAG functionality) * Input and Output Guardrails * Concurrent tool execution (`withExecuteToolsConcurrently`) for parallel Camel route tools and MCP tools within one LLM round trip +* Tool-calling control: round-trip limits, hallucinated tool handling, and error compensation +* AiServices builder customizer for advanced LangChain4j options + +==== Tool-calling options + +`AgentConfiguration` exposes the most common LangChain4j `AiServices` tool-calling settings as first-class fluent methods. These are applied automatically when Camel creates an agent from the configuration bean (inline agent mode or a registered `AgentWithMemory` / `AgentWithoutMemory` bean): + +[cols="1,2"] +|=== +| Method | Purpose + +| `withMaxToolCallingRoundTrips(int)` | Limits how many tool-calling round trips the LLM may perform per request (`0` = unset) +| `withHallucinatedToolNameStrategy(...)` | Handles requests for tool names that do not exist (for example a hallucinated `task_complete` tool) +| `withToolExecutionErrorHandler(...)` | Converts tool execution exceptions into tool result messages for the LLM +| `withToolArgumentsErrorHandler(...)` | Handles invalid or unparsable tool arguments +| `withCompensateOnToolErrors(Boolean)` | Sends tool errors back to the LLM so it can recover instead of failing the exchange +| `withExecuteToolsConcurrently()` / `withExecuteToolsConcurrently(Executor)` | Runs multiple tool calls from one LLM turn in parallel +|=== + +._Java-only: recover from a hallucinated tool name_ +[source,java] +---- +AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(chatModel) + .withMaxToolCallingRoundTrips(5) + .withHallucinatedToolNameStrategy(request -> + ToolExecutionResultMessage.from(request, "Unknown tool: " + request.name())); +---- + +Some LangChain4j builder options (for example `beforeToolExecution`, `afterToolExecution`, or `toolSearchStrategy`) are not wrapped individually. Use the customizer escape hatch instead: + +._Java-only: configure any remaining AiServices builder option_ +[source,java] +---- +AgentConfiguration configuration = new AgentConfiguration() + .withChatModel(chatModel) + .withAiServicesCustomizer(builder -> builder + .beforeToolExecution(ctx -> log.info("Calling tool {}", ctx.toolExecutionRequest().name())) + .afterToolExecution(exec -> log.info("Tool finished: {}", exec.result()))); +---- + +The customizer runs in `AbstractAgent.configureBuilder()` after all standard Camel wiring (tool providers, MCP, guardrails, RAG, and the first-class tool-calling options above) and before `build()` is called. ==== Concurrent tool execution diff --git a/components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java b/components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java index 9fd01535e17c7..b6294beadb721 100644 --- a/components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java +++ b/components/camel-ai/camel-langchain4j-agent-api/src/test/java/org/apache/camel/component/langchain4j/agent/api/AgentConfigurationToolCallingBehaviorTest.java @@ -115,7 +115,8 @@ public ChatResponse doChat(ChatRequest request) { Agent agent = new AgentWithoutMemory(configuration); assertThatThrownBy(() -> agent.chat(new AiAgentBody<>("count"), null)) - .hasMessageContaining("exceeded 1 tool calling round trips"); + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("tool calling round trips"); } @Test