From 1a95175a7ffd0e6dc2c1dd4e02bd9713cb2a2cd5 Mon Sep 17 00:00:00 2001 From: guanxu <1510424541@qq.com> Date: Fri, 17 Oct 2025 12:36:26 +0800 Subject: [PATCH] Add Builder for FactCheckingEvaluator (#4652) Signed-off-by: guanxu <1510424541@qq.com> --- .../evaluation/FactCheckingEvaluator.java | 58 +++++++++++++++--- .../FactCheckingEvaluatorTests.java | 61 +++++++++++++++++++ 2 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 spring-ai-client-chat/src/test/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluatorTests.java diff --git a/spring-ai-client-chat/src/main/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluator.java b/spring-ai-client-chat/src/main/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluator.java index cb094422cd2..ef05d9eff13 100644 --- a/spring-ai-client-chat/src/main/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluator.java +++ b/spring-ai-client-chat/src/main/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluator.java @@ -16,12 +16,14 @@ package org.springframework.ai.chat.evaluation; -import java.util.Collections; - import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.evaluation.EvaluationRequest; import org.springframework.ai.evaluation.EvaluationResponse; import org.springframework.ai.evaluation.Evaluator; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +import java.util.Collections; /** * Implementation of {@link Evaluator} used to evaluate the factual accuracy of Large @@ -59,6 +61,7 @@ * * @author Eddú Meléndez * @author Mark Pollack + * @author guan xu * @see Evaluator * @see EvaluationRequest * @see EvaluationResponse @@ -69,13 +72,20 @@ public class FactCheckingEvaluator implements Evaluator { private static final String DEFAULT_EVALUATION_PROMPT_TEXT = """ Evaluate whether or not the following claim is supported by the provided document. Respond with "yes" if the claim is supported, or "no" if it is not. - Document: \\n {document}\\n - Claim: \\n {claim} + + Document: + {document} + + Claim: + {claim} """; private static final String BESPOKE_EVALUATION_PROMPT_TEXT = """ - Document: \\n {document}\\n - Claim: \\n {claim} + Document: + {document} + + Claim: + {claim} """; private final ChatClient.Builder chatClientBuilder; @@ -89,7 +99,7 @@ public class FactCheckingEvaluator implements Evaluator { * evaluation */ public FactCheckingEvaluator(ChatClient.Builder chatClientBuilder) { - this(chatClientBuilder, DEFAULT_EVALUATION_PROMPT_TEXT); + this(chatClientBuilder, null); } /** @@ -99,9 +109,10 @@ public FactCheckingEvaluator(ChatClient.Builder chatClientBuilder) { * evaluation * @param evaluationPrompt The prompt text to use for evaluation */ - public FactCheckingEvaluator(ChatClient.Builder chatClientBuilder, String evaluationPrompt) { + public FactCheckingEvaluator(ChatClient.Builder chatClientBuilder, @Nullable String evaluationPrompt) { + Assert.notNull(chatClientBuilder, "chatClientBuilder cannot be null"); this.chatClientBuilder = chatClientBuilder; - this.evaluationPrompt = evaluationPrompt; + this.evaluationPrompt = evaluationPrompt != null ? evaluationPrompt : DEFAULT_EVALUATION_PROMPT_TEXT; } /** @@ -138,4 +149,33 @@ public EvaluationResponse evaluate(EvaluationRequest evaluationRequest) { return new EvaluationResponse(passing, "", Collections.emptyMap()); } + public static FactCheckingEvaluator.Builder builder() { + return new FactCheckingEvaluator.Builder(); + } + + public static final class Builder { + + private ChatClient.Builder chatClientBuilder; + + private String evaluationPrompt; + + private Builder() { + } + + public FactCheckingEvaluator.Builder chatClientBuilder(ChatClient.Builder chatClientBuilder) { + this.chatClientBuilder = chatClientBuilder; + return this; + } + + public FactCheckingEvaluator.Builder evaluationPrompt(String evaluationPrompt) { + this.evaluationPrompt = evaluationPrompt; + return this; + } + + public FactCheckingEvaluator build() { + return new FactCheckingEvaluator(this.chatClientBuilder, this.evaluationPrompt); + } + + } + } diff --git a/spring-ai-client-chat/src/test/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluatorTests.java b/spring-ai-client-chat/src/test/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluatorTests.java new file mode 100644 index 00000000000..f4440edd2ce --- /dev/null +++ b/spring-ai-client-chat/src/test/java/org/springframework/ai/chat/evaluation/FactCheckingEvaluatorTests.java @@ -0,0 +1,61 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.evaluation; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.model.ChatModel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +/** + * Unit tests for {@link FactCheckingEvaluator}. + * + * @author guan xu + */ +class FactCheckingEvaluatorTests { + + @Test + void whenChatClientBuilderIsNullThenThrow() { + assertThatThrownBy(() -> new FactCheckingEvaluator(null)).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("chatClientBuilder cannot be null"); + + assertThatThrownBy(() -> FactCheckingEvaluator.builder().chatClientBuilder(null).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("chatClientBuilder cannot be null"); + } + + @Test + void whenEvaluationPromptIsNullThenUseDefaultEvaluationPromptText() { + FactCheckingEvaluator evaluator = new FactCheckingEvaluator(ChatClient.builder(mock(ChatModel.class))); + assertThat(evaluator).isNotNull(); + + evaluator = FactCheckingEvaluator.builder() + .chatClientBuilder(ChatClient.builder(mock(ChatModel.class))) + .build(); + assertThat(evaluator).isNotNull(); + } + + @Test + void whenForBespokeMinicheckThenUseBespokeEvaluationPromptText() { + FactCheckingEvaluator evaluator = FactCheckingEvaluator.forBespokeMinicheck(ChatClient.builder(mock(ChatModel.class))); + assertThat(evaluator).isNotNull(); + } + +}