fix: image read and json error (Agent)

packages/service/core/ai/llm/request.ts

@@ -85,11 +85,12 @@ export const createLLMResponse = async <T extends CompletionsBodyType>(
 
   const { throwError = true, body, custonHeaders, userKey, maxContinuations = 1 } = args;
   const { messages, useVision, requestOrigin, tools, toolCallMode } = body;
+  const model = getLLMModel(body.model);
 
   // Messages process
   const requestMessages = await loadRequestMessages({
     messages,
-    useVision,
+    useVision: useVision && model.vision,
     origin: requestOrigin
   });
   // Message process

packages/service/core/ai/utils.ts

@@ -4,6 +4,7 @@ import { getLLMDefaultUsage } from '@fastgpt/global/core/ai/constants';
 import { removeDatasetCiteText } from '@fastgpt/global/core/ai/llm/utils';
 import json5 from 'json5';
 import { sliceJsonStr } from '@fastgpt/global/common/string/tools';
+import { jsonrepair } from 'jsonrepair';
 
 /* 
   Count response max token
@@ -332,7 +333,7 @@ export const parseLLMStreamResponse = () => {
 
 export const parseJsonArgs = <T = Record<string, any>>(str: string) => {
   try {
-    return json5.parse(sliceJsonStr(str)) as T;
+    return json5.parse(jsonrepair(sliceJsonStr(str))) as T;
   } catch {
     return;
   }

packages/service/core/workflow/dispatch/ai/agent/index.ts

@@ -18,6 +18,7 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import {
   chats2GPTMessages,
   chatValue2RuntimePrompt,
+  runtimePrompt2ChatsValue,
   GPTMessages2Chats
 } from '@fastgpt/global/core/chat/adapt';
 import { getPlanCallResponseText } from '@fastgpt/global/core/chat/utils';
@@ -42,6 +43,7 @@ export type DispatchAgentModuleProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.history]?: ChatItemType[];
   [NodeInputKeyEnum.userChatInput]: string;
 
+  [NodeInputKeyEnum.aiChatVision]?: boolean;
   [NodeInputKeyEnum.fileUrlList]?: string[];
   [NodeInputKeyEnum.aiModel]: string;
   [NodeInputKeyEnum.aiSystemPrompt]: string;
@@ -87,6 +89,7 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise
       userChatInput, // 本次任务的输入
       history = 6,
       fileUrlList: fileLinks,
+      aiChatVision = true,
       agent_selectedTools: selectedTools = [],
       // Dataset search configuration
       agent_datasetParams: datasetParams,
@@ -130,16 +133,25 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise
     });
 
     // 交互模式进来的话，这个值才是交互输入的值
-    const queryInput = chatValue2RuntimePrompt(query).text;
+    const { text: queryInput, files: queryFiles } = chatValue2RuntimePrompt(query);
     const formatUserChatInput = fileInputPrompt
       ? `${fileInputPrompt}\n\n${userChatInput}`
       : userChatInput;
+    const currentUserMessage = chats2GPTMessages({
+      messages: [
+        {
+          obj: ChatRoleEnum.Human,
+          value: runtimePrompt2ChatsValue({
+            text: formatUserChatInput,
+            files: queryFiles
+          })
+        }
+      ],
+      reserveId: false
+    })[0];
 
     let {
-      masterMessages = historiesMessages.concat({
-        role: 'user',
-        content: formatUserChatInput
-      }),
+      masterMessages: restoredMasterMessages,
       planHistoryMessages,
       agentPlan,
       planBuffer
@@ -162,6 +174,16 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise
         planBuffer: undefined
       };
     })();
+    let masterMessages: ChatCompletionMessageParam[];
+    if (!restoredMasterMessages) {
+      masterMessages = historiesMessages.concat(currentUserMessage ? [currentUserMessage] : []);
+    } else if (planHistoryMessages?.length) {
+      masterMessages = restoredMasterMessages ?? historiesMessages;
+    } else {
+      masterMessages = currentUserMessage
+        ? restoredMasterMessages.concat(currentUserMessage)
+        : restoredMasterMessages;
+    }
 
     // Get sub apps
     const { completionTools: agentCompletionTools, subAppsMap: agentSubAppsMap } = await getSubapps(

packages/service/core/workflow/dispatch/ai/agent/master/call.ts

@@ -92,7 +92,8 @@ export const masterCall = async ({
       // Dataset search configuration
       agent_datasetParams: datasetParams,
       // Sandbox (Computer Use)
-      useAgentSandbox = false
+      useAgentSandbox = false,
+      aiChatVision
     }
   } = props;
 
@@ -215,6 +216,7 @@ export const masterCall = async ({
       messages: requestMessages,
       model: getLLMModel(model),
       stream: true,
+      useVision: aiChatVision,
       tools: isStepCall
         ? completionTools.filter((item) => item.function.name !== SubAppIds.plan)
         : completionTools
@@ -604,7 +606,7 @@ export const masterCall = async ({
         response,
         assistantMessages: [], // TODO
         usages,
-        stop
+        stop: stop || checkIsStopping()
       };
     },
     onToolCompress: ({ call, response, usage }) => {

packages/service/core/workflow/dispatch/ai/agent/sub/plan/index.ts

@@ -4,7 +4,12 @@ import type {
   ChatCompletionTool
 } from '@fastgpt/global/core/ai/type';
 import { createLLMResponse } from '../../../../../../ai/llm/request';
-import { getInitialPlanPrompt, getContinuePlanPrompt, getInitialPlanQuery } from './prompt';
+import {
+  getInitialPlanPrompt,
+  getContinuePlanPrompt,
+  getInitialPlanQuery,
+  reTryPlanPrompt
+} from './prompt';
 import { getLLMModel } from '../../../../../../ai/model';
 import { formatModelChars2Points } from '../../../../../../../support/wallet/usage/utils';
 import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
@@ -28,6 +33,8 @@ import type { PlanAgentParamsType } from './constants';
 import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
 import { getLogger, LogCategories } from '../../../../../../../common/logger';
 
+const agentLogger = getLogger(LogCategories.MODULE.AI.AGENT);
+
 type PlanAgentConfig = {
   systemPrompt?: string;
   model: string;
@@ -81,20 +88,22 @@ const parsePlan = async ({
     return;
   }
 
-  const result = parseJsonArgs(text);
+  const result = parseJsonArgs<{ steps: AgentPlanType['steps'] }>(text);
+
   if (!result) {
-    return;
+    return result;
   }
 
   const params = await AgentPlanSchema.safeParseAsync({
     ...result,
+    planId,
     task,
     description,
-    background,
-    planId
+    background
   });
+
   if (!params.success) {
-    getLogger(LogCategories.MODULE.AI.AGENT).warn(`[Plan Agent] Not plan`, { text });
+    agentLogger.warn(`[Plan Agent] Not plan`, { text });
     return;
   }
 
@@ -142,7 +151,7 @@ const parseAskInteractive = async (
       }
     };
   } else {
-    getLogger(LogCategories.MODULE.AI.AGENT).warn(`[Plan Agent] Ask tool params is not valid`, {
+    agentLogger.warn(`[Plan Agent] Ask tool params is not valid`, {
       tooCall
     });
     return;
@@ -204,7 +213,7 @@ export const dispatchPlanAgent = async ({
         content: props.queryInput
       });
     } else {
-      getLogger(LogCategories.MODULE.AI.AGENT).error('Plan interactive mode error', {
+      agentLogger.error('Plan interactive mode error', {
         planMessages: props.planMessages
       });
       return Promise.reject('Plan interactive mode error');
@@ -228,6 +237,14 @@ export const dispatchPlanAgent = async ({
   // console.dir({ requestMessages }, { depth: null });
   // console.log('userInput:', userInput, 'mode:', mode, 'interactive?.type:', interactive?.type);
 
+  const requestParams = {
+    model: modelData.model,
+    stream: true,
+    tools: props.mode === 'continue' ? undefined : [AIAskTool],
+    tool_choice: 'auto' as const,
+    toolCallMode: modelData.toolChoice ? ('toolChoice' as const) : ('prompt' as const),
+    parallel_tool_calls: false
+  };
   let {
     answerText,
     toolCalls = [],
@@ -238,36 +255,102 @@ export const dispatchPlanAgent = async ({
   } = await createLLMResponse({
     isAborted: checkIsStopping,
     body: {
-      model: modelData.model,
       messages: requestMessages,
-      stream: true,
-      tools: props.mode === 'continue' ? undefined : [AIAskTool],
-      tool_choice: 'auto',
-      toolCallMode: modelData.toolChoice ? 'toolChoice' : 'prompt',
-      parallel_tool_calls: false
+      ...requestParams
     }
   });
 
   if (responseEmptyTip) {
     return Promise.reject(responseEmptyTip);
   }
 
+  const llmRequestIds: string[] = [requestId];
   /* 
     正常输出情况：
     1. text: 正常生成plan
     2. toolCall: 调用ask工具
     3. text + confirm: 成功生成工具 + 确认操作
   */
-  // 获取生成的 plan
-  const plan = await parsePlan({
-    text: answerText,
-    planId,
-    task,
-    description,
-    background
-  });
-  // 获取交互结果
-  const askInteractive = await parseAskInteractive(toolCalls);
+  // 1. 首次获取交互结果
+  const { askInteractive, plan } = await (async () => {
+    // 1. 首次获取交互结果
+    let [askInteractive, plan] = await Promise.all([
+      parseAskInteractive(toolCalls),
+      parsePlan({
+        text: answerText,
+        planId,
+        task,
+        description,
+        background
+      })
+    ]);
+    if (plan || askInteractive) {
+      return {
+        askInteractive,
+        plan
+      };
+    }
+
+    // 2. 二次尝试生成 plan
+    agentLogger.warn('[Plan Agent] parse failed, try regenerate plan once', {
+      requestId,
+      mode: props.mode,
+      answerText: answerText.slice(0, 2000)
+    });
+
+    const regenerateResponse = await createLLMResponse({
+      isAborted: checkIsStopping,
+      body: {
+        messages: [
+          ...completeMessages,
+          {
+            role: 'user',
+            content: reTryPlanPrompt
+          }
+        ],
+        ...requestParams
+      }
+    });
+    usage.inputTokens += regenerateResponse.usage.inputTokens;
+    usage.outputTokens += regenerateResponse.usage.outputTokens;
+    llmRequestIds.push(regenerateResponse.requestId);
+    completeMessages = regenerateResponse.completeMessages;
+
+    [askInteractive, plan] = await Promise.all([
+      parseAskInteractive(regenerateResponse.toolCalls || []),
+      parsePlan({
+        text: regenerateResponse.answerText,
+        planId,
+        task,
+        description,
+        background
+      })
+    ]);
+    if (plan || askInteractive) {
+      return {
+        askInteractive,
+        plan
+      };
+    }
+
+    // 真的失败了
+    agentLogger.warn('[Plan Agent] plan regenerate failed', {
+      requestId,
+      regenerateRequestId: regenerateResponse.requestId,
+      mode: props.mode,
+      answerText: regenerateResponse.answerText.slice(0, 2000)
+    });
+    askInteractive = {
+      type: 'agentPlanAskQuery',
+      params: {
+        content: i18nT('chat:agent_plan_parse_retry_tip')
+      }
+    };
+
+    return {
+      askInteractive
+    };
+  })();
 
   const { totalPoints, modelName } = formatModelChars2Points({
     model: modelData.model,
@@ -288,7 +371,7 @@ export const dispatchPlanAgent = async ({
     totalPoints,
     model: modelName,
     runningTime: +((Date.now() - startTime) / 1000).toFixed(2),
-    llmRequestIds: [requestId]
+    llmRequestIds
   };
 
   return {

packages/service/core/workflow/dispatch/ai/agent/sub/plan/prompt.ts

@@ -1125,3 +1125,29 @@ ${response}
 ## 下一步任务
 请基于已执行步骤及结果，根据系统提示词来判断是否需要继续规划、生成总结报告步骤、还是任务已完成，或者遇到问题直接返回`;
 };
+
+export const reTryPlanPrompt = `上一轮 plan 输出不是合法 JSON，无法解析。
+
+请基于原始任务重新生成完整 plan，严格按 JSON 输出。
+
+要求：
+- 仅返回 JSON
+- 包含 task 和 steps 字段
+- 每个 step 必须包含 id/title/description
+
+JSON 格式示例（只参考格式，不要照抄内容）：
+{
+  "task": "深入了解 Rust 编程语言（系统编程方向）",
+  "steps": [
+    {
+      "id": "step1",
+      "title": "了解 Rust 的核心特性",
+      "description": "使用 @webSearch 搜索 Rust 的所有权、借用检查与并发安全机制"
+    },
+    {
+      "id": "step2",
+      "title": "调研 Rust 在系统编程的应用",
+      "description": "使用 @webSearch 搜索 Rust 在操作系统、网络编程、嵌入式中的典型项目"
+    }
+  ]
+}`;

packages/service/core/workflow/dispatch/index.ts

@@ -211,10 +211,15 @@ export async function dispatchWorkFlow({
   const checkStoppingTimer =
     apiVersion === 'v2'
       ? setInterval(async () => {
-          stopping = await shouldWorkflowStop({
+          if (stopping) return;
+
+          const shouldStop = await shouldWorkflowStop({
             appId: runningAppInfo.id,
             chatId
           });
+          if (shouldStop) {
+            stopping = true;
+          }
         }, 100)
       : undefined;
 

packages/service/package.json

@@ -36,6 +36,7 @@
     "ioredis": "^5.6.0",
     "joplin-turndown-plugin-gfm": "^1.0.12",
     "json5": "catalog:",
+    "jsonrepair": "^3.0.0",
     "jsonpath-plus": "^10.3.0",
     "jsonwebtoken": "^9.0.2",
     "lodash": "catalog:",