From 0650002419a97075953176bc990e0fdb312abff2 Mon Sep 17 00:00:00 2001 From: Stanislav Popov Date: Fri, 27 Jun 2025 22:10:25 +0500 Subject: [PATCH 1/2] test: improve coverage for gpt helpers --- tests/helpers/gptTools.test.ts | 110 +++++++++++++++++++++++++++++++++ tests/helpers/llmCall.test.ts | 59 ++++++++++++++++++ tests/helpers/stt.test.ts | 20 +++--- tests/helpers/useTools.test.ts | 8 +-- 4 files changed, 185 insertions(+), 12 deletions(-) create mode 100644 tests/helpers/llmCall.test.ts diff --git a/tests/helpers/gptTools.test.ts b/tests/helpers/gptTools.test.ts index b424481..b69c348 100644 --- a/tests/helpers/gptTools.test.ts +++ b/tests/helpers/gptTools.test.ts @@ -185,4 +185,114 @@ describe("executeTools", () => { const res = await tools.executeTools(toolCalls, [], cfg, baseMsg); expect(res).toEqual([{ content: "Tool not found: missing" }]); }); + + it("formats expertizeme search params and calls tool", async () => { + const toolCalls: ChatCompletionMessageToolCall[] = [ + { + id: "1", + type: "function", + function: { + name: "expertizeme_search_items", + arguments: JSON.stringify({ + query: "foo", + filters: [{ field: "title", operator: "not", value: ["a", "b"] }], + sortField: "date", + sortDirection: "desc", + groupBy: "author", + }), + }, + }, + ]; + + const toolFn = jest.fn().mockResolvedValue({ content: "ok" }); + const chatTools: ChatToolType[] = [ + { + name: "expertizeme_search_items", + module: { + description: "", + call: () => ({ + functions: { get: () => toolFn, toolSpecs: { function: {} } }, + }), + }, + }, + ]; + const cfg: ConfigChatType = { ...baseConfig, chatParams: {} }; + await tools.executeTools(toolCalls, chatTools, cfg, baseMsg); + expect(toolFn).toHaveBeenCalledTimes(1); + const callArgs = mockSendToHttp.mock.calls[0]; + expect(callArgs[1]).toContain("**Title**: not a or b"); + }); + + it("retries tool once on 400 error", async () => { + const toolCalls: ChatCompletionMessageToolCall[] = [ + { + id: "1", + type: "function", + function: { name: "foo", arguments: "{}" }, + }, + ]; + const error = new Error("Invalid parameter"); + (error as any).status = 400; + const toolFn = jest + .fn() + .mockRejectedValueOnce(error) + .mockResolvedValue({ content: "done" }); + const chatTools: ChatToolType[] = [ + { + name: "foo", + module: { + description: "", + call: () => ({ + functions: { get: () => toolFn, toolSpecs: { function: {} } }, + }), + }, + }, + ]; + const cfg: ConfigChatType = { ...baseConfig, chatParams: {} }; + const res = await tools.executeTools(toolCalls, chatTools, cfg, baseMsg); + expect(res[0].content).toBe("done"); + expect(toolFn).toHaveBeenCalledTimes(2); + }); +}); + +describe("chatAsTool", () => { + it("throws when agent missing", () => { + mockUseConfig.mockReturnValue({ chats: [] }); + const msg = { ...baseMsg }; + const chatTool = tools.chatAsTool({ + agent_name: "missing", + name: "tool", + description: "d", + msg, + prompt_append: "", + }); + expect(() => + chatTool.module.call(baseConfig, { id: 1 } as ThreadStateType), + ).toThrow("Agent not found: missing"); + }); + + it("sends answer and stops on first tool", async () => { + const agentCfg = { ...baseConfig, agent_name: "agent2" }; + mockUseConfig.mockReturnValue({ chats: [agentCfg] }); + mockRequestGptAnswer.mockResolvedValue({ content: "hi" }); + const msg: Message.TextMessage = { ...baseMsg, text: "q" }; + const chatTool = tools.chatAsTool({ + agent_name: "agent2", + name: "tool", + description: "d", + tool_use_behavior: "stop_on_first_tool", + prompt_append: "", + msg, + }); + const module = chatTool.module.call(baseConfig, { + id: 1, + } as ThreadStateType); + const fn = module.functions.get(); + const res = await fn('{"input":"hi"}'); + expect(res.content).toBe(""); + expect(mockRequestGptAnswer).toHaveBeenCalledWith(msg, agentCfg); + expect(mockSendTelegramMessage).toHaveBeenCalledTimes(3); + const lastCall = mockSendTelegramMessage.mock.calls.pop(); + expect(lastCall[4]).toBe(baseConfig); + }); }); diff --git a/tests/helpers/llmCall.test.ts b/tests/helpers/llmCall.test.ts new file mode 100644 index 0000000..edfc887 --- /dev/null +++ b/tests/helpers/llmCall.test.ts @@ -0,0 +1,59 @@ +import { jest, describe, it, beforeEach, expect } from "@jest/globals"; +import type { Message } from "telegraf/types"; +import type { ConfigChatType } from "../../src/types"; + +const mockApiCreate = jest.fn(); +const mockObserve = jest.fn(); +const mockUseApi = jest.fn(); +const mockUseLangfuse = jest.fn(); + +jest.unstable_mockModule("../../src/helpers/useApi.ts", () => ({ + useApi: (...args: unknown[]) => mockUseApi(...args), +})); + +jest.unstable_mockModule("../../src/helpers/useLangfuse.ts", () => ({ + default: () => mockUseLangfuse(), +})); + +jest.unstable_mockModule("langfuse", () => ({ + observeOpenAI: (...args: unknown[]) => mockObserve(...args), +})); + +let llmCall: typeof import("../../src/helpers/gpt/llm.ts").llmCall; + +beforeEach(async () => { + jest.resetModules(); + jest.clearAllMocks(); + mockUseApi.mockReturnValue({ + chat: { completions: { create: mockApiCreate } }, + }); + mockUseLangfuse.mockReturnValue({ trace: {} }); + mockObserve.mockImplementation((api) => api); + ({ llmCall } = await import("../../src/helpers/gpt/llm.ts")); +}); + +describe("llmCall", () => { + it("calls API and returns result", async () => { + mockApiCreate.mockResolvedValue({ id: 1 }); + const msg = { chat: { id: 1, type: "private" } } as Message.TextMessage; + const chatConfig = { + local_model: "m1", + completionParams: {}, + chatParams: {}, + toolParams: {}, + } as unknown as ConfigChatType; + const apiParams = { messages: [] } as any; + const res = await llmCall({ + apiParams, + msg, + chatConfig, + generationName: "gen", + localModel: "m1", + }); + expect(mockUseApi).toHaveBeenCalledWith("m1"); + expect(mockObserve).toHaveBeenCalled(); + expect(mockApiCreate).toHaveBeenCalledWith(apiParams); + expect(res.res).toEqual({ id: 1 }); + expect(res.trace).toEqual({}); + }); +}); diff --git a/tests/helpers/stt.test.ts b/tests/helpers/stt.test.ts index 244cd25..8fd7a45 100644 --- a/tests/helpers/stt.test.ts +++ b/tests/helpers/stt.test.ts @@ -2,7 +2,7 @@ import { jest, describe, it, expect, beforeEach } from "@jest/globals"; const mockTmpNameSync = jest.fn(); const mockExec = jest.fn( - (cmd: string, cb: (err: unknown, out: string) => void) => cb(null, "") + (cmd: string, cb: (err: unknown, out: string) => void) => cb(null, ""), ); const mockAccess = jest.fn(); const mockReadFile = jest.fn(); @@ -55,7 +55,7 @@ describe("convertToMp3", () => { expect(mockTmpNameSync).toHaveBeenCalled(); expect(mockExec).toHaveBeenCalledWith( expect.stringContaining("ffmpeg"), - expect.any(Function) + expect.any(Function), ); expect(res).toBe("/tmp/file.mp3"); }); @@ -73,7 +73,7 @@ describe("detectAudioFileLanguage", () => { const res = await stt.detectAudioFileLanguage("/tmp/file.mp3"); expect(fetch).toHaveBeenCalledWith( "http://base/detect-language", - expect.objectContaining({ method: "POST" }) + expect.objectContaining({ method: "POST" }), ); expect(res).toEqual({ lang: "en" }); }); @@ -89,7 +89,7 @@ describe("detectAudioFileLanguage", () => { headers: { entries: () => [] }, } as never); await expect(stt.detectAudioFileLanguage("/tmp/file.mp3")).rejects.toThrow( - "HTTP error! status: 400, body: err" + "HTTP error! status: 400, body: err", ); }); }); @@ -101,7 +101,9 @@ describe("sendAudioWhisper", () => { (fetch as unknown as jest.Mock) .mockResolvedValueOnce({ ok: true, - text: jest.fn().mockResolvedValue('{"detected_language":"en"}' as never), + text: jest + .fn() + .mockResolvedValue('{"detected_language":"en"}' as never), headers: { entries: () => [] }, } as never) .mockResolvedValueOnce({ @@ -112,7 +114,7 @@ describe("sendAudioWhisper", () => { const res = await stt.sendAudioWhisper({ mp3Path: "p", prompt: "hi" }); expect(fetch).toHaveBeenCalledWith( expect.stringContaining("/asr?"), - expect.objectContaining({ method: "POST" }) + expect.objectContaining({ method: "POST" }), ); expect(res).toEqual({ text: "ok" }); }); @@ -123,7 +125,9 @@ describe("sendAudioWhisper", () => { (fetch as unknown as jest.Mock) .mockResolvedValueOnce({ ok: true, - text: jest.fn().mockResolvedValue('{"detected_language":"en"}' as never), + text: jest + .fn() + .mockResolvedValue('{"detected_language":"en"}' as never), headers: { entries: () => [] }, } as never) .mockResolvedValueOnce({ @@ -133,7 +137,7 @@ describe("sendAudioWhisper", () => { headers: { entries: () => [] }, } as never); await expect( - stt.sendAudioWhisper({ mp3Path: "p", prompt: "hi" }) + stt.sendAudioWhisper({ mp3Path: "p", prompt: "hi" }), ).rejects.toThrow("HTTP error! status: 500, body: oops"); }); }); diff --git a/tests/helpers/useTools.test.ts b/tests/helpers/useTools.test.ts index 967af21..fca1c8b 100644 --- a/tests/helpers/useTools.test.ts +++ b/tests/helpers/useTools.test.ts @@ -48,7 +48,7 @@ const barPath = path.resolve("src/tools/bar.ts"); beforeAll(() => { fs.writeFileSync( fooPath, - "export function call() { return { content: 'foo' }; }" + "export function call() { return { content: 'foo' }; }", ); fs.writeFileSync(barPath, "export const notCall = true;\n"); }); @@ -69,7 +69,7 @@ beforeEach(async () => { description: string; properties: unknown; model: string; - }> + }>, ); ({ default: useTools, initTools } = await import( "../../src/helpers/useTools.ts" @@ -107,7 +107,7 @@ describe("initTools", () => { // Type the mock implementation (mockInitMcp as unknown as jest.Mock).mockImplementation(() => - Promise.resolve(mockTools) + Promise.resolve(mockTools), ); const tools = await initTools(); @@ -118,7 +118,7 @@ describe("initTools", () => { const instance = mcpTool.module.call( {} as unknown as ConfigChatType, - {} as unknown as ThreadStateType + {} as unknown as ThreadStateType, ); await instance.functions.get("foo")("{}"); expect(mockCallMcp).toHaveBeenCalledWith("m1", "foo", "{}"); From 00e6b396e3e01b95188525bfb85b6d67cf829632 Mon Sep 17 00:00:00 2001 From: Stanislav Popov Date: Fri, 27 Jun 2025 22:17:37 +0500 Subject: [PATCH 2/2] fixes --- tests/helpers/gptTools.test.ts | 2 +- tests/helpers/llm.extras.test.ts | 172 +++++++++++++++++++++++++++++++ tests/helpers/llmCall.test.ts | 59 ----------- 3 files changed, 173 insertions(+), 60 deletions(-) create mode 100644 tests/helpers/llm.extras.test.ts delete mode 100644 tests/helpers/llmCall.test.ts diff --git a/tests/helpers/gptTools.test.ts b/tests/helpers/gptTools.test.ts index b69c348..3cb2066 100644 --- a/tests/helpers/gptTools.test.ts +++ b/tests/helpers/gptTools.test.ts @@ -232,7 +232,7 @@ describe("executeTools", () => { }, ]; const error = new Error("Invalid parameter"); - (error as any).status = 400; + (error as Error & { status: number }).status = 400; const toolFn = jest .fn() .mockRejectedValueOnce(error) diff --git a/tests/helpers/llm.extras.test.ts b/tests/helpers/llm.extras.test.ts new file mode 100644 index 0000000..f79960f --- /dev/null +++ b/tests/helpers/llm.extras.test.ts @@ -0,0 +1,172 @@ +import { jest, describe, it, expect, beforeEach } from "@jest/globals"; +import type { Message } from "telegraf/types"; +import type { ConfigChatType, ThreadStateType } from "../../src/types"; +import OpenAI from "openai"; + +const mockUseApi = jest.fn(); +const mockUseLangfuse = jest.fn(); +const mockObserveOpenAI = jest.fn(); +const mockAddToHistory = jest.fn(); +const mockForgetHistory = jest.fn(); + +jest.unstable_mockModule("../../src/helpers/useApi.ts", () => ({ + useApi: (...args: unknown[]) => mockUseApi(...args), +})); + +jest.unstable_mockModule("../../src/helpers/useLangfuse.ts", () => ({ + default: (...args: unknown[]) => mockUseLangfuse(...args), +})); + +jest.unstable_mockModule("langfuse", () => ({ + observeOpenAI: (...args: unknown[]) => mockObserveOpenAI(...args), +})); + +jest.unstable_mockModule("../../src/helpers/history.ts", () => ({ + addToHistory: (...args: unknown[]) => mockAddToHistory(...args), + forgetHistory: (...args: unknown[]) => mockForgetHistory(...args), +})); + +let llm: typeof import("../../src/helpers/gpt/llm.ts"); + +const baseMsg: Message.TextMessage = { + chat: { id: 1, type: "private", title: "chat" }, + message_id: 1, + text: "hi", + from: { username: "u" }, +} as Message.TextMessage; + +const chatConfig: ConfigChatType = { + name: "chat", + completionParams: {}, + chatParams: {}, + toolParams: {}, + local_model: "model", +} as ConfigChatType; + +beforeEach(async () => { + jest.resetModules(); + mockUseApi.mockReset(); + mockUseLangfuse.mockReset(); + mockObserveOpenAI.mockReset(); + mockUseLangfuse.mockReturnValue({ trace: undefined }); + const api = { + chat: { + completions: { + create: jest + .fn() + .mockResolvedValue({ choices: [{ message: { content: "a" } }] }), + }, + }, + }; + const apiObserved = { + chat: { + completions: { + create: jest + .fn() + .mockResolvedValue({ choices: [{ message: { content: "b" } }] }), + }, + }, + }; + mockUseApi.mockReturnValue(api); + mockObserveOpenAI.mockReturnValue(apiObserved); + llm = await import("../../src/helpers/gpt/llm.ts"); +}); + +describe("llmCall", () => { + it("calls API directly when no trace", async () => { + mockUseLangfuse.mockReturnValue({ trace: undefined }); + const params = { messages: [], model: "m" } as OpenAI.ChatCompletionCreateParams; + const result = await llm.llmCall({ + apiParams: params, + msg: { ...baseMsg }, + chatConfig, + }); + expect(mockUseApi).toHaveBeenCalledWith("model"); + expect(mockObserveOpenAI).not.toHaveBeenCalled(); + expect(result.res).toEqual({ choices: [{ message: { content: "a" } }] }); + }); + + it("wraps api when trace exists", async () => { + mockUseLangfuse.mockReturnValue({ trace: {} }); + const params = { messages: [], model: "m" } as OpenAI.ChatCompletionCreateParams; + const result = await llm.llmCall({ + apiParams: params, + msg: { ...baseMsg }, + chatConfig, + generationName: "gen", + localModel: "other", + }); + expect(mockUseApi).toHaveBeenCalledWith("other"); + expect(mockObserveOpenAI).toHaveBeenCalled(); + expect(result.res).toEqual({ choices: [{ message: { content: "b" } }] }); + }); +}); + +describe("requestGptAnswer", () => { + const threads: Record = {}; + const mockUseThreads = jest.fn(() => threads); + const mockResolveChatTools = jest.fn(); + const mockGetToolsPrompts = jest.fn(); + const mockGetSystemMessage = jest.fn(); + const mockBuildMessages = jest.fn(); + const mockReplaceUrl = jest.fn((s: string) => Promise.resolve(s)); + const mockReplaceTool = jest.fn((s: string) => Promise.resolve(s)); + const mockForward = jest.fn(); + + jest.unstable_mockModule("../../src/threads.ts", () => ({ + useThreads: () => mockUseThreads(), + })); + jest.unstable_mockModule("../../src/helpers/gpt/tools.ts", () => ({ + executeTools: jest.fn(), + resolveChatTools: (...args: unknown[]) => mockResolveChatTools(...args), + getToolsPrompts: (...args: unknown[]) => mockGetToolsPrompts(...args), + getToolsSystemMessages: jest.fn(), + })); + jest.unstable_mockModule("../../src/helpers/gpt/messages.ts", () => ({ + getSystemMessage: (...args: unknown[]) => mockGetSystemMessage(...args), + buildMessages: (...args: unknown[]) => mockBuildMessages(...args), + })); + jest.unstable_mockModule("../../src/helpers/placeholders.ts", () => ({ + replaceUrlPlaceholders: (...args: unknown[]) => mockReplaceUrl(...args), + replaceToolPlaceholders: (...args: unknown[]) => mockReplaceTool(...args), + })); + jest.unstable_mockModule("../../src/telegram/send.ts", () => ({ + getTelegramForwardedUser: (...args: unknown[]) => mockForward(...args), + sendTelegramMessage: jest.fn(), + getFullName: jest.fn(), + isAdminUser: jest.fn(), + })); + + let requestGptAnswer: typeof llm.requestGptAnswer; + + beforeEach(async () => { + jest.resetModules(); + mockUseLangfuse.mockReturnValue({ trace: undefined }); + mockUseThreads.mockClear(); + mockResolveChatTools.mockResolvedValue([]); + mockGetToolsPrompts.mockResolvedValue([]); + mockGetSystemMessage.mockResolvedValue("sys {date}"); + mockBuildMessages.mockResolvedValue([]); + mockForward.mockReturnValue("Bob"); + const mod = await import("../../src/helpers/gpt/llm.ts"); + requestGptAnswer = mod.requestGptAnswer; + }); + + it("returns undefined when no text", async () => { + const msg = { + chat: { id: 1, type: "private" }, + message_id: 1, + } as Message.TextMessage; + const res = await requestGptAnswer(msg, chatConfig); + expect(res).toBeUndefined(); + }); + + it("adds forwarded name and creates thread", async () => { + const msg: Message.TextMessage = { ...baseMsg }; + const res = await requestGptAnswer(msg, chatConfig); + expect(mockForward).toHaveBeenCalledWith(msg, chatConfig); + expect(msg.text).toBe("Переслано от: Bob\nhi"); + expect(threads[1]).toBeDefined(); + expect(res).toEqual({ content: "a" }); + }); +}); \ No newline at end of file diff --git a/tests/helpers/llmCall.test.ts b/tests/helpers/llmCall.test.ts deleted file mode 100644 index edfc887..0000000 --- a/tests/helpers/llmCall.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { jest, describe, it, beforeEach, expect } from "@jest/globals"; -import type { Message } from "telegraf/types"; -import type { ConfigChatType } from "../../src/types"; - -const mockApiCreate = jest.fn(); -const mockObserve = jest.fn(); -const mockUseApi = jest.fn(); -const mockUseLangfuse = jest.fn(); - -jest.unstable_mockModule("../../src/helpers/useApi.ts", () => ({ - useApi: (...args: unknown[]) => mockUseApi(...args), -})); - -jest.unstable_mockModule("../../src/helpers/useLangfuse.ts", () => ({ - default: () => mockUseLangfuse(), -})); - -jest.unstable_mockModule("langfuse", () => ({ - observeOpenAI: (...args: unknown[]) => mockObserve(...args), -})); - -let llmCall: typeof import("../../src/helpers/gpt/llm.ts").llmCall; - -beforeEach(async () => { - jest.resetModules(); - jest.clearAllMocks(); - mockUseApi.mockReturnValue({ - chat: { completions: { create: mockApiCreate } }, - }); - mockUseLangfuse.mockReturnValue({ trace: {} }); - mockObserve.mockImplementation((api) => api); - ({ llmCall } = await import("../../src/helpers/gpt/llm.ts")); -}); - -describe("llmCall", () => { - it("calls API and returns result", async () => { - mockApiCreate.mockResolvedValue({ id: 1 }); - const msg = { chat: { id: 1, type: "private" } } as Message.TextMessage; - const chatConfig = { - local_model: "m1", - completionParams: {}, - chatParams: {}, - toolParams: {}, - } as unknown as ConfigChatType; - const apiParams = { messages: [] } as any; - const res = await llmCall({ - apiParams, - msg, - chatConfig, - generationName: "gen", - localModel: "m1", - }); - expect(mockUseApi).toHaveBeenCalledWith("m1"); - expect(mockObserve).toHaveBeenCalled(); - expect(mockApiCreate).toHaveBeenCalledWith(apiParams); - expect(res.res).toEqual({ id: 1 }); - expect(res.trace).toEqual({}); - }); -});