diff --git a/README.md b/README.md index 603551b..499e722 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,10 @@ Empty `config.yml` should be generated. Fill it with your data: - auth.chatgpt_api_key - stt.whisperBaseUrl - http.http_token (per-chat tokens use chat.http_token) -- useChatsDir (optional, default `false`) -- chatsDir (optional, default `data/chats`) +- useChatsDir (optional, default `false`) – when enabled, chat configs are loaded from separate files + inside `chatsDir` instead of the `chats` section of `config.yml`. +- chatsDir (optional, default `data/chats`) – directory where per-chat YAML files are stored when + `useChatsDir` is turned on. ### Multiple Bots / Secondary bot_token diff --git a/src/config.ts b/src/config.ts index 9946f23..decdce5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,13 @@ import * as yaml from "js-yaml"; -import { readFileSync, writeFileSync, existsSync, watchFile } from "fs"; +import { + readFileSync, + writeFileSync, + existsSync, + watchFile, + readdirSync, + mkdirSync, +} from "fs"; +import path from "path"; import { ChatParamsType, ConfigChatType, @@ -8,13 +16,43 @@ import { ButtonsSyncConfigType, ConfigChatButtonType, } from "./types.js"; -import { log } from "./helpers.ts"; +import { log, safeFilename } from "./helpers.ts"; import { readGoogleSheet } from "./helpers/readGoogleSheet"; import { OAuth2Client } from "google-auth-library/build/src/auth/oauth2client"; import { GoogleAuth } from "google-auth-library"; import debounce from "lodash.debounce"; import { useThreads } from "./threads"; +export function loadChatsFromDir(dir: string): ConfigChatType[] { + if (!existsSync(dir)) return []; + const files = readdirSync(dir).filter( + (f) => f.endsWith(".yml") || f.endsWith(".yaml"), + ); + const chats: ConfigChatType[] = []; + for (const file of files) { + const content = readFileSync(path.join(dir, file), "utf8"); + const chat = yaml.load(content) as ConfigChatType; + chats.push(chat); + } + return chats; +} + +export function saveChatsToDir(dir: string, chats: ConfigChatType[]) { + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + chats.forEach((chat) => { + const safe = safeFilename(`${chat.name || chat.id}`, `${chat.id || 0}`); + const filePath = path.join(dir, `${safe}.yml`); + const yamlRaw = yaml.dump(chat, { + lineWidth: -1, + noCompatMode: true, + quotingType: '"', + }); + writeFileSync(filePath, yamlRaw); + }); +} + export function readConfig(path?: string): ConfigType { if (!path) path = process.env.CONFIG || "config.yml"; if (!existsSync(path)) { @@ -27,6 +65,11 @@ export function readConfig(path?: string): ConfigType { } const config = yaml.load(readFileSync(path, "utf8")) as ConfigType; + if (config.useChatsDir) { + const dir = config.chatsDir || "data/chats"; + config.chats = loadChatsFromDir(dir); + } + if (config.auth.proxy_url === generateConfig().auth.proxy_url) { delete config.auth.proxy_url; } @@ -65,7 +108,15 @@ export function writeConfig( config: ConfigType, ): ConfigType { try { - const yamlRaw = yaml.dump(config, { + let cfgToSave: ConfigType | Omit = config; + if (config.useChatsDir) { + const dir = config.chatsDir || "data/chats"; + saveChatsToDir(dir, config.chats); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { chats, ...rest } = config; + cfgToSave = rest; + } + const yamlRaw = yaml.dump(cfgToSave, { lineWidth: -1, noCompatMode: true, quotingType: '"', diff --git a/tests/config.test.ts b/tests/config.test.ts index 804945d..1ee7658 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -1,4 +1,5 @@ import { jest } from "@jest/globals"; +import path from "path"; import { ConfigChatType } from "../src/types.ts"; // Mock the modules using jest.requireMock @@ -9,6 +10,8 @@ const mockWatchFile = jest.fn(); const mockAppendFileSync = jest.fn(); const mockDump = jest.fn(); const mockLoad = jest.fn(); +const mockReaddirSync = jest.fn(); +const mockMkdirSync = jest.fn(); // Mock the modules before importing the module under test jest.unstable_mockModule("fs", () => ({ @@ -19,12 +22,16 @@ jest.unstable_mockModule("fs", () => ({ writeFileSync: mockWriteFileSync, watchFile: mockWatchFile, appendFileSync: mockAppendFileSync, + readdirSync: mockReaddirSync, + mkdirSync: mockMkdirSync, }, existsSync: mockExistsSync, readFileSync: mockReadFileSync, writeFileSync: mockWriteFileSync, watchFile: mockWatchFile, appendFileSync: mockAppendFileSync, + readdirSync: mockReaddirSync, + mkdirSync: mockMkdirSync, })); jest.unstable_mockModule("js-yaml", () => ({ @@ -39,7 +46,13 @@ jest.unstable_mockModule("js-yaml", () => ({ // Import the module under test after setting up mocks const configMod = await import("../src/config.ts"); -const { readConfig, writeConfig, generateConfig } = configMod; +const { + readConfig, + writeConfig, + generateConfig, + loadChatsFromDir, + saveChatsToDir, +} = configMod; describe("generateConfig", () => { it("sets defaults for chat directory fields", () => { @@ -84,6 +97,29 @@ describe("readConfig", () => { expect(mockLoad).toHaveBeenCalledWith("mockYaml"); expect(config).toEqual(mockConfig); }); + + it("loads chats from directory when useChatsDir enabled", () => { + mockExistsSync.mockReturnValue(true); + const cfg = generateConfig(); + cfg.useChatsDir = true; + cfg.chatsDir = "chats"; + mockReadFileSync + .mockReturnValueOnce("cfgYaml") + .mockReturnValueOnce("c1yaml") + .mockReturnValueOnce("c2yaml"); + const chat1 = { name: "c1", agent_name: "c1" } as ConfigChatType; + const chat2 = { name: "c2", agent_name: "c2" } as ConfigChatType; + mockLoad + .mockReturnValueOnce(cfg) + .mockReturnValueOnce(chat1) + .mockReturnValueOnce(chat2); + mockReaddirSync.mockReturnValue(["c1.yml", "c2.yml"]); + + const res = readConfig("testConfig.yml"); + + expect(mockReaddirSync).toHaveBeenCalledWith("chats"); + expect(res.chats).toEqual([chat1, chat2]); + }); }); describe("writeConfig", () => { @@ -128,6 +164,100 @@ describe("writeConfig", () => { // Clean up consoleErrorSpy.mockRestore(); }); + + it("saves chats to directory when useChatsDir enabled", () => { + const cfg = generateConfig(); + cfg.useChatsDir = true; + cfg.chatsDir = "chats"; + cfg.chats = [ + { + name: "c1", + completionParams: {}, + chatParams: {}, + toolParams: {}, + } as ConfigChatType, + ]; + mockDump.mockReturnValueOnce("chatYaml").mockReturnValueOnce("mainYaml"); + mockExistsSync.mockReturnValue(true); + + const res = writeConfig("cfg.yml", cfg); + + expect(mockWriteFileSync).toHaveBeenCalledWith( + path.join("chats", "c1.yml"), + "chatYaml", + ); + expect(mockWriteFileSync).toHaveBeenCalledWith("cfg.yml", "mainYaml"); + const mainDumpCall = mockDump.mock.calls.find((c) => c[0].auth); + expect(mainDumpCall[0].chats).toBeUndefined(); + expect(res).toEqual(cfg); + }); +}); + +describe("loadChatsFromDir", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("reads yaml files from directory", () => { + mockExistsSync.mockReturnValue(true); + mockReaddirSync.mockReturnValue(["a.yml", "b.yaml", "c.txt"]); + mockReadFileSync.mockReturnValueOnce("ayaml").mockReturnValueOnce("byaml"); + const chatA = { name: "a" } as ConfigChatType; + const chatB = { name: "b" } as ConfigChatType; + mockLoad.mockReturnValueOnce(chatA).mockReturnValueOnce(chatB); + const res = loadChatsFromDir("dir"); + expect(res).toEqual([chatA, chatB]); + expect(mockReadFileSync).toHaveBeenCalledWith( + path.join("dir", "a.yml"), + "utf8", + ); + expect(mockReadFileSync).toHaveBeenCalledWith( + path.join("dir", "b.yaml"), + "utf8", + ); + }); + + it("returns empty array when directory missing", () => { + mockExistsSync.mockReturnValue(false); + const res = loadChatsFromDir("dir"); + expect(res).toEqual([]); + expect(mockReaddirSync).not.toHaveBeenCalled(); + }); +}); + +describe("saveChatsToDir", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("writes chats to files", () => { + mockExistsSync.mockReturnValue(false); + const chats = [ + { + name: "a", + completionParams: {}, + chatParams: {}, + toolParams: {}, + } as ConfigChatType, + { + name: "b", + completionParams: {}, + chatParams: {}, + toolParams: {}, + } as ConfigChatType, + ]; + mockDump.mockReturnValueOnce("ayaml").mockReturnValueOnce("byaml"); + saveChatsToDir("dir", chats); + expect(mockMkdirSync).toHaveBeenCalledWith("dir", { recursive: true }); + expect(mockWriteFileSync).toHaveBeenCalledWith( + path.join("dir", "a.yml"), + "ayaml", + ); + expect(mockWriteFileSync).toHaveBeenCalledWith( + path.join("dir", "b.yml"), + "byaml", + ); + }); }); describe("readConfig agent_name", () => { diff --git a/tests/configExtras.test.ts b/tests/configExtras.test.ts index d9c745a..32fcb04 100644 --- a/tests/configExtras.test.ts +++ b/tests/configExtras.test.ts @@ -10,6 +10,8 @@ const mockDebounce = jest.fn((fn) => fn); const mockUseThreads = jest.fn(() => ({})); const mockLoad = jest.fn(); const mockDump = jest.fn((obj) => JSON.stringify(obj)); +const mockReaddirSync = jest.fn(); +const mockMkdirSync = jest.fn(); jest.unstable_mockModule("../src/helpers.ts", () => ({ log: mockLog, @@ -43,11 +45,15 @@ jest.unstable_mockModule("fs", () => ({ existsSync: mockExistsSync, readFileSync: mockReadFileSync, watchFile: mockWatchFile, + readdirSync: mockReaddirSync, + mkdirSync: mockMkdirSync, }, writeFileSync: mockWriteFile, existsSync: mockExistsSync, readFileSync: mockReadFileSync, watchFile: mockWatchFile, + readdirSync: mockReaddirSync, + mkdirSync: mockMkdirSync, })); let mod: typeof import("../src/config.ts"); @@ -61,6 +67,8 @@ beforeEach(async () => { mockWatchFile.mockClear(); mockExistsSync.mockClear(); mockReadFileSync.mockClear(); + mockReaddirSync.mockClear(); + mockMkdirSync.mockClear(); mod = await import("../src/config.ts"); }); diff --git a/tests/googleHelpers.test.ts b/tests/googleHelpers.test.ts index 2198e62..a6f9377 100644 --- a/tests/googleHelpers.test.ts +++ b/tests/googleHelpers.test.ts @@ -6,6 +6,8 @@ const mockExistsSync = jest.fn(); const mockReadFileSync = jest.fn(); const mockWriteFileSync = jest.fn(); const mockWatchFile = jest.fn(); +const mockMkdirSync = jest.fn(); +const mockReaddirSync = jest.fn(); jest.unstable_mockModule("fs", () => ({ __esModule: true, @@ -14,11 +16,15 @@ jest.unstable_mockModule("fs", () => ({ readFileSync: mockReadFileSync, writeFileSync: mockWriteFileSync, watchFile: mockWatchFile, + mkdirSync: mockMkdirSync, + readdirSync: mockReaddirSync, }, existsSync: mockExistsSync, readFileSync: mockReadFileSync, writeFileSync: mockWriteFileSync, watchFile: mockWatchFile, + mkdirSync: mockMkdirSync, + readdirSync: mockReaddirSync, })); let googleHelpers: typeof import("../src/helpers/google.ts"); @@ -28,6 +34,8 @@ beforeEach(async () => { mockExistsSync.mockReset(); mockReadFileSync.mockReset(); mockWriteFileSync.mockReset(); + mockMkdirSync.mockReset(); + mockReaddirSync.mockReset(); googleHelpers = await import("../src/helpers/google.ts"); }); diff --git a/tests/telegram/sendMessageExtra.test.ts b/tests/telegram/sendMessageExtra.test.ts index dd3923f..4e48269 100644 --- a/tests/telegram/sendMessageExtra.test.ts +++ b/tests/telegram/sendMessageExtra.test.ts @@ -13,6 +13,7 @@ jest.unstable_mockModule("../../src/bot.ts", () => ({ jest.unstable_mockModule("../../src/helpers.ts", () => ({ log: jest.fn(), + safeFilename: jest.fn((v) => v), })); // mock splitBigMessage so we control number of parts