diff --git a/.dockerignore b/.dockerignore index cd4afbe92f2..6b127780882 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,5 @@ node_modules/ .svelte-kit/ .env* !.env -.env.local \ No newline at end of file +.env.local +db \ No newline at end of file diff --git a/.gitignore b/.gitignore index d37a04b1b91..dcd5b727cc3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ SECRET_CONFIG .idea !.env.ci !.env -gcp-*.json \ No newline at end of file +gcp-*.json +db \ No newline at end of file diff --git a/src/lib/jobs/refresh-assistants-counts.ts b/src/lib/jobs/refresh-assistants-counts.ts index f4caf71bade..d9c79a3d454 100644 --- a/src/lib/jobs/refresh-assistants-counts.ts +++ b/src/lib/jobs/refresh-assistants-counts.ts @@ -26,49 +26,49 @@ async function refreshAssistantsCountsHelper() { } try { - await Database.getInstance() - .getClient() - .withSession((session) => - session.withTransaction(async () => { + await (await Database.getInstance()).getClient().withSession((session) => + session.withTransaction(async () => { + await ( await Database.getInstance() - .getCollections() - .assistants.aggregate([ - { $project: { _id: 1 } }, - { $set: { last24HoursCount: 0 } }, - { - $unionWith: { - coll: "assistants.stats", - pipeline: [ - { - $match: { "date.at": { $gte: subDays(new Date(), 1) }, "date.span": "hour" }, + ) + .getCollections() + .assistants.aggregate([ + { $project: { _id: 1 } }, + { $set: { last24HoursCount: 0 } }, + { + $unionWith: { + coll: "assistants.stats", + pipeline: [ + { + $match: { "date.at": { $gte: subDays(new Date(), 1) }, "date.span": "hour" }, + }, + { + $group: { + _id: "$assistantId", + last24HoursCount: { $sum: "$count" }, }, - { - $group: { - _id: "$assistantId", - last24HoursCount: { $sum: "$count" }, - }, - }, - ], - }, + }, + ], }, - { - $group: { - _id: "$_id", - last24HoursCount: { $sum: "$last24HoursCount" }, - }, + }, + { + $group: { + _id: "$_id", + last24HoursCount: { $sum: "$last24HoursCount" }, }, - { - $merge: { - into: "assistants", - on: "_id", - whenMatched: "merge", - whenNotMatched: "discard", - }, + }, + { + $merge: { + into: "assistants", + on: "_id", + whenMatched: "merge", + whenNotMatched: "discard", }, - ]) - .next(); - }) - ); + }, + ]) + .next(); + }) + ); } catch (e) { logger.error(e, "Refresh assistants counter failed!"); } diff --git a/src/lib/migrations/migrations.ts b/src/lib/migrations/migrations.ts index 2331644d8b8..bfe7152ef86 100644 --- a/src/lib/migrations/migrations.ts +++ b/src/lib/migrations/migrations.ts @@ -13,7 +13,7 @@ export async function checkAndRunMigrations() { } // check if all migrations have already been run - const migrationResults = await Database.getInstance() + const migrationResults = await (await Database.getInstance()) .getCollections() .migrationResults.find() .toArray(); @@ -21,7 +21,7 @@ export async function checkAndRunMigrations() { logger.info("[MIGRATIONS] Begin check..."); // connect to the database - const connectedClient = await Database.getInstance().getClient().connect(); + const connectedClient = await (await Database.getInstance()).getClient().connect(); const lockId = await acquireLock(LOCK_KEY); @@ -74,25 +74,23 @@ export async function checkAndRunMigrations() { }. Applying...` ); - await Database.getInstance() - .getCollections() - .migrationResults.updateOne( - { _id: migration._id }, - { - $set: { - name: migration.name, - status: "ongoing", - }, + await (await Database.getInstance()).getCollections().migrationResults.updateOne( + { _id: migration._id }, + { + $set: { + name: migration.name, + status: "ongoing", }, - { upsert: true } - ); + }, + { upsert: true } + ); const session = connectedClient.startSession(); let result = false; try { await session.withTransaction(async () => { - result = await migration.up(Database.getInstance()); + result = await migration.up(await Database.getInstance()); }); } catch (e) { logger.info(`[MIGRATIONS] "${migration.name}" failed!`); @@ -101,18 +99,16 @@ export async function checkAndRunMigrations() { await session.endSession(); } - await Database.getInstance() - .getCollections() - .migrationResults.updateOne( - { _id: migration._id }, - { - $set: { - name: migration.name, - status: result ? "success" : "failure", - }, + await (await Database.getInstance()).getCollections().migrationResults.updateOne( + { _id: migration._id }, + { + $set: { + name: migration.name, + status: result ? "success" : "failure", }, - { upsert: true } - ); + }, + { upsert: true } + ); } } diff --git a/src/lib/server/database.ts b/src/lib/server/database.ts index d0926dcfa32..2c452accc76 100644 --- a/src/lib/server/database.ts +++ b/src/lib/server/database.ts @@ -14,29 +14,69 @@ import type { MigrationResult } from "$lib/types/MigrationResult"; import type { Semaphore } from "$lib/types/Semaphore"; import type { AssistantStats } from "$lib/types/AssistantStats"; import type { CommunityToolDB } from "$lib/types/Tool"; - +import { MongoMemoryServer } from "mongodb-memory-server"; import { logger } from "$lib/server/logger"; import { building } from "$app/environment"; import type { TokenCache } from "$lib/types/TokenCache"; import { onExit } from "./exitHandler"; +import { fileURLToPath } from "url"; +import { dirname, join } from "path"; +import { existsSync, mkdirSync } from "fs"; export const CONVERSATION_STATS_COLLECTION = "conversations.stats"; +function findRepoRoot(startPath: string): string { + let currentPath = startPath; + while (currentPath !== "/") { + if (existsSync(join(currentPath, "package.json"))) { + return currentPath; + } + currentPath = dirname(currentPath); + } + throw new Error("Could not find repository root (no package.json found)"); +} + export class Database { - private client: MongoClient; + private client?: MongoClient; + private mongoServer?: MongoMemoryServer; private static instance: Database; - private constructor() { + private async init() { if (!env.MONGODB_URL) { - throw new Error( - "Please specify the MONGODB_URL environment variable inside .env.local. Set it to mongodb://localhost:27017 if you are running MongoDB locally, or to a MongoDB Atlas free instance for example." - ); - } + logger.warn("No MongoDB URL found, using in-memory server"); - this.client = new MongoClient(env.MONGODB_URL, { - directConnection: env.MONGODB_DIRECT_CONNECTION === "true", - }); + // Find repo root by looking for package.json + const currentFilePath = fileURLToPath(import.meta.url); + const repoRoot = findRepoRoot(dirname(currentFilePath)); + + // Use MONGO_STORAGE_PATH from env if set, otherwise use db/ in repo root + const dbPath = env.MONGO_STORAGE_PATH || join(repoRoot, "db"); + + logger.info(`Using database path: ${dbPath}`); + // Create db directory if it doesn't exist + if (!existsSync(dbPath)) { + logger.info(`Creating database directory at ${dbPath}`); + mkdirSync(dbPath, { recursive: true }); + } + + this.mongoServer = await MongoMemoryServer.create({ + instance: { + dbName: env.MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : ""), + dbPath, + }, + binary: { + version: "7.0.18", + }, + }); + this.client = new MongoClient(this.mongoServer.getUri(), { + directConnection: env.MONGODB_DIRECT_CONNECTION === "true", + }); + } else { + this.client = new MongoClient(env.MONGODB_URL, { + directConnection: env.MONGODB_DIRECT_CONNECTION === "true", + }); + } this.client.connect().catch((err) => { logger.error(err, "Connection error"); @@ -46,12 +86,17 @@ export class Database { this.client.on("open", () => this.initDatabase()); // Disconnect DB on exit - onExit(() => this.client.close(true)); + onExit(async () => { + logger.info("Closing database connection"); + await this.client?.close(true); + await this.mongoServer?.stop(); + }); } - public static getInstance(): Database { + public static async getInstance(): Promise { if (!Database.instance) { Database.instance = new Database(); + await Database.instance.init(); } return Database.instance; @@ -61,6 +106,10 @@ export class Database { * Return mongoClient */ public getClient(): MongoClient { + if (!this.client) { + throw new Error("Database not initialized"); + } + return this.client; } @@ -68,6 +117,10 @@ export class Database { * Return map of database's collections */ public getCollections() { + if (!this.client) { + throw new Error("Database not initialized"); + } + const db = this.client.db( env.MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : "") ); @@ -247,4 +300,4 @@ export class Database { export const collections = building ? ({} as unknown as ReturnType) - : Database.getInstance().getCollections(); + : await Database.getInstance().then((db) => db.getCollections()); diff --git a/src/routes/assistants/+page.server.ts b/src/routes/assistants/+page.server.ts index cc5ab3ab195..c185bf198be 100644 --- a/src/routes/assistants/+page.server.ts +++ b/src/routes/assistants/+page.server.ts @@ -1,6 +1,6 @@ import { base } from "$app/paths"; import { env } from "$env/dynamic/private"; -import { Database, collections } from "$lib/server/database.js"; +import { collections } from "$lib/server/database.js"; import { SortKey, type Assistant } from "$lib/types/Assistant"; import type { User } from "$lib/types/User"; import { generateQueryTokens } from "$lib/utils/searchTokens.js"; @@ -58,9 +58,8 @@ export const load = async ({ url, locals }) => { ...shouldBeFeatured, }; - const assistants = await Database.getInstance() - .getCollections() - .assistants.find(filter) + const assistants = await collections.assistants + .find(filter) .sort({ ...(sort === SortKey.TRENDING && { last24HoursCount: -1 }), userCount: -1, @@ -70,9 +69,7 @@ export const load = async ({ url, locals }) => { .limit(NUM_PER_PAGE) .toArray(); - const numTotalItems = await Database.getInstance() - .getCollections() - .assistants.countDocuments(filter); + const numTotalItems = await collections.assistants.countDocuments(filter); return { assistants: JSON.parse(JSON.stringify(assistants)) as Array, diff --git a/src/routes/tools/+page.server.ts b/src/routes/tools/+page.server.ts index 7aca65796ec..c7b0ee3e016 100644 --- a/src/routes/tools/+page.server.ts +++ b/src/routes/tools/+page.server.ts @@ -1,6 +1,6 @@ import { env } from "$env/dynamic/private"; import { authCondition } from "$lib/server/auth.js"; -import { Database, collections } from "$lib/server/database.js"; +import { collections } from "$lib/server/database.js"; import { toolFromConfigs } from "$lib/server/tools/index.js"; import { SortKey } from "$lib/types/Assistant.js"; import { ReviewStatus } from "$lib/types/Review"; @@ -60,9 +60,8 @@ export const load = async ({ url, locals }) => { }), }; - const communityTools = await Database.getInstance() - .getCollections() - .tools.find(filter) + const communityTools = await collections.tools + .find(filter) .skip(NUM_PER_PAGE * pageIndex) .sort({ ...(sort === SortKey.TRENDING && { last24HoursUseCount: -1 }), @@ -84,9 +83,7 @@ export const load = async ({ url, locals }) => { const tools = [...(pageIndex == 0 && !username ? configTools : []), ...communityTools]; - const numTotalItems = - (await Database.getInstance().getCollections().tools.countDocuments(filter)) + - toolFromConfigs.length; + const numTotalItems = (await collections.tools.countDocuments(filter)) + toolFromConfigs.length; return { tools: JSON.parse(JSON.stringify(tools)) as CommunityToolDB[],