feat(core): Add support for SQLite connection pooling (#8722)

.github/workflows/ci-postgres-mysql.yml

@@ -34,6 +34,33 @@ jobs:
           path: ./packages/**/dist
           key: ${{ github.sha }}:db-tests
 
+  sqlite-pooled:
+    name: SQLite Pooled
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 20
+    env:
+      DB_TYPE: sqlite
+      DB_SQLITE_POOL_SIZE: 4
+    steps:
+      - uses: actions/checkout@v4.1.1
+      - run: corepack enable
+      - uses: actions/setup-node@v4.0.1
+        with:
+          node-version: 18.x
+          cache: 'pnpm'
+      - run: pnpm install --frozen-lockfile
+
+      - name: Restore cached build artifacts
+        uses: actions/cache/restore@v4.0.0
+        with:
+          path: ./packages/**/dist
+          key: ${{ github.sha }}:db-tests
+
+      - name: Test SQLite Pooled
+        working-directory: packages/cli
+        run: pnpm jest --coverage
+
   mysql:
     name: MySQL
     runs-on: ubuntu-latest

packages/@n8n/nodes-langchain/package.json

@@ -140,7 +140,7 @@
     "@langchain/openai": "^0.0.16",
     "@langchain/pinecone": "^0.0.3",
     "@langchain/redis": "^0.0.2",
-    "@n8n/typeorm": "0.3.20-3",
+    "@n8n/typeorm": "0.3.20-7",
     "@n8n/vm2": "3.9.20",
     "@pinecone-database/pinecone": "2.1.0",
     "@qdrant/js-client-rest": "1.7.0",

packages/cli/package.json

@@ -97,7 +97,7 @@
     "@n8n/localtunnel": "2.1.0",
     "@n8n/n8n-nodes-langchain": "workspace:*",
     "@n8n/permissions": "workspace:*",
-    "@n8n/typeorm": "0.3.20-3",
+    "@n8n/typeorm": "0.3.20-7",
     "@n8n_io/license-sdk": "2.10.0",
     "@oclif/core": "3.18.1",
     "@rudderstack/rudder-sdk-node": "2.0.7",

packages/cli/src/config/schema.ts

@@ -171,11 +171,17 @@ export const schema = {
 				env: 'DB_SQLITE_DATABASE',
 			},
 			enableWAL: {
-				doc: 'Enable SQLite WAL mode',
+				doc: 'Enable SQLite WAL mode (Always enabled for pool-size > 1)',
 				format: Boolean,
 				default: false,
 				env: 'DB_SQLITE_ENABLE_WAL',
 			},
+			poolSize: {
+				doc: 'SQLite Pool Size (Setting this to 0 disables pooling)',
+				format: Number,
+				default: 0,
+				env: 'DB_SQLITE_POOL_SIZE',
+			},
 			executeVacuumOnStartup: {
 				doc: 'Runs VACUUM operation on startup to rebuild the database. Reduces filesize and optimizes indexes. WARNING: This is a long running blocking operation. Will increase start-up time.',
 				format: Boolean,

packages/cli/src/databases/config.ts

@@ -3,6 +3,7 @@ import { Container } from 'typedi';
 import type { TlsOptions } from 'tls';
 import type { DataSourceOptions, LoggerOptions } from '@n8n/typeorm';
 import type { SqliteConnectionOptions } from '@n8n/typeorm/driver/sqlite/SqliteConnectionOptions';
+import type { SqlitePooledConnectionOptions } from '@n8n/typeorm/driver/sqlite-pooled/SqlitePooledConnectionOptions';
 import type { PostgresConnectionOptions } from '@n8n/typeorm/driver/postgres/PostgresConnectionOptions';
 import type { MysqlConnectionOptions } from '@n8n/typeorm/driver/mysql/MysqlConnectionOptions';
 import { InstanceSettings } from 'n8n-core';
@@ -47,16 +48,26 @@ export const getOptionOverrides = (dbType: 'postgresdb' | 'mysqldb') => ({
 	password: config.getEnv(`database.${dbType}.password`),
 });
 
-const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({
-	type: 'sqlite',
-	...getCommonOptions(),
-	database: path.resolve(
-		Container.get(InstanceSettings).n8nFolder,
-		config.getEnv('database.sqlite.database'),
-	),
-	enableWAL: config.getEnv('database.sqlite.enableWAL'),
-	migrations: sqliteMigrations,
-});
+const getSqliteConnectionOptions = (): SqliteConnectionOptions | SqlitePooledConnectionOptions => {
+	const poolSize = config.getEnv('database.sqlite.poolSize');
+	const commonOptions = {
+		...getCommonOptions(),
+		database: path.resolve(
+			Container.get(InstanceSettings).n8nFolder,
+			config.getEnv('database.sqlite.database'),
+		),
+		migrations: sqliteMigrations,
+	};
+	if (poolSize > 0) {
+		return { type: 'sqlite-pooled', poolSize, enableWAL: true, ...commonOptions };
+	} else {
+		return {
+			type: 'sqlite',
+			enableWAL: config.getEnv('database.sqlite.enableWAL'),
+			...commonOptions,
+		};
+	}
+};
 
 const getPostgresConnectionOptions = (): PostgresConnectionOptions => {
 	const sslCa = config.getEnv('database.postgresdb.ssl.ca');