From c4c319d7cfb30772cca248a0039fd8e2b1c99eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Thu, 14 Mar 2024 13:45:16 +0100 Subject: [PATCH] feat(core): Add support for SQLite connection pooling (#8722) --- .github/workflows/ci-postgres-mysql.yml | 27 +++++++++++++++ packages/@n8n/nodes-langchain/package.json | 2 +- packages/cli/package.json | 2 +- packages/cli/src/config/schema.ts | 8 ++++- packages/cli/src/databases/config.ts | 31 ++++++++++++------ pnpm-lock.yaml | 38 +++++++++++++++++----- 6 files changed, 87 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci-postgres-mysql.yml b/.github/workflows/ci-postgres-mysql.yml index b8d0395277286..32275c65289c6 100644 --- a/.github/workflows/ci-postgres-mysql.yml +++ b/.github/workflows/ci-postgres-mysql.yml @@ -34,6 +34,33 @@ jobs: path: ./packages/**/dist key: ${{ github.sha }}:db-tests + sqlite-pooled: + name: SQLite Pooled + runs-on: ubuntu-latest + needs: build + timeout-minutes: 20 + env: + DB_TYPE: sqlite + DB_SQLITE_POOL_SIZE: 4 + steps: + - uses: actions/checkout@v4.1.1 + - run: corepack enable + - uses: actions/setup-node@v4.0.1 + with: + node-version: 18.x + cache: 'pnpm' + - run: pnpm install --frozen-lockfile + + - name: Restore cached build artifacts + uses: actions/cache/restore@v4.0.0 + with: + path: ./packages/**/dist + key: ${{ github.sha }}:db-tests + + - name: Test SQLite Pooled + working-directory: packages/cli + run: pnpm jest --coverage + mysql: name: MySQL runs-on: ubuntu-latest diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index 9b924a1267022..b1a37943f1503 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -140,7 +140,7 @@ "@langchain/openai": "^0.0.16", "@langchain/pinecone": "^0.0.3", "@langchain/redis": "^0.0.2", - "@n8n/typeorm": "0.3.20-3", + "@n8n/typeorm": "0.3.20-7", "@n8n/vm2": "3.9.20", "@pinecone-database/pinecone": "2.1.0", "@qdrant/js-client-rest": "1.7.0", diff --git a/packages/cli/package.json b/packages/cli/package.json index 79ef58cde31c0..837fa2d9bba6d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -97,7 +97,7 @@ "@n8n/localtunnel": "2.1.0", "@n8n/n8n-nodes-langchain": "workspace:*", "@n8n/permissions": "workspace:*", - "@n8n/typeorm": "0.3.20-3", + "@n8n/typeorm": "0.3.20-7", "@n8n_io/license-sdk": "2.10.0", "@oclif/core": "3.18.1", "@rudderstack/rudder-sdk-node": "2.0.7", diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 84bf6d37fba7d..8cae0d847a030 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -171,11 +171,17 @@ export const schema = { env: 'DB_SQLITE_DATABASE', }, enableWAL: { - doc: 'Enable SQLite WAL mode', + doc: 'Enable SQLite WAL mode (Always enabled for pool-size > 1)', format: Boolean, default: false, env: 'DB_SQLITE_ENABLE_WAL', }, + poolSize: { + doc: 'SQLite Pool Size (Setting this to 0 disables pooling)', + format: Number, + default: 0, + env: 'DB_SQLITE_POOL_SIZE', + }, executeVacuumOnStartup: { doc: 'Runs VACUUM operation on startup to rebuild the database. Reduces filesize and optimizes indexes. WARNING: This is a long running blocking operation. Will increase start-up time.', format: Boolean, diff --git a/packages/cli/src/databases/config.ts b/packages/cli/src/databases/config.ts index fe08b3d4b40a8..31336f81153f4 100644 --- a/packages/cli/src/databases/config.ts +++ b/packages/cli/src/databases/config.ts @@ -3,6 +3,7 @@ import { Container } from 'typedi'; import type { TlsOptions } from 'tls'; import type { DataSourceOptions, LoggerOptions } from '@n8n/typeorm'; import type { SqliteConnectionOptions } from '@n8n/typeorm/driver/sqlite/SqliteConnectionOptions'; +import type { SqlitePooledConnectionOptions } from '@n8n/typeorm/driver/sqlite-pooled/SqlitePooledConnectionOptions'; import type { PostgresConnectionOptions } from '@n8n/typeorm/driver/postgres/PostgresConnectionOptions'; import type { MysqlConnectionOptions } from '@n8n/typeorm/driver/mysql/MysqlConnectionOptions'; import { InstanceSettings } from 'n8n-core'; @@ -47,16 +48,26 @@ export const getOptionOverrides = (dbType: 'postgresdb' | 'mysqldb') => ({ password: config.getEnv(`database.${dbType}.password`), }); -const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({ - type: 'sqlite', - ...getCommonOptions(), - database: path.resolve( - Container.get(InstanceSettings).n8nFolder, - config.getEnv('database.sqlite.database'), - ), - enableWAL: config.getEnv('database.sqlite.enableWAL'), - migrations: sqliteMigrations, -}); +const getSqliteConnectionOptions = (): SqliteConnectionOptions | SqlitePooledConnectionOptions => { + const poolSize = config.getEnv('database.sqlite.poolSize'); + const commonOptions = { + ...getCommonOptions(), + database: path.resolve( + Container.get(InstanceSettings).n8nFolder, + config.getEnv('database.sqlite.database'), + ), + migrations: sqliteMigrations, + }; + if (poolSize > 0) { + return { type: 'sqlite-pooled', poolSize, enableWAL: true, ...commonOptions }; + } else { + return { + type: 'sqlite', + enableWAL: config.getEnv('database.sqlite.enableWAL'), + ...commonOptions, + }; + } +}; const getPostgresConnectionOptions = (): PostgresConnectionOptions => { const sslCa = config.getEnv('database.postgresdb.ssl.ca'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 50f5e0083576d..34276f10406af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -218,8 +218,8 @@ importers: specifier: ^0.0.2 version: 0.0.2 '@n8n/typeorm': - specifier: 0.3.20-3 - version: 0.3.20-3(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7) + specifier: 0.3.20-7 + version: 0.3.20-7(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7) '@n8n/vm2': specifier: 3.9.20 version: 3.9.20 @@ -443,8 +443,8 @@ importers: specifier: workspace:* version: link:../@n8n/permissions '@n8n/typeorm': - specifier: 0.3.20-3 - version: 0.3.20-3(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7) + specifier: 0.3.20-7 + version: 0.3.20-7(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7) '@n8n_io/license-sdk': specifier: 2.10.0 version: 2.10.0 @@ -4642,6 +4642,11 @@ packages: resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} engines: {node: '>=0.1.90'} + /@common.js/is-network-error@1.0.1: + resolution: {integrity: sha512-dkk7FX8L/JLia5pi+IQ11lCw2D6FTmbWL2iWTHgCbP40/deeXgknlkEQcQ/rOkjwQbqp8RZ4ey/anR17K66sqw==} + engines: {node: '>=16'} + dev: false + /@ctrl/tinycolor@3.6.0: resolution: {integrity: sha512-/Z3l6pXthq0JvMYdUFyX9j0MaCltlIn6mfh9jLyQwg5aPKxkyNa0PTHtU1AlFXLNk55ZuAeJRcpvq+tmLfKmaQ==} engines: {node: '>=10'} @@ -6437,6 +6442,15 @@ packages: - supports-color dev: false + /@n8n/p-retry@6.2.0-2: + resolution: {integrity: sha512-rbnMnSdEwq2yuYMgzOQ4jTXm+oH7yjN/0ISfB/7O6pUcEPsZt9UW60BYfQ1WWHkKa/evI8vgER2zV5/RC1BupQ==} + engines: {node: '>=18.10'} + dependencies: + '@common.js/is-network-error': 1.0.1 + '@types/retry': 0.12.5 + retry: 0.13.1 + dev: false + /@n8n/tournament@1.0.2: resolution: {integrity: sha512-fTpi7F8ra5flGSVfRzohPyG7czAAKCZPlLjdKdwbLJivLoI/Ekhgodov1jfVSCVFVbwQ06gRQRxLEDzl2jl8ig==} engines: {node: '>=18.10', pnpm: '>=8.6'} @@ -6447,8 +6461,8 @@ packages: recast: 0.22.0 dev: false - /@n8n/typeorm@0.3.20-3(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7): - resolution: {integrity: sha512-eTaEBvaJZZCStqiLKSUIvFLMZJ3WMOFc+pnF+jedcQOc6AunWM57FqhPvnuB/Z6c67JmgIyINlnjbLDHtPgKZQ==} + /@n8n/typeorm@0.3.20-7(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7): + resolution: {integrity: sha512-f4A9RGOnB3kCkusNAr1QDCGOVq1HU1YCBKoIGr2of+P3CVS3I+1vW7neOhlr/ic5S1F14Qy5TU8Lb78mRBYRSw==} engines: {node: '>=16.13.0'} hasBin: true peerDependencies: @@ -6511,6 +6525,7 @@ packages: typeorm-aurora-data-api-driver: optional: true dependencies: + '@n8n/p-retry': 6.2.0-2 '@sentry/node': 7.87.0 '@sqltools/formatter': 1.2.5 app-root-path: 3.1.0 @@ -6527,6 +6542,7 @@ packages: reflect-metadata: 0.2.1 sha.js: 2.4.11 sqlite3: 5.1.7 + tarn: 3.0.2 tslib: 2.6.1 uuid: 9.0.0 yargs: 17.7.2 @@ -6534,8 +6550,8 @@ packages: - supports-color dev: false - /@n8n/typeorm@0.3.20-3(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7): - resolution: {integrity: sha512-eTaEBvaJZZCStqiLKSUIvFLMZJ3WMOFc+pnF+jedcQOc6AunWM57FqhPvnuB/Z6c67JmgIyINlnjbLDHtPgKZQ==} + /@n8n/typeorm@0.3.20-7(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7): + resolution: {integrity: sha512-f4A9RGOnB3kCkusNAr1QDCGOVq1HU1YCBKoIGr2of+P3CVS3I+1vW7neOhlr/ic5S1F14Qy5TU8Lb78mRBYRSw==} engines: {node: '>=16.13.0'} hasBin: true peerDependencies: @@ -6598,6 +6614,7 @@ packages: typeorm-aurora-data-api-driver: optional: true dependencies: + '@n8n/p-retry': 6.2.0-2 '@sqltools/formatter': 1.2.5 app-root-path: 3.1.0 buffer: 6.0.3 @@ -6612,6 +6629,7 @@ packages: reflect-metadata: 0.2.1 sha.js: 2.4.11 sqlite3: 5.1.7 + tarn: 3.0.2 tslib: 2.6.1 uuid: 9.0.0 yargs: 17.7.2 @@ -9703,6 +9721,10 @@ packages: resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} dev: false + /@types/retry@0.12.5: + resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} + dev: false + /@types/rfc2047@2.0.1: resolution: {integrity: sha512-slgtykv+XXME7EperkdqfdBBUGcs28ru+a21BK0zOQY4IoxE7tEqvIcvAFAz5DJVxyOmoAUXo30Oxpm3KS+TBQ==} dev: true