feat(core): Lazy-load nodes and credentials to reduce baseline memory…

… usage (#4577)

package.json

@@ -11,6 +11,7 @@
   "scripts": {
     "preinstall": "node scripts/block-npm-install.js",
     "build": "turbo run build",
+    "typecheck": "turbo run typecheck",
     "dev": "turbo run dev --parallel",
     "clean": "turbo run clean --parallel",
     "format": "turbo run format && node scripts/format.mjs",
@@ -53,6 +54,7 @@
     "start-server-and-test": "^1.14.0",
     "supertest": "^6.2.2",
     "ts-jest": "^29.0.3",
+    "tsc-watch": "^5.0.3",
     "turbo": "1.5.5",
     "typescript": "^4.8.4"
   },

packages/cli/package.json

@@ -135,6 +135,7 @@
     "google-timezones-json": "^1.0.2",
     "handlebars": "4.7.7",
     "inquirer": "^7.0.1",
+    "ioredis": "^4.28.5",
     "json-diff": "^0.5.4",
     "jsonschema": "^1.4.1",
     "jsonwebtoken": "^8.5.1",
@@ -158,6 +159,7 @@
     "open": "^7.0.0",
     "openapi-types": "^10.0.0",
     "p-cancelable": "^2.0.0",
+    "parseurl": "^1.3.3",
     "passport": "^0.6.0",
     "passport-cookie": "^1.0.9",
     "passport-jwt": "^4.0.0",

packages/cli/src/ActiveExecutions.ts

@@ -14,7 +14,6 @@ import {
 
 import { ChildProcess } from 'child_process';
 import { stringify } from 'flatted';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import PCancelable from 'p-cancelable';
 import * as Db from '@/Db';
 import {

packages/cli/src/ActiveWorkflowRunner.ts

@@ -86,7 +86,7 @@ export class ActiveWorkflowRunner {
 			relations: ['shared', 'shared.user', 'shared.user.globalRole'],
 		})) as IWorkflowDb[];
 
-		if (!config.getEnv('endpoints.skipWebhoooksDeregistrationOnShutdown')) {
+		if (!config.getEnv('endpoints.skipWebhooksDeregistrationOnShutdown')) {
 			// Do not clean up database when skip registration is done.
 			// This flag is set when n8n is running in scaled mode.
 			// Impact is minimal, but for a short while, n8n will stop accepting requests.
@@ -401,7 +401,6 @@ export class ActiveWorkflowRunner {
 
 	/**
 	 * Adds all the webhooks of the workflow
-	 *
 	 */
 	async addWorkflowWebhooks(
 		workflow: Workflow,
@@ -462,7 +461,7 @@ export class ActiveWorkflowRunner {
 			} catch (error) {
 				if (
 					activation === 'init' &&
-					config.getEnv('endpoints.skipWebhoooksDeregistrationOnShutdown') &&
+					config.getEnv('endpoints.skipWebhooksDeregistrationOnShutdown') &&
 					error.name === 'QueryFailedError'
 				) {
 					// When skipWebhooksDeregistrationOnShutdown is enabled,
@@ -487,7 +486,10 @@ export class ActiveWorkflowRunner {
 				// TODO check if there is standard error code for duplicate key violation that works
 				// with all databases
 				if (error.name === 'QueryFailedError') {
-					error.message = `The URL path that the "${webhook.node}" node uses is already taken. Please change it to something else.`;
+					error = new Error(
+						`The URL path that the "${webhook.node}" node uses is already taken. Please change it to something else.`,
+						{ cause: error },
+					);
 				} else if (error.detail) {
 					// it's a error running the webhook methods (checkExists, create)
 					error.message = error.detail;

packages/cli/src/CommunityNodes/helpers.ts

@@ -3,7 +3,6 @@
 import { promisify } from 'util';
 import { exec } from 'child_process';
 import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';
-import { createContext, Script } from 'vm';
 import axios from 'axios';
 import { UserSettings } from 'n8n-core';
 import { LoggerProxy, PublicInstalledPackage } from 'n8n-workflow';
@@ -234,13 +233,3 @@ export const isClientError = (error: Error): boolean => {
 export function isNpmError(error: unknown): error is { code: number; stdout: string } {
 	return typeof error === 'object' && error !== null && 'code' in error && 'stdout' in error;
 }
-
-const context = createContext({ require });
-export const loadClassInIsolation = (filePath: string, className: string) => {
-	if (process.platform === 'win32') {
-		filePath = filePath.replace(/\\/g, '/');
-	}
-	const script = new Script(`new (require('${filePath}').${className})()`);
-	// eslint-disable-next-line @typescript-eslint/no-unsafe-return
-	return script.runInContext(context);
-};

packages/cli/src/CredentialTypes.ts

@@ -1,36 +1,58 @@
-import {
+import { loadClassInIsolation } from 'n8n-core';
+import type {
 	ICredentialType,
-	ICredentialTypeData,
-	ICredentialTypes as ICredentialTypesInterface,
+	ICredentialTypes,
+	INodesAndCredentials,
+	LoadedClass,
 } from 'n8n-workflow';
-import { RESPONSE_ERROR_MESSAGES } from '@/constants';
+import { RESPONSE_ERROR_MESSAGES } from './constants';
 
-class CredentialTypesClass implements ICredentialTypesInterface {
-	credentialTypes: ICredentialTypeData = {};
+class CredentialTypesClass implements ICredentialTypes {
+	constructor(private nodesAndCredentials: INodesAndCredentials) {}
 
-	async init(credentialTypes: ICredentialTypeData): Promise<void> {
-		this.credentialTypes = credentialTypes;
+	recognizes(type: string) {
+		return type in this.knownCredentials || type in this.loadedCredentials;
 	}
 
-	getAll(): ICredentialType[] {
-		return Object.values(this.credentialTypes).map((data) => data.type);
+	getByName(credentialType: string): ICredentialType {
+		return this.getCredential(credentialType).type;
 	}
 
-	getByName(credentialType: string): ICredentialType {
-		try {
-			return this.credentialTypes[credentialType].type;
-		} catch (error) {
-			throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_CREDENTIAL}: ${credentialType}`);
+	private getCredential(type: string): LoadedClass<ICredentialType> {
+		const loadedCredentials = this.loadedCredentials;
+		if (type in loadedCredentials) {
+			return loadedCredentials[type];
 		}
+
+		const knownCredentials = this.knownCredentials;
+		if (type in knownCredentials) {
+			const { className, sourcePath } = knownCredentials[type];
+			const loaded: ICredentialType = loadClassInIsolation(sourcePath, className);
+			loadedCredentials[type] = { sourcePath, type: loaded };
+			return loadedCredentials[type];
+		}
+		throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_CREDENTIAL}: ${type}`);
+	}
+
+	private get loadedCredentials() {
+		return this.nodesAndCredentials.loaded.credentials;
+	}
+
+	private get knownCredentials() {
+		return this.nodesAndCredentials.known.credentials;
 	}
 }
 
 let credentialTypesInstance: CredentialTypesClass | undefined;
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
-export function CredentialTypes(): CredentialTypesClass {
-	if (credentialTypesInstance === undefined) {
-		credentialTypesInstance = new CredentialTypesClass();
+export function CredentialTypes(nodesAndCredentials?: INodesAndCredentials): CredentialTypesClass {
+	if (!credentialTypesInstance) {
+		if (nodesAndCredentials) {
+			credentialTypesInstance = new CredentialTypesClass(nodesAndCredentials);
+		} else {
+			throw new Error('CredentialTypes not initialized yet');
+		}
 	}
 
 	return credentialTypesInstance;

packages/cli/src/CredentialsHelper.ts

@@ -7,8 +7,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 
 import { Credentials, NodeExecuteFunctions } from 'n8n-core';
-// eslint-disable-next-line import/no-extraneous-dependencies
-import { get } from 'lodash';
+import get from 'lodash.get';
 
 import {
 	ICredentialDataDecryptedObject,
@@ -25,8 +24,6 @@ import {
 	INodeParameters,
 	INodeProperties,
 	INodeType,
-	INodeTypeData,
-	INodeTypes,
 	IVersionedNodeType,
 	VersionedNodeType,
 	IRequestOptionsSimplified,
@@ -40,6 +37,8 @@ import {
 	LoggerProxy as Logger,
 	ErrorReporterProxy as ErrorReporter,
 	IHttpRequestHelper,
+	INodeTypeData,
+	INodeTypes,
 } from 'n8n-workflow';
 
 import * as Db from '@/Db';
@@ -52,19 +51,16 @@ import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 import { CredentialTypes } from '@/CredentialTypes';
 import { whereClause } from './UserManagement/UserManagementHelper';
 
+const mockNodesData: INodeTypeData = {};
 const mockNodeTypes: INodeTypes = {
-	nodeTypes: {} as INodeTypeData,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	init: async (nodeTypes?: INodeTypeData): Promise<void> => {},
 	getAll(): Array<INodeType | IVersionedNodeType> {
-		// @ts-ignore
-		return Object.values(this.nodeTypes).map((data) => data.type);
+		return Object.values(mockNodesData).map((data) => data.type);
 	},
 	getByNameAndVersion(nodeType: string, version?: number): INodeType | undefined {
-		if (this.nodeTypes[nodeType] === undefined) {
+		if (mockNodesData[nodeType] === undefined) {
 			return undefined;
 		}
-		return NodeHelpers.getVersionedNodeType(this.nodeTypes[nodeType].type, version);
+		return NodeHelpers.getVersionedNodeType(mockNodesData[nodeType].type, version);
 	},
 };
 
@@ -623,21 +619,16 @@ export class CredentialsHelper extends ICredentialsHelper {
 			},
 		};
 
-		const nodeTypes: INodeTypes = {
-			...mockNodeTypes,
-			nodeTypes: {
-				[nodeTypeCopy.description.name]: {
-					sourcePath: '',
-					type: nodeTypeCopy,
-				},
-			},
+		mockNodesData[nodeTypeCopy.description.name] = {
+			sourcePath: '',
+			type: nodeTypeCopy,
 		};
 
 		const workflow = new Workflow({
 			nodes: workflowData.nodes,
 			connections: workflowData.connections,
 			active: false,
-			nodeTypes,
+			nodeTypes: mockNodeTypes,
 		});
 
 		const mode = 'internal';
@@ -719,6 +710,8 @@ export class CredentialsHelper extends ICredentialsHelper {
 				status: 'Error',
 				message: error.message.toString(),
 			};
+		} finally {
+			delete mockNodesData[nodeTypeCopy.description.name];
 		}
 
 		if (

packages/cli/src/CredentialsOverwrites.ts

@@ -1,51 +1,36 @@
-/* eslint-disable @typescript-eslint/no-unsafe-argument */
-/* eslint-disable no-underscore-dangle */
-import { deepCopy, ICredentialDataDecryptedObject } from 'n8n-workflow';
-import { CredentialTypes } from '@/CredentialTypes';
+import type { ICredentialDataDecryptedObject, ICredentialTypes } from 'n8n-workflow';
+import { deepCopy, LoggerProxy as Logger, jsonParse } from 'n8n-workflow';
 import type { ICredentialsOverwrite } from '@/Interfaces';
 import * as GenericHelpers from '@/GenericHelpers';
 
 class CredentialsOverwritesClass {
-	private credentialTypes = CredentialTypes();
-
 	private overwriteData: ICredentialsOverwrite = {};
 
 	private resolvedTypes: string[] = [];
 
-	async init(overwriteData?: ICredentialsOverwrite) {
-		// If data gets reinitialized reset the resolved types cache
-		this.resolvedTypes.length = 0;
-
-		if (overwriteData !== undefined) {
-			// If data is already given it can directly be set instead of
-			// loaded from environment
-			this.__setData(deepCopy(overwriteData));
-			return;
-		}
+	constructor(private credentialTypes: ICredentialTypes) {}
 
+	async init() {
 		const data = (await GenericHelpers.getConfigValue('credentials.overwrite.data')) as string;
 
-		try {
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-shadow
-			const overwriteData = JSON.parse(data);
-			this.__setData(overwriteData);
-		} catch (error) {
-			throw new Error(`The credentials-overwrite is not valid JSON.`);
-		}
+		const overwriteData = jsonParse<ICredentialsOverwrite>(data, {
+			errorMessage: 'The credentials-overwrite is not valid JSON.',
+		});
+
+		this.setData(overwriteData);
 	}
 
-	__setData(overwriteData: ICredentialsOverwrite) {
-		this.overwriteData = overwriteData;
+	setData(overwriteData: ICredentialsOverwrite) {
+		// If data gets reinitialized reset the resolved types cache
+		this.resolvedTypes.length = 0;
 
-		// eslint-disable-next-line no-restricted-syntax
-		for (const credentialTypeData of this.credentialTypes.getAll()) {
-			const type = credentialTypeData.name;
+		this.overwriteData = overwriteData;
 
-			const overwrites = this.__getExtended(type);
+		for (const type in overwriteData) {
+			const overwrites = this.getOverwrites(type);
 
 			if (overwrites && Object.keys(overwrites).length) {
 				this.overwriteData[type] = overwrites;
-				credentialTypeData.__overwrittenProperties = Object.keys(overwrites);
 			}
 		}
 	}
@@ -70,18 +55,19 @@ class CredentialsOverwritesClass {
 		return returnData;
 	}
 
-	__getExtended(type: string): ICredentialDataDecryptedObject | undefined {
+	private getOverwrites(type: string): ICredentialDataDecryptedObject | undefined {
 		if (this.resolvedTypes.includes(type)) {
 			// Type got already resolved and can so returned directly
 			return this.overwriteData[type];
 		}
 
-		const credentialTypeData = this.credentialTypes.getByName(type);
-
-		if (credentialTypeData === undefined) {
-			throw new Error(`The credentials of type "${type}" are not known.`);
+		if (!this.credentialTypes.recognizes(type)) {
+			Logger.warn(`Unknown credential type ${type} in Credential overwrites`);
+			return;
 		}
 
+		const credentialTypeData = this.credentialTypes.getByName(type);
+
 		if (credentialTypeData.extends === undefined) {
 			this.resolvedTypes.push(type);
 			return this.overwriteData[type];
@@ -90,7 +76,7 @@ class CredentialsOverwritesClass {
 		const overwrites: ICredentialDataDecryptedObject = {};
 		// eslint-disable-next-line no-restricted-syntax
 		for (const credentialsTypeName of credentialTypeData.extends) {
-			Object.assign(overwrites, this.__getExtended(credentialsTypeName));
+			Object.assign(overwrites, this.getOverwrites(credentialsTypeName));
 		}
 
 		if (this.overwriteData[type] !== undefined) {
@@ -102,7 +88,7 @@ class CredentialsOverwritesClass {
 		return overwrites;
 	}
 
-	get(type: string): ICredentialDataDecryptedObject | undefined {
+	private get(type: string): ICredentialDataDecryptedObject | undefined {
 		return this.overwriteData[type];
 	}
 
@@ -114,9 +100,15 @@ class CredentialsOverwritesClass {
 let credentialsOverwritesInstance: CredentialsOverwritesClass | undefined;
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
-export function CredentialsOverwrites(): CredentialsOverwritesClass {
-	if (credentialsOverwritesInstance === undefined) {
-		credentialsOverwritesInstance = new CredentialsOverwritesClass();
+export function CredentialsOverwrites(
+	credentialTypes?: ICredentialTypes,
+): CredentialsOverwritesClass {
+	if (!credentialsOverwritesInstance) {
+		if (credentialTypes) {
+			credentialsOverwritesInstance = new CredentialsOverwritesClass(credentialTypes);
+		} else {
+			throw new Error('CredentialsOverwrites not initialized yet');
+		}
 	}
 
 	return credentialsOverwritesInstance;