From b6c57e19fc5683dd7fb9eabb60ec4e89359c59eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Wed, 23 Nov 2022 16:20:28 +0100 Subject: [PATCH] feat(core): Lazy-load nodes and credentials to reduce baseline memory usage (#4577) --- package.json | 2 + packages/cli/package.json | 2 + packages/cli/src/ActiveExecutions.ts | 1 - packages/cli/src/ActiveWorkflowRunner.ts | 10 +- packages/cli/src/CommunityNodes/helpers.ts | 11 - packages/cli/src/CredentialTypes.ts | 58 +- packages/cli/src/CredentialsHelper.ts | 33 +- packages/cli/src/CredentialsOverwrites.ts | 72 ++- packages/cli/src/GenericHelpers.ts | 7 +- packages/cli/src/Interfaces.ts | 27 +- packages/cli/src/LoadNodesAndCredentials.ts | 541 ++++++------------ packages/cli/src/NodeTypes.ts | 82 +-- packages/cli/src/Server.ts | 177 +----- packages/cli/src/UserManagement/Interfaces.ts | 6 - packages/cli/src/WebhookHelpers.ts | 3 +- packages/cli/src/WebhookServer.ts | 1 - .../cli/src/WorkflowExecuteAdditionalData.ts | 6 +- packages/cli/src/WorkflowHelpers.ts | 181 +----- packages/cli/src/WorkflowRunner.ts | 44 -- packages/cli/src/WorkflowRunnerProcess.ts | 59 +- packages/cli/src/api/executions.api.ts | 5 +- packages/cli/src/api/nodeTypes.api.ts | 52 +- packages/cli/src/commands/execute.ts | 12 +- packages/cli/src/commands/executeBatch.ts | 14 +- packages/cli/src/commands/start.ts | 24 +- packages/cli/src/commands/webhook.ts | 16 +- packages/cli/src/commands/worker.ts | 14 +- packages/cli/src/config/schema.ts | 4 +- packages/cli/src/constants.ts | 4 +- .../cli/src/databases/entities/Settings.ts | 2 +- .../src/databases/utils/migrationHelpers.ts | 2 +- packages/cli/src/requests.d.ts | 2 +- packages/cli/test/integration/shared/utils.ts | 24 +- .../cli/test/unit/CredentialTypes.test.ts | 64 +-- .../cli/test/unit/CredentialsHelper.test.ts | 9 +- packages/cli/test/unit/Helpers.ts | 18 +- .../cli/test/unit/PermissionChecker.test.ts | 9 +- packages/core/.eslintrc.js | 2 + packages/core/bin/common.js | 19 + packages/core/bin/generate-known | 47 ++ packages/core/bin/generate-ui-types | 29 + packages/core/package.json | 10 +- packages/core/src/ClassLoader.ts | 10 + packages/core/src/Constants.ts | 2 + packages/core/src/DirectoryLoader.ts | 339 +++++++++++ packages/core/src/Interfaces.ts | 15 + packages/core/src/NodeExecuteFunctions.ts | 4 +- packages/core/src/WorkflowExecute.ts | 3 +- packages/core/src/index.ts | 2 + packages/core/test/Helpers.ts | 5 +- packages/design-system/package.json | 4 +- packages/design-system/tsconfig.json | 1 + packages/editor-ui/package.json | 3 +- packages/editor-ui/src/Interface.ts | 1 + packages/editor-ui/src/api/credentials.ts | 6 +- packages/editor-ui/src/api/nodeTypes.ts | 10 +- .../src/components/CredentialIcon.vue | 13 +- .../editor-ui/src/components/NodeIcon.vue | 8 +- packages/editor-ui/src/stores/credentials.ts | 2 +- packages/editor-ui/src/stores/n8nRootStore.ts | 4 + packages/editor-ui/src/stores/nodeTypes.ts | 2 +- packages/editor-ui/tsconfig.json | 1 + .../NetlifyOAuth2Api.credentials.ts | 60 -- packages/nodes-base/package.json | 5 +- packages/workflow/package.json | 3 +- packages/workflow/src/Interfaces.ts | 49 +- packages/workflow/src/NodeHelpers.ts | 10 +- packages/workflow/src/ObservableObject.ts | 2 +- packages/workflow/test/Helpers.ts | 2 - pnpm-lock.yaml | 87 ++- turbo.json | 5 +- 71 files changed, 1093 insertions(+), 1270 deletions(-) create mode 100644 packages/core/bin/common.js create mode 100755 packages/core/bin/generate-known create mode 100755 packages/core/bin/generate-ui-types create mode 100644 packages/core/src/ClassLoader.ts create mode 100644 packages/core/src/DirectoryLoader.ts delete mode 100644 packages/nodes-base/credentials/NetlifyOAuth2Api.credentials.ts diff --git a/package.json b/package.json index 1df8bbee3ac27..e6fb0998da9ae 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "scripts": { "preinstall": "node scripts/block-npm-install.js", "build": "turbo run build", + "typecheck": "turbo run typecheck", "dev": "turbo run dev --parallel", "clean": "turbo run clean --parallel", "format": "turbo run format && node scripts/format.mjs", @@ -53,6 +54,7 @@ "start-server-and-test": "^1.14.0", "supertest": "^6.2.2", "ts-jest": "^29.0.3", + "tsc-watch": "^5.0.3", "turbo": "1.5.5", "typescript": "^4.8.4" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index 10248f7516f19..f5f5a11d3e447 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -135,6 +135,7 @@ "google-timezones-json": "^1.0.2", "handlebars": "4.7.7", "inquirer": "^7.0.1", + "ioredis": "^4.28.5", "json-diff": "^0.5.4", "jsonschema": "^1.4.1", "jsonwebtoken": "^8.5.1", @@ -158,6 +159,7 @@ "open": "^7.0.0", "openapi-types": "^10.0.0", "p-cancelable": "^2.0.0", + "parseurl": "^1.3.3", "passport": "^0.6.0", "passport-cookie": "^1.0.9", "passport-jwt": "^4.0.0", diff --git a/packages/cli/src/ActiveExecutions.ts b/packages/cli/src/ActiveExecutions.ts index d3f7b69a1d965..7781f2e614c93 100644 --- a/packages/cli/src/ActiveExecutions.ts +++ b/packages/cli/src/ActiveExecutions.ts @@ -14,7 +14,6 @@ import { import { ChildProcess } from 'child_process'; import { stringify } from 'flatted'; -// eslint-disable-next-line import/no-extraneous-dependencies import PCancelable from 'p-cancelable'; import * as Db from '@/Db'; import { diff --git a/packages/cli/src/ActiveWorkflowRunner.ts b/packages/cli/src/ActiveWorkflowRunner.ts index 2e2fdc69ce6a6..b212dd5464ffa 100644 --- a/packages/cli/src/ActiveWorkflowRunner.ts +++ b/packages/cli/src/ActiveWorkflowRunner.ts @@ -86,7 +86,7 @@ export class ActiveWorkflowRunner { relations: ['shared', 'shared.user', 'shared.user.globalRole'], })) as IWorkflowDb[]; - if (!config.getEnv('endpoints.skipWebhoooksDeregistrationOnShutdown')) { + if (!config.getEnv('endpoints.skipWebhooksDeregistrationOnShutdown')) { // Do not clean up database when skip registration is done. // This flag is set when n8n is running in scaled mode. // Impact is minimal, but for a short while, n8n will stop accepting requests. @@ -401,7 +401,6 @@ export class ActiveWorkflowRunner { /** * Adds all the webhooks of the workflow - * */ async addWorkflowWebhooks( workflow: Workflow, @@ -462,7 +461,7 @@ export class ActiveWorkflowRunner { } catch (error) { if ( activation === 'init' && - config.getEnv('endpoints.skipWebhoooksDeregistrationOnShutdown') && + config.getEnv('endpoints.skipWebhooksDeregistrationOnShutdown') && error.name === 'QueryFailedError' ) { // When skipWebhooksDeregistrationOnShutdown is enabled, @@ -487,7 +486,10 @@ export class ActiveWorkflowRunner { // TODO check if there is standard error code for duplicate key violation that works // with all databases if (error.name === 'QueryFailedError') { - error.message = `The URL path that the "${webhook.node}" node uses is already taken. Please change it to something else.`; + error = new Error( + `The URL path that the "${webhook.node}" node uses is already taken. Please change it to something else.`, + { cause: error }, + ); } else if (error.detail) { // it's a error running the webhook methods (checkExists, create) error.message = error.detail; diff --git a/packages/cli/src/CommunityNodes/helpers.ts b/packages/cli/src/CommunityNodes/helpers.ts index 22022aceced40..0c0770b88f86e 100644 --- a/packages/cli/src/CommunityNodes/helpers.ts +++ b/packages/cli/src/CommunityNodes/helpers.ts @@ -3,7 +3,6 @@ import { promisify } from 'util'; import { exec } from 'child_process'; import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises'; -import { createContext, Script } from 'vm'; import axios from 'axios'; import { UserSettings } from 'n8n-core'; import { LoggerProxy, PublicInstalledPackage } from 'n8n-workflow'; @@ -234,13 +233,3 @@ export const isClientError = (error: Error): boolean => { export function isNpmError(error: unknown): error is { code: number; stdout: string } { return typeof error === 'object' && error !== null && 'code' in error && 'stdout' in error; } - -const context = createContext({ require }); -export const loadClassInIsolation = (filePath: string, className: string) => { - if (process.platform === 'win32') { - filePath = filePath.replace(/\\/g, '/'); - } - const script = new Script(`new (require('${filePath}').${className})()`); - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return script.runInContext(context); -}; diff --git a/packages/cli/src/CredentialTypes.ts b/packages/cli/src/CredentialTypes.ts index 9f744cc9d32ab..7185560b00bf2 100644 --- a/packages/cli/src/CredentialTypes.ts +++ b/packages/cli/src/CredentialTypes.ts @@ -1,36 +1,58 @@ -import { +import { loadClassInIsolation } from 'n8n-core'; +import type { ICredentialType, - ICredentialTypeData, - ICredentialTypes as ICredentialTypesInterface, + ICredentialTypes, + INodesAndCredentials, + LoadedClass, } from 'n8n-workflow'; -import { RESPONSE_ERROR_MESSAGES } from '@/constants'; +import { RESPONSE_ERROR_MESSAGES } from './constants'; -class CredentialTypesClass implements ICredentialTypesInterface { - credentialTypes: ICredentialTypeData = {}; +class CredentialTypesClass implements ICredentialTypes { + constructor(private nodesAndCredentials: INodesAndCredentials) {} - async init(credentialTypes: ICredentialTypeData): Promise { - this.credentialTypes = credentialTypes; + recognizes(type: string) { + return type in this.knownCredentials || type in this.loadedCredentials; } - getAll(): ICredentialType[] { - return Object.values(this.credentialTypes).map((data) => data.type); + getByName(credentialType: string): ICredentialType { + return this.getCredential(credentialType).type; } - getByName(credentialType: string): ICredentialType { - try { - return this.credentialTypes[credentialType].type; - } catch (error) { - throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_CREDENTIAL}: ${credentialType}`); + private getCredential(type: string): LoadedClass { + const loadedCredentials = this.loadedCredentials; + if (type in loadedCredentials) { + return loadedCredentials[type]; } + + const knownCredentials = this.knownCredentials; + if (type in knownCredentials) { + const { className, sourcePath } = knownCredentials[type]; + const loaded: ICredentialType = loadClassInIsolation(sourcePath, className); + loadedCredentials[type] = { sourcePath, type: loaded }; + return loadedCredentials[type]; + } + throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_CREDENTIAL}: ${type}`); + } + + private get loadedCredentials() { + return this.nodesAndCredentials.loaded.credentials; + } + + private get knownCredentials() { + return this.nodesAndCredentials.known.credentials; } } let credentialTypesInstance: CredentialTypesClass | undefined; // eslint-disable-next-line @typescript-eslint/naming-convention -export function CredentialTypes(): CredentialTypesClass { - if (credentialTypesInstance === undefined) { - credentialTypesInstance = new CredentialTypesClass(); +export function CredentialTypes(nodesAndCredentials?: INodesAndCredentials): CredentialTypesClass { + if (!credentialTypesInstance) { + if (nodesAndCredentials) { + credentialTypesInstance = new CredentialTypesClass(nodesAndCredentials); + } else { + throw new Error('CredentialTypes not initialized yet'); + } } return credentialTypesInstance; diff --git a/packages/cli/src/CredentialsHelper.ts b/packages/cli/src/CredentialsHelper.ts index 771ca5bdddc7d..cde1530b5118d 100644 --- a/packages/cli/src/CredentialsHelper.ts +++ b/packages/cli/src/CredentialsHelper.ts @@ -7,8 +7,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ import { Credentials, NodeExecuteFunctions } from 'n8n-core'; -// eslint-disable-next-line import/no-extraneous-dependencies -import { get } from 'lodash'; +import get from 'lodash.get'; import { ICredentialDataDecryptedObject, @@ -25,8 +24,6 @@ import { INodeParameters, INodeProperties, INodeType, - INodeTypeData, - INodeTypes, IVersionedNodeType, VersionedNodeType, IRequestOptionsSimplified, @@ -40,6 +37,8 @@ import { LoggerProxy as Logger, ErrorReporterProxy as ErrorReporter, IHttpRequestHelper, + INodeTypeData, + INodeTypes, } from 'n8n-workflow'; import * as Db from '@/Db'; @@ -52,19 +51,16 @@ import { CredentialsOverwrites } from '@/CredentialsOverwrites'; import { CredentialTypes } from '@/CredentialTypes'; import { whereClause } from './UserManagement/UserManagementHelper'; +const mockNodesData: INodeTypeData = {}; const mockNodeTypes: INodeTypes = { - nodeTypes: {} as INodeTypeData, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - init: async (nodeTypes?: INodeTypeData): Promise => {}, getAll(): Array { - // @ts-ignore - return Object.values(this.nodeTypes).map((data) => data.type); + return Object.values(mockNodesData).map((data) => data.type); }, getByNameAndVersion(nodeType: string, version?: number): INodeType | undefined { - if (this.nodeTypes[nodeType] === undefined) { + if (mockNodesData[nodeType] === undefined) { return undefined; } - return NodeHelpers.getVersionedNodeType(this.nodeTypes[nodeType].type, version); + return NodeHelpers.getVersionedNodeType(mockNodesData[nodeType].type, version); }, }; @@ -623,21 +619,16 @@ export class CredentialsHelper extends ICredentialsHelper { }, }; - const nodeTypes: INodeTypes = { - ...mockNodeTypes, - nodeTypes: { - [nodeTypeCopy.description.name]: { - sourcePath: '', - type: nodeTypeCopy, - }, - }, + mockNodesData[nodeTypeCopy.description.name] = { + sourcePath: '', + type: nodeTypeCopy, }; const workflow = new Workflow({ nodes: workflowData.nodes, connections: workflowData.connections, active: false, - nodeTypes, + nodeTypes: mockNodeTypes, }); const mode = 'internal'; @@ -719,6 +710,8 @@ export class CredentialsHelper extends ICredentialsHelper { status: 'Error', message: error.message.toString(), }; + } finally { + delete mockNodesData[nodeTypeCopy.description.name]; } if ( diff --git a/packages/cli/src/CredentialsOverwrites.ts b/packages/cli/src/CredentialsOverwrites.ts index c70454559561c..92a003b29b60e 100644 --- a/packages/cli/src/CredentialsOverwrites.ts +++ b/packages/cli/src/CredentialsOverwrites.ts @@ -1,51 +1,36 @@ -/* eslint-disable @typescript-eslint/no-unsafe-argument */ -/* eslint-disable no-underscore-dangle */ -import { deepCopy, ICredentialDataDecryptedObject } from 'n8n-workflow'; -import { CredentialTypes } from '@/CredentialTypes'; +import type { ICredentialDataDecryptedObject, ICredentialTypes } from 'n8n-workflow'; +import { deepCopy, LoggerProxy as Logger, jsonParse } from 'n8n-workflow'; import type { ICredentialsOverwrite } from '@/Interfaces'; import * as GenericHelpers from '@/GenericHelpers'; class CredentialsOverwritesClass { - private credentialTypes = CredentialTypes(); - private overwriteData: ICredentialsOverwrite = {}; private resolvedTypes: string[] = []; - async init(overwriteData?: ICredentialsOverwrite) { - // If data gets reinitialized reset the resolved types cache - this.resolvedTypes.length = 0; - - if (overwriteData !== undefined) { - // If data is already given it can directly be set instead of - // loaded from environment - this.__setData(deepCopy(overwriteData)); - return; - } + constructor(private credentialTypes: ICredentialTypes) {} + async init() { const data = (await GenericHelpers.getConfigValue('credentials.overwrite.data')) as string; - try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-shadow - const overwriteData = JSON.parse(data); - this.__setData(overwriteData); - } catch (error) { - throw new Error(`The credentials-overwrite is not valid JSON.`); - } + const overwriteData = jsonParse(data, { + errorMessage: 'The credentials-overwrite is not valid JSON.', + }); + + this.setData(overwriteData); } - __setData(overwriteData: ICredentialsOverwrite) { - this.overwriteData = overwriteData; + setData(overwriteData: ICredentialsOverwrite) { + // If data gets reinitialized reset the resolved types cache + this.resolvedTypes.length = 0; - // eslint-disable-next-line no-restricted-syntax - for (const credentialTypeData of this.credentialTypes.getAll()) { - const type = credentialTypeData.name; + this.overwriteData = overwriteData; - const overwrites = this.__getExtended(type); + for (const type in overwriteData) { + const overwrites = this.getOverwrites(type); if (overwrites && Object.keys(overwrites).length) { this.overwriteData[type] = overwrites; - credentialTypeData.__overwrittenProperties = Object.keys(overwrites); } } } @@ -70,18 +55,19 @@ class CredentialsOverwritesClass { return returnData; } - __getExtended(type: string): ICredentialDataDecryptedObject | undefined { + private getOverwrites(type: string): ICredentialDataDecryptedObject | undefined { if (this.resolvedTypes.includes(type)) { // Type got already resolved and can so returned directly return this.overwriteData[type]; } - const credentialTypeData = this.credentialTypes.getByName(type); - - if (credentialTypeData === undefined) { - throw new Error(`The credentials of type "${type}" are not known.`); + if (!this.credentialTypes.recognizes(type)) { + Logger.warn(`Unknown credential type ${type} in Credential overwrites`); + return; } + const credentialTypeData = this.credentialTypes.getByName(type); + if (credentialTypeData.extends === undefined) { this.resolvedTypes.push(type); return this.overwriteData[type]; @@ -90,7 +76,7 @@ class CredentialsOverwritesClass { const overwrites: ICredentialDataDecryptedObject = {}; // eslint-disable-next-line no-restricted-syntax for (const credentialsTypeName of credentialTypeData.extends) { - Object.assign(overwrites, this.__getExtended(credentialsTypeName)); + Object.assign(overwrites, this.getOverwrites(credentialsTypeName)); } if (this.overwriteData[type] !== undefined) { @@ -102,7 +88,7 @@ class CredentialsOverwritesClass { return overwrites; } - get(type: string): ICredentialDataDecryptedObject | undefined { + private get(type: string): ICredentialDataDecryptedObject | undefined { return this.overwriteData[type]; } @@ -114,9 +100,15 @@ class CredentialsOverwritesClass { let credentialsOverwritesInstance: CredentialsOverwritesClass | undefined; // eslint-disable-next-line @typescript-eslint/naming-convention -export function CredentialsOverwrites(): CredentialsOverwritesClass { - if (credentialsOverwritesInstance === undefined) { - credentialsOverwritesInstance = new CredentialsOverwritesClass(); +export function CredentialsOverwrites( + credentialTypes?: ICredentialTypes, +): CredentialsOverwritesClass { + if (!credentialsOverwritesInstance) { + if (credentialTypes) { + credentialsOverwritesInstance = new CredentialsOverwritesClass(credentialTypes); + } else { + throw new Error('CredentialsOverwrites not initialized yet'); + } } return credentialsOverwritesInstance; diff --git a/packages/cli/src/GenericHelpers.ts b/packages/cli/src/GenericHelpers.ts index b04ad0a1de547..af5fb3731c3ce 100644 --- a/packages/cli/src/GenericHelpers.ts +++ b/packages/cli/src/GenericHelpers.ts @@ -7,6 +7,7 @@ import express from 'express'; import { join as pathJoin } from 'path'; import { readFile as fsReadFile } from 'fs/promises'; +import type { n8n } from 'n8n-core'; import { ExecutionError, IDataObject, @@ -25,7 +26,6 @@ import { IExecutionFlattedDb, IPackageVersions, IWorkflowDb, - IN8nNodePackageJson, } from '@/Interfaces'; import * as ResponseHelper from '@/ResponseHelper'; // eslint-disable-next-line import/order @@ -64,7 +64,6 @@ export function getSessionId(req: express.Request): string | undefined { /** * Returns information which version of the packages are installed - * */ export async function getVersions(): Promise { if (versionCache !== undefined) { @@ -72,11 +71,9 @@ export async function getVersions(): Promise { } const packageFile = await fsReadFile(pathJoin(CLI_DIR, 'package.json'), 'utf8'); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const packageData = jsonParse(packageFile); + const packageData = jsonParse(packageFile); versionCache = { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment cli: packageData.version, }; diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 59ae83395f59b..c2b623b19703b 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { +import type { ExecutionError, ICredentialDataDecryptedObject, ICredentialsDecrypted, @@ -15,6 +15,7 @@ import { ITelemetrySettings, ITelemetryTrackProperties, IWorkflowBase as IWorkflowBaseWorkflow, + LoadingDetails, Workflow, WorkflowActivateMode, WorkflowExecuteMode, @@ -22,7 +23,6 @@ import { import { WorkflowExecute } from 'n8n-core'; -// eslint-disable-next-line import/no-extraneous-dependencies import PCancelable from 'p-cancelable'; import type { FindOperator, Repository } from 'typeorm'; @@ -59,10 +59,7 @@ export interface ICustomRequest extends Request { } export interface ICredentialsTypeData { - [key: string]: { - className: string; - sourcePath: string; - }; + [key: string]: LoadingDetails; } export interface ICredentialsOverwrite { @@ -451,19 +448,6 @@ export interface IVersionNotificationSettings { infoUrl: string; } -export interface IN8nNodePackageJson { - name: string; - version: string; - n8n?: { - credentials?: string[]; - nodes?: string[]; - }; - author?: { - name?: string; - email?: string; - }; -} - export interface IN8nUISettings { endpointWebhook: string; endpointWebhookTest: string; @@ -649,7 +633,7 @@ export interface IResponseCallbackData { responseCode?: number; } -export interface ITransferNodeTypes { +export interface INodesTypeData { [key: string]: { className: string; sourcePath: string; @@ -697,10 +681,7 @@ export interface IWorkflowExecutionDataProcess { } export interface IWorkflowExecutionDataProcessWithExecution extends IWorkflowExecutionDataProcess { - credentialsOverwrite: ICredentialsOverwrite; - credentialsTypeData: ICredentialsTypeData; executionId: string; - nodeTypeData: ITransferNodeTypes; userId: string; } diff --git a/packages/cli/src/LoadNodesAndCredentials.ts b/packages/cli/src/LoadNodesAndCredentials.ts index 373769e4a2c2b..f4d8eb6dafc05 100644 --- a/packages/cli/src/LoadNodesAndCredentials.ts +++ b/packages/cli/src/LoadNodesAndCredentials.ts @@ -1,132 +1,108 @@ -/* eslint-disable no-underscore-dangle */ -/* eslint-disable @typescript-eslint/naming-convention */ -/* eslint-disable no-prototype-builtins */ -/* eslint-disable no-param-reassign */ -/* eslint-disable @typescript-eslint/prefer-optional-chain */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable no-await-in-loop */ -/* eslint-disable no-continue */ -/* eslint-disable no-restricted-syntax */ -import { CUSTOM_EXTENSION_ENV, UserSettings } from 'n8n-core'; import { - CodexData, - ICredentialType, - ICredentialTypeData, + CUSTOM_EXTENSION_ENV, + UserSettings, + CustomDirectoryLoader, + DirectoryLoader, + PackageDirectoryLoader, + LazyPackageDirectoryLoader, + Types, +} from 'n8n-core'; +import type { ILogger, - INodeType, - INodeTypeData, - INodeTypeNameVersion, - IVersionedNodeType, - LoggerProxy, - jsonParse, - ErrorReporterProxy as ErrorReporter, + INodesAndCredentials, + KnownNodesAndCredentials, + LoadedNodesAndCredentials, } from 'n8n-workflow'; +import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; import { access as fsAccess, + copyFile, + mkdir, readdir as fsReaddir, - readFile as fsReadFile, stat as fsStat, + writeFile, } from 'fs/promises'; -import glob from 'fast-glob'; import path from 'path'; -import pick from 'lodash.pick'; -import { IN8nNodePackageJson } from '@/Interfaces'; -import { getLogger } from '@/Logger'; import config from '@/config'; -import { NodeTypes } from '@/NodeTypes'; import { InstalledPackages } from '@db/entities/InstalledPackages'; import { InstalledNodes } from '@db/entities/InstalledNodes'; -import { executeCommand, loadClassInIsolation } from '@/CommunityNodes/helpers'; -import { CLI_DIR, RESPONSE_ERROR_MESSAGES } from '@/constants'; +import { executeCommand } from '@/CommunityNodes/helpers'; +import { CLI_DIR, GENERATED_STATIC_DIR, RESPONSE_ERROR_MESSAGES } from '@/constants'; import { persistInstalledPackageData, removePackageFromDatabase, } from '@/CommunityNodes/packageModel'; +import { CredentialsOverwrites } from '@/CredentialsOverwrites'; -const CUSTOM_NODES_CATEGORY = 'Custom Nodes'; +export class LoadNodesAndCredentialsClass implements INodesAndCredentials { + known: KnownNodesAndCredentials = { nodes: {}, credentials: {} }; -function toJSON() { - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return { - ...this, - authenticate: typeof this.authenticate === 'function' ? {} : this.authenticate, - }; -} - -class LoadNodesAndCredentialsClass { - nodeTypes: INodeTypeData = {}; + loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} }; - credentialTypes: ICredentialTypeData = {}; + types: Types = { nodes: [], credentials: [] }; - excludeNodes: string | undefined = undefined; + excludeNodes = config.getEnv('nodes.exclude'); - includeNodes: string | undefined = undefined; + includeNodes = config.getEnv('nodes.include'); logger: ILogger; async init() { - this.logger = getLogger(); - LoggerProxy.init(this.logger); - // Make sure the imported modules can resolve dependencies fine. const delimiter = process.platform === 'win32' ? ';' : ':'; process.env.NODE_PATH = module.paths.join(delimiter); + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/no-unsafe-call module.constructor._initPaths(); - const nodeModulesPath = await this.getNodeModulesFolderLocation(); - - this.excludeNodes = config.getEnv('nodes.exclude'); - this.includeNodes = config.getEnv('nodes.include'); + await mkdir(path.join(GENERATED_STATIC_DIR, 'icons/nodes'), { recursive: true }); + await mkdir(path.join(GENERATED_STATIC_DIR, 'icons/credentials'), { recursive: true }); - // Get all the installed packages which contain n8n nodes - const nodePackages = await this.getN8nNodePackages(nodeModulesPath); + await this.loadNodesFromBasePackages(); + await this.loadNodesFromDownloadedPackages(); + await this.loadNodesFromCustomDirectories(); + } - for (const packagePath of nodePackages) { - await this.loadDataFromPackage(packagePath); + async generateTypesForFrontend() { + const credentialsOverwrites = CredentialsOverwrites().getAll(); + for (const credential of this.types.credentials) { + if (credential.name in credentialsOverwrites) { + credential.__overwrittenProperties = Object.keys(credentialsOverwrites[credential.name]); + } } - await this.loadNodesFromDownloadedPackages(); + // pre-render all the node and credential types as static json files + await mkdir(path.join(GENERATED_STATIC_DIR, 'types'), { recursive: true }); + + const writeStaticJSON = async (name: string, data: any[]) => { + const filePath = path.join(GENERATED_STATIC_DIR, `types/${name}.json`); + const payload = `[\n${data.map((entry) => JSON.stringify(entry)).join(',\n')}\n]`; + await writeFile(filePath, payload, { encoding: 'utf-8' }); + }; - await this.loadNodesFromCustomFolders(); + await writeStaticJSON('nodes', this.types.nodes); + await writeStaticJSON('credentials', this.types.credentials); } - async getNodeModulesFolderLocation(): Promise { - // Get the path to the node-modules folder to be later able - // to load the credentials and nodes - const checkPaths = [ - // In case "n8n" package is in same node_modules folder. - path.join(CLI_DIR, '..', 'n8n-workflow'), - // In case "n8n" package is the root and the packages are - // in the "node_modules" folder underneath it. - path.join(CLI_DIR, 'node_modules', 'n8n-workflow'), - // In case "n8n" package is installed using npm/yarn workspaces - // the node_modules folder is in the root of the workspace. - path.join(CLI_DIR, '..', '..', 'node_modules', 'n8n-workflow'), - ]; - for (const checkPath of checkPaths) { - try { - await fsAccess(checkPath); - // Folder exists, so use it. - return path.dirname(checkPath); - } catch (_) { - // Folder does not exist so get next one - } + async loadNodesFromBasePackages() { + const nodeModulesPath = await this.getNodeModulesPath(); + const nodePackagePaths = await this.getN8nNodePackages(nodeModulesPath); + + for (const packagePath of nodePackagePaths) { + await this.runDirectoryLoader(LazyPackageDirectoryLoader, packagePath); } - throw new Error('Could not find "node_modules" folder!'); } async loadNodesFromDownloadedPackages(): Promise { const nodePackages = []; try { // Read downloaded nodes and credentials - const downloadedNodesFolder = UserSettings.getUserN8nFolderDownloadedNodesPath(); - const downloadedNodesFolderModules = path.join(downloadedNodesFolder, 'node_modules'); - await fsAccess(downloadedNodesFolderModules); - const downloadedPackages = await this.getN8nNodePackages(downloadedNodesFolderModules); + const downloadedNodesDir = UserSettings.getUserN8nFolderDownloadedNodesPath(); + const downloadedNodesDirModules = path.join(downloadedNodesDir, 'node_modules'); + await fsAccess(downloadedNodesDirModules); + const downloadedPackages = await this.getN8nNodePackages(downloadedNodesDirModules); nodePackages.push(...downloadedPackages); } catch (error) { // Folder does not exist so ignore and return @@ -135,15 +111,14 @@ class LoadNodesAndCredentialsClass { for (const packagePath of nodePackages) { try { - await this.loadDataFromPackage(packagePath); - // eslint-disable-next-line no-empty + await this.runDirectoryLoader(PackageDirectoryLoader, packagePath); } catch (error) { ErrorReporter.error(error); } } } - async loadNodesFromCustomFolders(): Promise { + async loadNodesFromCustomDirectories(): Promise { // Read nodes and credentials from custom directories const customDirectories = []; @@ -158,7 +133,7 @@ class LoadNodesAndCredentialsClass { } for (const directory of customDirectories) { - await this.loadDataFromDirectory('CUSTOM', directory); + await this.runDirectoryLoader(CustomDirectoryLoader, directory); } } @@ -192,46 +167,6 @@ class LoadNodesAndCredentialsClass { return getN8nNodePackagesRecursive(''); } - /** - * Loads credentials from a file - * - * @param {string} credentialName The name of the credentials - * @param {string} filePath The file to read credentials from - */ - loadCredentialsFromFile(credentialName: string, filePath: string): void { - let tempCredential: ICredentialType; - try { - tempCredential = loadClassInIsolation(filePath, credentialName); - - // Add serializer method "toJSON" to the class so that authenticate method (if defined) - // gets mapped to the authenticate attribute before it is sent to the client. - // The authenticate property is used by the client to decide whether or not to - // include the credential type in the predefined credentials (HTTP node) - Object.assign(tempCredential, { toJSON }); - - if (tempCredential.icon && tempCredential.icon.startsWith('file:')) { - // If a file icon gets used add the full path - tempCredential.icon = `file:${path.join( - path.dirname(filePath), - tempCredential.icon.substr(5), - )}`; - } - } catch (e) { - if (e instanceof TypeError) { - throw new Error( - `Class with name "${credentialName}" could not be found. Please check if the class is named correctly!`, - ); - } else { - throw e; - } - } - - this.credentialTypes[tempCredential.name] = { - type: tempCredential, - sourcePath: filePath, - }; - } - async loadNpmModule(packageName: string, version?: string): Promise { const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath(); const command = `npm install ${packageName}${version ? `@${version}` : ''}`; @@ -240,24 +175,30 @@ class LoadNodesAndCredentialsClass { const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName); - const loadedNodes = await this.loadDataFromPackage(finalNodeUnpackedPath); + const { loadedNodes, packageJson } = await this.runDirectoryLoader( + PackageDirectoryLoader, + finalNodeUnpackedPath, + ); if (loadedNodes.length > 0) { - const packageFile = await this.readPackageJson(finalNodeUnpackedPath); // Save info to DB try { const installedPackage = await persistInstalledPackageData( - packageFile.name, - packageFile.version, + packageJson.name, + packageJson.version, loadedNodes, - this.nodeTypes, - packageFile.author?.name, - packageFile.author?.email, + this.loaded.nodes, + packageJson.author?.name, + packageJson.author?.email, ); this.attachNodesToNodeTypes(installedPackage.installedNodes); + await this.generateTypesForFrontend(); return installedPackage; } catch (error) { - LoggerProxy.error('Failed to save installed packages and nodes', { error, packageName }); + LoggerProxy.error('Failed to save installed packages and nodes', { + error: error as Error, + packageName, + }); throw error; } } else { @@ -265,9 +206,7 @@ class LoadNodesAndCredentialsClass { const removeCommand = `npm remove ${packageName}`; try { await executeCommand(removeCommand); - } catch (error) { - // Do nothing - } + } catch (_) {} throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES); } @@ -278,7 +217,9 @@ class LoadNodesAndCredentialsClass { await executeCommand(command); - void (await removePackageFromDatabase(installedPackage)); + await removePackageFromDatabase(installedPackage); + + await this.generateTypesForFrontend(); this.unloadNodes(installedPackage.installedNodes); } @@ -294,7 +235,7 @@ class LoadNodesAndCredentialsClass { try { await executeCommand(command); } catch (error) { - if (error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) { + if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) { throw new Error(`The npm package "${packageName}" could not be found.`); } throw error; @@ -304,29 +245,35 @@ class LoadNodesAndCredentialsClass { const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName); - const loadedNodes = await this.loadDataFromPackage(finalNodeUnpackedPath); + const { loadedNodes, packageJson } = await this.runDirectoryLoader( + PackageDirectoryLoader, + finalNodeUnpackedPath, + ); if (loadedNodes.length > 0) { - const packageFile = await this.readPackageJson(finalNodeUnpackedPath); - // Save info to DB try { await removePackageFromDatabase(installedPackage); const newlyInstalledPackage = await persistInstalledPackageData( - packageFile.name, - packageFile.version, + packageJson.name, + packageJson.version, loadedNodes, - this.nodeTypes, - packageFile.author?.name, - packageFile.author?.email, + this.loaded.nodes, + packageJson.author?.name, + packageJson.author?.email, ); this.attachNodesToNodeTypes(newlyInstalledPackage.installedNodes); + await this.generateTypesForFrontend(); + return newlyInstalledPackage; } catch (error) { - LoggerProxy.error('Failed to save installed packages and nodes', { error, packageName }); + LoggerProxy.error('Failed to save installed packages and nodes', { + error: error as Error, + packageName, + }); throw error; } } else { @@ -334,249 +281,119 @@ class LoadNodesAndCredentialsClass { const removeCommand = `npm remove ${packageName}`; try { await executeCommand(removeCommand); - } catch (error) { - // Do nothing - } + } catch (_) {} throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES); } } - /** - * Loads a node from a file - * - * @param {string} packageName The package name to set for the found nodes - * @param {string} nodeName Tha name of the node - * @param {string} filePath The file to read node from - */ - loadNodeFromFile( - packageName: string, - nodeName: string, - filePath: string, - ): INodeTypeNameVersion | undefined { - let tempNode: INodeType | IVersionedNodeType; - let nodeVersion = 1; - - try { - tempNode = loadClassInIsolation(filePath, nodeName); - this.addCodex({ node: tempNode, filePath, isCustom: packageName === 'CUSTOM' }); - } catch (error) { - // eslint-disable-next-line no-console, @typescript-eslint/restrict-template-expressions - console.error(`Error loading node "${nodeName}" from: "${filePath}" - ${error.message}`); - throw error; - } - - const fullNodeName = `${packageName}.${tempNode.description.name}`; - tempNode.description.name = fullNodeName; - - if (tempNode.description.icon !== undefined && tempNode.description.icon.startsWith('file:')) { - // If a file icon gets used add the full path - tempNode.description.icon = `file:${path.join( - path.dirname(filePath), - tempNode.description.icon.substr(5), - )}`; - } - - if (tempNode.hasOwnProperty('nodeVersions')) { - const versionedNodeType = (tempNode as IVersionedNodeType).getNodeType(); - this.addCodex({ node: versionedNodeType, filePath, isCustom: packageName === 'CUSTOM' }); - nodeVersion = (tempNode as IVersionedNodeType).currentVersion; - - if ( - versionedNodeType.description.icon !== undefined && - versionedNodeType.description.icon.startsWith('file:') - ) { - // If a file icon gets used add the full path - versionedNodeType.description.icon = `file:${path.join( - path.dirname(filePath), - versionedNodeType.description.icon.substr(5), - )}`; - } - - if (versionedNodeType.hasOwnProperty('executeSingle')) { - this.logger.warn( - `"executeSingle" will get deprecated soon. Please update the code of node "${packageName}.${nodeName}" to use "execute" instead!`, - { filePath }, - ); - } - } else { - // Short renaming to avoid type issues - const tmpNode = tempNode as INodeType; - nodeVersion = Array.isArray(tmpNode.description.version) - ? tmpNode.description.version.slice(-1)[0] - : tmpNode.description.version; - } - - if (this.includeNodes !== undefined && !this.includeNodes.includes(fullNodeName)) { - return; - } - - // Check if the node should be skipped - if (this.excludeNodes !== undefined && this.excludeNodes.includes(fullNodeName)) { - return; - } - - this.nodeTypes[fullNodeName] = { - type: tempNode, - sourcePath: filePath, - }; - - // eslint-disable-next-line consistent-return - return { - name: fullNodeName, - version: nodeVersion, - } as INodeTypeNameVersion; + private unloadNodes(installedNodes: InstalledNodes[]): void { + installedNodes.forEach((installedNode) => { + delete this.loaded.nodes[installedNode.type]; + }); } - /** - * Retrieves `categories`, `subcategories`, partial `resources` and - * alias (if defined) from the codex data for the node at the given file path. - * - * @param {string} filePath The file path to a `*.node.js` file - */ - getCodex(filePath: string): CodexData { - // eslint-disable-next-line global-require, import/no-dynamic-require, @typescript-eslint/no-var-requires - const { categories, subcategories, resources: allResources, alias } = require(`${filePath}on`); // .js to .json - - const resources = pick(allResources, ['primaryDocumentation', 'credentialDocumentation']); - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return { - ...(categories && { categories }), - ...(subcategories && { subcategories }), - ...(resources && { resources }), - ...(alias && { alias }), - }; + private attachNodesToNodeTypes(installedNodes: InstalledNodes[]): void { + const loadedNodes = this.loaded.nodes; + installedNodes.forEach((installedNode) => { + const { type, sourcePath } = loadedNodes[installedNode.type]; + loadedNodes[installedNode.type] = { type, sourcePath }; + }); } /** - * Adds a node codex `categories` and `subcategories` (if defined) - * to a node description `codex` property. - * - * @param obj.node Node to add categories to - * @param obj.filePath Path to the built node - * @param obj.isCustom Whether the node is custom + * Run a loader of source files of nodes and credentials in a directory. */ - addCodex({ - node, - filePath, - isCustom, - }: { - node: INodeType | IVersionedNodeType; - filePath: string; - isCustom: boolean; - }) { - try { - const codex = this.getCodex(filePath); - - if (isCustom) { - codex.categories = codex.categories - ? codex.categories.concat(CUSTOM_NODES_CATEGORY) - : [CUSTOM_NODES_CATEGORY]; - } - - node.description.codex = codex; - } catch (_) { - this.logger.debug(`No codex available for: ${filePath.split('/').pop() ?? ''}`); - - if (isCustom) { - node.description.codex = { - categories: [CUSTOM_NODES_CATEGORY], - }; + private async runDirectoryLoader( + constructor: new (...args: ConstructorParameters) => T, + dir: string, + ) { + const loader = new constructor(dir, this.excludeNodes, this.includeNodes); + await loader.loadAll(); + + // list of node & credential types that will be sent to the frontend + const { types } = loader; + this.types.nodes = this.types.nodes.concat(types.nodes); + this.types.credentials = this.types.credentials.concat(types.credentials); + + // Copy over all icons and set `iconUrl` for the frontend + const iconPromises: Array> = []; + for (const node of types.nodes) { + if (node.icon?.startsWith('file:')) { + const icon = node.icon.substring(5); + const iconUrl = `icons/nodes/${node.name}${path.extname(icon)}`; + delete node.icon; + node.iconUrl = iconUrl; + iconPromises.push(copyFile(path.join(dir, icon), path.join(GENERATED_STATIC_DIR, iconUrl))); } } - } - - /** - * Loads nodes and credentials from the given directory - * - * @param {string} setPackageName The package name to set for the found nodes - * @param {string} directory The directory to look in - */ - async loadDataFromDirectory(setPackageName: string, directory: string): Promise { - const files = await glob('**/*.@(node|credentials).js', { - cwd: directory, - absolute: true, - }); - - for (const filePath of files) { - const [fileName, type] = path.parse(filePath).name.split('.'); - - if (type === 'node') { - this.loadNodeFromFile(setPackageName, fileName, filePath); - } else if (type === 'credentials') { - this.loadCredentialsFromFile(fileName, filePath); + for (const credential of types.credentials) { + if (credential.icon?.startsWith('file:')) { + const icon = credential.icon.substring(5); + const iconUrl = `icons/credentials/${credential.name}${path.extname(icon)}`; + delete credential.icon; + credential.iconUrl = iconUrl; + iconPromises.push(copyFile(path.join(dir, icon), path.join(GENERATED_STATIC_DIR, iconUrl))); } } - } + await Promise.all(iconPromises); - async readPackageJson(packagePath: string): Promise { - // Get the absolute path of the package - const packageFileString = await fsReadFile(path.join(packagePath, 'package.json'), 'utf8'); - return jsonParse(packageFileString); - } + // Nodes and credentials that have been loaded immediately + for (const nodeTypeName in loader.nodeTypes) { + this.loaded.nodes[nodeTypeName] = loader.nodeTypes[nodeTypeName]; + } - /** - * Loads nodes and credentials from the package with the given name - * - * @param {string} packagePath The path to read data from - */ - async loadDataFromPackage(packagePath: string): Promise { - // Get the absolute path of the package - const packageFile = await this.readPackageJson(packagePath); - if (!packageFile.n8n) { - return []; + for (const credentialTypeName in loader.credentialTypes) { + this.loaded.credentials[credentialTypeName] = loader.credentialTypes[credentialTypeName]; } - const packageName = packageFile.name; - const { nodes, credentials } = packageFile.n8n; - const returnData: INodeTypeNameVersion[] = []; - - // Read all node types - if (Array.isArray(nodes)) { - for (const filePath of nodes) { - const tempPath = path.join(packagePath, filePath); - const [fileName] = path.parse(filePath).name.split('.'); - const loadData = this.loadNodeFromFile(packageName, fileName, tempPath); - if (loadData) { - returnData.push(loadData); - } + // Nodes and credentials that will be lazy loaded + if (loader instanceof LazyPackageDirectoryLoader) { + const { packageName, known } = loader; + + for (const type in known.nodes) { + const { className, sourcePath } = known.nodes[type]; + this.known.nodes[`${packageName}.${type}`] = { + className, + sourcePath: path.join(dir, sourcePath), + }; } - } - // Read all credential types - if (Array.isArray(credentials)) { - for (const filePath of credentials) { - const tempPath = path.join(packagePath, filePath); - const [fileName] = path.parse(filePath).name.split('.'); - this.loadCredentialsFromFile(fileName, tempPath); + for (const type in known.credentials) { + const { className, sourcePath } = known.credentials[type]; + this.known.credentials[type] = { className, sourcePath: path.join(dir, sourcePath) }; } } - return returnData; + return loader; } - unloadNodes(installedNodes: InstalledNodes[]): void { - const nodeTypes = NodeTypes(); - installedNodes.forEach((installedNode) => { - nodeTypes.removeNodeType(installedNode.type); - delete this.nodeTypes[installedNode.type]; - }); - } - - attachNodesToNodeTypes(installedNodes: InstalledNodes[]): void { - const nodeTypes = NodeTypes(); - installedNodes.forEach((installedNode) => { - nodeTypes.attachNodeType( - installedNode.type, - this.nodeTypes[installedNode.type].type, - this.nodeTypes[installedNode.type].sourcePath, - ); - }); + private async getNodeModulesPath(): Promise { + // Get the path to the node-modules folder to be later able + // to load the credentials and nodes + const checkPaths = [ + // In case "n8n" package is in same node_modules folder. + path.join(CLI_DIR, '..', 'n8n-workflow'), + // In case "n8n" package is the root and the packages are + // in the "node_modules" folder underneath it. + path.join(CLI_DIR, 'node_modules', 'n8n-workflow'), + // In case "n8n" package is installed using npm/yarn workspaces + // the node_modules folder is in the root of the workspace. + path.join(CLI_DIR, '..', '..', 'node_modules', 'n8n-workflow'), + ]; + for (const checkPath of checkPaths) { + try { + await fsAccess(checkPath); + // Folder exists, so use it. + return path.dirname(checkPath); + } catch (_) {} // Folder does not exist so get next one + } + throw new Error('Could not find "node_modules" folder!'); } } let packagesInformationInstance: LoadNodesAndCredentialsClass | undefined; +// eslint-disable-next-line @typescript-eslint/naming-convention export function LoadNodesAndCredentials(): LoadNodesAndCredentialsClass { if (packagesInformationInstance === undefined) { packagesInformationInstance = new LoadNodesAndCredentialsClass(); diff --git a/packages/cli/src/NodeTypes.ts b/packages/cli/src/NodeTypes.ts index 272ba15fb254c..cb43074fdb2d0 100644 --- a/packages/cli/src/NodeTypes.ts +++ b/packages/cli/src/NodeTypes.ts @@ -1,36 +1,28 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { +import { loadClassInIsolation } from 'n8n-core'; +import type { + INodesAndCredentials, INodeType, - INodeTypeData, INodeTypeDescription, INodeTypes, IVersionedNodeType, - NodeHelpers, + LoadedClass, } from 'n8n-workflow'; +import { NodeHelpers } from 'n8n-workflow'; +import { RESPONSE_ERROR_MESSAGES } from './constants'; class NodeTypesClass implements INodeTypes { - nodeTypes: INodeTypeData = {}; - - async init(nodeTypes: INodeTypeData): Promise { + constructor(private nodesAndCredentials: INodesAndCredentials) { // Some nodeTypes need to get special parameters applied like the // polling nodes the polling times // eslint-disable-next-line no-restricted-syntax - for (const nodeTypeData of Object.values(nodeTypes)) { + for (const nodeTypeData of Object.values(this.loadedNodes)) { const nodeType = NodeHelpers.getVersionedNodeType(nodeTypeData.type); - const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeType); - - if (applyParameters.length) { - nodeType.description.properties.unshift(...applyParameters); - } + this.applySpecialNodeParameters(nodeType); } - this.nodeTypes = nodeTypes; } getAll(): Array { - return Object.values(this.nodeTypes).map((data) => data.type); + return Object.values(this.loadedNodes).map(({ type }) => type); } /** @@ -40,7 +32,7 @@ class NodeTypesClass implements INodeTypes { nodeTypeName: string, version: number, ): { description: INodeTypeDescription } & { sourcePath: string } { - const nodeType = this.nodeTypes[nodeTypeName]; + const nodeType = this.getNode(nodeTypeName); if (!nodeType) { throw new Error(`Unknown node type: ${nodeTypeName}`); @@ -52,34 +44,52 @@ class NodeTypesClass implements INodeTypes { } getByNameAndVersion(nodeType: string, version?: number): INodeType { - if (this.nodeTypes[nodeType] === undefined) { - throw new Error(`The node-type "${nodeType}" is not known!`); + return NodeHelpers.getVersionedNodeType(this.getNode(nodeType).type, version); + } + + private getNode(type: string): LoadedClass { + const loadedNodes = this.loadedNodes; + if (type in loadedNodes) { + return loadedNodes[type]; + } + + const knownNodes = this.knownNodes; + if (type in knownNodes) { + const { className, sourcePath } = knownNodes[type]; + const loaded: INodeType = loadClassInIsolation(sourcePath, className); + this.applySpecialNodeParameters(loaded); + loadedNodes[type] = { sourcePath, type: loaded }; + return loadedNodes[type]; } - return NodeHelpers.getVersionedNodeType(this.nodeTypes[nodeType].type, version); + throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_NODE}: ${type}`); } - attachNodeType( - nodeTypeName: string, - nodeType: INodeType | IVersionedNodeType, - sourcePath: string, - ): void { - this.nodeTypes[nodeTypeName] = { - type: nodeType, - sourcePath, - }; + private applySpecialNodeParameters(nodeType: INodeType) { + const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeType); + if (applyParameters.length) { + nodeType.description.properties.unshift(...applyParameters); + } } - removeNodeType(nodeType: string): void { - delete this.nodeTypes[nodeType]; + private get loadedNodes() { + return this.nodesAndCredentials.loaded.nodes; + } + + private get knownNodes() { + return this.nodesAndCredentials.known.nodes; } } let nodeTypesInstance: NodeTypesClass | undefined; // eslint-disable-next-line @typescript-eslint/naming-convention -export function NodeTypes(): NodeTypesClass { - if (nodeTypesInstance === undefined) { - nodeTypesInstance = new NodeTypesClass(); +export function NodeTypes(nodesAndCredentials?: INodesAndCredentials): NodeTypesClass { + if (!nodeTypesInstance) { + if (nodesAndCredentials) { + nodeTypesInstance = new NodeTypesClass(nodesAndCredentials); + } else { + throw new Error('NodeTypes not initialized yet'); + } } return nodeTypesInstance; diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 5e21982f931ac..b3346a8b053d3 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -9,7 +9,6 @@ /* eslint-disable no-return-assign */ /* eslint-disable no-param-reassign */ /* eslint-disable consistent-return */ -/* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable id-denylist */ @@ -29,7 +28,7 @@ /* eslint-disable no-await-in-loop */ import { exec as callbackExec } from 'child_process'; -import { existsSync, readFileSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; import { access as fsAccess, readFile, writeFile, mkdir } from 'fs/promises'; import os from 'os'; import { dirname as pathDirname, join as pathJoin, resolve as pathResolve } from 'path'; @@ -38,7 +37,6 @@ import { promisify } from 'util'; import cookieParser from 'cookie-parser'; import express from 'express'; import { FindManyOptions, getConnectionManager, In } from 'typeorm'; -// eslint-disable-next-line import/no-extraneous-dependencies import axios, { AxiosRequestConfig } from 'axios'; import clientOAuth1, { RequestOptions } from 'oauth-1.0a'; // IMPORTANT! Do not switch to anther bcrypt library unless really necessary and @@ -54,22 +52,20 @@ import { } from 'n8n-core'; import { - ICredentialType, INodeCredentials, INodeCredentialsDetails, INodeListSearchResult, INodeParameters, INodePropertyOptions, - INodeType, - INodeTypeDescription, INodeTypeNameVersion, ITelemetrySettings, LoggerProxy, - NodeHelpers, jsonParse, WebhookHttpMethod, WorkflowExecuteMode, ErrorReporterProxy as ErrorReporter, + INodeTypes, + ICredentialTypes, } from 'n8n-workflow'; import basicAuth from 'basic-auth'; @@ -95,6 +91,7 @@ import { nodesController } from '@/api/nodes.api'; import { workflowsController } from '@/workflows/workflows.controller'; import { AUTH_COOKIE_NAME, + GENERATED_STATIC_DIR, NODES_BASE_DIR, RESPONSE_ERROR_MESSAGES, TEMPLATES_DIR, @@ -151,6 +148,7 @@ import { ExternalHooks } from '@/ExternalHooks'; import * as GenericHelpers from '@/GenericHelpers'; import { NodeTypes } from '@/NodeTypes'; import * as Push from '@/Push'; +import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; import * as ResponseHelper from '@/ResponseHelper'; import * as TestWebhooks from '@/TestWebhooks'; import { WaitTracker, WaitTrackerClass } from '@/WaitTracker'; @@ -226,6 +224,10 @@ class App { webhookMethods: WebhookHttpMethod[]; + nodeTypes: INodeTypes; + + credentialTypes: ICredentialTypes; + constructor() { this.app = express(); this.app.disable('x-powered-by'); @@ -251,6 +253,9 @@ class App { this.testWebhooks = TestWebhooks.getInstance(); this.push = Push.getInstance(); + this.nodeTypes = NodeTypes(); + this.credentialTypes = CredentialTypes(); + this.activeExecutionsInstance = ActiveExecutions.getInstance(); this.waitTracker = WaitTracker(); @@ -424,6 +429,8 @@ class App { 'assets', 'healthz', 'metrics', + 'icons', + 'types', this.endpointWebhook, this.endpointWebhookTest, this.endpointPresetCredentials, @@ -824,7 +831,7 @@ class App { const loadDataInstance = new LoadNodeParameterOptions( nodeTypeAndVersion, - NodeTypes(), + this.nodeTypes, path, currentNodeParameters, credentials, @@ -885,7 +892,7 @@ class App { const listSearchInstance = new LoadNodeListSearch( nodeTypeAndVersion, - NodeTypes(), + this.nodeTypes, path, currentNodeParameters, credentials, @@ -910,47 +917,6 @@ class App { ), ); - // Returns all the node-types - this.app.get( - `/${this.restEndpoint}/node-types`, - ResponseHelper.send( - async (req: express.Request, res: express.Response): Promise => { - const returnData: INodeTypeDescription[] = []; - const onlyLatest = req.query.onlyLatest === 'true'; - - const nodeTypes = NodeTypes(); - const allNodes = nodeTypes.getAll(); - - const getNodeDescription = (nodeType: INodeType): INodeTypeDescription => { - const nodeInfo: INodeTypeDescription = { ...nodeType.description }; - if (req.query.includeProperties !== 'true') { - // @ts-ignore - delete nodeInfo.properties; - } - return nodeInfo; - }; - - if (onlyLatest) { - allNodes.forEach((nodeData) => { - const nodeType = NodeHelpers.getVersionedNodeType(nodeData); - const nodeInfo: INodeTypeDescription = getNodeDescription(nodeType); - returnData.push(nodeInfo); - }); - } else { - allNodes.forEach((nodeData) => { - const allNodeTypes = NodeHelpers.getVersionedNodeTypeAll(nodeData); - allNodeTypes.forEach((element) => { - const nodeInfo: INodeTypeDescription = getNodeDescription(element); - returnData.push(nodeInfo); - }); - }); - } - - return returnData; - }, - ), - ); - this.app.get( `/${this.restEndpoint}/credential-translation`, ResponseHelper.send( @@ -999,49 +965,6 @@ class App { this.app.use(`/${this.restEndpoint}/node-types`, nodeTypesController); - // Returns the node icon - this.app.get( - [ - `/${this.restEndpoint}/node-icon/:nodeType`, - `/${this.restEndpoint}/node-icon/:scope/:nodeType`, - ], - async (req: express.Request, res: express.Response): Promise => { - try { - const nodeTypeName = `${req.params.scope ? `${req.params.scope}/` : ''}${ - req.params.nodeType - }`; - - const nodeTypes = NodeTypes(); - const nodeType = nodeTypes.getByNameAndVersion(nodeTypeName); - - if (nodeType === undefined) { - res.status(404).send('The nodeType is not known.'); - return; - } - - if (nodeType.description.icon === undefined) { - res.status(404).send('No icon found for node.'); - return; - } - - if (!nodeType.description.icon.startsWith('file:')) { - res.status(404).send('Node does not have a file icon.'); - return; - } - - const filepath = nodeType.description.icon.substr(5); - - const maxAge = 7 * 24 * 60 * 60 * 1000; // 7 days - res.setHeader('Cache-control', `private max-age=${maxAge}`); - - res.sendFile(filepath); - } catch (error) { - // Error response - return ResponseHelper.sendErrorResponse(res, error); - } - }, - ); - // ---------------------------------------- // Active Workflows // ---------------------------------------- @@ -1107,63 +1030,6 @@ class App { }, ), ); - // ---------------------------------------- - // Credential-Types - // ---------------------------------------- - - // Returns all the credential types which are defined in the loaded n8n-modules - this.app.get( - `/${this.restEndpoint}/credential-types`, - ResponseHelper.send( - async (req: express.Request, res: express.Response): Promise => { - const returnData: ICredentialType[] = []; - - const credentialTypes = CredentialTypes(); - - credentialTypes.getAll().forEach((credentialData) => { - returnData.push(credentialData); - }); - - return returnData; - }, - ), - ); - - this.app.get( - `/${this.restEndpoint}/credential-icon/:credentialType`, - async (req: express.Request, res: express.Response): Promise => { - try { - const credentialName = req.params.credentialType; - - const credentialType = CredentialTypes().getByName(credentialName); - - if (credentialType === undefined) { - res.status(404).send('The credentialType is not known.'); - return; - } - - if (credentialType.icon === undefined) { - res.status(404).send('No icon found for credential.'); - return; - } - - if (!credentialType.icon.startsWith('file:')) { - res.status(404).send('Credential does not have a file icon.'); - return; - } - - const filepath = credentialType.icon.substr(5); - - const maxAge = 7 * 24 * 60 * 60 * 1000; // 7 days - res.setHeader('Cache-control', `private max-age=${maxAge}`); - - res.sendFile(filepath); - } catch (error) { - // Error response - return ResponseHelper.sendErrorResponse(res, error); - } - }, - ); // ---------------------------------------- // OAuth1-Credential/Auth @@ -1750,9 +1616,9 @@ class App { return; } - const credentialsOverwrites = CredentialsOverwrites(); + CredentialsOverwrites().setData(body); - await credentialsOverwrites.init(body); + await LoadNodesAndCredentials().generateTypesForFrontend(); this.presetCredentialsLoaded = true; @@ -1792,7 +1658,6 @@ class App { } const editorUiDistDir = pathJoin(pathDirname(require.resolve('n8n-editor-ui')), 'dist'); - const generatedStaticDir = pathJoin(UserSettings.getUserHome(), '.cache/n8n/public'); const closingTitleTag = ''; const compileFile = async (fileName: string) => { @@ -1805,7 +1670,7 @@ class App { if (filePath.endsWith('index.html')) { payload = payload.replace(closingTitleTag, closingTitleTag + scriptsString); } - const destFile = pathJoin(generatedStaticDir, fileName); + const destFile = pathJoin(GENERATED_STATIC_DIR, fileName); await mkdir(pathDirname(destFile), { recursive: true }); await writeFile(destFile, payload, 'utf-8'); } @@ -1815,13 +1680,15 @@ class App { const files = await glob('**/*.{css,js}', { cwd: editorUiDistDir }); await Promise.all(files.map(compileFile)); - this.app.use('/', express.static(generatedStaticDir), express.static(editorUiDistDir)); + this.app.use('/', express.static(GENERATED_STATIC_DIR), express.static(editorUiDistDir)); const startTime = new Date().toUTCString(); this.app.use('/index.html', (req, res, next) => { res.setHeader('Last-Modified', startTime); next(); }); + } else { + this.app.use('/', express.static(GENERATED_STATIC_DIR)); } } } diff --git a/packages/cli/src/UserManagement/Interfaces.ts b/packages/cli/src/UserManagement/Interfaces.ts index cf1c5307b8533..68d93b1c78f60 100644 --- a/packages/cli/src/UserManagement/Interfaces.ts +++ b/packages/cli/src/UserManagement/Interfaces.ts @@ -1,5 +1,4 @@ import { Application } from 'express'; -import { JwtFromRequestFunction } from 'passport-jwt'; import type { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner'; import type { IExternalHooksClass, IPersonalizationSurveyAnswers } from '@/Interfaces'; @@ -8,11 +7,6 @@ export interface JwtToken { expiresIn: number; } -export interface JwtOptions { - secretOrKey: string; - jwtFromRequest: JwtFromRequestFunction; -} - export interface JwtPayload { id: string; email: string | null; diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index f1b4d02744bb9..c2c74ae8a8f7c 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -14,8 +14,7 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable prefer-destructuring */ import express from 'express'; -// eslint-disable-next-line import/no-extraneous-dependencies -import { get } from 'lodash'; +import get from 'lodash.get'; import { BINARY_ENCODING, BinaryDataManager, NodeExecuteFunctions } from 'n8n-core'; diff --git a/packages/cli/src/WebhookServer.ts b/packages/cli/src/WebhookServer.ts index b8580b59ea306..5fbfbaa3e9874 100644 --- a/packages/cli/src/WebhookServer.ts +++ b/packages/cli/src/WebhookServer.ts @@ -13,7 +13,6 @@ import { getConnectionManager } from 'typeorm'; import bodyParser from 'body-parser'; import compression from 'compression'; -// eslint-disable-next-line import/no-extraneous-dependencies import parseUrl from 'parseurl'; import { WebhookHttpMethod } from 'n8n-workflow'; diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 83da7da11ae3e..abcc58306947f 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -880,10 +880,8 @@ export async function getWorkflowData( /** * Executes the workflow with the given ID - * - * @param {string} workflowId The id of the workflow to execute */ -export async function executeWorkflow( +async function executeWorkflow( workflowInfo: IExecuteWorkflowInfo, additionalData: IWorkflowExecuteAdditionalData, options?: { @@ -1111,7 +1109,7 @@ export async function getBase( * Returns WorkflowHooks instance for running integrated workflows * (Workflows which get started inside of another workflow) */ -export function getWorkflowHooksIntegrated( +function getWorkflowHooksIntegrated( mode: WorkflowExecuteMode, executionId: string, workflowData: IWorkflowBase, diff --git a/packages/cli/src/WorkflowHelpers.ts b/packages/cli/src/WorkflowHelpers.ts index f7902f492af1d..d0498e11febb1 100644 --- a/packages/cli/src/WorkflowHelpers.ts +++ b/packages/cli/src/WorkflowHelpers.ts @@ -1,13 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -/* eslint-disable no-underscore-dangle */ -/* eslint-disable no-continue */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable no-restricted-syntax */ -/* eslint-disable no-param-reassign */ import { In } from 'typeorm'; import { IDataObject, @@ -25,15 +15,8 @@ import { WorkflowExecuteMode, } from 'n8n-workflow'; import { v4 as uuid } from 'uuid'; -import { CredentialTypes } from '@/CredentialTypes'; import * as Db from '@/Db'; -import { - ICredentialsDb, - ICredentialsTypeData, - ITransferNodeTypes, - IWorkflowErrorData, - IWorkflowExecutionDataProcess, -} from '@/Interfaces'; +import { ICredentialsDb, IWorkflowErrorData, IWorkflowExecutionDataProcess } from '@/Interfaces'; import { NodeTypes } from '@/NodeTypes'; import { WorkflowRunner } from '@/WorkflowRunner'; @@ -183,6 +166,7 @@ export async function executeErrorWorkflow( if (workflowStartNode === undefined) { Logger.error( + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions `Calling Error Workflow for "${workflowErrorData.workflow.id}". Could not find "${ERROR_TRIGGER_TYPE}" in workflow "${workflowId}"`, ); return; @@ -231,170 +215,15 @@ export async function executeErrorWorkflow( } catch (error) { ErrorReporter.error(error); Logger.error( + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions, @typescript-eslint/no-unsafe-member-access `Calling Error Workflow for "${workflowErrorData.workflow.id}": "${error.message}"`, { workflowId: workflowErrorData.workflow.id }, ); } } -/** - * Returns all the defined NodeTypes - * - */ -export function getAllNodeTypeData(): ITransferNodeTypes { - const nodeTypes = NodeTypes(); - - // Get the data of all the node types that they - // can be loaded again in the process - const returnData: ITransferNodeTypes = {}; - for (const nodeTypeName of Object.keys(nodeTypes.nodeTypes)) { - if (nodeTypes.nodeTypes[nodeTypeName] === undefined) { - throw new Error(`The NodeType "${nodeTypeName}" could not be found!`); - } - - returnData[nodeTypeName] = { - className: nodeTypes.nodeTypes[nodeTypeName].type.constructor.name, - sourcePath: nodeTypes.nodeTypes[nodeTypeName].sourcePath, - }; - } - - return returnData; -} - -/** - * Returns all the defined CredentialTypes - * - */ -export function getAllCredentalsTypeData(): ICredentialsTypeData { - const credentialTypes = CredentialTypes(); - - // Get the data of all the credential types that they - // can be loaded again in the subprocess - const returnData: ICredentialsTypeData = {}; - for (const credentialTypeName of Object.keys(credentialTypes.credentialTypes)) { - if (credentialTypes.credentialTypes[credentialTypeName] === undefined) { - throw new Error(`The CredentialType "${credentialTypeName}" could not be found!`); - } - - returnData[credentialTypeName] = { - className: credentialTypes.credentialTypes[credentialTypeName].type.constructor.name, - sourcePath: credentialTypes.credentialTypes[credentialTypeName].sourcePath, - }; - } - - return returnData; -} - -/** - * Returns the data of the node types that are needed - * to execute the given nodes - * - */ -export function getNodeTypeData(nodes: INode[]): ITransferNodeTypes { - const nodeTypes = NodeTypes(); - - // Check which node-types have to be loaded - // eslint-disable-next-line @typescript-eslint/no-use-before-define - const neededNodeTypes = getNeededNodeTypes(nodes); - - // Get all the data of the needed node types that they - // can be loaded again in the process - const returnData: ITransferNodeTypes = {}; - for (const nodeTypeName of neededNodeTypes) { - if (nodeTypes.nodeTypes[nodeTypeName.type] === undefined) { - throw new Error(`The NodeType "${nodeTypeName.type}" could not be found!`); - } - - returnData[nodeTypeName.type] = { - className: nodeTypes.nodeTypes[nodeTypeName.type].type.constructor.name, - sourcePath: nodeTypes.nodeTypes[nodeTypeName.type].sourcePath, - }; - } - - return returnData; -} - -/** - * Returns the credentials data of the given type and its parent types - * it extends - * - * @param {string} type The credential type to return data off - */ -export function getCredentialsDataWithParents(type: string): ICredentialsTypeData { - const credentialTypes = CredentialTypes(); - const credentialType = credentialTypes.getByName(type); - - const credentialTypeData: ICredentialsTypeData = {}; - credentialTypeData[type] = { - className: credentialTypes.credentialTypes[type].type.constructor.name, - sourcePath: credentialTypes.credentialTypes[type].sourcePath, - }; - - if (credentialType === undefined || credentialType.extends === undefined) { - return credentialTypeData; - } - - for (const typeName of credentialType.extends) { - if (credentialTypeData[typeName] !== undefined) { - continue; - } - - credentialTypeData[typeName] = { - className: credentialTypes.credentialTypes[typeName].type.constructor.name, - sourcePath: credentialTypes.credentialTypes[typeName].sourcePath, - }; - Object.assign(credentialTypeData, getCredentialsDataWithParents(typeName)); - } - - return credentialTypeData; -} - -/** - * Returns all the credentialTypes which are needed to resolve - * the given workflow credentials - * - * @param {IWorkflowCredentials} credentials The credentials which have to be able to be resolved - */ -export function getCredentialsDataByNodes(nodes: INode[]): ICredentialsTypeData { - const credentialTypeData: ICredentialsTypeData = {}; - - for (const node of nodes) { - const credentialsUsedByThisNode = node.credentials; - if (credentialsUsedByThisNode) { - // const credentialTypesUsedByThisNode = Object.keys(credentialsUsedByThisNode!); - for (const credentialType of Object.keys(credentialsUsedByThisNode)) { - if (credentialTypeData[credentialType] !== undefined) { - continue; - } - - Object.assign(credentialTypeData, getCredentialsDataWithParents(credentialType)); - } - } - } - - return credentialTypeData; -} - -/** - * Returns the names of the NodeTypes which are are needed - * to execute the gives nodes - * - */ -export function getNeededNodeTypes(nodes: INode[]): Array<{ type: string; version: number }> { - // Check which node-types have to be loaded - const neededNodeTypes: Array<{ type: string; version: number }> = []; - for (const node of nodes) { - if (neededNodeTypes.find((neededNodes) => node.type === neededNodes.type) === undefined) { - neededNodeTypes.push({ type: node.type, version: node.typeVersion }); - } - } - - return neededNodeTypes; -} - /** * Saves the static data if it changed - * */ export async function saveStaticData(workflow: Workflow): Promise { if (workflow.staticData.__dataChanged === true) { @@ -402,12 +231,13 @@ export async function saveStaticData(workflow: Workflow): Promise { if (isWorkflowIdValid(workflow.id)) { // Workflow is saved so update in database try { - // eslint-disable-next-line @typescript-eslint/no-use-before-define + // eslint-disable-next-line @typescript-eslint/no-use-before-define, @typescript-eslint/no-non-null-assertion await saveStaticDataById(workflow.id!, workflow.staticData); workflow.staticData.__dataChanged = false; } catch (error) { ErrorReporter.error(error); Logger.error( + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions, @typescript-eslint/no-unsafe-member-access `There was a problem saving the workflow with id "${workflow.id}" to save changed staticData: "${error.message}"`, { workflowId: workflow.id }, ); @@ -452,7 +282,6 @@ export async function getStaticDataById(workflowId: string | number) { /** * Set node ids if not already set - * */ export function addNodeIds(workflow: WorkflowEntity) { const { nodes } = workflow; diff --git a/packages/cli/src/WorkflowRunner.ts b/packages/cli/src/WorkflowRunner.ts index 5dc546bf25c7d..0568ab2fbe0fa 100644 --- a/packages/cli/src/WorkflowRunner.ts +++ b/packages/cli/src/WorkflowRunner.ts @@ -26,22 +26,17 @@ import { WorkflowOperationError, } from 'n8n-workflow'; -// eslint-disable-next-line import/no-extraneous-dependencies import PCancelable from 'p-cancelable'; import { join as pathJoin } from 'path'; import { fork } from 'child_process'; import * as ActiveExecutions from '@/ActiveExecutions'; import config from '@/config'; -import { CredentialsOverwrites } from '@/CredentialsOverwrites'; import * as Db from '@/Db'; import { ExternalHooks } from '@/ExternalHooks'; import { - ICredentialsOverwrite, - ICredentialsTypeData, IExecutionFlattedDb, IProcessMessageDataHook, - ITransferNodeTypes, IWorkflowExecutionDataProcess, IWorkflowExecutionDataProcessWithExecution, } from '@/Interfaces'; @@ -60,8 +55,6 @@ import { PermissionChecker } from '@/UserManagement/PermissionChecker'; export class WorkflowRunner { activeExecutions: ActiveExecutions.ActiveExecutions; - credentialsOverwrites: ICredentialsOverwrite; - push: Push.Push; jobQueue: Queue.JobQueue; @@ -69,7 +62,6 @@ export class WorkflowRunner { constructor() { this.push = Push.getInstance(); this.activeExecutions = ActiveExecutions.getInstance(); - this.credentialsOverwrites = CredentialsOverwrites().getAll(); const executionsMode = config.getEnv('executions.mode'); @@ -618,43 +610,7 @@ export class WorkflowRunner { // Register the active execution const executionId = await this.activeExecutions.add(data, subprocess, restartExecutionId); - // Check if workflow contains a "executeWorkflow" Node as in this - // case we can not know which nodeTypes and credentialTypes will - // be needed and so have to load all of them in the workflowRunnerProcess - let loadAllNodeTypes = false; - for (const node of data.workflowData.nodes) { - if (node.type === 'n8n-nodes-base.executeWorkflow' && node.disabled !== true) { - loadAllNodeTypes = true; - break; - } - } - let nodeTypeData: ITransferNodeTypes; - let credentialTypeData: ICredentialsTypeData; - // eslint-disable-next-line prefer-destructuring - let credentialsOverwrites = this.credentialsOverwrites; - if (loadAllNodeTypes) { - // Supply all nodeTypes and credentialTypes - nodeTypeData = WorkflowHelpers.getAllNodeTypeData(); - credentialTypeData = WorkflowHelpers.getAllCredentalsTypeData(); - } else { - // Supply only nodeTypes, credentialTypes and overwrites that the workflow needs - nodeTypeData = WorkflowHelpers.getNodeTypeData(data.workflowData.nodes); - credentialTypeData = WorkflowHelpers.getCredentialsDataByNodes(data.workflowData.nodes); - - credentialsOverwrites = {}; - for (const credentialName of Object.keys(credentialTypeData)) { - if (this.credentialsOverwrites[credentialName] !== undefined) { - credentialsOverwrites[credentialName] = this.credentialsOverwrites[credentialName]; - } - } - } - (data as unknown as IWorkflowExecutionDataProcessWithExecution).executionId = executionId; - (data as unknown as IWorkflowExecutionDataProcessWithExecution).nodeTypeData = nodeTypeData; - (data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsOverwrite = - this.credentialsOverwrites; - (data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsTypeData = - credentialTypeData; const workflowHooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId); diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index 8271bee426240..f22a2d0d45592 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -11,15 +11,11 @@ import { BinaryDataManager, IProcessMessage, UserSettings, WorkflowExecute } fro import { ErrorReporterProxy as ErrorReporter, ExecutionError, - ICredentialType, - ICredentialTypeData, IDataObject, IExecuteResponsePromiseData, IExecuteWorkflowInfo, ILogger, INodeExecutionData, - INodeType, - INodeTypeData, IRun, ITaskData, IWorkflowExecuteAdditionalData, @@ -39,6 +35,7 @@ import { ExternalHooks } from '@/ExternalHooks'; import * as GenericHelpers from '@/GenericHelpers'; import { IWorkflowExecuteProcess, IWorkflowExecutionDataProcessWithExecution } from '@/Interfaces'; import { NodeTypes } from '@/NodeTypes'; +import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials'; import * as WebhookHelpers from '@/WebhookHelpers'; import * as WorkflowHelpers from '@/WorkflowHelpers'; import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; @@ -46,12 +43,11 @@ import { getLogger } from '@/Logger'; import config from '@/config'; import { InternalHooksManager } from '@/InternalHooksManager'; -import { loadClassInIsolation } from '@/CommunityNodes/helpers'; import { generateFailedExecutionFromError } from '@/WorkflowHelpers'; import { initErrorHandling } from '@/ErrorReporting'; import { PermissionChecker } from '@/UserManagement/PermissionChecker'; -export class WorkflowRunnerProcess { +class WorkflowRunnerProcess { data: IWorkflowExecutionDataProcessWithExecution | undefined; logger: ILogger; @@ -99,54 +95,15 @@ export class WorkflowRunnerProcess { this.startedAt = new Date(); - // Load the required nodes - const nodeTypesData: INodeTypeData = {}; - // eslint-disable-next-line no-restricted-syntax - for (const nodeTypeName of Object.keys(this.data.nodeTypeData)) { - let tempNode: INodeType; - const { className, sourcePath } = this.data.nodeTypeData[nodeTypeName]; - - try { - tempNode = loadClassInIsolation(sourcePath, className); - } catch (error) { - throw new Error(`Error loading node "${nodeTypeName}" from: "${sourcePath}"`); - } - - nodeTypesData[nodeTypeName] = { - type: tempNode, - sourcePath, - }; - } - - const nodeTypes = NodeTypes(); - await nodeTypes.init(nodeTypesData); - - // Load the required credentials - const credentialsTypeData: ICredentialTypeData = {}; - // eslint-disable-next-line no-restricted-syntax - for (const credentialTypeName of Object.keys(this.data.credentialsTypeData)) { - let tempCredential: ICredentialType; - const { className, sourcePath } = this.data.credentialsTypeData[credentialTypeName]; - - try { - tempCredential = loadClassInIsolation(sourcePath, className); - } catch (error) { - throw new Error(`Error loading credential "${credentialTypeName}" from: "${sourcePath}"`); - } - - credentialsTypeData[credentialTypeName] = { - type: tempCredential, - sourcePath, - }; - } + const loadNodesAndCredentials = LoadNodesAndCredentials(); + await loadNodesAndCredentials.init(); - // Init credential types the workflow uses (is needed to apply default values to credentials) - const credentialTypes = CredentialTypes(); - await credentialTypes.init(credentialsTypeData); + const nodeTypes = NodeTypes(loadNodesAndCredentials); + const credentialTypes = CredentialTypes(loadNodesAndCredentials); // Load the credentials overwrites if any exist - const credentialsOverwrites = CredentialsOverwrites(); - await credentialsOverwrites.init(inputData.credentialsOverwrite); + const credentialsOverwrites = CredentialsOverwrites(credentialTypes); + await credentialsOverwrites.init(); // Load all external hooks const externalHooks = ExternalHooks(); diff --git a/packages/cli/src/api/executions.api.ts b/packages/cli/src/api/executions.api.ts index 0940b5f36292e..8229e5557aafd 100644 --- a/packages/cli/src/api/executions.api.ts +++ b/packages/cli/src/api/executions.api.ts @@ -1,5 +1,4 @@ /* eslint-disable no-restricted-syntax */ -/* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ @@ -9,9 +8,9 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import express from 'express'; import { validate as jsonSchemaValidate } from 'jsonschema'; -import _, { cloneDeep } from 'lodash'; import { BinaryDataManager } from 'n8n-core'; import { + deepCopy, IDataObject, IWorkflowBase, JsonObject, @@ -259,7 +258,7 @@ executionsController.get( query = query.andWhere(filter); } - const countFilter = cloneDeep(filter ?? {}); + const countFilter = deepCopy(filter ?? {}); countFilter.id = Not(In(executingWorkflowIds)); const executions = await query.getMany(); diff --git a/packages/cli/src/api/nodeTypes.api.ts b/packages/cli/src/api/nodeTypes.api.ts index cec7676080271..1e0eb7c6227b4 100644 --- a/packages/cli/src/api/nodeTypes.api.ts +++ b/packages/cli/src/api/nodeTypes.api.ts @@ -1,15 +1,8 @@ -/* eslint-disable import/no-extraneous-dependencies */ import express from 'express'; import { readFile } from 'fs/promises'; -import _ from 'lodash'; +import get from 'lodash.get'; -import { - ICredentialType, - INodeType, - INodeTypeDescription, - INodeTypeNameVersion, - NodeHelpers, -} from 'n8n-workflow'; +import type { ICredentialType, INodeTypeDescription, INodeTypeNameVersion } from 'n8n-workflow'; import { CredentialTypes } from '@/CredentialTypes'; import config from '@/config'; @@ -74,50 +67,11 @@ function injectCustomApiCallOption(description: INodeTypeDescription) { export const nodeTypesController = express.Router(); -// Returns all the node-types -nodeTypesController.get( - '/', - ResponseHelper.send(async (req: express.Request): Promise => { - const returnData: INodeTypeDescription[] = []; - const onlyLatest = req.query.onlyLatest === 'true'; - - const nodeTypes = NodeTypes(); - const allNodes = nodeTypes.getAll(); - - const getNodeDescription = (nodeType: INodeType): INodeTypeDescription => { - const nodeInfo: INodeTypeDescription = { ...nodeType.description }; - if (req.query.includeProperties !== 'true') { - // @ts-ignore - delete nodeInfo.properties; - } - return nodeInfo; - }; - - if (onlyLatest) { - allNodes.forEach((nodeData) => { - const nodeType = NodeHelpers.getVersionedNodeType(nodeData); - const nodeInfo: INodeTypeDescription = getNodeDescription(nodeType); - returnData.push(nodeInfo); - }); - } else { - allNodes.forEach((nodeData) => { - const allNodeTypes = NodeHelpers.getVersionedNodeTypeAll(nodeData); - allNodeTypes.forEach((element) => { - const nodeInfo: INodeTypeDescription = getNodeDescription(element); - returnData.push(nodeInfo); - }); - }); - } - - return returnData; - }), -); - // Returns node information based on node names and versions nodeTypesController.post( '/', ResponseHelper.send(async (req: express.Request): Promise => { - const nodeInfos = _.get(req, 'body.nodeInfos', []) as INodeTypeNameVersion[]; + const nodeInfos = get(req, 'body.nodeInfos', []) as INodeTypeNameVersion[]; const defaultLocale = config.getEnv('defaultLocale'); diff --git a/packages/cli/src/commands/execute.ts b/packages/cli/src/commands/execute.ts index 1c6ee01f6526f..883c04d12bf4d 100644 --- a/packages/cli/src/commands/execute.ts +++ b/packages/cli/src/commands/execute.ts @@ -123,19 +123,19 @@ export class Execute extends Command { // Wait till the n8n-packages have been read await loadNodesAndCredentialsPromise; + NodeTypes(loadNodesAndCredentials); + const credentialTypes = CredentialTypes(loadNodesAndCredentials); + // Load the credentials overwrites if any exist - const credentialsOverwrites = CredentialsOverwrites(); - await credentialsOverwrites.init(); + await CredentialsOverwrites(credentialTypes).init(); // Load all external hooks const externalHooks = ExternalHooks(); await externalHooks.init(); // Add the found types to an instance other parts of the application can use - const nodeTypes = NodeTypes(); - await nodeTypes.init(loadNodesAndCredentials.nodeTypes); - const credentialTypes = CredentialTypes(); - await credentialTypes.init(loadNodesAndCredentials.credentialTypes); + const nodeTypes = NodeTypes(loadNodesAndCredentials); + CredentialTypes(loadNodesAndCredentials); const instanceId = await UserSettings.getInstanceId(); const { cli } = await GenericHelpers.getVersions(); diff --git a/packages/cli/src/commands/executeBatch.ts b/packages/cli/src/commands/executeBatch.ts index 26c9955a07703..960cf6912315a 100644 --- a/packages/cli/src/commands/executeBatch.ts +++ b/packages/cli/src/commands/executeBatch.ts @@ -17,8 +17,7 @@ import { sep } from 'path'; import { diff } from 'json-diff'; -// eslint-disable-next-line import/no-extraneous-dependencies -import { pick } from 'lodash'; +import pick from 'lodash.pick'; import { getLogger } from '@/Logger'; import * as ActiveExecutions from '@/ActiveExecutions'; @@ -312,18 +311,19 @@ export class ExecuteBatch extends Command { // Wait till the n8n-packages have been read await loadNodesAndCredentialsPromise; + NodeTypes(loadNodesAndCredentials); + const credentialTypes = CredentialTypes(loadNodesAndCredentials); + // Load the credentials overwrites if any exist - await CredentialsOverwrites().init(); + await CredentialsOverwrites(credentialTypes).init(); // Load all external hooks const externalHooks = ExternalHooks(); await externalHooks.init(); // Add the found types to an instance other parts of the application can use - const nodeTypes = NodeTypes(); - await nodeTypes.init(loadNodesAndCredentials.nodeTypes); - const credentialTypes = CredentialTypes(); - await credentialTypes.init(loadNodesAndCredentials.credentialTypes); + const nodeTypes = NodeTypes(loadNodesAndCredentials); + CredentialTypes(loadNodesAndCredentials); const instanceId = await UserSettings.getInstanceId(); const { cli } = await GenericHelpers.getVersions(); diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 92e3c71161886..fb2acba89007f 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -9,7 +9,6 @@ import localtunnel from 'localtunnel'; import { BinaryDataManager, TUNNEL_SUBDOMAIN_ENV, UserSettings } from 'n8n-core'; import { Command, flags } from '@oclif/command'; -// eslint-disable-next-line import/no-extraneous-dependencies import Redis from 'ioredis'; import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow'; @@ -115,7 +114,7 @@ export class Start extends Command { await InternalHooksManager.getInstance().onN8nStop(); const skipWebhookDeregistration = config.getEnv( - 'endpoints.skipWebhoooksDeregistrationOnShutdown', + 'endpoints.skipWebhooksDeregistrationOnShutdown', ); const removePromises = []; @@ -210,14 +209,13 @@ export class Start extends Command { await externalHooks.init(); // Add the found types to an instance other parts of the application can use - const nodeTypes = NodeTypes(); - await nodeTypes.init(loadNodesAndCredentials.nodeTypes); - const credentialTypes = CredentialTypes(); - await credentialTypes.init(loadNodesAndCredentials.credentialTypes); + const nodeTypes = NodeTypes(loadNodesAndCredentials); + const credentialTypes = CredentialTypes(loadNodesAndCredentials); // Load the credentials overwrites if any exist - const credentialsOverwrites = CredentialsOverwrites(); - await credentialsOverwrites.init(); + await CredentialsOverwrites(credentialTypes).init(); + + await loadNodesAndCredentials.generateTypesForFrontend(); // Wait till the database is ready await startDbInitPromise; @@ -227,13 +225,13 @@ export class Start extends Command { packageName: string; version: string; }>(); - installedPackages.forEach((installedpackage) => { - installedpackage.installedNodes.forEach((installedNode) => { - if (!loadNodesAndCredentials.nodeTypes[installedNode.type]) { + installedPackages.forEach((installedPackage) => { + installedPackage.installedNodes.forEach((installedNode) => { + if (!loadNodesAndCredentials.known.nodes[installedNode.type]) { // Leave the list ready for installing in case we need. missingPackages.add({ - packageName: installedpackage.packageName, - version: installedpackage.installedVersion, + packageName: installedPackage.packageName, + version: installedPackage.installedVersion, }); } }); diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index 8faf2f5192ea4..25cacb59d6917 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -6,7 +6,6 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { BinaryDataManager, UserSettings } from 'n8n-core'; import { Command, flags } from '@oclif/command'; -// eslint-disable-next-line import/no-extraneous-dependencies import Redis from 'ioredis'; import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow'; @@ -132,26 +131,23 @@ export class Webhook extends Command { // Make sure the settings exist // eslint-disable-next-line @typescript-eslint/no-unused-vars - const userSettings = await UserSettings.prepareUserSettings(); + await UserSettings.prepareUserSettings(); // Load all node and credential types const loadNodesAndCredentials = LoadNodesAndCredentials(); await loadNodesAndCredentials.init(); + // Add the found types to an instance other parts of the application can use + const nodeTypes = NodeTypes(loadNodesAndCredentials); + const credentialTypes = CredentialTypes(loadNodesAndCredentials); + // Load the credentials overwrites if any exist - const credentialsOverwrites = CredentialsOverwrites(); - await credentialsOverwrites.init(); + await CredentialsOverwrites(credentialTypes).init(); // Load all external hooks const externalHooks = ExternalHooks(); await externalHooks.init(); - // Add the found types to an instance other parts of the application can use - const nodeTypes = NodeTypes(); - await nodeTypes.init(loadNodesAndCredentials.nodeTypes); - const credentialTypes = CredentialTypes(); - await credentialTypes.init(loadNodesAndCredentials.credentialTypes); - // Wait till the database is ready await startDbInitPromise; diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index 7f3bd6e7ac18d..3be38ccb615e3 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -7,7 +7,6 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/no-unused-vars */ -// eslint-disable-next-line import/no-extraneous-dependencies import express from 'express'; import http from 'http'; import PCancelable from 'p-cancelable'; @@ -283,20 +282,17 @@ export class Worker extends Command { const loadNodesAndCredentials = LoadNodesAndCredentials(); await loadNodesAndCredentials.init(); + // Add the found types to an instance other parts of the application can use + const nodeTypes = NodeTypes(loadNodesAndCredentials); + const credentialTypes = CredentialTypes(loadNodesAndCredentials); + // Load the credentials overwrites if any exist - const credentialsOverwrites = CredentialsOverwrites(); - await credentialsOverwrites.init(); + await CredentialsOverwrites(credentialTypes).init(); // Load all external hooks const externalHooks = ExternalHooks(); await externalHooks.init(); - // Add the found types to an instance other parts of the application can use - const nodeTypes = NodeTypes(); - await nodeTypes.init(loadNodesAndCredentials.nodeTypes); - const credentialTypes = CredentialTypes(); - await credentialTypes.init(loadNodesAndCredentials.credentialTypes); - // Wait till the database is ready await startDbInitPromise; diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 5b779e3511248..e6ef47ed6c7e0 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -580,9 +580,9 @@ export const schema = { env: 'N8N_DISABLE_PRODUCTION_MAIN_PROCESS', doc: 'Disable production webhooks from main process. This helps ensures no http traffic load to main process when using webhook-specific processes.', }, - skipWebhoooksDeregistrationOnShutdown: { + skipWebhooksDeregistrationOnShutdown: { /** - * Longer explanation: n8n deregisters webhooks on shutdown / deactivation + * Longer explanation: n8n de-registers webhooks on shutdown / deactivation * and registers on startup / activation. If we skip * deactivation on shutdown, webhooks will remain active on 3rd party services. * We don't have to worry about startup as it always diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 9911a43c7a89e..749d82135f4cc 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -2,11 +2,12 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/naming-convention */ import { resolve, join } from 'path'; -import { RESPONSE_ERROR_MESSAGES as CORE_RESPONSE_ERROR_MESSAGES } from 'n8n-core'; +import { RESPONSE_ERROR_MESSAGES as CORE_RESPONSE_ERROR_MESSAGES, UserSettings } from 'n8n-core'; export const CLI_DIR = resolve(__dirname, '..'); export const TEMPLATES_DIR = join(CLI_DIR, 'templates'); export const NODES_BASE_DIR = join(CLI_DIR, '..', 'nodes-base'); +export const GENERATED_STATIC_DIR = join(UserSettings.getUserHome(), '.cache/n8n/public'); export const NODE_PACKAGE_PREFIX = 'n8n-nodes-'; @@ -14,6 +15,7 @@ export const STARTER_TEMPLATE_NAME = `${NODE_PACKAGE_PREFIX}starter`; export const RESPONSE_ERROR_MESSAGES = { NO_CREDENTIAL: 'Credential not found', + NO_NODE: 'Node not found', NO_ENCRYPTION_KEY: CORE_RESPONSE_ERROR_MESSAGES.NO_ENCRYPTION_KEY, PACKAGE_NAME_NOT_PROVIDED: 'Package name is required', PACKAGE_NAME_NOT_VALID: `Package name is not valid - it must start with "${NODE_PACKAGE_PREFIX}"`, diff --git a/packages/cli/src/databases/entities/Settings.ts b/packages/cli/src/databases/entities/Settings.ts index c42421e2991d0..7f1f99cd1e1e3 100644 --- a/packages/cli/src/databases/entities/Settings.ts +++ b/packages/cli/src/databases/entities/Settings.ts @@ -1,7 +1,7 @@ import type { IDataObject } from 'n8n-workflow'; import { Column, Entity, PrimaryColumn } from 'typeorm'; -export interface ISettingsDb { +interface ISettingsDb { key: string; value: string | boolean | IDataObject | number; loadOnStartup: boolean; diff --git a/packages/cli/src/databases/utils/migrationHelpers.ts b/packages/cli/src/databases/utils/migrationHelpers.ts index 9ffb0884ae6d3..daabab7b3d63d 100644 --- a/packages/cli/src/databases/utils/migrationHelpers.ts +++ b/packages/cli/src/databases/utils/migrationHelpers.ts @@ -65,7 +65,7 @@ export function logMigrationEnd( }, 100); } -export function batchQuery(query: string, limit: number, offset = 0): string { +function batchQuery(query: string, limit: number, offset = 0): string { return ` ${query} LIMIT ${limit} diff --git a/packages/cli/src/requests.d.ts b/packages/cli/src/requests.d.ts index 569a30709ce2b..973b94b33bffc 100644 --- a/packages/cli/src/requests.d.ts +++ b/packages/cli/src/requests.d.ts @@ -10,7 +10,7 @@ import { IWorkflowSettings, } from 'n8n-workflow'; -import type { IExecutionDeleteFilter, IWorkflowDb } from '.'; +import type { IExecutionDeleteFilter, IWorkflowDb } from '@/Interfaces'; import type { Role } from '@db/entities/Role'; import type { User } from '@db/entities/User'; import * as UserManagementMailer from '@/UserManagement/email/UserManagementMailer'; diff --git a/packages/cli/test/integration/shared/utils.ts b/packages/cli/test/integration/shared/utils.ts index c716ebe69817a..1f20a965c8a59 100644 --- a/packages/cli/test/integration/shared/utils.ts +++ b/packages/cli/test/integration/shared/utils.ts @@ -4,7 +4,7 @@ import { existsSync } from 'fs'; import bodyParser from 'body-parser'; import { CronJob } from 'cron'; import express from 'express'; -import { set } from 'lodash'; +import set from 'lodash.set'; import { BinaryDataManager, UserSettings } from 'n8n-core'; import { ICredentialType, @@ -13,8 +13,7 @@ import { INode, INodeExecutionData, INodeParameters, - INodeTypeData, - INodeTypes, + INodesAndCredentials, ITriggerFunctions, ITriggerResponse, LoggerProxy, @@ -67,6 +66,14 @@ import type { PostgresSchemaSection, } from './types'; +const loadNodesAndCredentials: INodesAndCredentials = { + loaded: { nodes: {}, credentials: {} }, + known: { nodes: {}, credentials: {} }, +}; + +const mockNodeTypes = NodeTypes(loadNodesAndCredentials); +CredentialTypes(loadNodesAndCredentials); + /** * Initialize a test server. * @@ -149,8 +156,6 @@ export async function initTestServer({ * Pre-requisite: Mock the telemetry module before calling. */ export function initTestTelemetry() { - const mockNodeTypes = { nodeTypes: {} } as INodeTypes; - void InternalHooksManager.init('test-instance-id', 'test-version', mockNodeTypes); } @@ -217,20 +222,19 @@ export function gitHubCredentialType(): ICredentialType { * Initialize node types. */ export async function initCredentialsTypes(): Promise { - const credentialTypes = CredentialTypes(); - await credentialTypes.init({ + loadNodesAndCredentials.loaded.credentials = { githubApi: { type: gitHubCredentialType(), sourcePath: '', }, - }); + }; } /** * Initialize node types. */ export async function initNodeTypes() { - const types: INodeTypeData = { + loadNodesAndCredentials.loaded.nodes = { 'n8n-nodes-base.start': { sourcePath: '', type: { @@ -524,8 +528,6 @@ export async function initNodeTypes() { }, }, }; - - await NodeTypes().init(types); } /** diff --git a/packages/cli/test/unit/CredentialTypes.test.ts b/packages/cli/test/unit/CredentialTypes.test.ts index 3fdb88a894b7b..06bc84a15d520 100644 --- a/packages/cli/test/unit/CredentialTypes.test.ts +++ b/packages/cli/test/unit/CredentialTypes.test.ts @@ -1,62 +1,46 @@ -import type { ICredentialTypeData, ICredentialTypes } from 'n8n-workflow'; +import type { ICredentialTypes, INodesAndCredentials } from 'n8n-workflow'; import { CredentialTypes } from '@/CredentialTypes'; describe('ActiveExecutions', () => { let credentialTypes: ICredentialTypes; beforeEach(() => { - credentialTypes = CredentialTypes(); - }); - - test('Should start with empty credential list', () => { - expect(credentialTypes.getAll()).toEqual([]); - }); - - test('Should initialize credential types', () => { - credentialTypes.init(mockCredentialTypes()); - expect(credentialTypes.getAll()).toHaveLength(2); - }); - - test('Should return all credential types', () => { - credentialTypes.init(mockCredentialTypes()); - const mockedCredentialTypes = mockCredentialTypes(); - expect(credentialTypes.getAll()).toStrictEqual([ - mockedCredentialTypes.fakeFirstCredential.type, - mockedCredentialTypes.fakeSecondCredential.type, - ]); + credentialTypes = CredentialTypes(mockNodesAndCredentials()); }); test('Should throw error when calling invalid credential name', () => { - credentialTypes.init(mockCredentialTypes()); expect(() => credentialTypes.getByName('fakeThirdCredential')).toThrowError(); }); test('Should return correct credential type for valid name', () => { - credentialTypes.init(mockCredentialTypes()); - const mockedCredentialTypes = mockCredentialTypes(); + const mockedCredentialTypes = mockNodesAndCredentials().loaded.credentials; expect(credentialTypes.getByName('fakeFirstCredential')).toStrictEqual( mockedCredentialTypes.fakeFirstCredential.type, ); }); }); -function mockCredentialTypes(): ICredentialTypeData { - return { - fakeFirstCredential: { - type: { - name: 'fakeFirstCredential', - displayName: 'Fake First Credential', - properties: [], +const mockNodesAndCredentials = (): INodesAndCredentials => ({ + loaded: { + nodes: {}, + credentials: { + fakeFirstCredential: { + type: { + name: 'fakeFirstCredential', + displayName: 'Fake First Credential', + properties: [], + }, + sourcePath: '', }, - sourcePath: '', - }, - fakeSecondCredential: { - type: { - name: 'fakeSecondCredential', - displayName: 'Fake Second Credential', - properties: [], + fakeSecondCredential: { + type: { + name: 'fakeSecondCredential', + displayName: 'Fake Second Credential', + properties: [], + }, + sourcePath: '', }, - sourcePath: '', }, - }; -} + }, + known: { nodes: {}, credentials: {} }, +}); diff --git a/packages/cli/test/unit/CredentialsHelper.test.ts b/packages/cli/test/unit/CredentialsHelper.test.ts index 7ad8c9fc494e8..2062329c7ee88 100644 --- a/packages/cli/test/unit/CredentialsHelper.test.ts +++ b/packages/cli/test/unit/CredentialsHelper.test.ts @@ -6,6 +6,7 @@ import { IHttpRequestOptions, INode, INodeProperties, + INodesAndCredentials, Workflow, } from 'n8n-workflow'; import { CredentialsHelper } from '@/CredentialsHelper'; @@ -13,6 +14,10 @@ import { CredentialTypes } from '@/CredentialTypes'; import * as Helpers from './Helpers'; const TEST_ENCRYPTION_KEY = 'test'; +const mockNodesAndCredentials: INodesAndCredentials = { + loaded: { nodes: {}, credentials: {} }, + known: { nodes: {}, credentials: {} }, +}; describe('CredentialsHelper', () => { describe('authenticate', () => { @@ -222,14 +227,14 @@ describe('CredentialsHelper', () => { for (const testData of tests) { test(testData.description, async () => { - const credentialTypes: ICredentialTypeData = { + mockNodesAndCredentials.loaded.credentials = { [testData.input.credentialType.name]: { type: testData.input.credentialType, sourcePath: '', }, }; - await CredentialTypes().init(credentialTypes); + CredentialTypes(mockNodesAndCredentials); const credentialsHelper = new CredentialsHelper(TEST_ENCRYPTION_KEY); diff --git a/packages/cli/test/unit/Helpers.ts b/packages/cli/test/unit/Helpers.ts index 64ca1c2679427..8b4065d491fd4 100644 --- a/packages/cli/test/unit/Helpers.ts +++ b/packages/cli/test/unit/Helpers.ts @@ -1,4 +1,10 @@ -import { INodeType, INodeTypeData, INodeTypes, NodeHelpers } from 'n8n-workflow'; +import { + INodesAndCredentials, + INodeType, + INodeTypeData, + INodeTypes, + NodeHelpers, +} from 'n8n-workflow'; class NodeTypesClass implements INodeTypes { nodeTypes: INodeTypeData = { @@ -36,8 +42,10 @@ class NodeTypesClass implements INodeTypes { }, }; - async init(nodeTypes: INodeTypeData): Promise { - this.nodeTypes = nodeTypes; + constructor(nodesAndCredentials?: INodesAndCredentials) { + if (nodesAndCredentials?.loaded?.nodes) { + this.nodeTypes = nodesAndCredentials?.loaded?.nodes; + } } getAll(): INodeType[] { @@ -55,9 +63,9 @@ class NodeTypesClass implements INodeTypes { let nodeTypesInstance: NodeTypesClass | undefined; -export function NodeTypes(): NodeTypesClass { +export function NodeTypes(nodesAndCredentials?: INodesAndCredentials): NodeTypesClass { if (nodeTypesInstance === undefined) { - nodeTypesInstance = new NodeTypesClass(); + nodeTypesInstance = new NodeTypesClass(nodesAndCredentials); } return nodeTypesInstance; diff --git a/packages/cli/test/unit/PermissionChecker.test.ts b/packages/cli/test/unit/PermissionChecker.test.ts index 2c029e6bdc30f..10b4fbe35579c 100644 --- a/packages/cli/test/unit/PermissionChecker.test.ts +++ b/packages/cli/test/unit/PermissionChecker.test.ts @@ -23,8 +23,13 @@ beforeAll(async () => { const initResult = await testDb.init(); testDbName = initResult.testDbName; - mockNodeTypes = MockNodeTypes(); - await mockNodeTypes.init(MOCK_NODE_TYPES_DATA); + mockNodeTypes = MockNodeTypes({ + loaded: { + nodes: MOCK_NODE_TYPES_DATA, + credentials: {}, + }, + known: { nodes: {}, credentials: {} }, + }); credentialOwnerRole = await testDb.getCredentialOwnerRole(); workflowOwnerRole = await testDb.getWorkflowOwnerRole(); diff --git a/packages/core/.eslintrc.js b/packages/core/.eslintrc.js index 9ce181979fdee..02dd3f467fd71 100644 --- a/packages/core/.eslintrc.js +++ b/packages/core/.eslintrc.js @@ -9,6 +9,8 @@ module.exports = { tsconfigRootDir: __dirname, }, + ignorePatterns: ['bin/*.js'], + rules: { // TODO: Remove this 'import/order': 'off', diff --git a/packages/core/bin/common.js b/packages/core/bin/common.js new file mode 100644 index 0000000000000..4148c59fda69d --- /dev/null +++ b/packages/core/bin/common.js @@ -0,0 +1,19 @@ +const path = require('path'); +const { mkdir, writeFile } = require('fs/promises'); + +const packageDir = process.cwd(); +const distDir = path.join(packageDir, 'dist'); + +const writeJSON = async (file, data) => { + const filePath = path.resolve(distDir, file); + await mkdir(path.dirname(filePath), { recursive: true }); + const payload = Array.isArray(data) + ? `[\n${data.map((entry) => JSON.stringify(entry)).join(',\n')}\n]` + : JSON.stringify(data, null, 2); + await writeFile(filePath, payload, { encoding: 'utf-8' }); +}; + +module.exports = { + packageDir, + writeJSON, +}; diff --git a/packages/core/bin/generate-known b/packages/core/bin/generate-known new file mode 100755 index 0000000000000..a274d552399fc --- /dev/null +++ b/packages/core/bin/generate-known @@ -0,0 +1,47 @@ +#!/usr/bin/env node + +const path = require('path'); +const glob = require('fast-glob'); +const { createContext, Script } = require('vm'); +const { LoggerProxy } = require('n8n-workflow'); +const { packageDir, writeJSON } = require('./common'); + +LoggerProxy.init({ + log: console.log.bind(console), + warn: console.warn.bind(console), +}); + +const context = Object.freeze(createContext({ require })); +const loadClass = (sourcePath) => { + try { + const [className] = path.parse(sourcePath).name.split('.'); + const absolutePath = path.resolve(packageDir, sourcePath); + const script = new Script(`new (require('${absolutePath}').${className})()`); + const instance = script.runInContext(context); + return { instance, sourcePath, className }; + } catch (e) { + LoggerProxy.warn('Failed to load %s: %s', sourcePath, e.message); + } +}; + +const generate = (kind) => { + const data = glob + .sync(`dist/${kind}/**/*.${kind === 'nodes' ? 'node' : kind}.js`, { + cwd: packageDir, + }) + .filter((filePath) => !/[vV]\d.node\.js$/.test(filePath)) + .map(loadClass) + .filter((data) => !!data) + .reduce((obj, { className, sourcePath, instance }) => { + const name = kind === 'nodes' ? instance.description.name : instance.name; + if (name in obj) console.error('already loaded', kind, name, sourcePath); + else obj[name] = { className, sourcePath }; + return obj; + }, {}); + LoggerProxy.info(`Detected ${Object.keys(data).length} ${kind}`); + return writeJSON(`known/${kind}.json`, data); +}; + +(async () => { + await Promise.all([generate('credentials'), generate('nodes')]); +})(); diff --git a/packages/core/bin/generate-ui-types b/packages/core/bin/generate-ui-types new file mode 100755 index 0000000000000..f8377a21b130d --- /dev/null +++ b/packages/core/bin/generate-ui-types @@ -0,0 +1,29 @@ +#!/usr/bin/env node + +const { LoggerProxy, NodeHelpers } = require('n8n-workflow'); +const { PackageDirectoryLoader } = require('../dist/DirectoryLoader'); +const { packageDir, writeJSON } = require('./common'); + +LoggerProxy.init({ + log: console.log.bind(console), + warn: console.warn.bind(console), +}); + +(async () => { + const loader = new PackageDirectoryLoader(packageDir); + await loader.loadAll(); + + const credentialTypes = Object.values(loader.credentialTypes).map((data) => data.type); + + const nodeTypes = Object.values(loader.nodeTypes) + .map((data) => data.type) + .flatMap((nodeData) => { + const allNodeTypes = NodeHelpers.getVersionedNodeTypeAll(nodeData); + return allNodeTypes.map((element) => element.description); + }); + + await Promise.all([ + writeJSON('types/credentials.json', credentialTypes), + writeJSON('types/nodes.json', nodeTypes), + ]); +})(); diff --git a/packages/core/package.json b/packages/core/package.json index 226f7b8d7ba05..b52dd30f82156 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -14,6 +14,10 @@ }, "main": "dist/index", "types": "dist/index.d.ts", + "bin": { + "n8n-generate-known": "./bin/generate-known", + "n8n-generate-ui-types": "./bin/generate-ui-types" + }, "scripts": { "clean": "rimraf dist .turbo", "typecheck": "tsc", @@ -22,11 +26,12 @@ "format": "prettier --write . --ignore-path ../../.prettierignore", "lint": "eslint .", "lintfix": "eslint . --fix", - "watch": "tsc --watch", + "watch": "tsc -p tsconfig.build.json --watch", "test": "jest" }, "files": [ - "dist" + "dist", + "bin" ], "devDependencies": { "@types/cron": "~1.7.1", @@ -42,6 +47,7 @@ "client-oauth2": "^4.2.5", "cron": "~1.7.2", "crypto-js": "~4.1.1", + "fast-glob": "^3.2.5", "file-type": "^16.5.4", "flatted": "^3.2.4", "form-data": "^4.0.0", diff --git a/packages/core/src/ClassLoader.ts b/packages/core/src/ClassLoader.ts new file mode 100644 index 0000000000000..4c7dd6ec22b42 --- /dev/null +++ b/packages/core/src/ClassLoader.ts @@ -0,0 +1,10 @@ +import { createContext, Script } from 'vm'; + +const context = createContext({ require }); +export const loadClassInIsolation = (filePath: string, className: string) => { + if (process.platform === 'win32') { + filePath = filePath.replace(/\\/g, '/'); + } + const script = new Script(`new (require('${filePath}').${className})()`); + return script.runInContext(context) as T; +}; diff --git a/packages/core/src/Constants.ts b/packages/core/src/Constants.ts index 6332229926a2d..6d2fe99321336 100644 --- a/packages/core/src/Constants.ts +++ b/packages/core/src/Constants.ts @@ -15,3 +15,5 @@ export const WAIT_TIME_UNLIMITED = '3000-01-01T00:00:00.000Z'; export const RESPONSE_ERROR_MESSAGES = { NO_ENCRYPTION_KEY: 'Encryption key is missing or was not set', }; + +export const CUSTOM_NODES_CATEGORY = 'Custom Nodes'; diff --git a/packages/core/src/DirectoryLoader.ts b/packages/core/src/DirectoryLoader.ts new file mode 100644 index 0000000000000..854e6742a6194 --- /dev/null +++ b/packages/core/src/DirectoryLoader.ts @@ -0,0 +1,339 @@ +import * as path from 'node:path'; +import { readFile } from 'node:fs/promises'; +import glob from 'fast-glob'; +import { jsonParse, KnownNodesAndCredentials, LoggerProxy as Logger } from 'n8n-workflow'; +import type { + CodexData, + DocumentationLink, + ICredentialType, + ICredentialTypeData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, + INodeTypeData, + INodeTypeNameVersion, + IVersionedNodeType, +} from 'n8n-workflow'; +import { CUSTOM_NODES_CATEGORY } from './Constants'; +import type { n8n } from './Interfaces'; +import { loadClassInIsolation } from './ClassLoader'; + +function toJSON(this: ICredentialType) { + return { + ...this, + authenticate: typeof this.authenticate === 'function' ? {} : this.authenticate, + }; +} + +export type Types = { + nodes: INodeTypeBaseDescription[]; + credentials: ICredentialType[]; +}; + +export abstract class DirectoryLoader { + readonly loadedNodes: INodeTypeNameVersion[] = []; + + readonly nodeTypes: INodeTypeData = {}; + + readonly credentialTypes: ICredentialTypeData = {}; + + readonly known: KnownNodesAndCredentials = { nodes: {}, credentials: {} }; + + readonly types: Types = { nodes: [], credentials: [] }; + + constructor( + protected readonly directory: string, + private readonly excludeNodes?: string, + private readonly includeNodes?: string, + ) {} + + abstract loadAll(): Promise; + + protected resolvePath(file: string) { + return path.resolve(this.directory, file); + } + + protected loadNodeFromFile(packageName: string, nodeName: string, filePath: string) { + let tempNode: INodeType | IVersionedNodeType; + let nodeVersion = 1; + + try { + tempNode = loadClassInIsolation(filePath, nodeName); + this.addCodex({ node: tempNode, filePath, isCustom: packageName === 'CUSTOM' }); + } catch (error) { + Logger.error( + `Error loading node "${nodeName}" from: "${filePath}" - ${(error as Error).message}`, + ); + throw error; + } + + const fullNodeName = `${packageName}.${tempNode.description.name}`; + + if (this.includeNodes !== undefined && !this.includeNodes.includes(fullNodeName)) { + return; + } + + if (this.excludeNodes?.includes(fullNodeName)) { + return; + } + + tempNode.description.name = fullNodeName; + + this.fixIconPath(tempNode.description, filePath); + + if ('nodeVersions' in tempNode) { + for (const versionNode of Object.values(tempNode.nodeVersions)) { + this.fixIconPath(versionNode.description, filePath); + } + + const currentVersionNode = tempNode.nodeVersions[tempNode.currentVersion]; + this.addCodex({ node: currentVersionNode, filePath, isCustom: packageName === 'CUSTOM' }); + nodeVersion = tempNode.currentVersion; + + if (currentVersionNode.hasOwnProperty('executeSingle')) { + Logger.warn( + `"executeSingle" will get deprecated soon. Please update the code of node "${packageName}.${nodeName}" to use "execute" instead!`, + { filePath }, + ); + } + } else { + // Short renaming to avoid type issues + const tmpNode = tempNode; + nodeVersion = Array.isArray(tmpNode.description.version) + ? tmpNode.description.version.slice(-1)[0] + : tmpNode.description.version; + } + + this.nodeTypes[fullNodeName] = { + type: tempNode, + sourcePath: filePath, + }; + + this.loadedNodes.push({ + name: fullNodeName, + version: nodeVersion, + }); + + this.types.nodes.push(tempNode.description); + } + + protected loadCredentialFromFile(credentialName: string, filePath: string): void { + let tempCredential: ICredentialType; + try { + tempCredential = loadClassInIsolation(filePath, credentialName); + + // Add serializer method "toJSON" to the class so that authenticate method (if defined) + // gets mapped to the authenticate attribute before it is sent to the client. + // The authenticate property is used by the client to decide whether or not to + // include the credential type in the predefined credentials (HTTP node) + Object.assign(tempCredential, { toJSON }); + + this.fixIconPath(tempCredential, filePath); + } catch (e) { + if (e instanceof TypeError) { + throw new Error( + `Class with name "${credentialName}" could not be found. Please check if the class is named correctly!`, + ); + } else { + throw e; + } + } + + this.credentialTypes[tempCredential.name] = { + type: tempCredential, + sourcePath: filePath, + }; + + this.types.credentials.push(tempCredential); + } + + /** + * Retrieves `categories`, `subcategories` and alias (if defined) + * from the codex data for the node at the given file path. + */ + private getCodex(filePath: string): CodexData { + type Codex = { + categories: string[]; + subcategories: { [subcategory: string]: string[] }; + resources: { + primaryDocumentation: DocumentationLink[]; + credentialDocumentation: DocumentationLink[]; + }; + alias: string[]; + }; + + const codexFilePath = `${filePath}on`; // .js to .json + + const { + categories, + subcategories, + resources: allResources, + alias, + } = module.require(codexFilePath) as Codex; + + const resources = { + primaryDocumentation: allResources.primaryDocumentation, + credentialDocumentation: allResources.credentialDocumentation, + }; + + return { + ...(categories && { categories }), + ...(subcategories && { subcategories }), + ...(resources && { resources }), + ...(alias && { alias }), + }; + } + + /** + * Adds a node codex `categories` and `subcategories` (if defined) + * to a node description `codex` property. + */ + private addCodex({ + node, + filePath, + isCustom, + }: { + node: INodeType | IVersionedNodeType; + filePath: string; + isCustom: boolean; + }) { + try { + const codex = this.getCodex(filePath); + + if (isCustom) { + codex.categories = codex.categories + ? codex.categories.concat(CUSTOM_NODES_CATEGORY) + : [CUSTOM_NODES_CATEGORY]; + } + + node.description.codex = codex; + } catch (_) { + Logger.debug(`No codex available for: ${filePath.split('/').pop() ?? ''}`); + + if (isCustom) { + node.description.codex = { + categories: [CUSTOM_NODES_CATEGORY], + }; + } + } + } + + private fixIconPath( + obj: INodeTypeDescription | INodeTypeBaseDescription | ICredentialType, + filePath: string, + ) { + if (obj.icon?.startsWith('file:')) { + const iconPath = path.join(path.dirname(filePath), obj.icon.substring(5)); + const relativePath = path.relative(this.directory, iconPath); + obj.icon = `file:${relativePath}`; + } + } +} + +/** + * Loader for source files of nodes and credentials located in a custom dir, + * e.g. `~/.n8n/custom` + */ +export class CustomDirectoryLoader extends DirectoryLoader { + override async loadAll() { + const filePaths = await glob('**/*.@(node|credentials).js', { + cwd: this.directory, + absolute: true, + }); + + for (const filePath of filePaths) { + const [fileName, type] = path.parse(filePath).name.split('.'); + + if (type === 'node') { + this.loadNodeFromFile('CUSTOM', fileName, filePath); + } else if (type === 'credentials') { + this.loadCredentialFromFile(fileName, filePath); + } + } + } +} + +/** + * Loader for source files of nodes and credentials located in a package dir, + * e.g. /nodes-base or community packages. + */ +export class PackageDirectoryLoader extends DirectoryLoader { + packageName = ''; + + packageJson!: n8n.PackageJson; + + async readPackageJson() { + this.packageJson = await this.readJSON('package.json'); + this.packageName = this.packageJson.name; + } + + override async loadAll() { + await this.readPackageJson(); + + const { n8n } = this.packageJson; + if (!n8n) return; + + const { nodes, credentials } = n8n; + + if (Array.isArray(credentials)) { + for (const credential of credentials) { + const filePath = this.resolvePath(credential); + const [credentialName] = path.parse(credential).name.split('.'); + + this.loadCredentialFromFile(credentialName, filePath); + } + } + + if (Array.isArray(nodes)) { + for (const node of nodes) { + const filePath = this.resolvePath(node); + const [nodeName] = path.parse(node).name.split('.'); + + this.loadNodeFromFile(this.packageName, nodeName, filePath); + } + } + + Logger.debug(`Loaded all credentials and nodes from ${this.packageName}`, { + credentials: credentials?.length ?? 0, + nodes: nodes?.length ?? 0, + }); + } + + protected async readJSON(file: string): Promise { + const filePath = this.resolvePath(file); + const fileString = await readFile(filePath, 'utf8'); + + try { + return jsonParse(fileString); + } catch (error) { + throw new Error(`Failed to parse JSON from ${filePath}`); + } + } +} + +/** + * This loader extends PackageDirectoryLoader to load node and credentials lazily, if possible + */ +export class LazyPackageDirectoryLoader extends PackageDirectoryLoader { + override async loadAll() { + await this.readPackageJson(); + + try { + this.known.nodes = await this.readJSON('dist/known/nodes.json'); + this.known.credentials = await this.readJSON('dist/known/credentials.json'); + + this.types.nodes = await this.readJSON('dist/types/nodes.json'); + this.types.credentials = await this.readJSON('dist/types/credentials.json'); + + Logger.debug(`Lazy Loading credentials and nodes from ${this.packageJson.name}`, { + credentials: this.types.credentials?.length ?? 0, + nodes: this.types.nodes?.length ?? 0, + }); + + return; // We can load nodes and credentials lazily now + } catch { + Logger.debug("Can't enable lazy-loading"); + await super.loadAll(); + } + } +} diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/Interfaces.ts index c048ad7b79858..da1b3e6f1b9ed 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/Interfaces.ts @@ -317,3 +317,18 @@ export interface IBinaryDataManager { deleteBinaryDataByExecutionId(executionId: string): Promise; persistBinaryDataForExecutionId(executionId: string): Promise; } + +export namespace n8n { + export interface PackageJson { + name: string; + version: string; + n8n?: { + credentials?: string[]; + nodes?: string[]; + }; + author?: { + name?: string; + email?: string; + }; + } +} diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index f93b1664d1bfd..0e2d10e7b2b70 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -71,9 +71,7 @@ import { stringify } from 'qs'; import clientOAuth1, { Token } from 'oauth-1.0a'; import clientOAuth2 from 'client-oauth2'; import crypto, { createHmac } from 'crypto'; -// eslint-disable-next-line import/no-extraneous-dependencies -import { get } from 'lodash'; -// eslint-disable-next-line import/no-extraneous-dependencies +import get from 'lodash.get'; import type { Request, Response } from 'express'; import FormData from 'form-data'; import path from 'path'; diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 8eed16f04ff52..351fae1e10ef6 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -37,8 +37,7 @@ import { WorkflowExecuteMode, WorkflowOperationError, } from 'n8n-workflow'; -// eslint-disable-next-line import/no-extraneous-dependencies -import { get } from 'lodash'; +import get from 'lodash.get'; import * as NodeExecuteFunctions from './NodeExecuteFunctions'; export class WorkflowExecute { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 95ff90e6eb620..aa0fc95e75d4c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -4,8 +4,10 @@ import * as UserSettings from './UserSettings'; export * from './ActiveWorkflows'; export * from './ActiveWebhooks'; export * from './BinaryDataManager'; +export * from './ClassLoader'; export * from './Constants'; export * from './Credentials'; +export * from './DirectoryLoader'; export * from './Interfaces'; export * from './LoadNodeParameterOptions'; export * from './LoadNodeListSearch'; diff --git a/packages/core/test/Helpers.ts b/packages/core/test/Helpers.ts index 7d43f1ade12a7..41d92f7d624c3 100644 --- a/packages/core/test/Helpers.ts +++ b/packages/core/test/Helpers.ts @@ -1,4 +1,4 @@ -import { set } from 'lodash'; +import set from 'lodash.set'; import { ICredentialDataDecryptedObject, @@ -805,8 +805,6 @@ class NodeTypesClass implements INodeTypes { }, }; - async init(nodeTypes: INodeTypeData): Promise {} - getAll(): INodeType[] { return Object.values(this.nodeTypes).map((data) => NodeHelpers.getVersionedNodeType(data.type)); } @@ -825,7 +823,6 @@ let nodeTypesInstance: NodeTypesClass | undefined; export function NodeTypes(): NodeTypesClass { if (nodeTypesInstance === undefined) { nodeTypesInstance = new NodeTypesClass(); - nodeTypesInstance.init({}); } return nodeTypesInstance; diff --git a/packages/design-system/package.json b/packages/design-system/package.json index 06a02923a4fba..178f33ac57173 100644 --- a/packages/design-system/package.json +++ b/packages/design-system/package.json @@ -15,7 +15,7 @@ "scripts": { "clean": "rimraf dist .turbo", "build": "vite build", - "build:vue:typecheck": "vue-tsc --emitDeclarationOnly", + "typecheck": "vue-tsc --emitDeclarationOnly", "test": "vitest run", "test:ci": "vitest run --coverage", "test:dev": "vitest", @@ -69,7 +69,7 @@ "vue-loader": "^15.9.7", "vue-property-decorator": "^9.1.2", "vue-template-compiler": "^2.7", - "vue-tsc": "^0.34.8", + "vue-tsc": "^0.35.0", "vue2-boring-avatars": "0.3.4", "webpack": "^4.46.0" }, diff --git a/packages/design-system/tsconfig.json b/packages/design-system/tsconfig.json index 8e1af0eab81e8..9d40abae50b69 100644 --- a/packages/design-system/tsconfig.json +++ b/packages/design-system/tsconfig.json @@ -8,6 +8,7 @@ "importHelpers": true, "skipLibCheck": true, "allowJs": true, + "incremental": false, "allowSyntheticDefaultImports": true, "baseUrl": ".", "types": ["webpack-env", "vitest/globals"], diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index 7d829cdd9ba3f..351388ff5efaf 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -16,6 +16,7 @@ "scripts": { "clean": "rimraf dist .turbo", "build": "cross-env VUE_APP_PUBLIC_PATH=\"/{{BASE_PATH}}/\" NODE_OPTIONS=\"--max-old-space-size=8192\" vite build", + "typecheck": "vue-tsc --emitDeclarationOnly", "dev": "pnpm serve", "lint": "tslint -p tsconfig.json -c tslint.json && eslint --ext .js,.ts,.vue src", "lintfix": "tslint --fix -p tsconfig.json -c tslint.json && eslint --ext .js,.ts,.vue src --fix", @@ -104,6 +105,6 @@ "vite-plugin-html": "^3.2.0", "vite-plugin-monaco-editor": "^1.0.10", "vitest": "0.9.3", - "vue-tsc": "^0.34.15" + "vue-tsc": "^0.35.0" } } diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index 9a1c4a592f165..e623057a92653 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -887,6 +887,7 @@ export interface IVersionNode { name: string; displayName: string; icon: string; + iconUrl?: string; defaults: INodeParameters; iconData: { type: string; diff --git a/packages/editor-ui/src/api/credentials.ts b/packages/editor-ui/src/api/credentials.ts index a4aa6d50d0fbd..539c0edf18dc2 100644 --- a/packages/editor-ui/src/api/credentials.ts +++ b/packages/editor-ui/src/api/credentials.ts @@ -7,9 +7,11 @@ import { INodeCredentialTestRequest, INodeCredentialTestResult, } from 'n8n-workflow'; +import axios from 'axios'; -export async function getCredentialTypes(context: IRestApiContext): Promise { - return await makeRestApiRequest(context, 'GET', '/credential-types'); +export async function getCredentialTypes(baseUrl: string): Promise { + const { data } = await axios.get(baseUrl + 'types/credentials.json'); + return data; } export async function getCredentialsNewName(context: IRestApiContext, name?: string): Promise<{name: string}> { diff --git a/packages/editor-ui/src/api/nodeTypes.ts b/packages/editor-ui/src/api/nodeTypes.ts index b5553c64cb6ab..0114e56c35eff 100644 --- a/packages/editor-ui/src/api/nodeTypes.ts +++ b/packages/editor-ui/src/api/nodeTypes.ts @@ -14,12 +14,11 @@ import type { INodeTypeDescription, INodeTypeNameVersion, } from 'n8n-workflow'; +import axios from 'axios'; -export async function getNodeTypes( - context: IRestApiContext, - { onlyLatest } = { onlyLatest: false }, -) { - return makeRestApiRequest(context, 'GET', '/node-types', { onlyLatest }); +export async function getNodeTypes(baseUrl: string) { + const { data } = await axios.get(baseUrl + 'types/nodes.json'); + return data; } export async function getNodeTranslationHeaders( @@ -55,4 +54,3 @@ export async function getResourceLocatorResults( ): Promise { return makeRestApiRequest(context, 'GET', '/nodes-list-search', sendData as unknown as IDataObject); } - diff --git a/packages/editor-ui/src/components/CredentialIcon.vue b/packages/editor-ui/src/components/CredentialIcon.vue index f4524cba59fb9..fd4bd9669a5f1 100644 --- a/packages/editor-ui/src/components/CredentialIcon.vue +++ b/packages/editor-ui/src/components/CredentialIcon.vue @@ -31,16 +31,15 @@ export default Vue.extend({ }, filePath(): string | null { - if (!this.credentialWithIcon || !this.credentialWithIcon.icon || !this.credentialWithIcon.icon.startsWith('file:')) { + const iconUrl = this.credentialWithIcon?.iconUrl; + if (!iconUrl) { return null; } - - const restUrl = this.rootStore.getRestUrl; - - return `${restUrl}/credential-icon/${this.credentialWithIcon.name}`; + return this.rootStore.getBaseUrl + iconUrl; }, + relevantNode(): INodeTypeDescription | null { - if (this.credentialWithIcon && this.credentialWithIcon.icon && this.credentialWithIcon.icon.startsWith('node:')) { + if (this.credentialWithIcon?.icon?.startsWith('node:')) { const nodeType = this.credentialWithIcon.icon.replace('node:', ''); return this.nodeTypesStore.getNodeType(nodeType); } @@ -65,7 +64,7 @@ export default Vue.extend({ return null; } - if (type.icon) { + if (type.icon || type.iconUrl) { return type; } diff --git a/packages/editor-ui/src/components/NodeIcon.vue b/packages/editor-ui/src/components/NodeIcon.vue index b162c06162c27..6e4b9d5e736a2 100644 --- a/packages/editor-ui/src/components/NodeIcon.vue +++ b/packages/editor-ui/src/components/NodeIcon.vue @@ -56,6 +56,7 @@ export default Vue.extend({ const nodeType = this.nodeType as INodeTypeDescription | IVersionNode | null; let iconType = 'unknown'; if (nodeType) { + if (nodeType.iconUrl) return 'file'; if ((nodeType as IVersionNode).iconData) { iconType = (nodeType as IVersionNode).iconData.type; } else if (nodeType.icon) { @@ -73,7 +74,7 @@ export default Vue.extend({ }, iconSource () : NodeIconSource { const nodeType = this.nodeType as INodeTypeDescription | IVersionNode | null; - const restUrl = this.rootStore.getRestUrl; + const baseUrl = this.rootStore.getBaseUrl; const iconSource = {} as NodeIconSource; if (nodeType) { @@ -84,11 +85,14 @@ export default Vue.extend({ fileBuffer: (nodeType as IVersionNode).iconData.fileBuffer, }; } + if (nodeType.iconUrl) { + return { path: baseUrl + nodeType.iconUrl }; + } // Otherwise, extract it from icon prop if (nodeType.icon) { const [type, path] = nodeType.icon.split(':'); if (type === 'file') { - iconSource.path = `${restUrl}/node-icon/${nodeType.name}`; + throw new Error(`Unexpected icon: ${nodeType.icon}`); } else { iconSource.icon = path; } diff --git a/packages/editor-ui/src/stores/credentials.ts b/packages/editor-ui/src/stores/credentials.ts index f6b805e9b9d7f..04231f073f4e5 100644 --- a/packages/editor-ui/src/stores/credentials.ts +++ b/packages/editor-ui/src/stores/credentials.ts @@ -154,7 +154,7 @@ export const useCredentialsStore = defineStore(STORES.CREDENTIALS, { return; } const rootStore = useRootStore(); - const credentialTypes = await getCredentialTypes(rootStore.getRestApiContext); + const credentialTypes = await getCredentialTypes(rootStore.getBaseUrl); this.setCredentialTypes(credentialTypes); }, async fetchAllCredentials(): Promise { diff --git a/packages/editor-ui/src/stores/n8nRootStore.ts b/packages/editor-ui/src/stores/n8nRootStore.ts index 77710ff73966c..1d6f9276de174 100644 --- a/packages/editor-ui/src/stores/n8nRootStore.ts +++ b/packages/editor-ui/src/stores/n8nRootStore.ts @@ -26,6 +26,10 @@ export const useRootStore = defineStore(STORES.ROOT, { instanceId: '', }), getters: { + getBaseUrl(): string { + return this.baseUrl; + }, + getWebhookUrl(): string { return `${this.urlBaseWebhook}${this.endpointWebhook}`; }, diff --git a/packages/editor-ui/src/stores/nodeTypes.ts b/packages/editor-ui/src/stores/nodeTypes.ts index 1b0064eecff2e..fdd4a2b930c67 100644 --- a/packages/editor-ui/src/stores/nodeTypes.ts +++ b/packages/editor-ui/src/stores/nodeTypes.ts @@ -120,7 +120,7 @@ export const useNodeTypesStore = defineStore(STORES.NODE_TYPES, { }, async getNodeTypes(): Promise { const rootStore = useRootStore(); - const nodeTypes = await getNodeTypes(rootStore.getRestApiContext); + const nodeTypes = await getNodeTypes(rootStore.getBaseUrl); if (nodeTypes.length) { this.setNodeTypes(nodeTypes); } diff --git a/packages/editor-ui/tsconfig.json b/packages/editor-ui/tsconfig.json index 8c623385b3d57..bdb0dd7307572 100644 --- a/packages/editor-ui/tsconfig.json +++ b/packages/editor-ui/tsconfig.json @@ -8,6 +8,7 @@ "skipLibCheck": true, "allowJs": true, "importHelpers": true, + "incremental": false, "allowSyntheticDefaultImports": true, "baseUrl": ".", "types": ["vitest/globals"], diff --git a/packages/nodes-base/credentials/NetlifyOAuth2Api.credentials.ts b/packages/nodes-base/credentials/NetlifyOAuth2Api.credentials.ts deleted file mode 100644 index 706dbc9a0356f..0000000000000 --- a/packages/nodes-base/credentials/NetlifyOAuth2Api.credentials.ts +++ /dev/null @@ -1,60 +0,0 @@ -// import { -// ICredentialType, -// NodePropertyTypes, -// } from 'n8n-workflow'; - -// export class NetlifyOAuth2Api implements ICredentialType { -// name = 'netlifyOAuth2Api'; -// extends = [ -// 'oAuth2Api', -// ]; -// displayName = 'Netlify OAuth2 API'; -// documentationUrl = 'netlify'; -// properties = [ -// { -// displayName: 'Authorization URL', -// name: 'authUrl', -// type: 'hidden' as NodePropertyTypes, -// default: 'https://app.netlify.com/authorize', -// required: true, -// }, -// { -// displayName: 'Client ID', -// name: 'clientId', -// type: 'string' as NodePropertyTypes, -// default: '', -// required: true, -// }, -// { -// displayName: 'Client Secret', -// name: 'clientSecret', -// type: 'string' as NodePropertyTypes, -// default: '', -// required: true, -// }, -// { -// displayName: 'Authentication', -// name: 'authentication', -// type: 'hidden' as NodePropertyTypes, -// default: 'body', -// }, -// { -// displayName: 'Access Token URL', -// name: 'accessTokenUrl', -// type: 'hidden' as NodePropertyTypes, -// default: 'https://api.netlify.com/api/v1/oauth/tickets', -// }, -// { -// displayName: 'Scope', -// name: 'scope', -// type: 'hidden' as NodePropertyTypes, -// default: '', -// }, -// { -// displayName: 'Auth URI Query Parameters', -// name: 'authQueryParameters', -// type: 'hidden' as NodePropertyTypes, -// default: '', -// } -// ]; -// } diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index d4781989a83df..14e3a6ae8ef32 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -15,12 +15,13 @@ "scripts": { "clean": "rimraf dist .turbo", "dev": "pnpm watch", - "build": "tsc && gulp build:icons && gulp build:translations", + "build": "tsc && gulp build:icons && gulp build:translations && pnpm build:metadata", "build:translations": "gulp build:translations", + "build:metadata": "pnpm n8n-generate-known && pnpm n8n-generate-ui-types", "format": "prettier --write . --ignore-path ../../.prettierignore", "lint": "tslint -p tsconfig.json -c tslint.json && eslint nodes credentials", "lintfix": "tslint --fix -p tsconfig.json -c tslint.json && eslint nodes credentials --fix", - "watch": "tsc --watch", + "watch": "tsc-watch --onSuccess \"pnpm n8n-generate-ui-types\"", "test": "jest" }, "files": [ diff --git a/packages/workflow/package.json b/packages/workflow/package.json index 2bb83d74b244c..f8f84c8ee968f 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -31,7 +31,7 @@ "format": "prettier --write . --ignore-path ../../.prettierignore", "lint": "eslint .", "lintfix": "eslint . --fix", - "watch": "tsc --watch", + "watch": "tsc -p tsconfig.build.json --watch", "test": "jest", "test:dev": "jest --watch" }, @@ -42,6 +42,7 @@ "@types/express": "^4.17.6", "@types/jmespath": "^0.15.0", "@types/lodash.get": "^4.4.6", + "@types/lodash.isequal": "^4.5.6", "@types/lodash.merge": "^4.6.6", "@types/lodash.set": "^4.3.6", "@types/luxon": "^2.0.9", diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index c4fe7a0230a69..02d9e521becfc 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -1,9 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable import/no-extraneous-dependencies */ -// eslint-disable-next-line import/no-extraneous-dependencies // eslint-disable-next-line max-classes-per-file -import * as express from 'express'; -import * as FormData from 'form-data'; +import type * as express from 'express'; +import type * as FormData from 'form-data'; import type { IncomingHttpHeaders } from 'http'; import type { URLSearchParams } from 'url'; import type { IDeferredPromise } from './DeferredPromise'; @@ -311,6 +309,7 @@ export interface ICredentialType { name: string; displayName: string; icon?: string; + iconUrl?: string; extends?: string[]; properties: INodeProperties[]; documentationUrl?: string; @@ -325,9 +324,7 @@ export interface ICredentialType { } export interface ICredentialTypes { - credentialTypes?: ICredentialTypeData; - init(credentialTypes?: ICredentialTypeData): Promise; - getAll(): ICredentialType[]; + recognizes(credentialType: string): boolean; getByName(credentialType: string): ICredentialType; } @@ -1257,6 +1254,7 @@ export interface INodeTypeBaseDescription { displayName: string; name: string; icon?: string; + iconUrl?: string; group: string[]; description: string; documentationUrl?: string; @@ -1473,24 +1471,37 @@ export type WebhookResponseData = 'allEntries' | 'firstEntryJson' | 'firstEntryB export type WebhookResponseMode = 'onReceived' | 'lastNode'; export interface INodeTypes { - nodeTypes: INodeTypeData; - init(nodeTypes?: INodeTypeData): Promise; getAll(): Array; getByNameAndVersion(nodeType: string, version?: number): INodeType | undefined; } -export interface ICredentialTypeData { - [key: string]: { - type: ICredentialType; - sourcePath: string; - }; +export type LoadingDetails = { + className: string; + sourcePath: string; +}; + +export type KnownNodesAndCredentials = { + nodes: Record; + credentials: Record; +}; + +export interface LoadedClass { + sourcePath: string; + type: T; } -export interface INodeTypeData { - [key: string]: { - type: INodeType | IVersionedNodeType; - sourcePath: string; - }; +type LoadedData = Record>; +export type ICredentialTypeData = LoadedData; +export type INodeTypeData = LoadedData; + +export type LoadedNodesAndCredentials = { + nodes: INodeTypeData; + credentials: ICredentialTypeData; +}; + +export interface INodesAndCredentials { + known: KnownNodesAndCredentials; + loaded: LoadedNodesAndCredentials; } export interface IRun { diff --git a/packages/workflow/src/NodeHelpers.ts b/packages/workflow/src/NodeHelpers.ts index 1d8cd4d5ee713..161038680c80a 100644 --- a/packages/workflow/src/NodeHelpers.ts +++ b/packages/workflow/src/NodeHelpers.ts @@ -12,8 +12,8 @@ /* eslint-disable prefer-spread */ /* eslint-disable no-restricted-syntax */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -// eslint-disable-next-line import/no-extraneous-dependencies -import { get, isEqual } from 'lodash'; +import get from 'lodash.get'; +import isEqual from 'lodash.isequal'; import { IContextObject, @@ -423,9 +423,7 @@ export function getContext( * Returns which parameters are dependent on which * */ -export function getParamterDependencies( - nodePropertiesArray: INodeProperties[], -): IParameterDependencies { +function getParameterDependencies(nodePropertiesArray: INodeProperties[]): IParameterDependencies { const dependencies: IParameterDependencies = {}; for (const nodeProperties of nodePropertiesArray) { @@ -548,7 +546,7 @@ export function getNodeParameters( parameterDependencies?: IParameterDependencies, ): INodeParameters | null { if (parameterDependencies === undefined) { - parameterDependencies = getParamterDependencies(nodePropertiesArray); + parameterDependencies = getParameterDependencies(nodePropertiesArray); } // Get the parameter names which get used multiple times as for this diff --git a/packages/workflow/src/ObservableObject.ts b/packages/workflow/src/ObservableObject.ts index d22bf26882a6b..a0b50608b5430 100644 --- a/packages/workflow/src/ObservableObject.ts +++ b/packages/workflow/src/ObservableObject.ts @@ -4,7 +4,7 @@ /* eslint-disable no-underscore-dangle */ import { IDataObject, IObservableObject } from './Interfaces'; -export interface IObservableOptions { +interface IObservableOptions { ignoreEmptyOnFirstChild?: boolean; } diff --git a/packages/workflow/test/Helpers.ts b/packages/workflow/test/Helpers.ts index bf0a00934ea91..289e6baa93289 100644 --- a/packages/workflow/test/Helpers.ts +++ b/packages/workflow/test/Helpers.ts @@ -673,8 +673,6 @@ class NodeTypesClass implements INodeTypes { }, }; - async init(nodeTypes: INodeTypeData): Promise {} - getAll(): INodeType[] { return Object.values(this.nodeTypes).map((data) => NodeHelpers.getVersionedNodeType(data.type)); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a18082a6f82b6..0bb816cebdc48 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,7 @@ importers: start-server-and-test: ^1.14.0 supertest: ^6.2.2 ts-jest: ^29.0.3 + tsc-watch: ^5.0.3 turbo: 1.5.5 typescript: ^4.8.4 dependencies: @@ -55,6 +56,7 @@ importers: start-server-and-test: 1.14.0 supertest: 6.3.0 ts-jest: 29.0.3_s73gpqhbuwbfokcbq32jn3f4zi + tsc-watch: 5.0.3_typescript@4.8.4 turbo: 1.5.5 typescript: 4.8.4 @@ -155,6 +157,7 @@ importers: google-timezones-json: ^1.0.2 handlebars: 4.7.7 inquirer: ^7.0.1 + ioredis: ^4.28.5 json-diff: ^0.5.4 jsonschema: ^1.4.1 jsonwebtoken: ^8.5.1 @@ -179,6 +182,7 @@ importers: open: ^7.0.0 openapi-types: ^10.0.0 p-cancelable: ^2.0.0 + parseurl: ^1.3.3 passport: ^0.6.0 passport-cookie: ^1.0.9 passport-jwt: ^4.0.0 @@ -237,6 +241,7 @@ importers: google-timezones-json: 1.0.2 handlebars: 4.7.7 inquirer: 7.3.3 + ioredis: 4.28.5 json-diff: 0.5.5 jsonschema: 1.4.1 jsonwebtoken: 8.5.1 @@ -260,6 +265,7 @@ importers: open: 7.4.2 openapi-types: 10.0.0 p-cancelable: 2.1.1 + parseurl: 1.3.3 passport: 0.6.0 passport-cookie: 1.0.9 passport-jwt: 4.0.0 @@ -274,7 +280,7 @@ importers: sse-channel: 3.1.1 swagger-ui-express: 4.5.0_express@4.18.2 tslib: 1.14.1 - typeorm: 0.2.45_tfktmxoxppkfsj4arg6322vdzq + typeorm: 0.2.45_6spgkqhramqg35yodisibk43rm uuid: 8.3.2 validator: 13.7.0 winston: 3.8.2 @@ -334,6 +340,7 @@ importers: client-oauth2: ^4.2.5 cron: ~1.7.2 crypto-js: ~4.1.1 + fast-glob: ^3.2.5 file-type: ^16.5.4 flatted: ^3.2.4 form-data: ^4.0.0 @@ -351,6 +358,7 @@ importers: client-oauth2: 4.3.3 cron: 1.7.2 crypto-js: 4.1.1 + fast-glob: 3.2.12 file-type: 16.5.4 flatted: 3.2.7 form-data: 4.0.0 @@ -411,7 +419,7 @@ importers: vue-loader: ^15.9.7 vue-property-decorator: ^9.1.2 vue-template-compiler: ^2.7 - vue-tsc: ^0.34.8 + vue-tsc: ^0.35.0 vue-typed-mixins: ^0.2.0 vue2-boring-avatars: 0.3.4 webpack: ^4.46.0 @@ -458,7 +466,7 @@ importers: vue-loader: 15.10.0_bmmfcdfkgwka5ige2hekgeknby vue-property-decorator: 9.1.2_lh5kvfzhejbphpoiiowdoloare vue-template-compiler: 2.7.13 - vue-tsc: 0.34.17_typescript@4.8.4 + vue-tsc: 0.35.2_typescript@4.8.4 webpack: 4.46.0 packages/editor-ui: @@ -534,7 +542,7 @@ importers: vue-prism-editor: ^0.3.0 vue-router: ^3.0.6 vue-template-compiler: ^2.7 - vue-tsc: ^0.34.15 + vue-tsc: ^0.35.0 vue-typed-mixins: ^0.2.0 vue2-boring-avatars: 0.3.4 vue2-teleport: ^1.0.1 @@ -618,7 +626,7 @@ importers: vite-plugin-html: 3.2.0_vite@2.9.5 vite-plugin-monaco-editor: 1.1.0_monaco-editor@0.33.0 vitest: 0.9.3_c8@7.12.0+sass@1.55.0 - vue-tsc: 0.34.17_typescript@4.8.4 + vue-tsc: 0.35.2_typescript@4.8.4 packages/node-dev: specifiers: @@ -840,6 +848,7 @@ importers: '@types/express': ^4.17.6 '@types/jmespath': ^0.15.0 '@types/lodash.get': ^4.4.6 + '@types/lodash.isequal': ^4.5.6 '@types/lodash.merge': ^4.6.6 '@types/lodash.set': ^4.3.6 '@types/luxon': ^2.0.9 @@ -864,6 +873,7 @@ importers: '@types/express': 4.17.14 '@types/jmespath': 0.15.0 '@types/lodash.get': 4.4.7 + '@types/lodash.isequal': 4.5.6 '@types/lodash.merge': 4.6.7 '@types/lodash.set': 4.3.7 '@types/luxon': 2.4.0 @@ -5754,6 +5764,12 @@ packages: '@types/lodash': 4.14.186 dev: true + /@types/lodash.isequal/4.5.6: + resolution: {integrity: sha512-Ww4UGSe3DmtvLLJm2F16hDwEQSv7U0Rr8SujLUA2wHI2D2dm8kPu6Et+/y303LfjTIwSBKXB/YTUcAKpem/XEg==} + dependencies: + '@types/lodash': 4.14.186 + dev: true + /@types/lodash.merge/4.6.7: resolution: {integrity: sha512-OwxUJ9E50gw3LnAefSHJPHaBLGEKmQBQ7CZe/xflHkyy/wH2zVyEIAKReHvVrrn7zKdF58p16We9kMfh7v0RRQ==} dependencies: @@ -6431,32 +6447,32 @@ packages: vue: 2.7.13 dev: true - /@volar/code-gen/0.34.17: - resolution: {integrity: sha512-rHR7BA71BJ/4S7xUOPMPiB7uk6iU9oTWpEMZxFi5VGC9iJmDncE82WzU5iYpcbOBCVHsOjMh0+5CGMgdO6SaPA==} + /@volar/code-gen/0.35.2: + resolution: {integrity: sha512-MoZHuNnPfUWnCNkQUI5+U+gvLTxrU+XlCTusdNOTFYUUAa+M68MH0RxFIS9Ybj4uAUWTcZx0Ow1q5t/PZozo+Q==} dependencies: - '@volar/source-map': 0.34.17 + '@volar/source-map': 0.35.2 dev: true - /@volar/source-map/0.34.17: - resolution: {integrity: sha512-3yn1IMXJGGWB/G817/VFlFMi8oh5pmE7VzUqvgMZMrppaZpKj6/juvJIEiXNxRsgWc0RxIO8OSp4htdPUg1Raw==} + /@volar/source-map/0.35.2: + resolution: {integrity: sha512-PFHh9wN/qMkOWYyvmB8ckvIzolrpNOvK5EBdxxdTpiPJhfYjW82rMDBnYf6RxCe7yQxrUrmve6BWVO7flxWNVQ==} dev: true - /@volar/vue-code-gen/0.34.17: - resolution: {integrity: sha512-17pzcK29fyFWUc+C82J3JYSnA+jy3QNrIldb9kPaP9Itbik05ZjEIyEue9FjhgIAuHeYSn4LDM5s6nGjxyfhsQ==} + /@volar/vue-code-gen/0.35.2: + resolution: {integrity: sha512-8H6P8EtN06eSVGjtcJhGqZzFIg6/nWoHVOlnhc5vKqC7tXwpqPbyMQae0tO7pLBd5qSb/dYU5GQcBAHsi2jgyA==} dependencies: - '@volar/code-gen': 0.34.17 - '@volar/source-map': 0.34.17 + '@volar/code-gen': 0.35.2 + '@volar/source-map': 0.35.2 '@vue/compiler-core': 3.2.40 '@vue/compiler-dom': 3.2.40 '@vue/shared': 3.2.40 dev: true - /@volar/vue-typescript/0.34.17: - resolution: {integrity: sha512-U0YSVIBPRWVPmgJHNa4nrfq88+oS+tmyZNxmnfajIw9A/GOGZQiKXHC0k09SVvbYXlsjgJ6NIjhm9NuAhGRQjg==} + /@volar/vue-typescript/0.35.2: + resolution: {integrity: sha512-PZI6Urb+Vr5Dvgf9xysM8X7TP09inWDy1wjDtprBoBhxS7r0Dg3V0qZuJa7sSGz7M0QMa5R/CBaZPhlxFCfJBw==} dependencies: - '@volar/code-gen': 0.34.17 - '@volar/source-map': 0.34.17 - '@volar/vue-code-gen': 0.34.17 + '@volar/code-gen': 0.35.2 + '@volar/source-map': 0.35.2 + '@volar/vue-code-gen': 0.35.2 '@vue/compiler-sfc': 3.2.40 '@vue/reactivity': 3.2.40 dev: true @@ -16015,6 +16031,10 @@ packages: resolution: {integrity: sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==} dev: false + /node-cleanup/2.1.2: + resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==} + dev: true + /node-dir/0.1.17: resolution: {integrity: sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==} engines: {node: '>= 0.10.5'} @@ -19679,6 +19699,11 @@ packages: engines: {node: '>=4'} dev: false + /string-argv/0.1.2: + resolution: {integrity: sha512-mBqPGEOMNJKXRo7z0keX0wlAhbBAjilUdPW13nN0PecVryZxdHIeM7TqbsSUA7VYuS00HGC6mojP7DlQzfa9ZA==} + engines: {node: '>=0.6.19'} + dev: true + /string-length/4.0.2: resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==} engines: {node: '>=10'} @@ -20556,6 +20581,21 @@ packages: plimit-lit: 1.4.1 dev: true + /tsc-watch/5.0.3_typescript@4.8.4: + resolution: {integrity: sha512-Hz2UawwELMSLOf0xHvAFc7anLeMw62cMVXr1flYmhRuOhOyOljwmb1l/O60ZwRyy1k7N1iC1mrn1QYM2zITfuw==} + engines: {node: '>=8.17.0'} + hasBin: true + peerDependencies: + typescript: '*' + dependencies: + cross-spawn: 7.0.3 + node-cleanup: 2.1.2 + ps-tree: 1.2.0 + string-argv: 0.1.2 + strip-ansi: 6.0.1 + typescript: 4.8.4 + dev: true + /tsconfig-paths/3.14.1: resolution: {integrity: sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==} dependencies: @@ -20751,7 +20791,7 @@ packages: /typedarray/0.0.6: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} - /typeorm/0.2.45_tfktmxoxppkfsj4arg6322vdzq: + /typeorm/0.2.45_6spgkqhramqg35yodisibk43rm: resolution: {integrity: sha512-c0rCO8VMJ3ER7JQ73xfk0zDnVv0WDjpsP6Q1m6CVKul7DB9iVdWLRjPzc8v2eaeBuomsbZ2+gTaYr8k1gm3bYA==} hasBin: true peerDependencies: @@ -20810,6 +20850,7 @@ packages: debug: 4.3.4 dotenv: 8.6.0 glob: 7.2.3 + ioredis: 4.28.5 js-yaml: 4.1.0 mkdirp: 1.0.4 mysql2: 2.3.3 @@ -21794,13 +21835,13 @@ packages: resolution: {integrity: sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==} dev: true - /vue-tsc/0.34.17_typescript@4.8.4: - resolution: {integrity: sha512-jzUXky44ZLHC4daaJag7FQr3idlPYN719/K1eObGljz5KaS2UnVGTU/XSYCd7d6ampYYg4OsyalbHyJIxV0aEQ==} + /vue-tsc/0.35.2_typescript@4.8.4: + resolution: {integrity: sha512-aqY16VlODHzqtKGUkqdumNpH+s5ABCkufRyvMKQlL/mua+N2DfSVnHufzSNNUMr7vmOO0YsNg27jsspBMq4iGA==} hasBin: true peerDependencies: typescript: '*' dependencies: - '@volar/vue-typescript': 0.34.17 + '@volar/vue-typescript': 0.35.2 typescript: 4.8.4 dev: true diff --git a/turbo.json b/turbo.json index 46edb28e4d730..ff00cca9e4326 100644 --- a/turbo.json +++ b/turbo.json @@ -5,8 +5,11 @@ "cache": false }, "build": { - "dependsOn": ["^build"] + "dependsOn": [ + "^build" + ] }, + "typecheck": {}, "format": {}, "lint": {}, "lintfix": {},