refactor(core): Refactor community-packages endpoints (no-changelog)

This also fixes PAY-605

packages/cli/src/LoadNodesAndCredentials.ts

@@ -1,5 +1,7 @@
+import { EventEmitter } from 'events';
 import uniq from 'lodash/uniq';
 import glob from 'fast-glob';
+import { Service } from 'typedi';
 import type { DirectoryLoader, Types } from 'n8n-core';
 import {
 	CUSTOM_EXTENSION_ENV,
@@ -22,22 +24,18 @@ import { createWriteStream } from 'fs';
 import { mkdir } from 'fs/promises';
 import path from 'path';
 import config from '@/config';
-import type { InstalledPackages } from '@db/entities/InstalledPackages';
-import { CommunityPackageService } from './services/communityPackage.service';
 import {
 	GENERATED_STATIC_DIR,
-	RESPONSE_ERROR_MESSAGES,
 	CUSTOM_API_CALL_KEY,
 	CUSTOM_API_CALL_NAME,
 	inTest,
 	CLI_DIR,
 	inE2ETests,
 } from '@/constants';
 import { CredentialsOverwrites } from '@/CredentialsOverwrites';
-import Container, { Service } from 'typedi';
 
 @Service()
-export class LoadNodesAndCredentials implements INodesAndCredentials {
+export class LoadNodesAndCredentials extends EventEmitter implements INodesAndCredentials {
 	known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
 
 	loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} };
@@ -92,6 +90,12 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		await this.loadNodesFromCustomDirectories();
 		await this.postProcessLoaders();
 		this.injectCustomApiCallOptions();
+
+		this.on('postProcess:start', async () => {
+			await this.postProcessLoaders();
+			await this.generateTypesForFrontend();
+			this.emit('postProcess:done');
+		});
 	}
 
 	async generateTypesForFrontend() {
@@ -180,93 +184,16 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		}
 	}
 
-	private async installOrUpdateNpmModule(
-		packageName: string,
-		options: { version?: string } | { installedPackage: InstalledPackages },
-	) {
-		const isUpdate = 'installedPackage' in options;
-		const command = isUpdate
-			? `npm update ${packageName}`
-			: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;
-
-		const communityPackageService = Container.get(CommunityPackageService);
-
-		try {
-			await communityPackageService.executeNpmCommand(command);
-		} catch (error) {
-			if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
-				throw new Error(`The npm package "${packageName}" could not be found.`);
-			}
-			throw error;
-		}
-
+	async loadPackage(packageName: string) {
 		const finalNodeUnpackedPath = path.join(this.downloadFolder, 'node_modules', packageName);
-
-		let loader: PackageDirectoryLoader;
-		try {
-			loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
-		} catch (error) {
-			// Remove this package since loading it failed
-			const removeCommand = `npm remove ${packageName}`;
-			try {
-				await communityPackageService.executeNpmCommand(removeCommand);
-			} catch {}
-			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
-		}
-
-		if (loader.loadedNodes.length > 0) {
-			// Save info to DB
-			try {
-				if (isUpdate) {
-					await communityPackageService.removePackageFromDatabase(options.installedPackage);
-				}
-				const installedPackage = await communityPackageService.persistInstalledPackage(loader);
-				await this.postProcessLoaders();
-				await this.generateTypesForFrontend();
-				return installedPackage;
-			} catch (error) {
-				LoggerProxy.error('Failed to save installed packages and nodes', {
-					error: error as Error,
-					packageName,
-				});
-				throw error;
-			}
-		} else {
-			// Remove this package since it contains no loadable nodes
-			const removeCommand = `npm remove ${packageName}`;
-			try {
-				await communityPackageService.executeNpmCommand(removeCommand);
-			} catch {}
-
-			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
-		}
+		return this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
 	}
 
-	async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
-		return this.installOrUpdateNpmModule(packageName, { version });
-	}
-
-	async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
-		const communityPackageService = Container.get(CommunityPackageService);
-
-		await communityPackageService.executeNpmCommand(`npm remove ${packageName}`);
-
-		await communityPackageService.removePackageFromDatabase(installedPackage);
-
+	async unloadPackage(packageName: string) {
 		if (packageName in this.loaders) {
 			this.loaders[packageName].reset();
 			delete this.loaders[packageName];
 		}
-
-		await this.postProcessLoaders();
-		await this.generateTypesForFrontend();
-	}
-
-	async updateNpmModule(
-		packageName: string,
-		installedPackage: InstalledPackages,
-	): Promise<InstalledPackages> {
-		return this.installOrUpdateNpmModule(packageName, { installedPackage });
 	}
 
 	/**
@@ -336,7 +263,7 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		return loader;
 	}
 
-	async postProcessLoaders() {
+	private async postProcessLoaders() {
 		this.known = { nodes: {}, credentials: {} };
 		this.loaded = { nodes: {}, credentials: {} };
 		this.types = { nodes: [], credentials: [] };

packages/cli/src/NodeTypes.ts

@@ -22,6 +22,8 @@ export class NodeTypes implements INodeTypes {
 		// Some nodeTypes need to get special parameters applied like the
 		// polling nodes the polling times
 		this.applySpecialNodeParameters();
+
+		this.nodesAndCredentials.on('postProcess:done', () => this.applySpecialNodeParameters());
 	}
 
 	/**

packages/cli/src/ReloadNodesAndCredentials.ts

@@ -2,12 +2,10 @@ import path from 'path';
 import { realpath, access } from 'fs/promises';
 
 import type { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
-import type { NodeTypes } from '@/NodeTypes';
 import type { Push } from '@/push';
 
 export const reloadNodesAndCredentials = async (
 	loadNodesAndCredentials: LoadNodesAndCredentials,
-	nodeTypes: NodeTypes,
 	push: Push,
 ) => {
 	const { default: debounce } = await import('lodash/debounce');
@@ -33,10 +31,10 @@ export const reloadNodesAndCredentials = async (
 
 			loader.reset();
 			await loader.loadAll();
-			await loadNodesAndCredentials.postProcessLoaders();
-			await loadNodesAndCredentials.generateTypesForFrontend();
-			nodeTypes.applySpecialNodeParameters();
-			push.send('nodeDescriptionUpdated', undefined);
+			loadNodesAndCredentials.once('postProcess:done', () => {
+				push.send('nodeDescriptionUpdated', undefined);
+			});
+			loadNodesAndCredentials.emit('postProcess:start');
 		}, 100);
 
 		const toWatch = loader.isLazyLoaded

packages/cli/src/Server.ts

@@ -89,7 +89,6 @@ import {
 	LdapController,
 	MeController,
 	MFAController,
-	NodesController,
 	NodeTypesController,
 	OwnerController,
 	PasswordResetController,
@@ -414,7 +413,7 @@ export class Server extends AbstractServer {
 
 		if (inDevelopment && process.env.N8N_DEV_RELOAD === 'true') {
 			const { reloadNodesAndCredentials } = await import('@/ReloadNodesAndCredentials');
-			await reloadNodesAndCredentials(this.loadNodesAndCredentials, this.nodeTypes, this.push);
+			await reloadNodesAndCredentials(this.loadNodesAndCredentials, this.push);
 		}
 
 		void Db.collections.Workflow.findOne({
@@ -571,9 +570,11 @@ export class Server extends AbstractServer {
 		}
 
 		if (config.getEnv('nodes.communityPackages.enabled')) {
-			controllers.push(
-				new NodesController(config, this.loadNodesAndCredentials, this.push, internalHooks),
+			// eslint-disable-next-line @typescript-eslint/naming-convention
+			const { CommunityPackagesController } = await import(
+				'@/controllers/communityPackages.controller'
 			);
+			controllers.push(Container.get(CommunityPackagesController));
 		}
 
 		if (inE2ETests) {

packages/cli/src/controllers/nodes.controller.ts → packages/cli/src/controllers/communityPackages.controller.ts

@@ -1,4 +1,6 @@
+import { Service } from 'typedi';
 import { Request, Response, NextFunction } from 'express';
+import config from '@/config';
 import {
 	RESPONSE_ERROR_MESSAGES,
 	STARTER_TEMPLATE_NAME,
@@ -9,12 +11,9 @@ import { NodeRequest } from '@/requests';
 import { BadRequestError, InternalServerError } from '@/ResponseHelper';
 import type { InstalledPackages } from '@db/entities/InstalledPackages';
 import type { CommunityPackages } from '@/Interfaces';
-import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
 import { InternalHooks } from '@/InternalHooks';
 import { Push } from '@/push';
-import { Config } from '@/config';
 import { CommunityPackageService } from '@/services/communityPackage.service';
-import Container from 'typedi';
 
 const {
 	PACKAGE_NOT_INSTALLED,
@@ -33,24 +32,20 @@ export function isNpmError(error: unknown): error is { code: number; stdout: str
 	return typeof error === 'object' && error !== null && 'code' in error && 'stdout' in error;
 }
 
+@Service()
 @Authorized(['global', 'owner'])
-@RestController('/nodes')
-export class NodesController {
-	private communityPackageService: CommunityPackageService;
-
+@RestController('/community-packages')
+export class CommunityPackagesController {
 	constructor(
-		private config: Config,
-		private loadNodesAndCredentials: LoadNodesAndCredentials,
 		private push: Push,
 		private internalHooks: InternalHooks,
-	) {
-		this.communityPackageService = Container.get(CommunityPackageService);
-	}
+		private communityPackageService: CommunityPackageService,
+	) {}
 
 	// TODO: move this into a new decorator `@IfConfig('executions.mode', 'queue')`
 	@Middleware()
 	checkIfCommunityNodesEnabled(req: Request, res: Response, next: NextFunction) {
-		if (this.config.getEnv('executions.mode') === 'queue' && req.method !== 'GET')
+		if (config.getEnv('executions.mode') === 'queue' && req.method !== 'GET')
 			res.status(400).json({
 				status: 'error',
 				message: 'Package management is disabled when running in "queue" mode',
@@ -105,7 +100,7 @@ export class NodesController {
 
 		let installedPackage: InstalledPackages;
 		try {
-			installedPackage = await this.loadNodesAndCredentials.installNpmModule(
+			installedPackage = await this.communityPackageService.installNpmModule(
 				parsed.packageName,
 				parsed.version,
 			);
@@ -180,7 +175,7 @@ export class NodesController {
 		);
 
 		try {
-			const missingPackages = this.config.get('nodes.packagesMissing') as string | undefined;
+			const missingPackages = config.get('nodes.packagesMissing') as string | undefined;
 			if (missingPackages) {
 				hydratedPackages = this.communityPackageService.matchMissingPackages(
 					hydratedPackages,
@@ -215,7 +210,7 @@ export class NodesController {
 		}
 
 		try {
-			await this.loadNodesAndCredentials.removeNpmModule(name, installedPackage);
+			await this.communityPackageService.removeNpmModule(name, installedPackage);
 		} catch (error) {
 			const message = [
 				`Error removing package "${name}"`,
@@ -259,7 +254,7 @@ export class NodesController {
 		}
 
 		try {
-			const newInstalledPackage = await this.loadNodesAndCredentials.updateNpmModule(
+			const newInstalledPackage = await this.communityPackageService.updateNpmModule(
 				this.communityPackageService.parseNpmPackageName(name).packageName,
 				previouslyInstalledPackage,
 			);

packages/cli/src/controllers/index.ts

@@ -2,7 +2,6 @@ export { AuthController } from './auth.controller';
 export { LdapController } from './ldap.controller';
 export { MeController } from './me.controller';
 export { MFAController } from './mfa.controller';
-export { NodesController } from './nodes.controller';
 export { NodeTypesController } from './nodeTypes.controller';
 export { OwnerController } from './owner.controller';
 export { PasswordResetController } from './passwordReset.controller';