diff --git a/packages/cli/src/commands/import/workflow.ts b/packages/cli/src/commands/import/workflow.ts index 58aed9484d218..9f3ccfc682cfc 100644 --- a/packages/cli/src/commands/import/workflow.ts +++ b/packages/cli/src/commands/import/workflow.ts @@ -1,26 +1,18 @@ import { flags } from '@oclif/command'; -import type { INode, INodeCredentialsDetails } from 'n8n-workflow'; import { ApplicationError, jsonParse } from 'n8n-workflow'; import fs from 'fs'; import glob from 'fast-glob'; import { Container } from 'typedi'; -import type { EntityManager } from 'typeorm'; -import { v4 as uuid } from 'uuid'; -import * as Db from '@/Db'; -import { SharedWorkflow } from '@db/entities/SharedWorkflow'; import { WorkflowEntity } from '@db/entities/WorkflowEntity'; -import type { Role } from '@db/entities/Role'; -import type { User } from '@db/entities/User'; import { disableAutoGeneratedIds } from '@db/utils/commandHelpers'; -import type { ICredentialsDb, IWorkflowToImport } from '@/Interfaces'; -import { replaceInvalidCredentials } from '@/WorkflowHelpers'; +import type { IWorkflowToImport } from '@/Interfaces'; import { BaseCommand } from '../BaseCommand'; import { generateNanoId } from '@db/utils/generators'; import { RoleService } from '@/services/role.service'; -import { TagService } from '@/services/tag.service'; import { UM_FIX_INSTRUCTION } from '@/constants'; import { UserRepository } from '@db/repositories/user.repository'; -import { CredentialsRepository } from '@db/repositories/credentials.repository'; +import { ImportService } from '@/services/import.service'; +import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; function assertHasWorkflowsToImport(workflows: unknown): asserts workflows is IWorkflowToImport[] { if (!Array.isArray(workflows)) { @@ -64,16 +56,9 @@ export class ImportWorkflowsCommand extends BaseCommand { }), }; - private ownerWorkflowRole: Role; - - private transactionManager: EntityManager; - - private tagService: TagService; - async init() { disableAutoGeneratedIds(WorkflowEntity); await super.init(); - this.tagService = Container.get(TagService); } async run(): Promise { @@ -94,12 +79,8 @@ export class ImportWorkflowsCommand extends BaseCommand { } } - await this.initOwnerWorkflowRole(); const user = flags.userId ? await this.getAssignee(flags.userId) : await this.getOwner(); - const credentials = await Container.get(CredentialsRepository).find(); - const tags = await this.tagService.getAll(); - let totalImported = 0; if (flags.separate) { @@ -116,41 +97,17 @@ export class ImportWorkflowsCommand extends BaseCommand { totalImported = files.length; this.logger.info(`Importing ${totalImported} workflows...`); - await Db.getConnection().transaction(async (transactionManager) => { - this.transactionManager = transactionManager; - - for (const file of files) { - const workflow = jsonParse( - fs.readFileSync(file, { encoding: 'utf8' }), - ); - if (!workflow.id) { - workflow.id = generateNanoId(); - } - - if (credentials.length > 0) { - workflow.nodes.forEach((node: INode) => { - this.transformCredentials(node, credentials); - - if (!node.id) { - node.id = uuid(); - } - }); - } - - if (Object.prototype.hasOwnProperty.call(workflow, 'tags')) { - await this.tagService.setTagsForImport(transactionManager, workflow, tags); - } - - if (workflow.active) { - this.logger.info( - `Deactivating workflow "${workflow.name}" during import, remember to activate it later.`, - ); - workflow.active = false; - } - - await this.storeWorkflow(workflow, user); + + for (const file of files) { + const workflow = jsonParse(fs.readFileSync(file, { encoding: 'utf8' })); + if (!workflow.id) { + workflow.id = generateNanoId(); } - }); + + const _workflow = Container.get(WorkflowRepository).create(workflow); + + await Container.get(ImportService).importWorkflows([_workflow], user.id); + } this.reportSuccess(totalImported); process.exit(); @@ -160,46 +117,13 @@ export class ImportWorkflowsCommand extends BaseCommand { fs.readFileSync(flags.input, { encoding: 'utf8' }), ); + const _workflows = workflows.map((w) => Container.get(WorkflowRepository).create(w)); + assertHasWorkflowsToImport(workflows); totalImported = workflows.length; - await Db.getConnection().transaction(async (transactionManager) => { - this.transactionManager = transactionManager; - - for (const workflow of workflows) { - let oldCredentialFormat = false; - if (credentials.length > 0) { - workflow.nodes.forEach((node: INode) => { - this.transformCredentials(node, credentials); - if (!node.id) { - node.id = uuid(); - } - if (!node.credentials?.id) { - oldCredentialFormat = true; - } - }); - } - if (oldCredentialFormat) { - try { - await replaceInvalidCredentials(workflow as unknown as WorkflowEntity); - } catch (error) { - this.logger.error('Failed to replace invalid credential', error as Error); - } - } - if (Object.prototype.hasOwnProperty.call(workflow, 'tags')) { - await this.tagService.setTagsForImport(transactionManager, workflow, tags); - } - if (workflow.active) { - this.logger.info( - `Deactivating workflow "${workflow.name}" during import, remember to activate it later.`, - ); - workflow.active = false; - } - - await this.storeWorkflow(workflow, user); - } - }); + await Container.get(ImportService).importWorkflows(_workflows, user.id); this.reportSuccess(totalImported); } @@ -213,29 +137,6 @@ export class ImportWorkflowsCommand extends BaseCommand { this.logger.info(`Successfully imported ${total} ${total === 1 ? 'workflow.' : 'workflows.'}`); } - private async initOwnerWorkflowRole() { - const ownerWorkflowRole = await Container.get(RoleService).findWorkflowOwnerRole(); - - if (!ownerWorkflowRole) { - throw new ApplicationError(`Failed to find owner workflow role. ${UM_FIX_INSTRUCTION}`); - } - - this.ownerWorkflowRole = ownerWorkflowRole; - } - - private async storeWorkflow(workflow: object, user: User) { - const result = await this.transactionManager.upsert(WorkflowEntity, workflow, ['id']); - await this.transactionManager.upsert( - SharedWorkflow, - { - workflowId: result.identifiers[0].id as string, - userId: user.id, - roleId: this.ownerWorkflowRole.id, - }, - ['workflowId', 'userId'], - ); - } - private async getOwner() { const ownerGlobalRole = await Container.get(RoleService).findGlobalOwnerRole(); @@ -259,28 +160,4 @@ export class ImportWorkflowsCommand extends BaseCommand { return user; } - - private transformCredentials(node: INode, credentialsEntities: ICredentialsDb[]) { - if (node.credentials) { - const allNodeCredentials = Object.entries(node.credentials); - for (const [type, name] of allNodeCredentials) { - if (typeof name === 'string') { - const nodeCredentials: INodeCredentialsDetails = { - id: null, - name, - }; - - const matchingCredentials = credentialsEntities.filter( - (credentials) => credentials.name === name && credentials.type === type, - ); - - if (matchingCredentials.length === 1) { - nodeCredentials.id = matchingCredentials[0].id; - } - - node.credentials[type] = nodeCredentials; - } - } - } - } } diff --git a/packages/cli/src/services/import.service.ts b/packages/cli/src/services/import.service.ts new file mode 100644 index 0000000000000..9ba0715b77ecc --- /dev/null +++ b/packages/cli/src/services/import.service.ts @@ -0,0 +1,153 @@ +import { Service } from 'typedi'; +import { v4 as uuid } from 'uuid'; +import { type INode, type INodeCredentialsDetails } from 'n8n-workflow'; +import type { EntityManager } from 'typeorm'; + +import { Logger } from '@/Logger'; +import * as Db from '@/Db'; +import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; +import { TagRepository } from '@/databases/repositories/tag.repository'; +import { SharedWorkflow } from '@/databases/entities/SharedWorkflow'; +import { RoleService } from '@/services/role.service'; +import { replaceInvalidCredentials } from '@/WorkflowHelpers'; +import { WorkflowEntity } from '@/databases/entities/WorkflowEntity'; +import { WorkflowTagMapping } from '@/databases/entities/WorkflowTagMapping'; + +import type { TagEntity } from '@/databases/entities/TagEntity'; +import type { Role } from '@/databases/entities/Role'; +import type { ICredentialsDb } from '@/Interfaces'; + +@Service() +export class ImportService { + private dbCredentials: ICredentialsDb[] = []; + + private dbTags: TagEntity[] = []; + + private workflowOwnerRole: Role; + + constructor( + private readonly logger: Logger, + private readonly credentialsRepository: CredentialsRepository, + private readonly tagRepository: TagRepository, + private readonly roleService: RoleService, + ) {} + + async initRecords() { + this.dbCredentials = await this.credentialsRepository.find(); + this.dbTags = await this.tagRepository.find(); + this.workflowOwnerRole = await this.roleService.findWorkflowOwnerRole(); + } + + async importWorkflows(workflows: WorkflowEntity[], userId: string) { + await this.initRecords(); + + for (const workflow of workflows) { + workflow.nodes.forEach((node) => { + this.toNewCredentialFormat(node); + + if (!node.id) node.id = uuid(); + }); + + const hasInvalidCreds = workflow.nodes.some((node) => !node.credentials?.id); + + if (hasInvalidCreds) await this.replaceInvalidCreds(workflow); + } + + await Db.transaction(async (tx) => { + for (const workflow of workflows) { + if (workflow.active) { + workflow.active = false; + + this.logger.info(`Deactivating workflow "${workflow.name}". Remember to activate later.`); + } + + const upsertResult = await tx.upsert(WorkflowEntity, workflow, ['id']); + + const workflowId = upsertResult.identifiers.at(0)?.id as string; + + await tx.upsert(SharedWorkflow, { workflowId, userId, roleId: this.workflowOwnerRole.id }, [ + 'workflowId', + 'userId', + ]); + + if (!workflow.tags?.length) continue; + + await this.setTags(tx, workflow); + + for (const tag of workflow.tags) { + await tx.upsert(WorkflowTagMapping, { tagId: tag.id, workflowId }, [ + 'tagId', + 'workflowId', + ]); + } + } + }); + } + + async replaceInvalidCreds(workflow: WorkflowEntity) { + try { + await replaceInvalidCredentials(workflow); + } catch (e) { + const error = e instanceof Error ? e : new Error(`${e}`); + this.logger.error('Failed to replace invalid credential', error); + } + } + + /** + * Convert a node's credentials from old format `{ : }` + * to new format: `{ : { id: string | null, name: } }` + */ + private toNewCredentialFormat(node: INode) { + if (!node.credentials) return; + + for (const [type, name] of Object.entries(node.credentials)) { + if (typeof name !== 'string') continue; + + const nodeCredential: INodeCredentialsDetails = { id: null, name }; + + const match = this.dbCredentials.find((c) => c.name === name && c.type === type); + + if (match) nodeCredential.id = match.id; + + node.credentials[type] = nodeCredential; + } + } + + /** + * Set tags on workflow to import while ensuring all tags exist in the database, + * either by matching incoming to existing tags or by creating them first. + */ + private async setTags(tx: EntityManager, workflow: WorkflowEntity) { + if (!workflow?.tags?.length) return; + + for (let i = 0; i < workflow.tags.length; i++) { + const importTag = workflow.tags[i]; + + if (!importTag.name) continue; + + const identicalMatch = this.dbTags.find( + (dbTag) => + dbTag.id === importTag.id && + dbTag.createdAt && + importTag.createdAt && + dbTag.createdAt.getTime() === new Date(importTag.createdAt).getTime(), + ); + + if (identicalMatch) { + workflow.tags[i] = identicalMatch; + continue; + } + + const nameMatch = this.dbTags.find((dbTag) => dbTag.name === importTag.name); + + if (nameMatch) { + workflow.tags[i] = nameMatch; + continue; + } + + const tagEntity = this.tagRepository.create(importTag); + + workflow.tags[i] = await tx.save(tagEntity); + } + } +} diff --git a/packages/cli/src/services/tag.service.ts b/packages/cli/src/services/tag.service.ts index 8d3e40ad93251..63050ea14981c 100644 --- a/packages/cli/src/services/tag.service.ts +++ b/packages/cli/src/services/tag.service.ts @@ -1,9 +1,9 @@ import { TagRepository } from '@db/repositories/tag.repository'; import { Service } from 'typedi'; import { validateEntity } from '@/GenericHelpers'; -import type { ITagToImport, ITagWithCountDb, IWorkflowToImport } from '@/Interfaces'; +import type { ITagWithCountDb } from '@/Interfaces'; import type { TagEntity } from '@db/entities/TagEntity'; -import type { EntityManager, FindManyOptions, FindOneOptions } from 'typeorm'; +import type { FindManyOptions, FindOneOptions } from 'typeorm'; import type { UpsertOptions } from 'typeorm/repository/UpsertOptions'; import { ExternalHooks } from '@/ExternalHooks'; @@ -89,69 +89,4 @@ export class TagService { return requestOrder.map((tagId) => tagMap[tagId]); } - - /** - * Set tag IDs to use existing tags, creates a new tag if no matching tag could be found - */ - async setTagsForImport( - transactionManager: EntityManager, - workflow: IWorkflowToImport, - tags: TagEntity[], - ) { - if (!this.hasTags(workflow)) return; - - const workflowTags = workflow.tags; - const tagLookupPromises = []; - for (let i = 0; i < workflowTags.length; i++) { - if (workflowTags[i]?.name) { - const lookupPromise = this.findOrCreateTag(transactionManager, workflowTags[i], tags).then( - (tag) => { - workflowTags[i] = { - id: tag.id, - name: tag.name, - }; - }, - ); - tagLookupPromises.push(lookupPromise); - } - } - - await Promise.all(tagLookupPromises); - } - - private hasTags(workflow: IWorkflowToImport) { - return 'tags' in workflow && Array.isArray(workflow.tags) && workflow.tags.length > 0; - } - - private async findOrCreateTag( - transactionManager: EntityManager, - importTag: ITagToImport, - tagsEntities: TagEntity[], - ) { - // Assume tag is identical if createdAt date is the same to preserve a changed tag name - const identicalMatch = tagsEntities.find( - (existingTag) => - existingTag.id === importTag.id && - existingTag.createdAt && - importTag.createdAt && - existingTag.createdAt.getTime() === new Date(importTag.createdAt).getTime(), - ); - if (identicalMatch) { - return identicalMatch; - } - - const nameMatch = tagsEntities.find((existingTag) => existingTag.name === importTag.name); - if (nameMatch) { - return nameMatch; - } - - const created = await this.txCreateTag(transactionManager, importTag.name); - tagsEntities.push(created); - return created; - } - - private async txCreateTag(transactionManager: EntityManager, name: string) { - const tag = this.tagRepository.create({ name: name.trim() }); - return transactionManager.save(tag); - } } diff --git a/packages/cli/test/integration/import.service.test.ts b/packages/cli/test/integration/import.service.test.ts new file mode 100644 index 0000000000000..07b7f1a600478 --- /dev/null +++ b/packages/cli/test/integration/import.service.test.ts @@ -0,0 +1,185 @@ +import Container from 'typedi'; +import { mock } from 'jest-mock-extended'; +import { v4 as uuid } from 'uuid'; +import type { INode } from 'n8n-workflow'; + +import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; +import { TagRepository } from '@/databases/repositories/tag.repository'; +import { ImportService } from '@/services/import.service'; +import { RoleService } from '@/services/role.service'; +import { TagEntity } from '@/databases/entities/TagEntity'; +import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import { SharedWorkflowRepository } from '@/databases/repositories/sharedWorkflow.repository'; + +import * as testDb from './shared/testDb'; +import { mockInstance } from '../shared/mocking'; +import { createOwner } from './shared/db/users'; +import { createWorkflow, getWorkflowById } from './shared/db/workflows'; + +import type { User } from '@db/entities/User'; + +describe('ImportService', () => { + let importService: ImportService; + let tagRepository: TagRepository; + let owner: User; + + beforeAll(async () => { + await testDb.init(); + + owner = await createOwner(); + + tagRepository = Container.get(TagRepository); + + const credentialsRepository = mockInstance(CredentialsRepository); + + credentialsRepository.find.mockResolvedValue([]); + + importService = new ImportService( + mock(), + credentialsRepository, + tagRepository, + Container.get(RoleService), + ); + }); + + afterEach(async () => { + await testDb.truncate(['Workflow', 'SharedWorkflow', 'Tag', 'WorkflowTagMapping']); + }); + + afterAll(async () => { + await testDb.terminate(); + }); + + test('should import credless and tagless workflow', async () => { + const workflowToImport = await createWorkflow(); + + await importService.importWorkflows([workflowToImport], owner.id); + + const dbWorkflow = await getWorkflowById(workflowToImport.id); + + if (!dbWorkflow) fail('Expected to find workflow'); + + expect(dbWorkflow.id).toBe(workflowToImport.id); + }); + + test('should make user owner of imported workflow', async () => { + const workflowToImport = await createWorkflow(); + + await importService.importWorkflows([workflowToImport], owner.id); + + const workflowOwnerRole = await Container.get(RoleService).findWorkflowOwnerRole(); + + const dbSharing = await Container.get(SharedWorkflowRepository).findOneOrFail({ + where: { workflowId: workflowToImport.id, userId: owner.id, roleId: workflowOwnerRole.id }, + }); + + expect(dbSharing.userId).toBe(owner.id); + }); + + test('should deactivate imported workflow if active', async () => { + const workflowToImport = await createWorkflow({ active: true }); + + await importService.importWorkflows([workflowToImport], owner.id); + + const dbWorkflow = await getWorkflowById(workflowToImport.id); + + if (!dbWorkflow) fail('Expected to find workflow'); + + expect(dbWorkflow.active).toBe(false); + }); + + test('should leave intact new-format credentials', async () => { + const credential = { + n8nApi: { id: '123', name: 'n8n API' }, + }; + + const nodes: INode[] = [ + { + id: uuid(), + name: 'n8n', + parameters: {}, + position: [0, 0], + type: 'n8n-nodes-base.n8n', + typeVersion: 1, + credentials: credential, + }, + ]; + + const workflowToImport = await createWorkflow({ nodes }); + + await importService.importWorkflows([workflowToImport], owner.id); + + const dbWorkflow = await getWorkflowById(workflowToImport.id); + + if (!dbWorkflow) fail('Expected to find workflow'); + + expect(dbWorkflow.nodes.at(0)?.credentials).toMatchObject(credential); + }); + + test('should set tag by identical match', async () => { + const tag = Object.assign(new TagEntity(), { + id: '123', + createdAt: new Date(), + name: 'Test', + }); + + await tagRepository.save(tag); // tag stored + + const workflowToImport = await createWorkflow({ tags: [tag] }); + + await importService.importWorkflows([workflowToImport], owner.id); + + const dbWorkflow = await Container.get(WorkflowRepository).findOneOrFail({ + where: { id: workflowToImport.id }, + relations: ['tags'], + }); + + expect(dbWorkflow.tags).toStrictEqual([tag]); // workflow tagged + + const dbTags = await tagRepository.find(); + + expect(dbTags).toStrictEqual([tag]); // tag matched + }); + + test('should set tag by name match', async () => { + const tag = Object.assign(new TagEntity(), { name: 'Test' }); + + await tagRepository.save(tag); // tag stored + + const workflowToImport = await createWorkflow({ tags: [tag] }); + + await importService.importWorkflows([workflowToImport], owner.id); + + const dbWorkflow = await Container.get(WorkflowRepository).findOneOrFail({ + where: { id: workflowToImport.id }, + relations: ['tags'], + }); + + expect(dbWorkflow.tags).toStrictEqual([tag]); // workflow tagged + + const dbTags = await tagRepository.find(); + + expect(dbTags).toStrictEqual([tag]); // tag matched + }); + + test('should set tag by creating if no match', async () => { + const tag = Object.assign(new TagEntity(), { name: 'Test' }); // tag not stored + + const workflowToImport = await createWorkflow({ tags: [tag] }); + + await importService.importWorkflows([workflowToImport], owner.id); + + const dbWorkflow = await Container.get(WorkflowRepository).findOneOrFail({ + where: { id: workflowToImport.id }, + relations: ['tags'], + }); + + if (!dbWorkflow.tags) fail('No tags found on workflow'); + + expect(dbWorkflow.tags.at(0)?.name).toBe(tag.name); // workflow tagged + + const dbTag = await tagRepository.findOneOrFail({ where: { name: tag.name } }); + + expect(dbTag.name).toBe(tag.name); // tag created + }); +});