From f9555483661d43121663cb3cad2f2dd15ddfd1b4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 24 Oct 2025 15:30:31 +0100 Subject: [PATCH 01/11] changeset --- .changeset/fluffy-eggs-wink.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fluffy-eggs-wink.md diff --git a/.changeset/fluffy-eggs-wink.md b/.changeset/fluffy-eggs-wink.md new file mode 100644 index 000000000..7395a2356 --- /dev/null +++ b/.changeset/fluffy-eggs-wink.md @@ -0,0 +1,5 @@ +--- +'@openfn/project': patch +--- + +On merge, warn when projects may have diverged From d2e0651f9638f83e2c06f760ae8952ebbd0169e4 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Mon, 20 Oct 2025 10:14:18 +0000 Subject: [PATCH 02/11] feat: workflow merge compatibility --- packages/project/src/Workflow.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 721e51e43..2e720f7dc 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -20,6 +20,8 @@ class Workflow { id: string; openfn: OpenfnMeta; + history: string[] = []; + constructor(workflow: l.Workflow) { this.index = { steps: {}, // steps by id From 4e30d4b77bfbff9812f7a0606512a07ebe6bf8b1 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Mon, 20 Oct 2025 15:37:29 +0000 Subject: [PATCH 03/11] feat: add history to workflow property --- packages/project/src/Workflow.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 2e720f7dc..18e16d002 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -20,8 +20,6 @@ class Workflow { id: string; openfn: OpenfnMeta; - history: string[] = []; - constructor(workflow: l.Workflow) { this.index = { steps: {}, // steps by id @@ -179,6 +177,7 @@ class Workflow { // return true if the current workflow can be merged into the target workflow without losing any changes canMergeInto(target: Workflow) { +<<<<<<< HEAD const thisHistory = this.workflow.history?.concat(this.getVersionHash()); const targetHistory = target.workflow.history?.concat( target.getVersionHash() @@ -186,6 +185,12 @@ class Workflow { const targetHead = targetHistory[targetHistory.length - 1]; if (thisHistory.indexOf(targetHead) > -1) return true; +======= + if (!target.workflow.history.length) return true; + const targetHead = + target.workflow.history[target.workflow.history.length - 1]; + if (this.workflow.history.indexOf(targetHead) > -1) return true; +>>>>>>> 664dfbea (feat: add history to workflow property) return false; } } From 2cc1d63799f4a4d564686d945505897925652c76 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Tue, 21 Oct 2025 14:27:29 +0000 Subject: [PATCH 04/11] feat: error when there are incompatible workflows --- packages/project/src/merge/merge-project.ts | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index 09becaae5..2ee0ab9f9 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -1,4 +1,3 @@ -import { Workflow } from '@openfn/lexicon'; import { defaultsDeep, isEmpty } from 'lodash-es'; import { Project } from '../Project'; @@ -6,6 +5,7 @@ import { mergeWorkflows } from './merge-node'; import mapUuids from './map-uuids'; import baseMerge from '../util/base-merge'; import getDuplicates from '../util/get-duplicates'; +import Workflow from '../Workflow'; export type MergeProjectOptions = Partial<{ workflowMappings: Record; // @@ -56,6 +56,27 @@ export function merge( return !!options?.workflowMappings[w.id]; }); + // mergeability + const mergeMapping: Record = {}; + for (const sourceWorkflow of sourceWorkflows) { + const targetId = + options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id; + const targetWorkflow = target.getWorkflow(targetId); + if (!sourceWorkflow.canMergeInto(targetWorkflow)) { + mergeMapping[sourceWorkflow.name] = targetWorkflow?.name; + } + } + + if (Object.keys(mergeMapping).length) { + throw new Error( + `The below workflows can't merge directly without losing data.\n${Object.entries( + mergeMapping + ) + .map(([from, to]) => `${from} → ${to}`) + .join('\n')}` + ); + } + for (const sourceWorkflow of sourceWorkflows) { const targetId = options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id; From 261d5b8f8f6ebe120b6a669ca52129d524f259f2 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Tue, 21 Oct 2025 14:34:42 +0000 Subject: [PATCH 05/11] feat: ignore incompatibility when force passed --- packages/cli/src/merge/command.ts | 2 ++ packages/cli/src/merge/handler.ts | 1 + packages/project/src/merge/merge-project.ts | 5 ++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/merge/command.ts b/packages/cli/src/merge/command.ts index f61db6750..f81c57645 100644 --- a/packages/cli/src/merge/command.ts +++ b/packages/cli/src/merge/command.ts @@ -12,6 +12,7 @@ export type MergeOptions = Required< | 'removeUnmapped' | 'workflowMappings' | 'log' + | 'force' > >; @@ -21,6 +22,7 @@ const options = [ o.removeUnmapped, o.workflowMappings, o.log, + o.force, ]; const mergeCommand: yargs.CommandModule = { diff --git a/packages/cli/src/merge/handler.ts b/packages/cli/src/merge/handler.ts index 4c767df83..fe3ed2283 100644 --- a/packages/cli/src/merge/handler.ts +++ b/packages/cli/src/merge/handler.ts @@ -55,6 +55,7 @@ const mergeHandler = async (options: MergeOptions, logger: Logger) => { const final = Project.merge(sourceProject, targetProject, { removeUnmapped: options.removeUnmapped, workflowMappings: options.workflowMappings, + force: options.force, }); const yaml = final.serialize('state', { format: 'yaml' }); await fs.writeFile(finalPath, yaml); diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index 2ee0ab9f9..ad0335d79 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -10,8 +10,7 @@ import Workflow from '../Workflow'; export type MergeProjectOptions = Partial<{ workflowMappings: Record; // removeUnmapped: boolean; - - force: boolean; // TODO not implemented yet + force: boolean; }>; /** @@ -67,7 +66,7 @@ export function merge( } } - if (Object.keys(mergeMapping).length) { + if (Object.keys(mergeMapping).length && !options?.force) { throw new Error( `The below workflows can't merge directly without losing data.\n${Object.entries( mergeMapping From 94e5d937e86bccd872f08e3968d6c326d13b1f6b Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Tue, 21 Oct 2025 14:42:09 +0000 Subject: [PATCH 06/11] tests: fix workflow check --- packages/project/src/merge/merge-project.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index ad0335d79..db78b4691 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -61,7 +61,7 @@ export function merge( const targetId = options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id; const targetWorkflow = target.getWorkflow(targetId); - if (!sourceWorkflow.canMergeInto(targetWorkflow)) { + if (targetWorkflow && !sourceWorkflow.canMergeInto(targetWorkflow)) { mergeMapping[sourceWorkflow.name] = targetWorkflow?.name; } } From 54a12b8fa4fca9d4875b4fe03e3749b7c2c440d7 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 22 Oct 2025 12:01:17 +0000 Subject: [PATCH 07/11] tests: incompatibility merging via project class --- packages/project/test/project.test.ts | 33 ++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/project/test/project.test.ts b/packages/project/test/project.test.ts index ac5475322..ba29c746e 100644 --- a/packages/project/test/project.test.ts +++ b/packages/project/test/project.test.ts @@ -2,7 +2,7 @@ import test from 'ava'; import type { Provisioner } from '@openfn/lexicon/lightning'; import { Project } from '../src/Project'; -import generateWorkflow from '../src/gen/generator'; +import generateWorkflow, { generateProject } from '../src/gen/generator'; // TODO move to fixtures and re-use? // Or use util function instead? @@ -154,3 +154,34 @@ test('should return UUIDs for everything', (t) => { }, }); }); + +test('incompatible-merge: should throw error when merge is incompatible', (t) => { + const source = generateWorkflow('trigger-x'); + source.pushHistory(source.getVersionHash()); + const target = generateWorkflow('trigger-y'); + target.pushHistory(target.getVersionHash()); + + t.false(source.canMergeInto(target)); + + const sourceProject = new Project({ workflows: [source] }); + const targetProject = new Project({ workflows: [target] }); + t.throws(() => Project.merge(sourceProject, targetProject), { + message: `The below workflows can't be merged directly without losing data.\nWorkflow → Workflow`, + }); +}); + +test('incompatible-merge-force: should ignore incompatiblity and merge when forced', (t) => { + // same as the above test with force + const source = generateWorkflow('trigger-x'); + source.pushHistory(source.getVersionHash()); + const target = generateWorkflow('trigger-y'); + target.pushHistory(target.getVersionHash()); + + t.false(source.canMergeInto(target)); + + const sourceProject = new Project({ workflows: [source] }); + const targetProject = new Project({ workflows: [target] }); + t.notThrows(() => + Project.merge(sourceProject, targetProject, { force: true }) + ); +}); From 70bb12cfdcfa9f5d2d07b324904c9f6e90a82f5f Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 22 Oct 2025 12:07:04 +0000 Subject: [PATCH 08/11] tests: fix error wording --- packages/project/src/merge/merge-project.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index db78b4691..0ae87d4e8 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -68,7 +68,7 @@ export function merge( if (Object.keys(mergeMapping).length && !options?.force) { throw new Error( - `The below workflows can't merge directly without losing data.\n${Object.entries( + `The below workflows can't be merged directly without losing data.\n${Object.entries( mergeMapping ) .map(([from, to]) => `${from} → ${to}`) From 590ce7987c6a128e3ea237ef29dff44a442e8a61 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Fri, 24 Oct 2025 09:22:14 +0000 Subject: [PATCH 09/11] chore: updates --- packages/cli/src/merge/command.ts | 6 ++++-- packages/project/src/merge/merge-project.ts | 12 ++++++------ packages/project/test/project.test.ts | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/merge/command.ts b/packages/cli/src/merge/command.ts index f81c57645..a07447e1d 100644 --- a/packages/cli/src/merge/command.ts +++ b/packages/cli/src/merge/command.ts @@ -1,6 +1,6 @@ import yargs from 'yargs'; import { Opts } from '../options'; -import { ensure, build } from '../util/command-builders'; +import { ensure, build, override } from '../util/command-builders'; import * as o from '../options'; export type MergeOptions = Required< @@ -22,7 +22,9 @@ const options = [ o.removeUnmapped, o.workflowMappings, o.log, - o.force, + override(o.force, { + description: 'Force a merge even when workflows are incompatible', + }), ]; const mergeCommand: yargs.CommandModule = { diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index 0ae87d4e8..9d3b7b1a6 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -56,23 +56,23 @@ export function merge( }); // mergeability - const mergeMapping: Record = {}; + const potentialConflicts: Record = {}; for (const sourceWorkflow of sourceWorkflows) { const targetId = options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id; const targetWorkflow = target.getWorkflow(targetId); if (targetWorkflow && !sourceWorkflow.canMergeInto(targetWorkflow)) { - mergeMapping[sourceWorkflow.name] = targetWorkflow?.name; + potentialConflicts[sourceWorkflow.name] = targetWorkflow?.name; } } - if (Object.keys(mergeMapping).length && !options?.force) { + if (Object.keys(potentialConflicts).length && !options?.force) { throw new Error( - `The below workflows can't be merged directly without losing data.\n${Object.entries( - mergeMapping + `The below workflows can't be merged directly without losing data\n${Object.entries( + potentialConflicts ) .map(([from, to]) => `${from} → ${to}`) - .join('\n')}` + .join('\n')}\nPass --force to force the merge anyway` ); } diff --git a/packages/project/test/project.test.ts b/packages/project/test/project.test.ts index ba29c746e..8a0ac6bf1 100644 --- a/packages/project/test/project.test.ts +++ b/packages/project/test/project.test.ts @@ -166,7 +166,7 @@ test('incompatible-merge: should throw error when merge is incompatible', (t) => const sourceProject = new Project({ workflows: [source] }); const targetProject = new Project({ workflows: [target] }); t.throws(() => Project.merge(sourceProject, targetProject), { - message: `The below workflows can't be merged directly without losing data.\nWorkflow → Workflow`, + message: `The below workflows can't be merged directly without losing data\nWorkflow → Workflow\nPass --force to force the merge anyway`, }); }); From 9130cbc579dff25627438310754881e0bf002f1b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 24 Oct 2025 15:40:00 +0100 Subject: [PATCH 10/11] conflict --- packages/project/src/Workflow.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 18e16d002..721e51e43 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -177,7 +177,6 @@ class Workflow { // return true if the current workflow can be merged into the target workflow without losing any changes canMergeInto(target: Workflow) { -<<<<<<< HEAD const thisHistory = this.workflow.history?.concat(this.getVersionHash()); const targetHistory = target.workflow.history?.concat( target.getVersionHash() @@ -185,12 +184,6 @@ class Workflow { const targetHead = targetHistory[targetHistory.length - 1]; if (thisHistory.indexOf(targetHead) > -1) return true; -======= - if (!target.workflow.history.length) return true; - const targetHead = - target.workflow.history[target.workflow.history.length - 1]; - if (this.workflow.history.indexOf(targetHead) > -1) return true; ->>>>>>> 664dfbea (feat: add history to workflow property) return false; } } From b9cb060677cc8ea3e007441f3ee6214c6f8bcd28 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 24 Oct 2025 15:41:35 +0100 Subject: [PATCH 11/11] changeset --- .changeset/weak-gifts-work.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/weak-gifts-work.md diff --git a/.changeset/weak-gifts-work.md b/.changeset/weak-gifts-work.md new file mode 100644 index 000000000..ea45311f8 --- /dev/null +++ b/.changeset/weak-gifts-work.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +Warn when merging projects which may have diverged