Release: support multiple local adaptor repos

packages/cli/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/cli
 
+## 1.37.0
+
+### Minor Changes
+
+- ff1b1b6: `OPENFN_ADAPTORS_REPO` now supports multiple comma-separated paths.
+
 ## 1.36.3
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.36.3",
+  "version": "1.37.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/commands.ts

@@ -97,7 +97,7 @@ const parse = async (options: Opts, log?: Logger) => {
   const { monorepoPath } = options;
   if (monorepoPath) {
     // TODO how does this occur?
-    if (monorepoPath === 'ERR') {
+    if (monorepoPath[0] === 'ERR') {
       logger.error(
         'ERROR: --use-adaptors-monorepo was passed, but OPENFN_ADAPTORS_REPO env var is undefined'
       );

packages/cli/src/options.ts

@@ -46,7 +46,7 @@ export type Opts = {
   keepUnsupported?: boolean;
   log?: Record<string, LogLevel>;
   logJson?: boolean;
-  monorepoPath?: string;
+  monorepoPath?: string[];
   only?: string; // only run this workflow node
   operation?: string;
   outputPath?: string;
@@ -587,7 +587,16 @@ export const useAdaptorsMonorepo: CLIOption = {
   },
   ensure: (opts) => {
     if (opts.useAdaptorsMonorepo) {
-      opts.monorepoPath = process.env.OPENFN_ADAPTORS_REPO || 'ERR';
+      const repo = process.env.OPENFN_ADAPTORS_REPO;
+      // OPENFN_ADAPTORS_REPO is a comma-separated list of monorepo roots
+      // (a single path is just a one-element list)
+      opts.monorepoPath = repo
+        ? repo
+            .split(',')
+            .map((p) => p.trim())
+            .filter((p) => p.length > 0)
+            .map((p) => nodePath.resolve(p))
+        : ['ERR'];
     }
   },
 };

packages/cli/src/util/map-adaptors-to-monorepo.ts

@@ -1,6 +1,5 @@
-import { readFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
 import path from 'node:path';
-import assert from 'node:assert';
 import { Logger } from '@openfn/logger';
 import {
   getNameAndVersion,
@@ -10,19 +9,21 @@ import {
 
 import type { Opts } from '../options';
 
-export const validateMonoRepo = async (repoPath: string, log: Logger) => {
-  try {
-    const raw = await readFile(`${repoPath}/package.json`, 'utf8');
-    const pkg = JSON.parse(raw);
-    assert(pkg.name === 'adaptors');
-  } catch (e) {
-    log.error(`ERROR: Adaptors Monorepo not found at ${repoPath}`);
-    process.exit(9);
+export const validateMonoRepo = async (repoPaths: string[], log: Logger) => {
+  for (const repoPath of repoPaths) {
+    if (!existsSync(path.resolve(repoPath, 'packages'))) {
+      log.error(`ERROR: Adaptors Monorepo not found at ${repoPath}`);
+      process.exit(9);
+    }
   }
 };
 
 // Convert an adaptor name into a path to the adaptor in the monorepo
-export const updatePath = (adaptor: string, repoPath: string, log: Logger) => {
+export const updatePath = (
+  adaptor: string,
+  repoPaths: string[],
+  log: Logger
+) => {
   if (adaptor.match('=')) {
     // Should do nothing if a path is already provided
     return adaptor;
@@ -36,7 +37,22 @@ export const updatePath = (adaptor: string, repoPath: string, log: Logger) => {
     );
   }
   const shortName = name.replace('@openfn/language-', '');
-  const abspath = path.resolve(repoPath, 'packages', shortName);
+
+  // Find the first root in the monorepo list that contains the adaptor
+  // (order is precedence, so an earlier root overrides a later one)
+  const abspath = repoPaths
+    .map((repoPath) => path.join(repoPath, 'packages', shortName))
+    .find((candidate) => existsSync(candidate));
+
+  if (!abspath) {
+    if (repoPaths.length > 1) {
+      throw new Error(
+        `Adaptor ${name} not found in any provided adaptors monorepo`
+      );
+    } else {
+      throw new Error(`Adaptor ${name} not found in the adaptors monorepo`);
+    }
+  }
 
   log.info(`Mapped adaptor ${name} to monorepo: ${abspath}`);
   return `${name}=${abspath}`;
@@ -48,11 +64,11 @@ export type MapAdaptorsToMonorepoOptions = Pick<
 >;
 
 const mapAdaptorsToMonorepo = (
-  monorepoPath: string = '',
+  monorepoPath: string[] = [],
   input: string[] | ExecutionPlan = [],
   log: Logger
 ): string[] | ExecutionPlan => {
-  if (monorepoPath) {
+  if (monorepoPath.length) {
     if (Array.isArray(input)) {
       const adaptors = input as string[];
       return adaptors.map((a) => updatePath(a, monorepoPath, log));

packages/cli/test/options/ensure/useAdaptorsMonorepo.test.ts

@@ -1,3 +1,4 @@
+import path from 'node:path';
 import test from 'ava';
 import { useAdaptorsMonorepo, Opts } from '../../../src/options';
 
@@ -41,7 +42,19 @@ test('monorepoPath is set with a value from OPENFN_ADAPTORS_REPO', (t) => {
 
   useAdaptorsMonorepo.ensure!(opts);
 
-  t.is(opts.monorepoPath, 'a/b/c');
+  t.deepEqual(opts.monorepoPath, [path.resolve('a/b/c')]);
+  delete process.env.OPENFN_ADAPTORS_REPO;
+});
+
+test('monorepoPath is set with multiple comma-separated paths', (t) => {
+  process.env.OPENFN_ADAPTORS_REPO = 'a/b/c, d/e/f';
+  const opts = {
+    useAdaptorsMonorepo: true,
+  } as Opts;
+
+  useAdaptorsMonorepo.ensure!(opts);
+
+  t.deepEqual(opts.monorepoPath, [path.resolve('a/b/c'), path.resolve('d/e/f')]);
   delete process.env.OPENFN_ADAPTORS_REPO;
 });
 
@@ -54,5 +67,5 @@ test('monorepoPath is set to an error value if OPENFN_ADAPTORS_REPO is not set',
 
   useAdaptorsMonorepo.ensure!(opts);
 
-  t.is(opts.monorepoPath, 'ERR');
+  t.deepEqual(opts.monorepoPath, ['ERR']);
 });

packages/cli/test/util/load-plan.test.ts

@@ -331,7 +331,7 @@ test.serial('xplan: map to monorepo', async (t) => {
     workflowPath: 'test/wf.json',
     expandAdaptors: true,
     plan: {},
-    monorepoPath: '/repo/',
+    monorepoPath: ['/repo/'],
   } as Partial<Opts>;
 
   const plan = createPlan([
@@ -344,6 +344,7 @@ test.serial('xplan: map to monorepo', async (t) => {
 
   mock({
     'test/wf.json': JSON.stringify(plan),
+    '/repo/packages/common': {},
   });
 
   const result = await loadPlan(opts as Opts, logger);

packages/cli/test/util/map-adaptors-to-monorepo.test.ts

@@ -9,77 +9,166 @@ import mapAdaptorsToMonorepo, {
 } from '../../src/util/map-adaptors-to-monorepo';
 import { ExecutionPlan } from '@openfn/runtime';
 
-const REPO_PATH = 'a/b/c';
-const ABS_REPO_PATH = path.resolve(REPO_PATH);
+// Paths are resolved to absolute in the option's ensure block, so the util
+// always receives absolute roots
+const REPO_PATH = path.resolve('a/b/c');
+const REPO_PATH_2 = path.resolve('d/e/f');
 
 const logger = createMockLogger();
 
 test.afterEach(() => {
   logger._reset();
+  mock.restore();
 });
 
-test('updatePath: common', (t) => {
-  const result = updatePath('common', REPO_PATH, logger);
+test.serial('updatePath: common', (t) => {
+  mock({
+    [`${REPO_PATH}/packages/common`]: {},
+  });
+
+  const result = updatePath('common', [REPO_PATH], logger);
 
-  t.is(result, `common=${ABS_REPO_PATH}/packages/common`);
+  t.is(result, `common=${REPO_PATH}/packages/common`);
 });
 
-test('updatePath: @openfn/language-common', (t) => {
-  const result = updatePath('@openfn/language-common', REPO_PATH, logger);
+test.serial('updatePath: @openfn/language-common', (t) => {
+  mock({
+    [`${REPO_PATH}/packages/common`]: {},
+  });
+
+  const result = updatePath('@openfn/language-common', [REPO_PATH], logger);
 
-  t.is(result, `@openfn/language-common=${ABS_REPO_PATH}/packages/common`);
+  t.is(result, `@openfn/language-common=${REPO_PATH}/packages/common`);
 });
 
-test('updatePath: common@1.2.3 (with warning)', (t) => {
-  const result = updatePath('common@1.2.3', REPO_PATH, logger);
+test.serial('updatePath: common@1.2.3 (with warning)', (t) => {
+  mock({
+    [`${REPO_PATH}/packages/common`]: {},
+  });
 
-  t.is(result, `common=${ABS_REPO_PATH}/packages/common`);
+  const result = updatePath('common@1.2.3', [REPO_PATH], logger);
+
+  t.is(result, `common=${REPO_PATH}/packages/common`);
 
   const { level, message } = logger._parse(logger._last);
   t.is(level, 'warn');
   t.regex(message as string, /ignoring version specifier/i);
 });
 
 test('updatePath: common=x/y/z', (t) => {
-  const result = updatePath('common=x/y/z', REPO_PATH, logger);
+  const result = updatePath('common=x/y/z', [REPO_PATH], logger);
 
   t.is(result, `common=x/y/z`);
 });
 
+test.serial('updatePath: prefer the root which has the adaptor', (t) => {
+  mock({
+    [`${REPO_PATH_2}/packages/common`]: {},
+  });
+
+  // common only exists in the second root, so that path should be used
+  const result = updatePath('common', [REPO_PATH, REPO_PATH_2], logger);
+
+  t.is(result, `common=${REPO_PATH_2}/packages/common`);
+});
+
+test.serial('updatePath: earlier root wins when both have the adaptor', (t) => {
+  mock({
+    [`${REPO_PATH}/packages/common`]: {},
+    [`${REPO_PATH_2}/packages/common`]: {},
+  });
+
+  const result = updatePath('common', [REPO_PATH, REPO_PATH_2], logger);
+
+  t.is(result, `common=${REPO_PATH}/packages/common`);
+});
+
+test.serial('updatePath: throw if not found in the single root', (t) => {
+  mock({
+    [`${REPO_PATH}/packages`]: {},
+  });
+
+  t.throws(() => updatePath('common', [REPO_PATH], logger), {
+    message: /not found in the adaptors monorepo/,
+  });
+});
+
+test.serial('updatePath: throw if not found in any root', (t) => {
+  mock({
+    [`${REPO_PATH}/packages`]: {},
+    [`${REPO_PATH_2}/packages`]: {},
+  });
+
+  t.throws(() => updatePath('common', [REPO_PATH, REPO_PATH_2], logger), {
+    message: /not found in any provided adaptors monorepo/,
+  });
+});
+
 // TODO can't test this in ava, have to use an integration test
 test.skip('validate monorepo: log and exit early if repo not found', async (t) => {
   mock({
     a: {},
   });
 
-  await t.throwsAsync(async () => validateMonoRepo(REPO_PATH, logger), {
+  await t.throwsAsync(async () => validateMonoRepo([REPO_PATH], logger), {
     message: 'Monorepo not found',
   });
   const { level, message } = logger._parse(logger._last);
   t.is(level, 'error');
   t.is(message, `ERROR: Monorepo not found at ${REPO_PATH}`);
 });
 
-test('validate monorepo: all OK', async (t) => {
+test.serial('validate monorepo: all OK', async (t) => {
   mock({
-    [`${REPO_PATH}/package.json`]: '{ "name": "adaptors" }',
+    [`${REPO_PATH}/packages`]: {},
   });
 
-  await t.notThrowsAsync(async () => validateMonoRepo(REPO_PATH, logger));
+  await t.notThrowsAsync(async () => validateMonoRepo([REPO_PATH], logger));
+});
+
+test.serial('validate monorepo: all OK with multiple paths', async (t) => {
+  mock({
+    [`${REPO_PATH}/packages`]: {},
+    [`${REPO_PATH_2}/packages`]: {},
+  });
+
+  await t.notThrowsAsync(async () =>
+    validateMonoRepo([REPO_PATH, REPO_PATH_2], logger)
+  );
 });
 
 test.serial('mapAdaptorsToMonorepo: map adaptors', async (t) => {
   mock({
-    [`${REPO_PATH}/package.json`]: '{ "name": "adaptors" }',
+    [`${REPO_PATH}/packages/common`]: {},
   });
 
-  const result = await mapAdaptorsToMonorepo(REPO_PATH, ['common'], logger);
-  t.deepEqual(result, [`common=${ABS_REPO_PATH}/packages/common`]);
+  const result = await mapAdaptorsToMonorepo([REPO_PATH], ['common'], logger);
+  t.deepEqual(result, [`common=${REPO_PATH}/packages/common`]);
 });
 
+test.serial(
+  'mapAdaptorsToMonorepo: map adaptors across multiple roots',
+  async (t) => {
+    mock({
+      [`${REPO_PATH}/packages/http`]: {},
+      [`${REPO_PATH_2}/packages/common`]: {},
+    });
+
+    const result = await mapAdaptorsToMonorepo(
+      [REPO_PATH, REPO_PATH_2],
+      ['http', 'common'],
+      logger
+    );
+    t.deepEqual(result, [
+      `http=${REPO_PATH}/packages/http`,
+      `common=${REPO_PATH_2}/packages/common`,
+    ]);
+  }
+);
+
 test.serial('mapAdaptorsToMonorepo: map workflow', async (t) => {
   mock({
-    [`${REPO_PATH}/package.json`]: '{ "name": "adaptors" }',
+    [`${REPO_PATH}/packages/common`]: {},
   });
 
   const plan: ExecutionPlan = {
@@ -94,12 +183,12 @@ test.serial('mapAdaptorsToMonorepo: map workflow', async (t) => {
     options: {},
   };
 
-  await mapAdaptorsToMonorepo(REPO_PATH, plan, logger);
+  await mapAdaptorsToMonorepo([REPO_PATH], plan, logger);
   t.deepEqual(plan.workflow, {
     steps: [
       {
         expression: '.',
-        adaptors: [`common=${ABS_REPO_PATH}/packages/common`],
+        adaptors: [`common=${REPO_PATH}/packages/common`],
       },
     ],
   });