diff --git a/integration-tests/cli/CHANGELOG.md b/integration-tests/cli/CHANGELOG.md index 111bb2d75..30232317b 100644 --- a/integration-tests/cli/CHANGELOG.md +++ b/integration-tests/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/integration-tests-cli +## 1.0.21 + +### Patch Changes + +- @openfn/lightning-mock@2.4.16 +- @openfn/project@0.15.1 + ## 1.0.20 ### Patch Changes diff --git a/integration-tests/cli/package.json b/integration-tests/cli/package.json index 3987fac43..1ad6870f8 100644 --- a/integration-tests/cli/package.json +++ b/integration-tests/cli/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-cli", "private": true, - "version": "1.0.20", + "version": "1.0.21", "description": "CLI integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 72a7275d7..9f5c6bf5d 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,18 @@ # @openfn/cli +## 1.35.2 + +### Patch Changes + +Update type interfaces to match new portability spec. + +- Updated dependencies [728e6cb] +- Updated dependencies [380bccd] + - @openfn/lexicon@2.0.0 + - @openfn/runtime@1.9.3 + - @openfn/compiler@1.2.5 + - @openfn/project@0.15.1 + ## 1.35.1 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 56d359232..1c5b0d562 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.35.1", + "version": "1.35.2", "description": "CLI devtools for the OpenFn toolchain", "engines": { "node": ">=18", diff --git a/packages/cli/src/compile/compile.ts b/packages/cli/src/compile/compile.ts index ea5071ffb..822188702 100644 --- a/packages/cli/src/compile/compile.ts +++ b/packages/cli/src/compile/compile.ts @@ -3,12 +3,8 @@ import compile, { Options, getExports, } from '@openfn/compiler'; -import { getModulePath } from '@openfn/runtime'; -import type { - ExecutionPlan, - Job, - SourceMapWithOperations, -} from '@openfn/lexicon'; +import { getModulePath, type ExecutionPlan, type Job } from '@openfn/runtime'; +import type { SourceMapWithOperations } from '@openfn/lexicon'; import createLogger, { COMPILER, Logger } from '../util/logger'; import abort from '../util/abort'; diff --git a/packages/cli/src/execute/apply-credential-map.ts b/packages/cli/src/execute/apply-credential-map.ts index a4a0cf174..ebd96bae3 100644 --- a/packages/cli/src/execute/apply-credential-map.ts +++ b/packages/cli/src/execute/apply-credential-map.ts @@ -3,7 +3,7 @@ * and apply credentials to each step */ -import { ExecutionPlan } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; import { Logger } from '../util'; type JobId = string; diff --git a/packages/cli/src/execute/execute.ts b/packages/cli/src/execute/execute.ts index a01cf4d8a..5e23a7f46 100644 --- a/packages/cli/src/execute/execute.ts +++ b/packages/cli/src/execute/execute.ts @@ -1,6 +1,7 @@ import run, { NOTIFY_JOB_COMPLETE, getNameAndVersion } from '@openfn/runtime'; -import type { ExecutionPlan, Job } from '@openfn/lexicon'; import type { + ExecutionPlan, + Job, ModuleInfo, ModuleInfoMap, NotifyJobCompletePayload, diff --git a/packages/cli/src/execute/get-autoinstall-targets.ts b/packages/cli/src/execute/get-autoinstall-targets.ts index aea294366..8cbb62e9d 100644 --- a/packages/cli/src/execute/get-autoinstall-targets.ts +++ b/packages/cli/src/execute/get-autoinstall-targets.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; const getAutoinstallTargets = (plan: ExecutionPlan) => { const adaptors = {} as Record; diff --git a/packages/cli/src/execute/handler.ts b/packages/cli/src/execute/handler.ts index e005a5464..aba2893f5 100644 --- a/packages/cli/src/execute/handler.ts +++ b/packages/cli/src/execute/handler.ts @@ -1,4 +1,4 @@ -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import path from 'node:path'; import type { ExecuteOptions } from './command'; diff --git a/packages/cli/src/metadata/handler.ts b/packages/cli/src/metadata/handler.ts index 14935907b..1ccde3792 100644 --- a/packages/cli/src/metadata/handler.ts +++ b/packages/cli/src/metadata/handler.ts @@ -2,8 +2,11 @@ import { Logger } from '../util/logger'; import { MetadataOpts } from './command'; import loadState from '../util/load-state'; import * as cache from './cache'; -import { getModuleEntryPoint, registerEsmHook } from '@openfn/runtime'; -import { ExecutionPlan } from '@openfn/lexicon'; +import { + getModuleEntryPoint, + registerEsmHook, + type ExecutionPlan, +} from '@openfn/runtime'; import { install, removePackage } from '../repo/handler'; // Add created date stamp to the metadata object diff --git a/packages/cli/src/test/handler.ts b/packages/cli/src/test/handler.ts index 2f67d67c9..b1cfec062 100644 --- a/packages/cli/src/test/handler.ts +++ b/packages/cli/src/test/handler.ts @@ -1,4 +1,4 @@ -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import { TestOptions } from './command'; import { createNullLogger, Logger } from '../util/logger'; diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index 60e89ab0e..7529d2c17 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -1,43 +1,34 @@ -// the executionPLan for the CLI is a bit differnt to the runtime's perspective +import { WorkflowOptions } from '@openfn/lexicon'; +import type { ExecutionPlan, Job, Trigger } from '@openfn/runtime'; -import { Trigger, UUID, WorkflowOptions } from '@openfn/lexicon'; - -// Ie config can be a string export type JobNodeID = string; export type OldCLIWorkflow = { - id?: string; // UUID for this plan + id?: string; start?: JobNodeID; jobs: CLIJobNode[]; }; -export type CLIExecutionPlan = { - id?: UUID; +// Input-format wrapper around the runtime ExecutionPlan. +// Accepts singular `adaptor` on jobs (normalized to `adaptors[]` by ensureAdaptors +// before handing to the runtime) and adds the CLI-only collectionsEndpoint option. +export type CLIExecutionPlan = Omit & { options?: WorkflowOptions & { collectionsEndpoint?: string; }; - workflow: { - id?: string; - name?: string; + workflow: Omit & { steps: Array; - globals?: string; }; }; -export type CLIJobNode = { - id?: string; - expression?: string; // path or expression - configuration?: string | object; // path or credential object +// Loose input variant of Job: `adaptor` (singular) is normalized to `adaptors[]` +// by the CLI before the plan reaches the runtime. +export type CLIJobNode = Job & { data?: any; - next?: string | Record; - - // We can accept a single adaptor or multiple - // The CLI will convert it to adaptors as an array adaptor?: string; - adaptors?: string[]; }; export type CLIJobEdge = { - condition?: string; // Javascript expression (function body, not function) + condition?: string; label?: string; }; diff --git a/packages/cli/src/util/cache.ts b/packages/cli/src/util/cache.ts index 182ab0ce3..846cb9cf2 100644 --- a/packages/cli/src/util/cache.ts +++ b/packages/cli/src/util/cache.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { rmdir } from 'node:fs/promises'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import type { Opts } from '../options'; import type { Logger } from './logger'; diff --git a/packages/cli/src/util/expand-adaptors.ts b/packages/cli/src/util/expand-adaptors.ts index 3ccd72fd1..c61bcc050 100644 --- a/packages/cli/src/util/expand-adaptors.ts +++ b/packages/cli/src/util/expand-adaptors.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; const expand = (name: string) => { if (typeof name === 'string') { diff --git a/packages/cli/src/util/fuzzy-match-step.ts b/packages/cli/src/util/fuzzy-match-step.ts index 14ffda27e..06adbd809 100644 --- a/packages/cli/src/util/fuzzy-match-step.ts +++ b/packages/cli/src/util/fuzzy-match-step.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; export default (plan: ExecutionPlan, stepPattern?: string) => { if (stepPattern) { diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index 67180457d..a4f4e3d84 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -6,7 +6,8 @@ import { Workspace, yamlToJson } from '@openfn/project'; import abort from './abort'; import expandAdaptors from './expand-adaptors'; import mapAdaptorsToMonorepo from './map-adaptors-to-monorepo'; -import type { ExecutionPlan, Job, WorkflowOptions } from '@openfn/lexicon'; +import type { WorkflowOptions } from '@openfn/lexicon'; +import type { ExecutionPlan, Job } from '@openfn/runtime'; import type { Opts } from '../options'; import type { Logger } from './logger'; import type { CLIExecutionPlan, CLIJobNode, OldCLIWorkflow } from '../types'; diff --git a/packages/cli/src/util/load-state.ts b/packages/cli/src/util/load-state.ts index fd0c911f9..082b870a7 100644 --- a/packages/cli/src/util/load-state.ts +++ b/packages/cli/src/util/load-state.ts @@ -2,7 +2,7 @@ import fs from 'node:fs/promises'; import { getCachePath } from './cache'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; import type { Opts } from '../options'; diff --git a/packages/cli/src/util/map-adaptors-to-monorepo.ts b/packages/cli/src/util/map-adaptors-to-monorepo.ts index 8154a124f..833f36069 100644 --- a/packages/cli/src/util/map-adaptors-to-monorepo.ts +++ b/packages/cli/src/util/map-adaptors-to-monorepo.ts @@ -2,8 +2,11 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; import assert from 'node:assert'; import { Logger } from '@openfn/logger'; -import { getNameAndVersion } from '@openfn/runtime'; -import type { ExecutionPlan, Job } from '@openfn/lexicon'; +import { + getNameAndVersion, + type ExecutionPlan, + type Job, +} from '@openfn/runtime'; import type { Opts } from '../options'; diff --git a/packages/cli/src/util/override-plan-adaptors.ts b/packages/cli/src/util/override-plan-adaptors.ts index 5fc1b8263..92f97cc44 100644 --- a/packages/cli/src/util/override-plan-adaptors.ts +++ b/packages/cli/src/util/override-plan-adaptors.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan, Job, Step } from '@openfn/lexicon'; +import { ExecutionPlan, Job, Step } from '@openfn/runtime'; function overridePlanAdaptors( plan: ExecutionPlan, diff --git a/packages/cli/src/util/validate-plan.ts b/packages/cli/src/util/validate-plan.ts index db02da5a0..ae18a2444 100644 --- a/packages/cli/src/util/validate-plan.ts +++ b/packages/cli/src/util/validate-plan.ts @@ -1,4 +1,5 @@ -import { ExecutionPlan, Job, Trigger, WorkflowOptions } from '@openfn/lexicon'; +import { WorkflowOptions } from '@openfn/lexicon'; +import { ExecutionPlan, Job, Trigger } from '@openfn/runtime'; import { Logger } from '@openfn/logger'; const assertWorkflowStructure = (plan: ExecutionPlan, logger: Logger) => { diff --git a/packages/cli/test/compile/compile.test.ts b/packages/cli/test/compile/compile.test.ts index 158644d2d..1265fb44a 100644 --- a/packages/cli/test/compile/compile.test.ts +++ b/packages/cli/test/compile/compile.test.ts @@ -11,7 +11,7 @@ import compile, { import { CompileOptions } from '../../src/compile/command'; import { mockFs, resetMockFs } from '../util'; -import type { ExecutionPlan, Job } from '@openfn/lexicon'; +import type { ExecutionPlan, Job } from '@openfn/runtime'; const mockLog = createMockLogger(); diff --git a/packages/cli/test/execute/get-autoinstall-targets.test.ts b/packages/cli/test/execute/get-autoinstall-targets.test.ts index 38e525346..6469237b6 100644 --- a/packages/cli/test/execute/get-autoinstall-targets.test.ts +++ b/packages/cli/test/execute/get-autoinstall-targets.test.ts @@ -1,7 +1,7 @@ import test from 'ava'; import getAutoinstallTargets from '../../src/execute/get-autoinstall-targets'; -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; const getPlan = (steps: Job[]) => ({ diff --git a/packages/cli/test/execute/parse-adaptors.test.ts b/packages/cli/test/execute/parse-adaptors.test.ts index 8bbb0efdd..46c1a2f4a 100644 --- a/packages/cli/test/execute/parse-adaptors.test.ts +++ b/packages/cli/test/execute/parse-adaptors.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; import { parseAdaptors } from '../../src/execute/execute'; diff --git a/packages/cli/test/util.ts b/packages/cli/test/util.ts index 2fb7253f8..84312058c 100644 --- a/packages/cli/test/util.ts +++ b/packages/cli/test/util.ts @@ -4,7 +4,7 @@ import mock from 'mock-fs'; import path from 'node:path'; -import type { ExecutionPlan, Job, StepEdge } from '@openfn/lexicon'; +import type { ExecutionPlan, Job, StepEdge } from '@openfn/runtime'; export const mockFs = (files: Record) => { // We have to explicitly expose some modules paths so that dependencies can run in the tests diff --git a/packages/cli/test/util/load-plan.test.ts b/packages/cli/test/util/load-plan.test.ts index 932354981..68fbb8730 100644 --- a/packages/cli/test/util/load-plan.test.ts +++ b/packages/cli/test/util/load-plan.test.ts @@ -2,7 +2,7 @@ import test from 'ava'; import mock from 'mock-fs'; import { createMockLogger } from '@openfn/logger'; import { omit } from 'lodash-es'; -import type { Job } from '@openfn/lexicon'; +import type { Job } from '@openfn/runtime'; import loadPlan from '../../src/util/load-plan'; import { Opts } from '../../src/options'; diff --git a/packages/cli/test/util/map-adaptors-to-monorepo.test.ts b/packages/cli/test/util/map-adaptors-to-monorepo.test.ts index 97f71e60a..886bd942d 100644 --- a/packages/cli/test/util/map-adaptors-to-monorepo.test.ts +++ b/packages/cli/test/util/map-adaptors-to-monorepo.test.ts @@ -7,7 +7,7 @@ import mapAdaptorsToMonorepo, { validateMonoRepo, updatePath, } from '../../src/util/map-adaptors-to-monorepo'; -import { ExecutionPlan } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; const REPO_PATH = 'a/b/c'; const ABS_REPO_PATH = path.resolve(REPO_PATH); diff --git a/packages/cli/test/util/override-plan-adaptor.test.ts b/packages/cli/test/util/override-plan-adaptor.test.ts index 01866530e..befcf873d 100644 --- a/packages/cli/test/util/override-plan-adaptor.test.ts +++ b/packages/cli/test/util/override-plan-adaptor.test.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; import test from 'ava'; import overridePlanAdaptors, { isJob, diff --git a/packages/cli/test/util/validate-plan.test.ts b/packages/cli/test/util/validate-plan.test.ts index 81dad64a4..cae029e9f 100644 --- a/packages/cli/test/util/validate-plan.test.ts +++ b/packages/cli/test/util/validate-plan.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; import { createMockLogger } from '@openfn/logger'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import validate from '../../src/util/validate-plan'; const logger = createMockLogger('', { level: 'debug' }); diff --git a/packages/compiler/CHANGELOG.md b/packages/compiler/CHANGELOG.md index bcb26fbfd..edc11af03 100644 --- a/packages/compiler/CHANGELOG.md +++ b/packages/compiler/CHANGELOG.md @@ -1,5 +1,14 @@ # @openfn/compiler +## 1.2.5 + +### Patch Changes + +Update type interfaces to match new portability spec. + +- Updated dependencies [728e6cb] + - @openfn/lexicon@2.0.0 + ## 1.2.4 ### Patch Changes diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 7266be54a..21d60dc03 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/compiler", - "version": "1.2.4", + "version": "1.2.5", "description": "Compiler and language tooling for openfn jobs.", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md index 7cdd7b4c2..7687e6f44 100644 --- a/packages/engine-multi/CHANGELOG.md +++ b/packages/engine-multi/CHANGELOG.md @@ -1,5 +1,17 @@ # engine-multi +## 1.11.4 + +### Patch Changes + +Update type interfaces to match new portability spec. + +- Updated dependencies [728e6cb] +- Updated dependencies [380bccd] + - @openfn/lexicon@2.0.0 + - @openfn/runtime@1.9.3 + - @openfn/compiler@1.2.5 + ## 1.11.3 ### Patch Changes diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 228d90823..1e4a1f6c4 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/engine-multi", - "version": "1.11.3", + "version": "1.11.4", "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index bf4bae9a0..6d351d9f2 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -4,9 +4,10 @@ import { getNameAndVersion, getLatestVersion, loadRepoPkg, + install as runtimeInstall, + type ExecutionPlan, + type Job, } from '@openfn/runtime'; -import { install as runtimeInstall } from '@openfn/runtime'; -import type { ExecutionPlan, Job } from '@openfn/lexicon'; import type { Logger } from '@openfn/logger'; import { AUTOINSTALL_COMPLETE, AUTOINSTALL_ERROR } from '../events'; diff --git a/packages/engine-multi/src/api/preload-credentials.ts b/packages/engine-multi/src/api/preload-credentials.ts index 8bedbc298..7b1ea465f 100644 --- a/packages/engine-multi/src/api/preload-credentials.ts +++ b/packages/engine-multi/src/api/preload-credentials.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; import { CredentialErrorObj, CredentialLoadError } from '../errors'; diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 31ecc0774..9360d5c82 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -1,7 +1,8 @@ import { EventEmitter } from 'node:events'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import type { ExecutionPlan, State, UUID } from '@openfn/lexicon'; +import type { State, UUID } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; import { diff --git a/packages/engine-multi/src/test/util.ts b/packages/engine-multi/src/test/util.ts index 93916563a..7064717f2 100644 --- a/packages/engine-multi/src/test/util.ts +++ b/packages/engine-multi/src/test/util.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; export const createPlan = (job = {}) => ({ diff --git a/packages/engine-multi/src/test/worker-functions.ts b/packages/engine-multi/src/test/worker-functions.ts index dca335cea..8a0068827 100644 --- a/packages/engine-multi/src/test/worker-functions.ts +++ b/packages/engine-multi/src/test/worker-functions.ts @@ -2,7 +2,7 @@ import path from 'node:path'; import { register, publish, threadId } from '../worker/thread/runtime'; import { increment } from './counter.js'; -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; const tasks = { test: async (result = 42) => { diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index ae2a44b7a..5320857e8 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -1,5 +1,6 @@ import type { Logger, SanitizePolicies } from '@openfn/logger'; -import type { ExecutionPlan, State, UUID } from '@openfn/lexicon'; +import type { State, UUID } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import type { EventEmitter } from 'node:events'; import type { EngineOptions } from './engine'; diff --git a/packages/engine-multi/src/util/create-state.ts b/packages/engine-multi/src/util/create-state.ts index e640ff32f..827a1b69c 100644 --- a/packages/engine-multi/src/util/create-state.ts +++ b/packages/engine-multi/src/util/create-state.ts @@ -1,4 +1,5 @@ -import { ExecutionPlan, State } from '@openfn/lexicon'; +import { State } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; import { WorkflowState } from '../types'; // TODO should this be a weakmap for better memory efficiency? diff --git a/packages/engine-multi/src/worker/thread/compile.ts b/packages/engine-multi/src/worker/thread/compile.ts index b8bef0344..3a68f965e 100644 --- a/packages/engine-multi/src/worker/thread/compile.ts +++ b/packages/engine-multi/src/worker/thread/compile.ts @@ -1,6 +1,10 @@ import compile, { preloadAdaptorExports, Options } from '@openfn/compiler'; -import { getModulePath, getNameAndVersion } from '@openfn/runtime'; -import type { ExecutionPlan, Job } from '@openfn/lexicon'; +import { + getModulePath, + getNameAndVersion, + type ExecutionPlan, + type Job, +} from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; import { CompileError } from '../../errors'; diff --git a/packages/engine-multi/src/worker/thread/run.ts b/packages/engine-multi/src/worker/thread/run.ts index 483753ee1..8d6103192 100644 --- a/packages/engine-multi/src/worker/thread/run.ts +++ b/packages/engine-multi/src/worker/thread/run.ts @@ -1,8 +1,8 @@ // This is the run command that will be executed inside the worker thread // Most of the heavy lifting is actually handled by execute import run from '@openfn/runtime'; -import type { NotifyEvents } from '@openfn/runtime'; -import type { ExecutionPlan, State } from '@openfn/lexicon'; +import type { ExecutionPlan, NotifyEvents } from '@openfn/runtime'; +import type { State } from '@openfn/lexicon'; import type { LogLevel, SanitizePolicies } from '@openfn/logger'; import compile from './compile'; diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index 40759f5ed..d83fd6e6f 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; import { createMockLogger } from '@openfn/logger'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import createAPI from '../src/api'; import type { RuntimeEngine } from '../src/types'; diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index 1bced2a3a..36d550164 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; import { createMockLogger } from '@openfn/logger'; -import type { ExecutionPlan, Job } from '@openfn/lexicon'; +import type { ExecutionPlan, Job } from '@openfn/runtime'; import autoinstall, { AutoinstallOptions, diff --git a/packages/engine-multi/test/api/preload-credentials.test.ts b/packages/engine-multi/test/api/preload-credentials.test.ts index 9786c79b2..a52ee14a7 100644 --- a/packages/engine-multi/test/api/preload-credentials.test.ts +++ b/packages/engine-multi/test/api/preload-credentials.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; import preloadCredentials from '../../src/api/preload-credentials'; diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index fc0c032b1..370641c43 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -1,7 +1,7 @@ import test from 'ava'; import path from 'node:path'; import { createMockLogger } from '@openfn/logger'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import createAPI from '../src/api'; import type { RuntimeEngine } from '../src'; diff --git a/packages/lexicon/CHANGELOG.md b/packages/lexicon/CHANGELOG.md index 043e8b469..83eda5f90 100644 --- a/packages/lexicon/CHANGELOG.md +++ b/packages/lexicon/CHANGELOG.md @@ -1,5 +1,11 @@ # lexicon +## 2.0.0 + +### Major Changes + +- 728e6cb: Introduce a formal Portabilty schema + ## 1.5.0 ### Minor Changes diff --git a/packages/lexicon/README.md b/packages/lexicon/README.md index baa736f5f..978ef8207 100644 --- a/packages/lexicon/README.md +++ b/packages/lexicon/README.md @@ -1,27 +1,25 @@ -The lexicon (aka the OpenFunctionicon) is a central repositoty of key type and word definitions. +The lexicon is a central repository of key type and word definitions. It's a types repo and glossary at the same time. -It's a types repo and glossary at the same time. +The most important part of it is the Portability Spec: which describes the portable Project and Workflow interfaces which are common not just to this repo, but to the whole OpenFn platform and toolchain. ## Overview The OpenFunction stack is built on the concepts of Workflows, Runs, Jobs and Expressions (and more). Some of these terms can be used interchangable, or used differently in certain contexts. -Here are the key concepts +Here are the key concepts: - An **Expression** is a string of Javascript (or Javascript-like code) written to be run in the CLI or Lightning. -- A **Job** is an expression plus some metadata required to run it - typically an adaptor and credentials. - The terms Job and Expression are often used interchangeably. -- A **Workflow** is a series of steps to be executed in sequence. Steps are usually Jobs (and so job and step are often used - interchangeably), but can be Triggers. +- A **Step** is an expression plus some metadata required to run it - typically an adaptor and credentials. Also known as a Job. +- A **Workflow** is a series of steps to be executed in sequence - An **Execution Plan** is a Workflow plus some options which inform how it should be executed (ie, start node, timeout). The term "Execution plan" is mostly used internally and not exposed to users, and is usually interchangeable with Workflow. You can find formal type definition of these and more in `src/core.d.ts`. -Lightning also introduces it's own terminolgy as it is standalone application and has features that the runtime itself does not. +Lightning also introduces its own terminology for platform-specific features not share by the runtime. -In Lightning, a Step can be a Job or a Trigger. Jobs are connected by Paths (also known sometimes as Edges), which may be conditional. +In Lightning, a Step can be a Job or a Trigger. Jobs are connected by Edges , which may be conditional. You can find lightning-specific typings in `src/lightning.d.ts` diff --git a/packages/lexicon/core.d.ts b/packages/lexicon/core.d.ts index f8a0433c4..bdf6be58c 100644 --- a/packages/lexicon/core.d.ts +++ b/packages/lexicon/core.d.ts @@ -1,32 +1,25 @@ import { SanitizePolicies } from '@openfn/logger'; import type { RawSourceMap } from 'source-map'; -/** UUID v4 (or numbers, when running in dev mode*/ -export type UUID = string | number; - -export type SourceMap = RawSourceMap; - -export type SourceMapWithOperations = RawSourceMap & { - operations: [{ line: number; order: number; name: string }]; -}; - -export type SandboxMeta = { - parentId?: string; - parentName?: string; // not supported yet -}; - -// The serialised shape of of a project, as JSON -// this is what is saved to project.yaml -export type Project = { - /** Single-word identifier */ - id: string; - - /** human readable name */ - name?: string; - - description?: string; +import { Credential, Job, ProjectSpec, WorkflowSpec } from './portability'; +export { + Step, + StepId, + Job, + Trigger, + StepEdge, + ConditionalStepEdge, + Credential, + WorkflowSpec, +} from './portability'; - workflows: Workflow[]; +/** + * Canonical internal stateful Project representation. + * This is what gets serialized in a v2 @.yaml file + */ +export interface ProjectState extends WithState { + // override Workflows to include state + workflows: WorkflowState[]; options?: { env?: string; @@ -37,16 +30,63 @@ export type Project = { sandbox?: SandboxMeta; - credentials: any; - collections: string[]; - - // metadata about the app for sync - openfn?: Partial; - config: WorkspaceConfig; + credentials?: Array; + /** Stuff only used by the CLI for this project */ cli?: LocalMeta; +} + +export interface WorkflowState extends WithState { + /** holds version history information of a workflow **/ + history?: string[]; + + /** global credentials (gets applied to every configuration object) */ + credentials?: Record; +} + +export interface JobState extends Job { + /** Some of this stuff is more like internal runtime options and needs moving */ + + state?: Omit | string; + + // Internal use only + // Allow module paths and versions to be overridden in the linker + // Maps to runtime.ModuleInfoMap + linker?: Record< + string, + { + path?: string; + version?: string; + } + >; +} + +/** UUID v4 (or numbers, when running in dev mode*/ +export type UUID = string | number; + +export type SourceMap = RawSourceMap; + +export type SourceMapWithOperations = RawSourceMap & { + operations: [{ line: number; order: number; name: string }]; +}; + +export type SandboxMeta = { + parentId?: string; + parentName?: string; // not supported yet + [key: string]: any; +}; + +/** + * Utility to append a .openfn state object to + * another object + */ +export type WithState = P & { + openfn?: { + uuid?: UUID; + [key: string]: any; + } & S; }; export interface LocalMeta { @@ -62,6 +102,10 @@ export interface OpenFnMetadata { uuid?: UUID; } +export interface CredentialState extends Credential { + uuid?: UUID; +} + type FileFormats = 'yaml' | 'json'; // This is the old workspace config file, up to 0.6 @@ -136,125 +180,6 @@ export interface NodeMeta { [key: string]: unknown; } -/** - * An execution plan is a portable definition of a Work Order, - * or, a unit of work to execute - * This definition represents the external format - the shape of - * the plan pre-compilation before it's passed into the runtime manager - * (ie, the CLI or Worker) - */ -export type ExecutionPlan = { - id?: UUID; // TODO make required - workflow: Workflow; - options?: WorkflowOptions; -}; - -/** - * A workflow is just a series of steps, executed start to finish - */ -export type Workflow = { - /** The ID is the primary internal identifier for a Workflow */ - id?: string; - - /** Human readable name, like display. Can be used to generate an internal id */ - name?: string; - - /** Local shorthand name for use in CLI commands. Not used by Lightning */ - alias?: string; - - steps: Array; - - // global credentials - // (gets applied to every configuration object) - credentials?: Record; - - // a path to a file where functions are defined - globals?: string; - - openfn?: WorkflowMeta; - - // holds history information of a workflow - history?: string[]; - - /** The default start node - the one the workflow was designed for (the trigger) */ - start?: string; - - /** extra options from the app. Not really used */ - options?: any; -}; - -export type StepId = string; - -/** - * A thing to be run as part of a workflow - * (usually a job) - */ -export interface Step { - // TODO a Step must ALWAYS have an id (util functions can default it) - id?: StepId; - name?: string; // user-friendly name used in logging - next?: string | Record; - previous?: StepId; -} - -/** - * Not actually keen on the node/edge semantics here - * Maybe StepLink? - */ -export type StepEdge = boolean | string | ConditionalStepEdge; - -export type ConditionalStepEdge = { - condition?: string; // Javascript expression (function body, not function) - label?: string; // TODO this is probably the name - disabled?: boolean; -}; - -/** - * A no-op type of Step - */ -export interface Trigger extends Step { - enabled?: boolean; -} - -/** - * An expression which has been compiled, and so includes import and export statements - */ -export type CompiledExpression = Expression; - -/** - * A type of Step which executes code - * This is some openfn expression plus metadata (adaptor, credentials) - */ -export interface Job extends Step { - adaptors?: string[]; - expression?: Expression; - configuration?: object | string; - state?: Omit | string; - - sourceMap?: SourceMapWithOperations; - - // Internal use only - // Allow module paths and versions to be overridden in the linker - // Maps to runtime.ModuleInfoMap - linker?: Record< - string, - { - path?: string; - version?: string; - } - >; -} - -/** - * A raw openfn-js script to be executed by the runtime - * - * Can be compiled as part of a job. - * - * The expression itself has no metadata. It likely needs - * an adaptor and input state to run - */ -export type Expression = string; - /** * State is an object passed into a workflow and returned from a workflow */ diff --git a/packages/lexicon/package.json b/packages/lexicon/package.json index 80df4b8b0..3b51706dd 100644 --- a/packages/lexicon/package.json +++ b/packages/lexicon/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lexicon", - "version": "1.5.0", + "version": "2.0.0", "description": "Central repo of names and type definitions", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/lexicon/portability.d.ts b/packages/lexicon/portability.d.ts new file mode 100644 index 000000000..8db5f1ee0 --- /dev/null +++ b/packages/lexicon/portability.d.ts @@ -0,0 +1,96 @@ +/** + * Typings which define the Portability Spec + * See https://docs.openfn.org/documentation/deploy/portability + */ + +/** + * Schema for portable representation of a Project + * AKA Project Spec + * Can serialize to JSON or YAML or whatever you like + * + * If you serialize a v2 project file without state, this is what you get + */ +export interface ProjectSpec { + /** Single-word identifier */ + id: string; + + /** human readable name */ + name?: string; + + description?: string; + + workflows: WorkflowSpec[]; + + credentials?: Credential[]; + + collections?: string[]; +} + +export interface WorkflowSpec { + /** The primary internal identifier for a Workflow (not a UUID) */ + id?: string; + + /** Human readable label. Can be used to generate an internal id */ + name?: string; + + steps: Array; + + /** Global functions (path to a js file, unsupported in app) */ + globals?: string; + + /** The default start node - the one the workflow was designed for (the trigger) */ + start?: string; + + /** extra options used to configure the workflow (unused?)*/ + options?: any; +} + +export type StepId = string; + +/** + * A thing to be run as part of a workflow + * (usually a job) + */ +export interface Step { + // TODO a Step must ALWAYS have an id (util functions can default it) + id?: StepId; + name?: string; // user-friendly name used in logging + + // TODO remove next: string (next should always be an object) + next?: string | Record; +} + +export type StepEdge = boolean | string | ConditionalStepEdge; + +export type ConditionalStepEdge = { + condition?: string; // Javascript expression (function body, not function) + label?: string; // TODO this is probably the name + disabled?: boolean; +}; + +export interface Trigger extends Step { + type?: 'webhook' | 'cron' | 'kafka'; + + /** cron schedule, only meaningful when type is 'cron' */ + cron_expression?: string; + + enabled?: boolean; + + webhook_reply?: string; + cron_cursor_job_id?: string; + + /** Allow arbitrary properties on trigger nodes (as configuration options) */ + [option: string]: any; +} + +// TODO credential should just be an id string in the near future +export interface Credential { + name: string; + owner: string; +} + +export interface Job extends Step { + adaptor?: string; + expression?: string; + configuration?: object | string; +} diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index a045e37d4..c7010cc9e 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -1,5 +1,16 @@ # @openfn/lightning-mock +## 2.4.16 + +### Patch Changes + +- Updated dependencies [728e6cb] +- Updated dependencies [380bccd] + - @openfn/lexicon@2.0.0 + - @openfn/runtime@1.9.3 + - @openfn/engine-multi@1.11.4 + - @openfn/project@0.15.1 + ## 2.4.15 ### Patch Changes diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 744725e68..b944e47ab 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "2.4.15", + "version": "2.4.16", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/packages/project/CHANGELOG.md b/packages/project/CHANGELOG.md index edd478aad..a7d7dc036 100644 --- a/packages/project/CHANGELOG.md +++ b/packages/project/CHANGELOG.md @@ -1,5 +1,13 @@ # @openfn/project +## 0.15.1 + +### Patch Changes + +- Update type interfaces to match new portability spec. +- Updated dependencies [728e6cb] + - @openfn/lexicon@2.0.0 + ## 0.15.0 ### Minor Changes diff --git a/packages/project/package.json b/packages/project/package.json index aeb9fd69b..c46f42b9a 100644 --- a/packages/project/package.json +++ b/packages/project/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/project", - "version": "0.15.0", + "version": "0.15.1", "description": "Read, serialize, replicate and sync OpenFn projects", "scripts": { "test": "pnpm ava", diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index e9b5e025a..3691bacde 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -14,7 +14,12 @@ import { diff as projectDiff } from './util/project-diff'; import { Workspace } from './Workspace'; import { buildConfig, extractConfig } from './util/config'; import { Provisioner } from '@openfn/lexicon/lightning'; -import { SandboxMeta, UUID, WorkspaceConfig } from '@openfn/lexicon'; +import { + SandboxMeta, + UUID, + WorkspaceConfig, + Credential, +} from '@openfn/lexicon'; import { parse } from './util/get-credential-name'; const maybeCreateWorkflow = (wf: any) => @@ -35,14 +40,6 @@ type CLIMeta = { forked_from?: Record; }; -export type Credential = { - uuid: string; - name: string; - - // TODO having the owner in the credential is controvertial and we may need to rethink this later - owner: string; -}; - export class Project { // what schema version is this? // And how are we tracking this? @@ -138,7 +135,7 @@ export class Project { // TODO maybe the constructor is (data, Workspace) constructor( - data: Partial = {}, + data: Partial = {}, meta?: Partial & CLIMeta ) { this.id = @@ -164,7 +161,7 @@ export class Project { this.options = data.options; this.workflows = data.workflows?.map(maybeCreateWorkflow) ?? []; this.collections = data.collections; - this.credentials = data.credentials; + this.credentials = data.credentials ?? []; this.sandbox = data.sandbox; } diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 2d323bfbb..8fcce5868 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -9,7 +9,7 @@ export type WithMeta = T & { }; class Workflow { - workflow: l.Workflow; // this is the raw workflow JSON representation + workflow: l.WorkflowState; // this is the raw workflow JSON representation index: any; name?: string; @@ -17,7 +17,7 @@ class Workflow { openfn?: l.WorkflowMeta; options: any; // TODO - constructor(workflow: l.Workflow) { + constructor(workflow: l.WorkflowState) { this.index = { steps: {}, // steps by id edges: {}, // edges by from-id id diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index 20311bee6..47e0d62d0 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -1,6 +1,6 @@ import { defaultsDeep, isEmpty } from 'lodash-es'; - -import { Credential, Project } from '../Project'; +import { CredentialState } from '@openfn/lexicon'; +import { Project } from '../Project'; import { mergeWorkflows } from './merge-workflow'; import mapUuids from './map-uuids'; import baseMerge from '../util/base-merge'; @@ -178,9 +178,9 @@ export function merge( } export const replaceCredentials = ( - sourceCreds: Credential[] = [], - targetCreds: Credential[] = [] -): Credential[] => { + sourceCreds: CredentialState[] = [], + targetCreds: CredentialState[] = [] +): CredentialState[] => { const result = [...targetCreds]; // Build an object of existing target credential names for quick lookup @@ -196,7 +196,7 @@ export const replaceCredentials = ( // This is a new credential - add it without the source uuid // (a new UUID will be generated elsewhere) const { uuid, ...credWithoutUuid } = sourceCred; - result.push(credWithoutUuid as Credential); + result.push(credWithoutUuid as CredentialState); } } diff --git a/packages/project/src/merge/merge-workflow.ts b/packages/project/src/merge/merge-workflow.ts index ed7aa7202..2f2cc714c 100644 --- a/packages/project/src/merge/merge-workflow.ts +++ b/packages/project/src/merge/merge-workflow.ts @@ -61,9 +61,7 @@ export function mergeWorkflows( newNode = baseMerge(targetNodes[preservedId], sourceStep, [ 'id', 'name', - // @ts-ignore 'adaptor', - 'adaptors', 'expression', 'next', ]); diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index 4348aeb57..b6ed9324c 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -3,7 +3,7 @@ import * as l from '@openfn/lexicon'; import { Provisioner } from '@openfn/lexicon/lightning'; -import { Project, Credential } from '../Project'; +import { Project } from '../Project'; import renameKeys from '../util/rename-keys'; import slugify from '../util/slugify'; import ensureJson from '../util/ensure-json'; @@ -42,7 +42,7 @@ export default ( owner: c.owner, })); - const proj: Partial = { + const proj: Partial = { name, description: description ?? undefined, collections, @@ -71,7 +71,7 @@ export default ( mapWorkflow(w, proj.credentials) ); - return new Project(proj as l.Project, config); + return new Project(proj as l.ProjectState, config); }; // TODO maybe this is a util and moved out of this file @@ -103,11 +103,11 @@ export const mapEdge = (edge: Provisioner.Edge) => { // TODO this probably gets easier if I index everything by name export const mapWorkflow = ( workflow: Provisioner.Workflow, - credentials: Credential[] = [] + credentials: l.CredentialState[] = [] ) => { const { jobs, edges, triggers, name, version_history, ...remoteProps } = workflow; - const mapped: l.Workflow = { + const mapped: l.WorkflowState = { name: workflow.name, steps: [], history: workflow.version_history ?? [], diff --git a/packages/project/src/parse/from-fs.ts b/packages/project/src/parse/from-fs.ts index 99e6a750c..1382eecba 100644 --- a/packages/project/src/parse/from-fs.ts +++ b/packages/project/src/parse/from-fs.ts @@ -114,7 +114,7 @@ export const parseProject = async (options: FromFsConfig) => { } } - return new Project(proj as l.Project, { + return new Project(proj as l.ProjectState, { alias, ...context.workspace, }); diff --git a/packages/project/src/parse/from-project.ts b/packages/project/src/parse/from-project.ts index cd1bd8bb9..716df4381 100644 --- a/packages/project/src/parse/from-project.ts +++ b/packages/project/src/parse/from-project.ts @@ -11,7 +11,7 @@ import { WithMeta } from '../Workflow'; // But is really designed for v2 project.yaml files // TODO move these types to a common types.ts, or maybe Project.ts -export type SerializedProject = Omit, 'workflows'> & { +export type SerializedProject = Omit, 'workflows'> & { version: number; workflows: SerializedWorkflow[]; }; @@ -26,7 +26,7 @@ export type SerializedWorkflow = { }; export default ( - data: l.Project | SerializedProject | string, + data: l.ProjectState | SerializedProject | string, config?: Partial & { alias?: string; version?: number } ) => { // first ensure the data is in JSON format diff --git a/packages/project/src/serialize/to-app-state.ts b/packages/project/src/serialize/to-app-state.ts index ebf9362d7..644147bb2 100644 --- a/packages/project/src/serialize/to-app-state.ts +++ b/packages/project/src/serialize/to-app-state.ts @@ -1,8 +1,9 @@ import { pick, omitBy, isNil, sortBy } from 'lodash-es'; +import { CredentialState } from '@openfn/lexicon'; import { Provisioner } from '@openfn/lexicon/lightning'; import { randomUUID } from 'node:crypto'; -import { Credential, Project } from '../Project'; +import { Project } from '../Project'; import renameKeys from '../util/rename-keys'; import { jsonToYaml } from '../util/yaml'; import Workflow from '../Workflow'; @@ -44,12 +45,12 @@ export default function ( const credentialsWithUuids = project.credentials?.map((c) => ({ ...c, - uuid: c.uuid ?? randomUUID(), + uuid: (c as CredentialState).uuid ?? randomUUID(), })) ?? []; state.project_credentials = credentialsWithUuids.map((c) => ({ // note the subtle conversion here - id: c.uuid, + id: c.uuid as string, name: c.name, owner: c.owner, })); @@ -74,7 +75,7 @@ export default function ( export const mapWorkflow = ( workflow: Workflow, - credentials: Credential[] = [] + credentials: CredentialState[] = [] ) => { if (workflow instanceof Workflow) { // @ts-ignore diff --git a/packages/project/src/util/get-credential-name.ts b/packages/project/src/util/get-credential-name.ts index 964eaf44a..df7910cec 100644 --- a/packages/project/src/util/get-credential-name.ts +++ b/packages/project/src/util/get-credential-name.ts @@ -1,4 +1,4 @@ -import { Credential } from '../Project'; +import { Credential } from '@openfn/lexicon'; export const DELIMETER = '|'; diff --git a/packages/project/src/util/uuid.ts b/packages/project/src/util/uuid.ts index 1e284bf83..cba1f12d0 100644 --- a/packages/project/src/util/uuid.ts +++ b/packages/project/src/util/uuid.ts @@ -1,8 +1,8 @@ // bunch of common utilities for comparing state and project files // just a sketch/idea right now -import { Workflow } from '@openfn/lexicon'; import { Project } from '../Project'; +import Workflow from '../Workflow'; // Given a workflow step, this will find the UUID for it in state // TODO We probably need the workflow name too? diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index fd1106b96..1e6fbba07 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/runtime +## 1.9.3 + +### Patch Changes + +- 380bccd: Refactor internal workflow types to be independent of the portability spec in the lexicon + ## 1.9.2 ### Patch Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 4cd8172f5..32967ce27 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/runtime", - "version": "1.9.2", + "version": "1.9.3", "description": "Job processing runtime.", "type": "module", "exports": { diff --git a/packages/runtime/src/execute/compile-plan.ts b/packages/runtime/src/execute/compile-plan.ts index 00f958a36..ea27b9835 100644 --- a/packages/runtime/src/execute/compile-plan.ts +++ b/packages/runtime/src/execute/compile-plan.ts @@ -1,12 +1,15 @@ import type { + ExecutionPlan, + Workflow, CompiledEdge, CompiledExecutionPlan, CompiledStep, + Job, + StepEdge, } from '../types'; import compileFunction from '../modules/compile-function'; import { conditionContext, Context } from './context'; -import { ExecutionPlan, Job, StepEdge, Workflow } from '@openfn/lexicon'; import { getNameAndVersion } from '../modules/repo'; // map special condition strings to JS expressions diff --git a/packages/runtime/src/execute/plan.ts b/packages/runtime/src/execute/plan.ts index 879ed8c15..94f753d88 100644 --- a/packages/runtime/src/execute/plan.ts +++ b/packages/runtime/src/execute/plan.ts @@ -1,5 +1,5 @@ import type { Logger } from '@openfn/logger'; -import type { ExecutionPlan, State, Lazy } from '@openfn/lexicon'; +import type { State } from '@openfn/lexicon'; import executeStep from './step'; import compilePlan from './compile-plan'; @@ -9,7 +9,12 @@ import validatePlan from '../util/validate-plan'; import createErrorReporter from '../util/log-error'; import createProfiler from '../util/profile-memory'; import { NOTIFY_STATE_LOAD } from '../events'; -import { CompiledExecutionPlan, ExecutionContext } from '../types'; +import { + CompiledExecutionPlan, + ExecutionContext, + ExecutionPlan, + Lazy, +} from '../types'; const executePlan = async ( plan: ExecutionPlan, diff --git a/packages/runtime/src/execute/step.ts b/packages/runtime/src/execute/step.ts index a9d1e26e8..e5ef79a52 100644 --- a/packages/runtime/src/execute/step.ts +++ b/packages/runtime/src/execute/step.ts @@ -1,4 +1,4 @@ -import type { Job, State, StepId } from '@openfn/lexicon'; +import type { State } from '@openfn/lexicon'; import type { Logger } from '@openfn/logger'; import executeExpression, { ExecutionErrorWrapper } from './expression'; @@ -7,6 +7,8 @@ import assembleState from '../util/assemble-state'; import type { CompiledStep, ExecutionContext, + Job, + StepId, NotifyJobCompletePayload, } from '../types'; import { EdgeConditionError } from '../errors'; diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index 06e541663..f0072a404 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -1,10 +1,6 @@ import { createMockLogger, Logger } from '@openfn/logger'; -import type { - ExecutionPlan, - State, - SourceMapWithOperations, -} from '@openfn/lexicon'; -import type { ExecutionCallbacks } from './types'; +import type { State, SourceMapWithOperations } from '@openfn/lexicon'; +import type { ExecutionCallbacks, ExecutionPlan } from './types'; import type { LinkerOptions } from './modules/linker'; import executePlan from './execute/plan'; import { defaultState, parseRegex, clone } from './util/index'; diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 957696233..de67ce942 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -1,8 +1,13 @@ -import { Operation, StepId, WorkflowOptions, Step } from '@openfn/lexicon'; - +import { + Operation, + SourceMapWithOperations, + State, + UUID, + WorkflowOptions, +} from '@openfn/lexicon'; import { Logger } from '@openfn/logger'; import { Options } from './runtime'; -import { ErrorReporter } from './util/log-error'; +import type { ErrorReporter } from './util/log-error'; import { NOTIFY_INIT_COMPLETE, NOTIFY_JOB_COMPLETE, @@ -13,10 +18,60 @@ import { } from './events'; import { ModuleInfoMap } from './modules/linker'; -export type ErrorPosition = { - line: number; - column: number; - src?: string; // the source line for this error +export type StepId = string; + +export type ConditionalStepEdge = { + condition?: string; // Javascript expression (function body, not function) + label?: string; + disabled?: boolean; +}; + +export type StepEdge = boolean | string | ConditionalStepEdge; + +export interface Step { + id?: StepId; + name?: string; + next?: string | Record; + previous?: StepId; +} + +export interface Trigger extends Step { + enabled?: boolean; + + // The trigger is allowed extra keys, but they will be ignored + [key: string]: unknown; +} + +export interface Job extends Step { + // Spec-compliant props + expression?: string; + configuration?: object | string; + + // internal runtime props + adaptors?: string[]; + state?: Omit | string; + sourceMap?: SourceMapWithOperations; + linker?: ModuleInfoMap; +} + +// Runtime-internal mirror of the portability schema. Adds runtime-only +// fields (linker, sourceMap, adaptors[], state, etc) that aren't portable. +export interface Workflow { + // Spec-compliant props + id?: string; + name?: string; + steps: Array; + globals?: string; + start?: StepId; + + // internal runtime props + credentials?: Record; +} + +export type ExecutionPlan = { + id?: UUID; + workflow: Workflow; + options?: WorkflowOptions; }; export type CompiledEdge = @@ -29,30 +84,47 @@ export type CompiledEdge = export type CompiledStep = Omit & { id: StepId; next?: Record; - - // custom overrides for the linker - // This lets us set version or even path per job + previous?: StepId; linker?: ModuleInfoMap; [other: string]: any; }; -export type Lazy = string | T; +export interface CompiledWorkflow { + globals?: string; + steps: Record; + sourceMap?: SourceMapWithOperations; + credentials?: Record; + start?: StepId; +} export type CompiledExecutionPlan = { - workflow: { - globals?: string; - steps: Record; - credentials?: Record; - /** The default start node - the one the workflow was designed for (the trigger) */ - start?: StepId; - }; + id?: UUID; + workflow: CompiledWorkflow; options: WorkflowOptions & { - /** User-specified start node */ start: StepId; }; }; +export type Lazy = string | T; + +export type ErrorPosition = { + line: number; + column: number; + src?: string; // the source line for this error +}; + +export type ErrorReport = { + type: string; // The name/type of error, ie Error, TypeError + message: string; // simple human readable message + stepId: StepId; // ID of the associated job + error: Error; // the original underlying error object + + code?: string; // The error code, if any (found on node errors) + stack?: string; + data?: any; +}; + export type JobModule = { operations: Operation[]; execute?: (...operations: Operation[]) => (state: any) => any; diff --git a/packages/runtime/src/util/log-error.ts b/packages/runtime/src/util/log-error.ts index 14ecc342c..155999c23 100644 --- a/packages/runtime/src/util/log-error.ts +++ b/packages/runtime/src/util/log-error.ts @@ -1,5 +1,6 @@ import { Logger } from '@openfn/logger'; -import type { State, ErrorReport, StepId } from '@openfn/lexicon'; +import type { State } from '@openfn/lexicon'; +import type { ErrorReport, StepId } from '../types'; export type ErrorReporter = ( state: State, diff --git a/packages/runtime/src/util/sourcemap-errors.ts b/packages/runtime/src/util/sourcemap-errors.ts index f50b2d75f..1bb9f993f 100644 --- a/packages/runtime/src/util/sourcemap-errors.ts +++ b/packages/runtime/src/util/sourcemap-errors.ts @@ -1,7 +1,7 @@ import { SourceMapConsumer } from 'source-map'; import { extractPositionForFrame, RTError } from '../errors'; import pick from './pick'; -import type { Job } from '@openfn/lexicon'; +import type { Job } from '../types'; // This function takes an error and a job and updates the error with sourcemapped metadata export default async (job: Job, error: RTError) => { diff --git a/packages/runtime/src/util/validate-plan.ts b/packages/runtime/src/util/validate-plan.ts index fd1541b9e..8e946049f 100644 --- a/packages/runtime/src/util/validate-plan.ts +++ b/packages/runtime/src/util/validate-plan.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan, Step } from '@openfn/lexicon'; +import { ExecutionPlan, Step } from '../types'; import { ValidationError } from '../errors'; type ModelNode = { diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md index acaf667d6..6bae7c6ab 100644 --- a/packages/ws-worker/CHANGELOG.md +++ b/packages/ws-worker/CHANGELOG.md @@ -1,5 +1,16 @@ # ws-worker +## 1.24.2 + +### Patch Changes + +- Update type interfaces to match new portability spec. +- Updated dependencies [728e6cb] +- Updated dependencies [380bccd] + - @openfn/lexicon@2.0.0 + - @openfn/runtime@1.9.3 + - @openfn/engine-multi@1.11.4 + ## 1.24.1 ### Patch Changes diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 18ce60881..53f331221 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/ws-worker", - "version": "1.24.1", + "version": "1.24.2", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "dist/index.js", "type": "module", diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 422c68f1f..7d348438c 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -1,4 +1,5 @@ -import type { ExecutionPlan, Lazy, State, UUID } from '@openfn/lexicon'; +import type { State, UUID } from '@openfn/lexicon'; +import type { ExecutionPlan, Lazy } from '@openfn/runtime'; import * as Sentry from '@sentry/node'; import type { Logger } from '@openfn/logger'; diff --git a/packages/ws-worker/src/api/reasons.ts b/packages/ws-worker/src/api/reasons.ts index 2651eeb0f..ab0c20c63 100644 --- a/packages/ws-worker/src/api/reasons.ts +++ b/packages/ws-worker/src/api/reasons.ts @@ -1,4 +1,5 @@ -import { State, Step } from '@openfn/lexicon'; +import { State } from '@openfn/lexicon'; +import { Step } from '@openfn/runtime'; import { ExitReason, ExitReasonStrings } from '@openfn/lexicon/lightning'; import type { RunState } from '../types'; diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 2d76b0b74..5ab25734c 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -1,8 +1,8 @@ import { EventEmitter } from 'node:events'; import crypto from 'node:crypto'; -import run from '@openfn/runtime'; +import run, { type ExecutionPlan, type Job } from '@openfn/runtime'; import * as engine from '@openfn/engine-multi'; -import type { ExecutionPlan, Job, State } from '@openfn/lexicon'; +import type { State } from '@openfn/lexicon'; import mockResolvers from './resolvers'; import { RuntimeEngine } from '@openfn/engine-multi'; diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index 877fd6910..4d9a27b6a 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -1,4 +1,5 @@ -import type { ExecutionPlan, Lazy, State } from '@openfn/lexicon'; +import type { State } from '@openfn/lexicon'; +import type { ExecutionPlan, Lazy } from '@openfn/runtime'; import type { Channel as PhxChannel } from 'phoenix'; export type { Socket } from 'node:ws'; diff --git a/packages/ws-worker/src/util/convert-lightning-plan.ts b/packages/ws-worker/src/util/convert-lightning-plan.ts index 2d24190da..1f8f575ec 100644 --- a/packages/ws-worker/src/util/convert-lightning-plan.ts +++ b/packages/ws-worker/src/util/convert-lightning-plan.ts @@ -1,18 +1,20 @@ import crypto from 'node:crypto'; import path from 'node:path'; +import type { State, WorkflowOptions } from '@openfn/lexicon'; import type { Step, StepId, ExecutionPlan, - State, Job, - Trigger, StepEdge, - WorkflowOptions, Lazy, -} from '@openfn/lexicon'; +} from '@openfn/runtime'; import type { LogLevel } from '@openfn/logger'; -import { LightningPlan, LightningEdge } from '@openfn/lexicon/lightning'; +import { + LightningPlan, + LightningEdge, + LightningTrigger, +} from '@openfn/lexicon/lightning'; import { ExecuteOptions } from '@openfn/engine-multi'; import { getNameAndVersion } from '@openfn/runtime'; @@ -138,7 +140,7 @@ export default ( // We don't really care about triggers, it's mostly just a empty node if (run.triggers?.length) { - run.triggers.forEach((trigger: Trigger) => { + run.triggers.forEach((trigger: LightningTrigger) => { const id = trigger.id || 'trigger'; nodes[id] = { @@ -149,9 +151,8 @@ export default ( const connectedEdges = edges.filter((e) => e.source_trigger_id === id); if (connectedEdges.length) { nodes[id].next = connectedEdges.reduce( - (obj: Partial, edge) => { + (obj: Record, edge) => { if (edge.enabled !== false) { - // @ts-ignore obj[edge.target_job_id] = mapTriggerEdgeCondition(edge); } return obj; diff --git a/packages/ws-worker/src/util/create-run-state.ts b/packages/ws-worker/src/util/create-run-state.ts index 5fca6a74a..dfa078597 100644 --- a/packages/ws-worker/src/util/create-run-state.ts +++ b/packages/ws-worker/src/util/create-run-state.ts @@ -1,4 +1,5 @@ -import type { ExecutionPlan, Job, Lazy, State } from '@openfn/lexicon'; +import type { State } from '@openfn/lexicon'; +import type { ExecutionPlan, Job, Lazy } from '@openfn/runtime'; import type { RunState } from '../types'; export default (plan: ExecutionPlan, input?: Lazy): RunState => { diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 6c1f67489..404a003ab 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; import { createMockLogger } from '@openfn/logger'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import { STEP_START, diff --git a/packages/ws-worker/test/api/process-event.test.ts b/packages/ws-worker/test/api/process-event.test.ts index b7199caad..4b84de6be 100644 --- a/packages/ws-worker/test/api/process-event.test.ts +++ b/packages/ws-worker/test/api/process-event.test.ts @@ -12,7 +12,7 @@ import { import { eventProcessor } from '../../src/api/process-events'; import createMockEngine from '../../src/mock/runtime-engine'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import { createMockLogger } from '@openfn/logger'; import { EventEmitter } from 'node:events'; diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index a876baf07..d51424ace 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import type { ExecutionPlan } from '@openfn/lexicon'; +import type { ExecutionPlan } from '@openfn/runtime'; import create from '../../src/mock/runtime-engine'; import { waitForEvent, clone, createPlan } from '../util'; diff --git a/packages/ws-worker/test/reasons.test.ts b/packages/ws-worker/test/reasons.test.ts index 0e1bc797d..605ff497a 100644 --- a/packages/ws-worker/test/reasons.test.ts +++ b/packages/ws-worker/test/reasons.test.ts @@ -14,7 +14,7 @@ import { RUN_COMPLETE, GET_CREDENTIAL, } from '../src/events'; -import { ExecutionPlan } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; let engine: any; let logger: any; diff --git a/packages/ws-worker/test/util.ts b/packages/ws-worker/test/util.ts index 54000ddc9..5fcf1e63a 100644 --- a/packages/ws-worker/test/util.ts +++ b/packages/ws-worker/test/util.ts @@ -1,4 +1,4 @@ -import { ExecutionPlan, Job } from '@openfn/lexicon'; +import { ExecutionPlan, Job } from '@openfn/runtime'; import * as Sentry from '@sentry/node'; import sentryTestkit from 'sentry-testkit'; import { diff --git a/packages/ws-worker/test/util/convert-lightning-plan.test.ts b/packages/ws-worker/test/util/convert-lightning-plan.test.ts index 7cf1e3132..20622b534 100644 --- a/packages/ws-worker/test/util/convert-lightning-plan.test.ts +++ b/packages/ws-worker/test/util/convert-lightning-plan.test.ts @@ -5,7 +5,7 @@ import type { LightningTrigger, } from '@openfn/lexicon/lightning'; import convertPlan from '../../src/util/convert-lightning-plan'; -import { Job } from '@openfn/lexicon'; +import { Job } from '@openfn/runtime'; // Creates a lightning node (job or trigger) const createNode = (props = {}) => diff --git a/packages/ws-worker/test/util/create-run-state.test.ts b/packages/ws-worker/test/util/create-run-state.test.ts index 9b36f811a..4e75e68e7 100644 --- a/packages/ws-worker/test/util/create-run-state.test.ts +++ b/packages/ws-worker/test/util/create-run-state.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import type { ExecutionPlan, Job } from '@openfn/lexicon'; +import type { ExecutionPlan, Job } from '@openfn/runtime'; import { createRunState } from '../../src/util'; diff --git a/packages/ws-worker/test/worker.test.ts b/packages/ws-worker/test/worker.test.ts index 643376839..6c845a2cf 100644 --- a/packages/ws-worker/test/worker.test.ts +++ b/packages/ws-worker/test/worker.test.ts @@ -17,7 +17,7 @@ import { RUN_COMPLETE, GET_CREDENTIAL, } from '../src/events'; -import { ExecutionPlan } from '@openfn/lexicon'; +import { ExecutionPlan } from '@openfn/runtime'; let engine: any; let logger: any;