Runtime Engine

.changeset/cyan-worms-clap.md

@@ -0,0 +1,5 @@
+---
+'@openfn/logger': patch
+---
+
+Export SanitizePolicies type

.changeset/short-pens-punch.md

@@ -0,0 +1,5 @@
+---
+'@openfn/runtime': patch
+---
+
+Support a cacheKey to bust cached modules in long-running processes

.changeset/tough-coats-unite.md

@@ -0,0 +1,5 @@
+---
+'@openfn/runtime': patch
+---
+
+Trigger callbacks on job start, complete and init

.circleci/config.yml

@@ -9,7 +9,7 @@ jobs:
     # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
     # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor
     docker:
-      - image: cimg/node:18.12
+      - image: cimg/node:18.18
     resource_class: medium
     # Add steps to the job
     # See: https://circleci.com/docs/2.0/configuration-reference/#steps

.tool-versions

@@ -1 +1 @@
-nodejs 18.12.1
+nodejs 18.18.2

integration-tests/cli/package.json

@@ -1,7 +1,7 @@
 {
-  "name": "@openfn/integration-tests",
+  "name": "@openfn/integration-tests-cli",
   "private": true,
-  "version": "0.0.1",
+  "version": "1.0.0",
   "description": "CLI integration tests",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",

integration-tests/worker/.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+repo

integration-tests/worker/Dockerfile

@@ -0,0 +1,15 @@
+# This dockerfile allows us to run tests against the build in isolation
+# (I'm not really sure we need it but it's been a useful dev tool)
+FROM node:18.12.0
+
+ENV NODE_ENV=production
+
+WORKDIR /app
+
+COPY . .
+
+COPY ./package.json.container ./package.json
+
+RUN npm install --production
+
+CMD ["npm", "test"]

integration-tests/worker/ava.config.js

@@ -0,0 +1,13 @@
+export default {
+  extensions: {
+    ts: 'module',
+  },
+
+  environmentVariables: {
+    TS_NODE_TRANSPILE_ONLY: 'true',
+  },
+
+  nodeArguments: ['--loader=ts-node/esm', '--no-warnings'],
+
+  files: ['test/**/*test.ts'],
+};

integration-tests/worker/dummy-repo/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "test-repo",
+  "private": true,
+  "version": "1.0.0",
+  "dependencies": {
+    "@openfn/stateful-test_1.0.0": "@npm:@openfn/stateful-test@1.0.0"
+  }
+}

integration-tests/worker/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@openfn/integration-tests-worker",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Lightning WOrker integration tests",
+  "author": "Open Function Group <admin@openfn.org>",
+  "license": "ISC",
+  "type": "module",
+  "scripts": {
+    "clean": "rimraf dist tmp/repo/*",
+    "build:pack": "pnpm clean && cd ../.. && pnpm pack:local integration-tests/worker/dist --no-version",
+    "build": "pnpm build:pack && docker build --tag worker-integration-tests .",
+    "start": "docker run worker-integration-tests",
+    "test": "pnpm clean && npx ava -s --timeout 2m"
+  },
+  "dependencies": {
+    "@openfn/engine-multi": "workspace:^",
+    "@openfn/lightning-mock": "workspace:^",
+    "@openfn/logger": "workspace:^",
+    "@openfn/ws-worker": "workspace:^",
+    "@types/node": "^18.15.13",
+    "@types/rimraf": "^3.0.2",
+    "ava": "5.3.1",
+    "date-fns": "^2.30.0",
+    "koa": "^2.13.4",
+    "rimraf": "^3.0.2",
+    "ts-node": "10.8.1",
+    "tslib": "^2.4.0",
+    "typescript": "^5.1.6"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ]
+}

integration-tests/worker/package.json.container

@@ -0,0 +1,34 @@
+{
+  "name": "@openfn/integration-tests-worker",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Lightning WOrker integration tests",
+  "author": "Open Function Group <admin@openfn.org>",
+  "license": "ISC",
+  "type": "module",
+  "scripts": {
+    "clean": "rimraf dist tmp/repo/*",
+    "build:pack": "npm run clean && cd ../.. && npm run pack:local integration-tests/worker/dist --no-version",
+    "build": "npm run build:pack && docker build --tag worker-integration-tests .",
+    "start": "docker run worker-integration-tests",
+    "test": "npm run clean && npx ava -s --timeout 2m"
+  },
+  "dependencies": {
+    "@openfn/engine-multi": "./dist/openfn-engine-multi.tgz",
+    "@openfn/lightning-mock": "./dist/openfn-lightning-mock.tgz",
+    "@openfn/logger": "./dist/openfn-logger.tgz",
+    "@openfn/ws-worker": "./dist/openfn-ws-worker.tgz",
+    "@types/node": "^18.15.13",
+    "ava": "5.3.1",
+    "date-fns": "^2.30.0",
+    "rimraf": "^3.0.2",
+    "ts-node": "10.8.1",
+    "tslib": "^2.4.0",
+    "typescript": "^5.1.6",
+    "@types/rimraf": "^3.0.2"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ]
+}

integration-tests/worker/readme.md

@@ -0,0 +1 @@
+Integration tests for Lightning Worker architecture.