-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Start a basic server which runs a job on POST requests
- Loading branch information
1 parent
0ffbf66
commit b9394eb
Showing
13 changed files
with
806 additions
and
121 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,67 @@ | ||
import koa from 'koa'; | ||
import fs from 'node:fs/promises'; | ||
import path from 'node:path'; | ||
import Manager from '../Manager'; | ||
|
||
const loadJobs = async () => { | ||
for (const name of ['slow-random']) { | ||
const source = await fs.readFile(path.resolve(`src/server/jobs/${name}.js`), { encoding: 'utf8' }); | ||
runtime.registerJob(name, source); | ||
} | ||
console.log('Jobs loaded:') | ||
console.log(runtime.getRegisteredJobs()); | ||
}; | ||
|
||
const app = new koa(); | ||
|
||
console.log('starting server') | ||
|
||
const runtime = Manager(); | ||
|
||
loadJobs(); | ||
|
||
// Create http server | ||
export default {} | ||
// GET works return alist of workers | ||
|
||
// on post to job/name we run that job | ||
|
||
// need a web socket to listen to and report changes | ||
|
||
const handlePost = (ctx: koa.Context) => { | ||
ctx; | ||
// start a job | ||
runJob('slow-random') | ||
}; | ||
|
||
const runJob = async (name: string) => { | ||
console.log(`Running job: ${name}...`) | ||
|
||
const result = await runtime.run(name, { | ||
configuration: { | ||
delay: 4000 | ||
} | ||
}); | ||
|
||
console.log('--') | ||
console.log(`Job ${name} finished`) | ||
console.log(result) | ||
console.log('--') | ||
report(); | ||
} | ||
|
||
const report = () => { | ||
const threadCount = runtime.getActiveJobs().length; | ||
console.log(`active threads: ${threadCount}`) | ||
} | ||
|
||
app.use((ctx) => { | ||
if (ctx.method === "POST") { | ||
handlePost(ctx); | ||
} | ||
}) | ||
|
||
app.listen(1234) | ||
|
||
report(); | ||
|
||
export default {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
// This job takes a random number of seconds and returns a random number | ||
const slowmo = (state) => new Promise((resolve) => { | ||
const done = () => { | ||
resolve({ data: { result: Math.random() * 100 }}) | ||
}; | ||
setTimeout(done, state.configuration?.delay ?? 500); | ||
}); | ||
|
||
export default [slowmo]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
// middleware to handle the worker manager | ||
const middleware = () => { | ||
|
||
} | ||
|
||
export default middleware; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,23 +1,11 @@ | ||
// // Dedicated worker for running jobs | ||
// // Interesting plot twist: how do we enable vm modules in the thread? | ||
// import workerpool from 'workerpool'; | ||
// import run from '@openfn/runtime'; | ||
|
||
// const runJob = async (src: string) => { | ||
// return await run(src); | ||
// }; | ||
|
||
// workerpool.worker({ | ||
// runJob | ||
// }) | ||
|
||
// Dedicated worker for running jobs | ||
// Security thoughts: the process inherits the node command arguments | ||
// (it has to for experimental modules to work) | ||
// Is this a concern? If secrets are passed in they could be visible | ||
// The sandbox should hep | ||
import run from '@openfn/runtime'; | ||
|
||
export default async (src: string) => { | ||
return await run(src); | ||
export default async (args: [string, any]) => { | ||
const [src, state] = args; | ||
return await run(src, state); | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
import test from 'ava'; | ||
import execute from '@openfn/runtime'; | ||
import slowmo from '../src/server/jobs/slow-random'; | ||
|
||
type SlowMoState = { | ||
data: { | ||
result: number; | ||
} | ||
} | ||
|
||
const wait = async(time: number) => new Promise(resolve => { | ||
setTimeout(resolve, time); | ||
}); | ||
|
||
test('slowmo should return a value', async (t) => { | ||
const result = await execute(slowmo) as SlowMoState; | ||
|
||
t.assert(result); | ||
t.assert(result.data.result); | ||
t.falsy(isNaN(result.data.result)) | ||
}); | ||
|
||
test('slowmo should return after 500ms', async (t) => { | ||
let result; | ||
|
||
execute(slowmo).then((r)=> { | ||
result = r; | ||
}); | ||
|
||
// Should not return immediately | ||
t.falsy(result); | ||
|
||
await wait(100) | ||
t.falsy(result); | ||
|
||
// Should have returned by now | ||
await wait(500); | ||
// @ts-ignore | ||
t.falsy(isNaN(result.data.result)) | ||
}); | ||
|
||
test('slowmo should accept a delay time as config', async (t) => { | ||
let result; | ||
|
||
const state = { | ||
configuration: { | ||
delay: 10 | ||
}, | ||
data: {} | ||
}; | ||
execute(slowmo, state).then((r)=> { | ||
result = r; | ||
}); | ||
|
||
// Should not return immediately | ||
t.falsy(result); | ||
|
||
// Should have data already | ||
await wait(100) | ||
// @ts-ignore | ||
t.falsy(isNaN(result.data.result)) | ||
}); | ||
|
||
test('slowmo should return random numbers', async (t) => { | ||
const state = { | ||
configuration: { | ||
delay: 1 | ||
}, | ||
data: {} | ||
}; | ||
const a = await execute(slowmo, state) | ||
const b = await execute(slowmo, state) | ||
t.assert(a !== b); | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
import test from 'ava'; | ||
import Manager from '../src/Manager'; | ||
|
||
test('Should create a new manager', (t) => { | ||
const m = Manager(); | ||
t.assert(m); | ||
t.assert(m.run); | ||
}); | ||
|
||
test('Should register a job', (t) => { | ||
const m = Manager(); | ||
m.registerJob('my_job', 'x'); | ||
t.assert(m); | ||
}); | ||
|
||
test('Should throw if registering a job that already exists', (t) => { | ||
const m = Manager(); | ||
m.registerJob('my_job', 'x'); | ||
t.throws(() => m.registerJob('my_job', 'x')); | ||
}); | ||
|
||
test('Should return a registered job list', (t) => { | ||
const m = Manager(); | ||
m.registerJob('my_job', 'x'); | ||
m.registerJob('my_other_job', 'x'); | ||
|
||
t.deepEqual(m.getRegisteredJobs(), ['my_job', 'my_other_job']); | ||
}); | ||
|
||
// The ava test runner doesn't seem to be getting the experimental_vm_modules flag and so this fails :( | ||
test.skip('Should run a simple job', async (t) => { | ||
const m = Manager(); | ||
m.registerJob('test', 'export default [() => 10];'); | ||
const result = await m.run('test'); | ||
// @ts-ignore | ||
t.assert(result === 10); | ||
}); | ||
|
||
// should publish an event when a job starts | ||
// should publish an event when a job stops | ||
// should return a job list | ||
// should return a list of active jobs |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.