From 2e74e61e5afb3e6c83df83d2d26f614eb5d43965 Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Fri, 2 Feb 2024 17:08:30 -0800 Subject: [PATCH] chore(endomoat): better scenario test running - Eschew use of `console.log` in lieu of `t.log` - Unwrap scenarios into their own tests! --- packages/endomoat/.eslintrc.cjs | 10 ++ packages/endomoat/test/helpers.js | 211 ++++++++++++++--------- packages/endomoat/test/scenarios.spec.js | 102 ++++++----- 3 files changed, 195 insertions(+), 128 deletions(-) diff --git a/packages/endomoat/.eslintrc.cjs b/packages/endomoat/.eslintrc.cjs index d2ca48cbd8..aea6f67122 100644 --- a/packages/endomoat/.eslintrc.cjs +++ b/packages/endomoat/.eslintrc.cjs @@ -13,4 +13,14 @@ module.exports = { '@endo/no-nullish-coalescing': 'off', }, extends: ['plugin:@endo/recommended'], + overrides: [ + { + files: ['./test/**/*.js'], + rules: { + 'no-console': 'error', + // of dubious value + 'ava/use-t-well': 'off', + }, + }, + ], } diff --git a/packages/endomoat/test/helpers.js b/packages/endomoat/test/helpers.js index 2c2d01e584..731d5912f1 100644 --- a/packages/endomoat/test/helpers.js +++ b/packages/endomoat/test/helpers.js @@ -1,48 +1,61 @@ import capcon from 'capture-console' import { mergePolicy } from 'lavamoat-core' +import util from 'node:util' // @ts-expect-error - needs types import { prepareScenarioOnDisk } from 'lavamoat-core/test/util.js' -import { Volume, createFsFromVolume } from 'memfs' +import { memfs } from 'memfs' import path from 'node:path' import { run, toEndoPolicy } from '../src/index.js' /** + * Dumps a bunch of information about an error and the virtual FS volume. + * Optionally, policies * * @param {unknown} err * @param {import('memfs').Volume} vol - * @param {import('lavamoat-core').LavaMoatPolicy} [lavamoatPolicy] - * @param {import('../src/policy-converter.js').LavaMoatEndoPolicy} [endoPolicy] + * @param {object} options + * @param {import('lavamoat-core').LavaMoatPolicy} [options.lavamoatPolicy] + * @param {import('../src/policy-converter.js').LavaMoatEndoPolicy} [options.endoPolicy] + * @param {(...args: any) => void} [options.log] */ -function dumpError(err, vol, lavamoatPolicy, endoPolicy) { - console.debug() - console.debug(err) +function dumpError( + err, + vol, + // eslint-disable-next-line no-console + { lavamoatPolicy, endoPolicy, log = console.error.bind(console) } = {} +) { + log() + log(err) if (endoPolicy) { - console.debug('Endo policy:') - console.dir(endoPolicy, { depth: null }) + log('Endo policy:') + log(util.inspect(endoPolicy, { depth: null })) } if (lavamoatPolicy) { - console.debug('Lavamoat policy:') - console.dir(lavamoatPolicy, { depth: null }) + log('Lavamoat policy:') + log(util.inspect(lavamoatPolicy, { depth: null })) } - console.debug('FS:') - console.debug(vol.toTree()) - console.debug() + log('Filesystem:') + log(vol.toTree()) + log() } /** * Reads a policy and policy overrides using a `ReadFn`, then merges the result. * - * @param {import('@endo/compartment-mapper').ReadFn} readPower - * @param {string} policyDir + * @param {import('@endo/compartment-mapper').ReadFn} readPower - File read + * power + * @param {string} policyDir - Path to the policy directory. If relative, will + * be resolved to cwd * @returns {Promise} + * @todo Make the stuff in `lavamoat-core`'s `loadPolicy` accept a `readPower`, + * and get rid of this. */ async function readPolicy(readPower, policyDir) { // console.debug('Reading policy from %s', policyDir) let [lavamoatPolicy, lavamoatPolicyOverrides] = await Promise.all( ['policy.json', 'policy-override.json'].map((filename) => - readPower(path.join(policyDir, filename)) - .then((bytes) => `${bytes}`) - .then(JSON.parse) + readPower(path.resolve(policyDir, filename)) + .then((bytes) => JSON.parse(`${bytes}`)) .catch((err) => { if (err.code !== 'ENOENT') { throw err @@ -54,17 +67,20 @@ async function readPolicy(readPower, policyDir) { if (!lavamoatPolicy) { throw new Error(`LavaMoat - policy not found in ${policyDir}`) } + if (lavamoatPolicyOverrides) { - // console.debug('Applying policy overrides; before:') - // console.dir(lavamoatPolicy, { depth: null }) lavamoatPolicy = mergePolicy(lavamoatPolicy, lavamoatPolicyOverrides) - // console.debug('after:') - // console.dir(lavamoatPolicy, { depth: null }) } return lavamoatPolicy } +/** + * Captures `process.stderr` & `process.stdout` and returns a function to stop + * capturing them. + * + * @returns {() => { stdout: string; stderr: string }} + */ function capture() { let stdout = '' let stderr = '' @@ -83,79 +99,104 @@ function capture() { return { stdout, stderr } } } -// @ts-expect-error - needs types -export async function runScenario({ scenario }) { - const vol = new Volume() - const fs = createFsFromVolume(vol) - const readFile = /** @type {import('@endo/compartment-mapper').ReadFn} */ ( - fs.promises.readFile - ) +/** + * Bootstraps the scenario runner. + * + * Return value should be provided to `lavamoat-core`'s `runAndTestScenario` + * + * @param {(...args: any) => void} log Logger + * @returns + */ +// eslint-disable-next-line no-console +export function createScenarioRunner(log = console.error.bind(console)) { + /** + * Runs a scenario from `lavamoat-core`. + * + * The idea here is to establishe feature-compatibility with `lavamoat-node`. + * + * @remarks + * The runner in e.g., `lavamoat-node` spawns a child process for each + * scenario; I don't feel that is necessary for our purposes. It _is_ useful + * and necessary to test the CLI, but I'm not convinced there's added value in + * doing so for _every scenario_. + * @param {{ scenario: any }} opts + * @returns {Promise} Result of the stdout of the scenario parsed as + * JSON + * @todo Scenario needs a type definition + */ + return async ({ scenario }) => { + const { fs, vol } = memfs() + + const readFile = /** @type {import('@endo/compartment-mapper').ReadFn} */ ( + fs.promises.readFile + ) - /** @type {string} */ - let projectDir - /** @type {string} */ - let policyDir - - // for eslint - await Promise.resolve() - - try { - ;({ projectDir, policyDir } = await prepareScenarioOnDisk({ - fs: fs.promises, - scenario, - policyName: 'endomoat', - })) - } catch (e) { - dumpError(e, vol) - throw e - } + /** @type {string} */ + let projectDir + /** @type {string} */ + let policyDir - /** @type {import('lavamoat-core').LavaMoatPolicy} */ - let lavamoatPolicy - try { - lavamoatPolicy = await readPolicy(readFile, policyDir) - } catch (e) { - dumpError(e, vol) - throw e - } + // for eslint + await Promise.resolve() - const endoPolicy = toEndoPolicy(lavamoatPolicy) + try { + ;({ projectDir, policyDir } = await prepareScenarioOnDisk({ + fs: fs.promises, + scenario, + policyName: 'endomoat', + })) + } catch (e) { + dumpError(e, vol, { log }) + throw e + } - const entryPath = new URL( - `file://` + path.join(projectDir, scenario.entries[0]) - ) + /** @type {import('lavamoat-core').LavaMoatPolicy} */ + let lavamoatPolicy + try { + lavamoatPolicy = await readPolicy(readFile, policyDir) + } catch (e) { + dumpError(e, vol, { log }) + throw e + } + + const endoPolicy = toEndoPolicy(lavamoatPolicy) - /** @type {import('@endo/compartment-mapper').ReadFn} */ - const readPower = async (location) => readFile(new URL(location).pathname) + const entryPath = new URL( + `file://` + path.join(projectDir, scenario.entries[0]) + ) - const stopCapture = capture() + /** @type {import('@endo/compartment-mapper').ReadFn} */ + const readPower = async (location) => readFile(new URL(location).pathname) - /** @type {string} */ - let stderr - /** @type {string} */ - let stdout - try { - await run(entryPath, { readPower, endoPolicy }) - ;({ stderr, stdout } = stopCapture()) - } catch (err) { - // this must happen prior to dumping the error - ;({ stderr, stdout } = stopCapture()) + const stopCapture = capture() - if (!scenario.expectedFailure) { - dumpError(err, vol, lavamoatPolicy, endoPolicy) + /** @type {string} */ + let stderr + /** @type {string} */ + let stdout + try { + await run(entryPath, { readPower, endoPolicy }) + ;({ stderr, stdout } = stopCapture()) + } catch (err) { + // this must happen prior to dumping the error + ;({ stderr, stdout } = stopCapture()) + + if (!scenario.expectedFailure) { + dumpError(err, vol, { lavamoatPolicy, endoPolicy, log }) + } + throw err } - throw err - } - let result - if (stderr) { - throw new Error(`Unexpected output in standard err: \n${stderr}`) - } - try { - result = JSON.parse(stdout) - } catch (e) { - throw new Error(`Unexpected output in standard out: \n${stdout}`) + let result + if (stderr) { + throw new Error(`Unexpected output in standard err: \n${stderr}`) + } + try { + result = JSON.parse(stdout) + } catch (e) { + throw new Error(`Unexpected output in standard out: \n${stdout}`) + } + return result } - return result } diff --git a/packages/endomoat/test/scenarios.spec.js b/packages/endomoat/test/scenarios.spec.js index ce31fd8c04..6a6fbd82ea 100644 --- a/packages/endomoat/test/scenarios.spec.js +++ b/packages/endomoat/test/scenarios.spec.js @@ -1,13 +1,21 @@ +/* eslint-disable ava/no-skip-test */ import test from 'ava' // @ts-expect-error - needs types import { loadScenarios } from 'lavamoat-core/test/scenarios/index.js' // @ts-expect-error - needs types import { runAndTestScenario } from 'lavamoat-core/test/util.js' -import { runScenario } from './helpers.js' +import { createScenarioRunner } from './helpers.js' const GLOBAL_WRITE_REGEX = /".+?":"write"/g -// @ts-expect-error - needs types +/** + * If the policy file contains a writable global we have to skip it until we've + * implemented it + * + * @param {any} scenario + * @returns + * @todo Implement & remove + */ function policyHasWritableGlobal(scenario) { return ( GLOBAL_WRITE_REGEX.test(JSON.stringify(scenario.config)) || @@ -15,51 +23,59 @@ function policyHasWritableGlobal(scenario) { ) } -const SKIP_SCENARIOS = new Set([ +const FAILING_SCENARIOS = new Set([ 'globalRef - check default containment', 'globalRef - Webpack code in the wild works', ]) -test('Run scenarios', async (t) => { - let count = 0 - let skipped = 0 - for await (const scenario of loadScenarios()) { - count++ - if ( - !( - Object.keys(scenario.context).length === 0 && - scenario.context.constructor === Object - ) - ) { - console.debug(`[SKIPPING] ${scenario.name} (???)`) - skipped++ - continue - } +/** + * Macro to test a scenario + */ +const testScenario = test.macro( + /** + * @param {import('ava').ExecutionContext} t + * @param {any} scenario + */ + async (t, scenario) => { + await runAndTestScenario(t, scenario, createScenarioRunner(t.log.bind(t))) + } +) - if (policyHasWritableGlobal(scenario)) { - console.debug(`[SKIPPING] ${scenario.name} (has writable global)`) - skipped++ - continue - } - if (scenario.name.startsWith('scuttle')) { - console.debug(`[SKIPPING] ${scenario.name} (scuttling unsupported)`) - skipped++ - continue - } - if (SKIP_SCENARIOS.has(scenario.name)) { - console.debug(`[SKIPPING] ${scenario.name} (fixme)`) - skipped++ - continue - } - // console.debug(`[SCENARIO] ${scenario.name}`) - try { - await runAndTestScenario(t, scenario, runScenario) - console.debug(`[OK] ${scenario.name}`) - } catch (err) { - console.debug(/** @type {Error} */ (err).message ?? err) - console.debug(`[FAIL] ${scenario.name}`) - } +// TIL: you can dynamically create tests in an async iterator +// as long as you use `test.serial()` +for await (const scenario of loadScenarios()) { + if ( + !( + Object.keys(scenario.context).length === 0 && + scenario.context.constructor === Object + ) + ) { + // these will never work. I don't know what they are, though. + test.serial.skip( + `${scenario.name} - incompatible platform`, + testScenario, + scenario + ) + continue } - console.debug('Ran %d/%d scenarios', count - skipped, count) -}) + // TODO fix + if (FAILING_SCENARIOS.has(scenario.name)) { + test.serial.failing(`${scenario.name} - `, testScenario, scenario) + continue + } + + // TODO implement writable globals + if (policyHasWritableGlobal(scenario)) { + test.todo(`${scenario.name} - has writable global`) + continue + } + + // TODO implement scuttling + if (scenario.name.startsWith('scuttle')) { + test.todo(`${scenario.name} - scuttling unsupported`) + continue + } + + test.serial(scenario.name, testScenario, scenario) +}