feat: introduce isolated execution

as-test.config.js

@@ -25,4 +25,6 @@ export default {
 
   /** optional: test result output format, default "table" */
   mode: ["html", "json", "table"],
+
+  isolated: false,
 };

bin/cli.js

@@ -23,7 +23,12 @@ program
   .option("--testcase <testcases...>", "run only specified test cases deprecated, use --testFiles instead")
   .option("--testFiles <testFiles...>", "run only specified test files")
   .option("--testNamePattern <test name pattern>", "run only tests with a name that matches the regex pattern")
-  .option("--onlyFailures", "Run tests that failed in the previous");
+  .option("--onlyFailures", "Run tests that failed in the previous")
+  .option("--isolated <boolean>", "Run tests in isolated mode")
+  .addHelpText(
+    "beforeAll",
+    "submit feature requests or issues: https://github.com/wasm-ecosystem/assemblyscript-unittest-framework/issues"
+  );
 
 program.parse(process.argv);
 const options = program.opts();
@@ -61,6 +66,33 @@ const collectCoverage =
   config.collectCoverage ||
   (testFiles === undefined && options.testNamePattern === undefined && !onlyFailures);
 
+const getBoolean = (optionValue, configValue) => {
+  if (optionValue !== undefined) {
+    if (optionValue == "true") {
+      return true;
+    } else if (optionValue == "false") {
+      return false;
+    }
+  }
+  if (configValue !== undefined) {
+    return Boolean(configValue);
+  }
+  return undefined;
+};
+const isolatedInConfig = getBoolean(options.isolated, config.isolated);
+if (isolatedInConfig === undefined) {
+  console.warn(
+    chalk.yellowBright(
+      "Warning: In the next version, the default value of isolated will change. Please specify isolated in config"
+    )
+  );
+}
+// TODO: switch to false default in 2.x
+const isolated = isolatedInConfig ?? true;
+
+/**
+ * @type {import("../dist/interface.d.ts").TestOption}
+ */
 const testOption = {
   includes,
   excludes,
@@ -77,6 +109,8 @@ const testOption = {
   mode: options.mode || config.mode || "table",
   warnLimit: Number(options.coverageLimit?.at(1)),
   errorLimit: Number(options.coverageLimit?.at(0)),
+
+  isolated,
 };
 
 start_unit_test(testOption)

docs/api-documents/options.md

@@ -1,20 +1,16 @@
+[[toc]]
+
 ## Options
 
-### Define Config
+### Config File
 
 ```
   --config <config file>                 path of config file (default: "as-test.config.js")
 ```
 
 ### Override Config File
 
-There are command line options which can override the configuration in `as-test.config.js`.
-
-```
-  --temp <path>                          test template file folder
-  --output <path>                        coverage report output folder
-  --mode <output mode>                   test result output format
-```
+Command line options have higher priority then config file, so that it can override the configuration in `as-test.config.js`.
 
 ### Warning Behavior
 
@@ -108,3 +104,24 @@ Provides `--onlyFailures` command line option to run the test cases that failed
 The framework collects coverage and generates reports by default, but it will be disablea while running partial test cases by `--testFiles` or `--testNamePattern`.
 
 You can control the coverage collection manually with `--collectCoverage` option.
+
+### Isolated Execution
+
+Isolated test execution helps isolate error propagation between different test scenarios and reduces the burden of restoring context, which is very helpful for rapid technical verification.
+
+However, as the project scales, isolated test execution will compile the source code multiple times, slowing down overall test performance. In this case, restoring the context in code and disabling the `isolated` option after testing can help reduce test time.
+
+- disable by config:
+
+  ```js
+  {
+    // ...
+    isolated: false
+  }
+  ```
+
+- disable by cli
+
+  ```bash
+  npx as-test ... --isolated false
+  ```

docs/release-note.md

@@ -1,5 +1,12 @@
 # Release Note
 
+## latest
+
+🚀 Highlight Features
+
+- Improved the as-test performances.
+- Introduce new features `isolated: false` to significantly reduce test execution time in large projects. ([#73](https://github.com/wasm-ecosystem/assemblyscript-unittest-framework/pull/73))
+
 ## 1.3.1
 
 🚀 Highlight Features

src/core/compile.ts

@@ -1,41 +1,64 @@
 import { join, relative } from "node:path";
 import { findRoot } from "../utils/pathResolver.js";
 import { ascMain } from "../utils/ascWrapper.js";
+import { TestOption } from "../interface.js";
 
-export async function compile(testCodePaths: string[], outputFolder: string, compileFlags: string): Promise<string[]> {
+export type CompileOption = Pick<TestOption, "isolated" | "outputFolder" | "flags">;
+
+export async function compile(testCodePaths: string[], option: CompileOption): Promise<string[]> {
+  const { isolated } = option;
+  return isolated ? await separatedCompile(testCodePaths, option) : [await unifiedCompile(testCodePaths, option)];
+}
+
+function getNewPath(newFolder: string, oldFolder: string, srcPath: string): string {
+  return join(newFolder, relative(oldFolder, srcPath)).replaceAll(/\\/g, "/");
+}
+
+function getAscArgs(sources: string[], outputWasm: string, outputWat: string, flags: string): string[] {
+  let ascArgv = [
+    ...sources,
+    "--outFile",
+    outputWasm,
+    "--textFile",
+    outputWat,
+    "--exportStart",
+    "_start",
+    "--sourceMap",
+    "--debug",
+    "-O0",
+  ];
+  if (flags.length > 0) {
+    const argv = flags.split(" ");
+    ascArgv = ascArgv.concat(argv);
+  }
+  return ascArgv;
+}
+
+async function unifiedCompile(testCodePaths: string[], option: CompileOption): Promise<string> {
+  const { outputFolder, flags } = option;
+  const outputWasm = join(outputFolder, "test.wasm");
+  const outputWat = join(outputFolder, "test.wat");
+  const ascArgv = getAscArgs(testCodePaths, outputWasm, outputWat, flags);
+  await ascMain(ascArgv, false);
+  return outputWasm;
+}
+
+async function separatedCompile(testCodePaths: string[], option: CompileOption): Promise<string[]> {
+  const { outputFolder, flags } = option;
   const wasm: string[] = [];
   const root = findRoot(testCodePaths);
-  const compile = async (testCodePath: string) => {
+  const compileOneFile = async (testCodePath: string) => {
     const outputWasm = getNewPath(outputFolder, root, testCodePath).slice(0, -2).concat("wasm");
     wasm.push(outputWasm);
     const outputWat = getNewPath(outputFolder, root, testCodePath).slice(0, -2).concat("wat");
-    let ascArgv = [
-      testCodePath,
-      "--outFile",
-      outputWasm,
-      "--textFile",
-      outputWat,
-      "--exportStart",
-      "_start",
-      "--sourceMap",
-      "--debug",
-      "-O0",
-    ];
-    if (compileFlags) {
-      const argv = compileFlags.split(" ");
-      ascArgv = ascArgv.concat(argv);
-    }
+    const ascArgv = getAscArgs([testCodePath], outputWasm, outputWat, flags);
     await ascMain(ascArgv, false);
   };
 
   // Here, for-await is more efficient and less memory cost than Promise.all()
   for (const codePath of testCodePaths) {
-    await compile(codePath);
+    await compileOneFile(codePath);
   }
 
   return wasm;
 }
-
-function getNewPath(newFolder: string, oldFolder: string, srcPath: string): string {
-  return join(newFolder, relative(oldFolder, srcPath)).replaceAll(/\\/g, "/");
-}

src/index.ts

@@ -59,7 +59,7 @@ async function startUniTestImpl(options: TestOption): Promise<number> {
   );
   console.log(chalk.blueBright("code analysis: ") + chalk.bold.greenBright("OK"));
 
-  const wasmPaths = await compile(unittestPackage.testCodePaths, options.tempFolder, options.flags);
+  const wasmPaths = await compile(unittestPackage.testCodePaths, options);
   console.log(chalk.blueBright("compile test files: ") + chalk.bold.greenBright("OK"));
 
   const sourcePaths = unittestPackage.sourceFunctions ? Array.from(unittestPackage.sourceFunctions.keys()) : [];

src/interface.ts

@@ -213,6 +213,8 @@ export interface TestOption {
   mode: OutputMode | OutputMode[];
   warnLimit?: number;
   errorLimit?: number;
+
+  isolated: boolean;
 }
 
 export type OutputMode = "html" | "json" | "table";

tests/as/mock.test.ts

@@ -18,7 +18,6 @@ describe("mock test", () => {
   });
 
   test("function with functionRef, but not mocked", () => {
-    expect(add.index).equal(2);
     expect(add(1, 1)).equal(2);
   });
 

tests/e2e/isolated-cli/as-test.config.js

@@ -0,0 +1,13 @@
+import path from "node:path";
+
+const __dirname = path.dirname(new URL(import.meta.url).pathname);
+
+/**
+ * @type {import("../../../dist/interface.d.ts").TestOption}
+ */
+export default {
+  include: [__dirname],
+  temp: path.join(__dirname, "tmp"),
+  output: path.join(__dirname, "tmp"),
+  mode: [],
+};

tests/e2e/isolated-cli/env.ts

@@ -0,0 +1,5 @@
+class Fn {
+  raw: (() => void) | null = null;
+}
+
+export let fn = new Fn();

tests/e2e/isolated-cli/stdout.txt

@@ -0,0 +1,6 @@
+code analysis: OK
+compile test files: OK
+instrument: OK
+execute test files: OK
+
+test case: 1/1 (success/total)

tests/e2e/isolated-cli/succeed_0.test.ts

@@ -0,0 +1,12 @@
+import { test, expect } from "../../../assembly";
+import { fn } from "./env";
+
+test("succeed 0", () => {
+  if (fn.raw == null) {
+    fn.raw = () => {
+      expect(true).equal(true);
+    };
+  } else {
+    fn.raw();
+  }
+});

tests/e2e/isolated-cli/succeed_1.test.ts

@@ -0,0 +1,12 @@
+import { test, expect } from "../../../assembly";
+import { fn } from "./env";
+
+test("succeed_1", () => {
+  if (fn.raw == null) {
+    fn.raw = () => {
+      expect(true).equal(true);
+    };
+  } else {
+    fn.raw();
+  }
+});

tests/e2e/isolated-cli/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "assemblyscript/std/assembly.json",
+  "include": ["./**/*.ts"]
+}

tests/e2e/isolated-false/as-test.config.js

@@ -0,0 +1,14 @@
+import path from "node:path";
+
+const __dirname = path.dirname(new URL(import.meta.url).pathname);
+
+/**
+ * @type {import("../../../dist/interface.d.ts").TestOption}
+ */
+export default {
+  include: [__dirname],
+  temp: path.join(__dirname, "tmp"),
+  output: path.join(__dirname, "tmp"),
+  mode: [],
+  isolated: false,
+};

tests/e2e/isolated-false/env.ts

@@ -0,0 +1,5 @@
+class Fn {
+  raw: (() => void) | null = null;
+}
+
+export let fn = new Fn();

tests/e2e/isolated-false/stdout.txt

@@ -0,0 +1,6 @@
+code analysis: OK
+compile test files: OK
+instrument: OK
+execute test files: OK
+
+test case: 1/1 (success/total)

tests/e2e/isolated-false/succeed_0.test.ts

@@ -0,0 +1,12 @@
+import { test, expect } from "../../../assembly";
+import { fn } from "./env";
+
+test("succeed 0", () => {
+  if (fn.raw == null) {
+    fn.raw = () => {
+      expect(true).equal(true);
+    };
+  } else {
+    fn.raw();
+  }
+});

tests/e2e/isolated-false/succeed_1.test.ts

@@ -0,0 +1,12 @@
+import { test, expect } from "../../../assembly";
+import { fn } from "./env";
+
+test("succeed_1", () => {
+  if (fn.raw == null) {
+    fn.raw = () => {
+      expect(true).equal(true);
+    };
+  } else {
+    fn.raw();
+  }
+});

tests/e2e/isolated-false/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "assemblyscript/std/assembly.json",
+  "include": ["./**/*.ts"]
+}

tests/e2e/isolated-true/as-test.config.js

@@ -0,0 +1,14 @@
+import path from "node:path";
+
+const __dirname = path.dirname(new URL(import.meta.url).pathname);
+
+/**
+ * @type {import("../../../dist/interface.d.ts").TestOption}
+ */
+export default {
+  include: [__dirname],
+  temp: path.join(__dirname, "tmp"),
+  output: path.join(__dirname, "tmp"),
+  mode: [],
+  isolated: true,
+};

tests/e2e/isolated-true/env.ts

@@ -0,0 +1,5 @@
+class Fn {
+  raw: (() => void) | null = null;
+}
+
+export let fn = new Fn();

tests/e2e/isolated-true/stdout.txt

@@ -0,0 +1,6 @@
+code analysis: OK
+compile test files: OK
+instrument: OK
+execute test files: OK
+
+test case: 0/0 (success/total)

tests/e2e/isolated-true/succeed_0.test.ts

@@ -0,0 +1,12 @@
+import { test, expect } from "../../../assembly";
+import { fn } from "./env";
+
+test("succeed 0", () => {
+  if (fn.raw == null) {
+    fn.raw = () => {
+      expect(true).equal(true);
+    };
+  } else {
+    fn.raw();
+  }
+});