multicore test suite

.github/workflows/performance-check.yml

@@ -31,6 +31,22 @@ jobs:
         run: |
           ./configure.sh
 
+      - name: Collect test suite timing
+        shell: bash
+        run: |
+          pushd tests
+          QUARTO_TEST_TIMING=timing.txt ./run-tests.sh
+          if [ "$?" == "0" ]; then          
+            git config --global user.name "GitHub Actions auto-commit"
+            git config --global user.email "username@users.noreply.github.com"
+            git add timing.txt
+            git commit -m "(auto) update timing"
+            git push origin main
+          else
+            echo "Test suite failed; skipping update of timing.txt"
+          fi
+          popd tests
+
       - name: Test Bundle
         id: create_bundle
         shell: bash
@@ -48,7 +64,7 @@ jobs:
 
           # move the TS file so quarto command will use bundled JS
           mv src/quarto.ts src/quarto1.ts
-          
+
           pushd package/pkg-working/bin
 
           # test a bare quarto command

tests/run-parallel-tests.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+quarto run run-parallel-tests.ts

tests/run-parallel-tests.ts

@@ -0,0 +1,104 @@
+import { expandGlobSync } from "https://deno.land/std/fs/expand_glob.ts";
+import { relative } from "https://deno.land/std/path/mod.ts";
+
+try {
+  Deno.readTextFileSync("timing.txt");
+} catch (e) {
+  console.log(e);
+  console.log(
+    "timing.txt missing. run ./tests.sh with QUARTO_TEST_TIMING=timing.txt",
+  );
+  Deno.exit(1);
+}
+
+const lines = Deno.readTextFileSync("timing.txt").trim().split("\n");
+const currentTests = new Set(
+  [...expandGlobSync("**/*.test.ts", { globstar: true })].map((entry) =>
+    `./${relative(Deno.cwd(), entry.path)}`
+  ),
+);
+const timedTests = new Set<string>();
+
+type Timing = {
+  real: number;
+  user: number;
+  sys: number;
+};
+type TestTiming = {
+  name: string;
+  timing: Timing;
+};
+
+const testTimings: TestTiming[] = [];
+
+for (let i = 0; i < lines.length; i += 2) {
+  const name = lines[i].trim();
+  const timingStrs = lines[i + 1].trim().replaceAll(/ +/g, " ").split(" ");
+  const timing = {
+    real: Number(timingStrs[0]),
+    user: Number(timingStrs[2]),
+    sys: Number(timingStrs[4]),
+  };
+  testTimings.push({ name, timing });
+  timedTests.add(name);
+}
+let failed = false;
+
+// console.log(
+//   testTimings.map((a) => (a.timing.real)).reduce((a, b) => a + b, 0),
+// );
+// console.log(testTimings.sort((a, b) => a.timing.real - b.timing.real));
+// Deno.exit(0);
+
+const buckets: TestTiming[][] = [];
+const nBuckets = navigator.hardwareConcurrency;
+const bucketSizes = (new Array(nBuckets)).fill(0);
+
+const argmin = (a: number[]): number => {
+  let best = -1, bestValue = Infinity;
+  for (let i = 0; i < a.length; ++i) {
+    if (a[i] < bestValue) {
+      best = i;
+      bestValue = a[i];
+    }
+  }
+  return best;
+};
+
+for (let i = 0; i < nBuckets; ++i) {
+  buckets.push([]);
+}
+
+for (const timing of testTimings) {
+  const ix = argmin(bucketSizes);
+  buckets[ix].push(timing);
+  bucketSizes[ix] += timing.timing.real;
+}
+
+for (const currentTest of currentTests) {
+  if (!timedTests.has(currentTest)) {
+    console.log(`Missing test ${currentTest} in timing.txt`);
+    failed = true;
+    bucketSizes[Math.floor(Math.random() * nBuckets)].push(currentTest);
+  }
+}
+
+console.log(`Will run in ${nBuckets} cores`);
+if (!failed) {
+  console.log(
+    `Expected speedup: ${
+      (bucketSizes.reduce((a, b) => a + b, 0) / Math.max(...bucketSizes))
+        .toFixed(
+          2,
+        )
+    }`,
+  );
+}
+
+Promise.all(buckets.map((bucket) => {
+  const cmd: string[] = ["./run-tests.sh"];
+  cmd.push(...bucket.map((tt) => tt.name));
+  return Deno.run({
+    cmd,
+  }).status();
+}));

tests/run-tests.sh

@@ -39,7 +39,20 @@ fi
 # Ensure that tinytex is installed
 quarto install tinytex --no-prompt
 
-"${DENO_DIR}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_ARGMAP}" $@
+if [ "$QUARTO_TEST_TIMING" != "" ]; then
+  QUARTO_DENO_OPTIONS="--config test-conf.json --unstable --allow-read --allow-write --allow-run --allow-env --allow-net --no-check"
+  rm -f timing.txt
+  FILES=$@
+  if [ "$FILES" == "" ]; then
+    FILES=`find . | grep \.test\.ts$`
+  fi
+  for i in $FILES; do
+    echo $i >> timing.txt
+    /usr/bin/time -a -o timing.txt "${DENO_DIR}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_ARGMAP}" $i
+  done
+else
+  "${DENO_DIR}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_ARGMAP}" $@
+fi
 
 SUCCESS=$?
 

tests/smoke/render/render-docx.test.ts

@@ -4,8 +4,10 @@
 * Copyright (C) 2020-2022 Posit Software, PBC
 *
 */
+
 import { docs } from "../../utils.ts";
-import { testRender } from "./render.ts";
+import { testRender, testSimpleIsolatedRender } from "./render.ts";
 
-testRender(docs("test.Rmd"), "html", false, []);
+//testRender(docs("test.Rmd"), "html", false, []);
+testSimpleIsolatedRender(docs("test.Rmd"), "html", false);
 testRender(docs("test.qmd"), "docx", true, []);

tests/smoke/render/render-r.test.ts

@@ -5,27 +5,38 @@
 *
 */
 
-import { docs, fileLoader } from "../../utils.ts";
+import { docs, fileLoader, inTempDirectory } from "../../utils.ts";
+import { join } from "path/mod.ts";
 import { ensureHtmlSelectorSatisfies, fileExists } from "../../verify.ts";
 import { testRender } from "./render.ts";
 
-const plotPath = "docs/test_files/figure-html";
+inTempDirectory((dir) => {
+  const tempInput = join(dir, "test.Rmd");
+  Deno.copyFileSync(docs("test.Rmd"), tempInput);
+  const thisPlotPath = join(dir, "test_files/figure-html");
 
-testRender(docs("test.Rmd"), "html", false, [
-  fileExists(plotPath),
-], {
-  teardown: () => {
-    return Deno.remove(plotPath, { recursive: true });
-  },
+  testRender(tempInput, "html", false, [
+    fileExists(thisPlotPath),
+  ], {
+    teardown: () => {
+      return Deno.remove(dir, { recursive: true });
+    },
+  });
 });
 
-testRender(docs("test.Rmd"), "html", false, [
-  fileExists(plotPath),
-], {
-  teardown: () => {
-    return Deno.remove(plotPath, { recursive: true });
-  },
-}, ["--execute-params", "docs/params.yml"]);
+inTempDirectory((dir) => {
+  const tempInput = join(dir, "test.Rmd");
+  Deno.copyFileSync(docs("test.Rmd"), tempInput);
+
+  const thisPlotPath = join(dir, "test_files/figure-html");
+  testRender(tempInput, "html", false, [
+    fileExists(thisPlotPath),
+  ], {
+    teardown: () => {
+      return Deno.remove(dir, { recursive: true });
+    },
+  }, ["--execute-params", "docs/params.yml"]);
+});
 
 const knitrOptions = fileLoader()("test-knitr-options.qmd", "html");
 testRender(knitrOptions.input, "html", false, [

tests/smoke/render/render.ts

@@ -5,6 +5,7 @@
 *
 */
 import { existsSync } from "fs/mod.ts";
+import { basename, join } from "path/mod.ts";
 
 import { outputForInput } from "../../utils.ts";
 import { TestContext, testQuartoCmd, Verify } from "../../test.ts";
@@ -15,6 +16,21 @@ import {
   outputCreated,
 } from "../../verify.ts";
 
+export function testSimpleIsolatedRender(
+  file: string,
+  to: string,
+  noSupporting: boolean,
+) {
+  const dir = Deno.makeTempDirSync();
+  const tempInput = join(dir, basename(file));
+  Deno.copyFileSync(file, tempInput);
+  testRender(tempInput, to, noSupporting, [], {
+    teardown: () => {
+      return Deno.remove(dir, { recursive: true });
+    },
+  });
+}
+
 export function testRender(
   input: string,
   to: string,

tests/test.ts

@@ -138,7 +138,7 @@ export function test(test: TestDescriptor) {
         };
 
         // Capture the output
-        const log = join(wd, "test-out.json");
+        const log = Deno.makeTempFileSync({ suffix: ".json" });
         await initializeLogger({
           log: log,
           level: "INFO",