Add startup benchmark helper

startup-helper/BabelCacheBuster.js

@@ -0,0 +1,27 @@
+// Babel plugin that adds CACHE_BUST_COMMENT to every function body.
+const CACHE_BUST_COMMENT = "ThouShaltNotCache";
+
+module.exports = function ({ types: t }) {
+  return {
+    visitor: {
+      Function(path) {
+        const bodyPath = path.get("body");
+        // Handle arrow functions: () => "value"
+        // Convert them to block statements: () => { return "value"; }
+        if (!bodyPath.isBlockStatement()) {
+          const newBody = t.blockStatement([t.returnStatement(bodyPath.node)]);
+          path.set("body", newBody);
+        }
+
+        // Handle empty function bodies: function foo() {}
+        // Add an empty statement so we have a first node to attach the comment to.
+        if (path.get("body.body").length === 0) {
+          path.get("body").pushContainer("body", t.emptyStatement());
+        }
+
+        const firstNode = path.node.body.body[0];
+        t.addComment(firstNode, "leading", CACHE_BUST_COMMENT);
+      },
+    },
+  };
+};

startup-helper/StartupBenchmark.js

@@ -0,0 +1,119 @@
+const CACHE_BUST_COMMENT = "/*ThouShaltNotCache*/";
+const CACHE_BUST_COMMENT_RE = new RegExp(
+  `\n${RegExp.escape(CACHE_BUST_COMMENT)}\n`,
+  "g"
+);
+
+class StartupBenchmark {
+  // Total iterations for this benchmark.
+  #iterationCount = 0;
+  // Original source code.
+  #sourceCode;
+  // quickHahs(this.#sourceCode) for use in custom validate() methods.
+  #sourceHash = 0;
+  // Number of no-cache comments in the original #sourceCode.
+  #expectedCacheCommentCount = 0;
+  // How many times (separate iterations) should we reuse the source code.
+  // Use 0 to skip and only use a single #sourceCode string.
+  #sourceCodeReuseCount = 1;
+  // #sourceCode for each iteration, number of unique sources is controlled
+  // by codeReuseCount;
+  #iterationSourceCodes = [];
+
+  constructor({
+    iterationCount,
+    expectedCacheCommentCount,
+    sourceCodeReuseCount = 1,
+  } = {}) {
+    console.assert(iterationCount > 0);
+    this.#iterationCount = iterationCount;
+    console.assert(expectedCacheCommentCount > 0);
+    this.#expectedCacheCommentCount = expectedCacheCommentCount;
+    console.assert(sourceCodeReuseCount >= 0);
+    this.#sourceCodeReuseCount = sourceCodeReuseCount;
+  }
+
+  get iterationCount() {
+    return this.#iterationCount;
+  }
+
+  get sourceCode() {
+    return this.#sourceCode;
+  }
+
+  get sourceHash() {
+    return this.#sourceHash;
+  }
+
+  get expectedCacheCommentCount() {
+    return this.#expectedCacheCommentCount;
+  }
+
+  get sourceCodeReuseCount() {
+    return this.#sourceCodeReuseCount;
+  }
+
+  get iterationSourceCodes() {
+    return this.#iterationSourceCodes;
+  }
+
+  async init() {
+    this.#sourceCode = await JetStream.getString(JetStream.preload.BUNDLE);
+    const cacheCommentCount = this.sourceCode.match(
+      CACHE_BUST_COMMENT_RE
+    ).length;
+    this.#sourceHash = this.quickHash(this.sourceCode);
+    this.validateSourceCacheComments(cacheCommentCount);
+    for (let i = 0; i < this.iterationCount; i++)
+      this.#iterationSourceCodes[i] = this.createIterationSourceCode(i);
+    this.validateIterationSourceCodes();
+  }
+
+  validateSourceCacheComments(cacheCommentCount) {
+    console.assert(
+      cacheCommentCount === this.expectedCacheCommentCount,
+      `Invalid cache comment count ${cacheCommentCount} expected ${this.expectedCacheCommentCount}.`
+    );
+  }
+
+  validateIterationSourceCodes() {
+    if (this.#iterationSourceCodes.some((each) => !each?.length))
+      throw new Error(`Got invalid iterationSourceCodes`);
+    let expectedSize = 1;
+    if (this.sourceCodeReuseCount !== 0)
+      expectedSize = Math.ceil(this.iterationCount / this.sourceCodeReuseCount);
+    const uniqueSources = new Set(this.iterationSourceCodes);
+    if (uniqueSources.size != expectedSize)
+      throw new Error(
+        `Expected ${expectedSize} unique sources, but got ${uniqueSources.size}.`
+      );
+  }
+
+  createIterationSourceCode(iteration) {
+    // Alter the code per iteration to prevent caching.
+    const cacheId =
+      Math.floor(iteration / this.sourceCodeReuseCount) *
+      this.sourceCodeReuseCount;
+    // Reuse existing sources if this.codeReuseCount > 1:
+    if (cacheId < this.iterationSourceCodes.length)
+      return this.iterationSourceCodes[cacheId];
+
+    const sourceCode = this.sourceCode.replaceAll(
+      CACHE_BUST_COMMENT_RE,
+      `/*${cacheId}*/`
+    );
+    // Warm up quickHash.
+    this.quickHash(sourceCode);
+    return sourceCode;
+  }
+
+  quickHash(str) {
+    let hash = 5381;
+    let i = str.length;
+    while (i > 0) {
+      hash = (hash * 33) ^ (str.charCodeAt(i) | 0);
+      i -= 919;
+    }
+    return hash | 0;
+  }
+}

tests/unit-tests.js

@@ -1,4 +1,5 @@
-load("shell-config.js")
+load("shell-config.js");
+load("startup-helper/StartupBenchmark.js");
 load("JetStreamDriver.js");
 
 function assertTrue(condition, message) {
@@ -19,17 +20,25 @@ function assertEquals(actual, expected, message) {
   }
 }
 
+function assertThrows(message, func) {
+  let didThrow = false;
+  try {
+    func();
+  } catch (e) {
+    didThrow = true;
+  }
+  assertTrue(didThrow, message);
+}
+
 (function testTagsAreLowerCaseStrings() {
   for (const benchmark of BENCHMARKS) {
-    benchmark.tags.forEach(tag => {
-        assertTrue(typeof(tag) == "string");
-        assertTrue(tag == tag.toLowerCase());
-    })
+    benchmark.tags.forEach((tag) => {
+      assertTrue(typeof tag == "string");
+      assertTrue(tag == tag.toLowerCase());
+    });
   }
 })();
 
-
-
 (function testTagsAll() {
   for (const benchmark of BENCHMARKS) {
     const tags = benchmark.tags;
@@ -41,53 +50,56 @@ function assertEquals(actual, expected, message) {
   }
 })();
 
-
 (function testDriverBenchmarksOrder() {
   const benchmarks = findBenchmarksByTag("all");
   const driver = new Driver(benchmarks);
   assertEquals(driver.benchmarks.length, BENCHMARKS.length);
-  const names = driver.benchmarks.map(b => b.name.toLowerCase()).sort().reverse();
+  const names = driver.benchmarks
+    .map((b) => b.name.toLowerCase())
+    .sort()
+    .reverse();
   for (let i = 0; i < names.length; i++) {
     assertEquals(driver.benchmarks[i].name.toLowerCase(), names[i]);
   }
 })();
 
-
 (function testEnableByTag() {
   const driverA = new Driver(findBenchmarksByTag("Default"));
   const driverB = new Driver(findBenchmarksByTag("default"));
   assertTrue(driverA.benchmarks.length > 0);
   assertEquals(driverA.benchmarks.length, driverB.benchmarks.length);
   const enabledBenchmarkNames = new Set(
-      Array.from(driverA.benchmarks).map(b => b.name));
+    Array.from(driverA.benchmarks).map((b) => b.name)
+  );
   for (const benchmark of BENCHMARKS) {
     if (benchmark.tags.has("default"))
       assertTrue(enabledBenchmarkNames.has(benchmark.name));
   }
 })();
 
-
 (function testDriverEnableDuplicateAndSort() {
-    const benchmarks = [...findBenchmarksByTag("wasm"), ...findBenchmarksByTag("wasm")];
-    assertTrue(benchmarks.length > 0);
-    const uniqueBenchmarks = new Set(benchmarks);
-    assertFalse(uniqueBenchmarks.size == benchmarks.length);
-    const driver = new Driver(benchmarks);
-    assertEquals(driver.benchmarks.length, uniqueBenchmarks.size);
+  const benchmarks = [
+    ...findBenchmarksByTag("wasm"),
+    ...findBenchmarksByTag("wasm"),
+  ];
+  assertTrue(benchmarks.length > 0);
+  const uniqueBenchmarks = new Set(benchmarks);
+  assertFalse(uniqueBenchmarks.size == benchmarks.length);
+  const driver = new Driver(benchmarks);
+  assertEquals(driver.benchmarks.length, uniqueBenchmarks.size);
 })();
 
-
 (function testBenchmarkSubScores() {
   for (const benchmark of BENCHMARKS) {
     const subScores = benchmark.subScores();
     assertTrue(subScores instanceof Object);
     assertTrue(Object.keys(subScores).length > 0);
     for (const [name, value] of Object.entries(subScores)) {
-      assertTrue(typeof(name) == "string");
+      assertTrue(typeof name == "string");
       // "Score" can only be part of allScores().
       assertFalse(name == "Score");
       // Without running values should be either null (or 0 for GroupedBenchmark)
-      assertFalse(value)
+      assertFalse(value);
     }
   }
 })();
@@ -98,7 +110,112 @@ function assertEquals(actual, expected, message) {
     const allScores = benchmark.allScores();
     assertTrue("Score" in allScores);
     // All subScore items are part of allScores.
-    for (const name of Object.keys(subScores))
-      assertTrue(name in allScores);
+    for (const name of Object.keys(subScores)) assertTrue(name in allScores);
   }
 })();
+
+function validateIterationSources(sources) {
+  for (const source of sources) {
+    assertTrue(typeof source == "string");
+    assertFalse(source.includes(CACHE_BUST_COMMENT));
+  }
+}
+
+(async function testStartupBenchmark() {
+  try {
+    JetStream.preload = { BUNDLE: "test-bundle.js" };
+    JetStream.getString = (file) => {
+      assertEquals(file, "test-bundle.js");
+      return `function test() { 
+${CACHE_BUST_COMMENT}
+        return 1;
+        }`;
+    };
+    await testStartupBenchmarkInnerTests();
+  } finally {
+    JetStream.preload = undefined;
+    JetStream.getString = undefined;
+  }
+})();
+
+async function testStartupBenchmarkInnerTests() {
+  const benchmark = new StartupBenchmark({
+    iterationCount: 12,
+    expectedCacheCommentCount: 1,
+  });
+  assertEquals(benchmark.iterationCount, 12);
+  assertEquals(benchmark.expectedCacheCommentCount, 1);
+  assertEquals(benchmark.iterationSourceCodes.length, 0);
+  assertEquals(benchmark.sourceCode, undefined);
+  assertEquals(benchmark.sourceHash, 0);
+  await benchmark.init();
+  assertEquals(benchmark.sourceHash, 177573);
+  assertEquals(benchmark.sourceCode.length, 68);
+  assertEquals(benchmark.iterationSourceCodes.length, 12);
+  assertEquals(new Set(benchmark.iterationSourceCodes).size, 12);
+  validateIterationSources(benchmark.iterationSourceCodes);
+
+  const noReuseBenchmark = new StartupBenchmark({
+    iterationCount: 12,
+    expectedCacheCommentCount: 1,
+    sourceCodeReuseCount: 0,
+  });
+  assertEquals(noReuseBenchmark.iterationSourceCodes.length, 0);
+  await noReuseBenchmark.init();
+  assertEquals(noReuseBenchmark.iterationSourceCodes.length, 12);
+  assertEquals(new Set(noReuseBenchmark.iterationSourceCodes).size, 1);
+  validateIterationSources(noReuseBenchmark.iterationSourceCodes);
+
+  const reuseBenchmark = new StartupBenchmark({
+    iterationCount: 12,
+    expectedCacheCommentCount: 1,
+    sourceCodeReuseCount: 3,
+  });
+  assertEquals(reuseBenchmark.iterationSourceCodes.length, 0);
+  await reuseBenchmark.init();
+  assertEquals(reuseBenchmark.iterationSourceCodes.length, 12);
+  assertEquals(new Set(reuseBenchmark.iterationSourceCodes).size, 4);
+  validateIterationSources(reuseBenchmark.iterationSourceCodes);
+
+  const reuseBenchmark2 = new StartupBenchmark({
+    iterationCount: 12,
+    expectedCacheCommentCount: 1,
+    sourceCodeReuseCount: 5,
+  });
+  assertEquals(reuseBenchmark2.iterationSourceCodes.length, 0);
+  await reuseBenchmark2.init();
+  assertEquals(reuseBenchmark2.iterationSourceCodes.length, 12);
+  assertEquals(new Set(reuseBenchmark2.iterationSourceCodes).size, 3);
+  validateIterationSources(reuseBenchmark2.iterationSourceCodes);
+}
+
+(function testStartupBenchmarkThrow() {
+  assertThrows(
+    "StartupBenchmark constructor should throw with no arguments.",
+    () => new StartupBenchmark()
+  );
+
+  assertThrows(
+    "StartupBenchmark constructor should throw with missing expectedCacheCommentCount.",
+    () => new StartupBenchmark({ iterationCount: 1 })
+  );
+
+  assertThrows(
+    "StartupBenchmark constructor should throw with missing iterationCount.",
+    () => new StartupBenchmark({ expectedCacheCommentCount: 1 })
+  );
+
+  assertThrows(
+    "StartupBenchmark constructor should throw with iterationCount=0.",
+    () => {
+      new StartupBenchmark({ iterationCount: 0, expectedCacheCommentCount: 1 });
+    }
+  );
+
+  assertThrows(
+    "StartupBenchmark constructor should throw with expectedCacheCommentCount=0.",
+    () => {
+      new StartupBenchmark({ iterationCount: 1, expectedCacheCommentCount: 0 });
+    }
+  );
+})();