feat(bundle-source): Zip original sources with --no-transforms

packages/bundle-source/cache.js

@@ -16,6 +16,7 @@ const { Fail, quote: q } = assert;
  * @property {string} bundleFileName
  * @property {string} bundleTime ISO format
  * @property {number} bundleSize
+ * @property {boolean} noTransforms
  * @property {{ relative: string, absolute: string }} moduleSource
  * @property {Array<{ relativePath: string, mtime: string, size: number }>} contents
  */
@@ -41,9 +42,18 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     ...bundleOptions
   } = opts || {};
 
-  const add = async (rootPath, targetName, log = defaultLog) => {
+  /**
+   * @param {string} rootPath
+   * @param {string} targetName
+   * @param {Logger} [log]
+   * @param {object} [options]
+   * @param {boolean} [options.noTransforms]
+   */
+  const add = async (rootPath, targetName, log = defaultLog, options = {}) => {
     const srcRd = cwd.neighbor(rootPath);
 
+    const { noTransforms = false } = options;
+
     const statsByPath = new Map();
 
     const loggedRead = async loc => {
@@ -69,10 +79,14 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     const bundleWr = wr.neighbor(bundleFileName);
     const metaWr = wr.neighbor(toBundleMeta(targetName));
 
-    const bundle = await bundleSource(rootPath, bundleOptions, {
-      ...readPowers,
-      read: loggedRead,
-    });
+    const bundle = await bundleSource(
+      rootPath,
+      { ...bundleOptions, noTransforms },
+      {
+        ...readPowers,
+        read: loggedRead,
+      },
+    );
 
     const { moduleFormat } = bundle;
     assert.equal(moduleFormat, 'endoZipBase64');
@@ -98,6 +112,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
           size,
         }),
       ),
+      noTransforms,
     };
 
     await metaWr.atomicWriteText(JSON.stringify(meta, null, 2));
@@ -200,9 +215,16 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {string} rootPath
    * @param {string} targetName
    * @param {Logger} [log]
+   * @param {object} [options]
+   * @param {boolean} [options.noTransforms]
    * @returns {Promise<BundleMeta>}
    */
-  const validateOrAdd = async (rootPath, targetName, log = defaultLog) => {
+  const validateOrAdd = async (
+    rootPath,
+    targetName,
+    log = defaultLog,
+    options = {},
+  ) => {
     const metaText = await loadMetaText(targetName, log);
 
     /** @type {BundleMeta | undefined} */
@@ -211,7 +233,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     if (meta !== undefined) {
       try {
         meta = await validate(targetName, rootPath, log, meta);
-        const { bundleTime, bundleSize, contents } = meta;
+        const { bundleTime, bundleSize, contents, noTransforms } = meta;
         log(
           `${wr}`,
           toBundleName(targetName),
@@ -221,6 +243,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
           bundleTime,
           'with size',
           bundleSize,
+          noTransforms ? 'w/o transforms' : 'with transforms',
         );
       } catch (invalid) {
         meta = undefined;
@@ -230,8 +253,8 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
 
     if (meta === undefined) {
       log(`${wr}`, 'add:', targetName, 'from', rootPath);
-      meta = await add(rootPath, targetName, log);
-      const { bundleFileName, bundleTime, contents } = meta;
+      meta = await add(rootPath, targetName, log, options);
+      const { bundleFileName, bundleTime, contents, noTransforms } = meta;
       log(
         `${wr}`,
         'bundled',
@@ -240,6 +263,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
         bundleFileName,
         'at',
         bundleTime,
+        noTransforms ? 'w/o transforms' : 'with transforms',
       );
     }
 
@@ -251,11 +275,13 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {string} rootPath
    * @param {string} [targetName]
    * @param {Logger} [log]
+   * @param {object} [options]
    */
   const load = async (
     rootPath,
     targetName = readPowers.basename(rootPath, '.js'),
     log = defaultLog,
+    options = {},
   ) => {
     const found = loaded.get(targetName);
     // console.log('load', { targetName, found: !!found, rootPath });
@@ -264,7 +290,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     }
     const todo = makePromiseKit();
     loaded.set(targetName, { rootPath, bundle: todo.promise });
-    const bundle = await validateOrAdd(rootPath, targetName, log)
+    const bundle = await validateOrAdd(rootPath, targetName, log, options)
       .then(
         ({ bundleFileName }) =>
           import(`${wr.readOnly().neighbor(bundleFileName)}`),

packages/bundle-source/package.json

@@ -39,6 +39,7 @@
   "devDependencies": {
     "@endo/lockdown": "^1.0.7",
     "@endo/ses-ava": "^1.2.2",
+    "@endo/zip": "^1.0.5",
     "ava": "^6.1.2",
     "c8": "^7.14.0",
     "typescript": "5.5.0-beta"

packages/bundle-source/src/main.js

@@ -13,6 +13,16 @@ const USAGE =
  * @returns {Promise<void>}
  */
 export const main = async (args, { loadModule, pid, log }) => {
+  let noTransforms = false;
+  while (args.length > 0) {
+    if (args[0] === '--no-transforms') {
+      noTransforms = true;
+      args.shift();
+    } else {
+      break;
+    }
+  }
+
   const [to, dest, ...pairs] = args;
   if (!(dest && pairs.length > 0 && pairs.length % 2 === 0)) {
     throw Error(USAGE);
@@ -41,6 +51,8 @@ export const main = async (args, { loadModule, pid, log }) => {
     const [bundleRoot, bundleName] = pairs.slice(ix, ix + 2);
 
     // eslint-disable-next-line no-await-in-loop
-    await cache.validateOrAdd(bundleRoot, bundleName);
+    await cache.validateOrAdd(bundleRoot, bundleName, undefined, {
+      noTransforms,
+    });
   }
 };

packages/bundle-source/src/types.js

@@ -64,6 +64,9 @@ export {};
  * @property {T} [format]
  * @property {boolean} [dev] - development mode, for test bundles that need
  * access to devDependencies of the entry package.
+ * @property {boolean} [noTransforms] - when true, generates a bundle with the
+ * original sources instead of SES-shim specific ESM and CJS. This may become
+ * default in a future major version.
  */
 
 /**

packages/bundle-source/src/zip-base64.js

@@ -7,7 +7,9 @@ import url from 'url';
 import fs from 'fs';
 import os from 'os';
 
-import { makeAndHashArchive } from '@endo/compartment-mapper/archive.js';
+import { parserForLanguage as transformingParserForLanguage } from '@endo/compartment-mapper/archive-parsers.js';
+import { parserForLanguage as transparentParserForLanguage } from '@endo/compartment-mapper/import-parsers.js';
+import { makeAndHashArchive } from '@endo/compartment-mapper/archive-lite.js';
 import { encodeBase64 } from '@endo/base64';
 import { whereEndoCache } from '@endo/where';
 import { makeReadPowers } from '@endo/compartment-mapper/node-powers.js';
@@ -22,7 +24,12 @@ export async function bundleZipBase64(
   options = {},
   grantedPowers = {},
 ) {
-  const { dev = false, cacheSourceMaps = false, commonDependencies } = options;
+  const {
+    dev = false,
+    cacheSourceMaps = false,
+    noTransforms = false,
+    commonDependencies,
+  } = options;
   const powers = { ...readPowers, ...grantedPowers };
   const {
     computeSha512,
@@ -137,9 +144,11 @@ export async function bundleZipBase64(
     return { bytes: objectBytes, parser, sourceMap };
   };
 
-  const { bytes, sha512 } = await makeAndHashArchive(powers, entry, {
-    dev,
-    moduleTransforms: {
+  let parserForLanguage = transparentParserForLanguage;
+  let moduleTransforms = {};
+  if (!noTransforms) {
+    parserForLanguage = transformingParserForLanguage;
+    moduleTransforms = {
       async mjs(
         sourceBytes,
         specifier,
@@ -170,7 +179,13 @@ export async function bundleZipBase64(
           sourceMap,
         );
       },
-    },
+    };
+  }
+
+  const { bytes, sha512 } = await makeAndHashArchive(powers, entry, {
+    dev,
+    parserForLanguage,
+    moduleTransforms,
     sourceMapHook(sourceMap, sourceDescriptor) {
       sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor));
     },

packages/bundle-source/test/no-transforms.test.js

@@ -0,0 +1,36 @@
+// @ts-check
+import test from '@endo/ses-ava/prepare-endo.js';
+
+import fs from 'fs';
+import url from 'url';
+import { decodeBase64 } from '@endo/base64';
+import { ZipReader } from '@endo/zip';
+import bundleSource from '../src/index.js';
+
+test('no-transforms applies no transforms', async t => {
+  const entryPath = url.fileURLToPath(
+    new URL(`../demo/circular/a.js`, import.meta.url),
+  );
+  const { endoZipBase64 } = await bundleSource(entryPath, {
+    moduleFormat: 'endoZipBase64',
+    noTransforms: true,
+  });
+  const endoZipBytes = decodeBase64(endoZipBase64);
+  const zipReader = new ZipReader(endoZipBytes);
+  const compartmentMapBytes = zipReader.read('compartment-map.json');
+  const compartmentMapText = new TextDecoder().decode(compartmentMapBytes);
+  const compartmentMap = JSON.parse(compartmentMapText);
+  const { entry, compartments } = compartmentMap;
+  const compartment = compartments[entry.compartment];
+  const module = compartment.modules[entry.module];
+  // Alleged module type is not precompiled (pre-mjs-json)
+  t.is(module.parser, 'mjs');
+
+  const moduleBytes = zipReader.read(
+    `${compartment.location}/${module.location}`,
+  );
+  const moduleText = new TextDecoder().decode(moduleBytes);
+  const originalModuleText = await fs.promises.readFile(entryPath, 'utf-8');
+  // And, just to be sure, the text in the bundle matches the original text.
+  t.is(moduleText, originalModuleText);
+});