From 05792e2125dcd5d231e27c24626b2343c65d9479 Mon Sep 17 00:00:00 2001
From: Christopher Hiller <boneskull@boneskull.com>
Date: Wed, 5 Jun 2024 11:23:21 -0700
Subject: [PATCH 1/5] fix(compartment-mapper): remove language-specific
 assertions from compartment map validation

---
 .../compartment-mapper/src/compartment-map.js | 28 -------------------
 1 file changed, 28 deletions(-)

diff --git a/packages/compartment-mapper/src/compartment-map.js b/packages/compartment-mapper/src/compartment-map.js
index bf85596733..8b23459fe1 100644
--- a/packages/compartment-mapper/src/compartment-map.js
+++ b/packages/compartment-mapper/src/compartment-map.js
@@ -11,16 +11,6 @@ import { assertPackagePolicy } from './policy-format.js';
 // this definition of `q` rather than `assert.quote`
 const q = JSON.stringify;
 
-const moduleLanguages = [
-  'cjs',
-  'mjs',
-  'json',
-  'text',
-  'bytes',
-  'pre-mjs-json',
-  'pre-cjs-json',
-];
-
 /** @type {(a: string, b: string) => number} */
 // eslint-disable-next-line no-nested-ternary
 export const stringCompare = (a, b) => (a === b ? 0 : a < b ? -1 : 1);
@@ -162,12 +152,6 @@ const assertFileModule = (allegedModule, path, url) => {
     'string',
     `${path}.parser must be a string, got ${q(parser)} in ${q(url)}`,
   );
-  assert(
-    moduleLanguages.includes(parser),
-    `${path}.parser must be one of ${q(moduleLanguages)}, got ${parser} in ${q(
-      url,
-    )}`,
-  );
 
   if (sha512 !== undefined) {
     assert.typeof(
@@ -275,12 +259,6 @@ const assertParsers = (allegedParsers, path, url) => {
       'string',
       `${path}.parsers[${q(key)}] must be a string, got ${value} in ${q(url)}`,
     );
-    assert(
-      moduleLanguages.includes(value),
-      `${path}.parsers[${q(key)}] must be one of ${q(
-        moduleLanguages,
-      )}, got ${value} in ${q(url)}`,
-    );
   }
 };
 
@@ -362,12 +340,6 @@ const assertTypes = (allegedTypes, path, url) => {
       'string',
       `${path}.types[${q(key)}] must be a string, got ${value} in ${q(url)}`,
     );
-    assert(
-      moduleLanguages.includes(value),
-      `${path}.types[${q(key)}] must be one of ${q(
-        moduleLanguages,
-      )}, got ${value} in ${q(url)}`,
-    );
   }
 };
 

From d1e17e3c2dd0bcd2428ee01f06d217a61c86c561 Mon Sep 17 00:00:00 2001
From: Christopher Hiller <boneskull@boneskull.com>
Date: Wed, 5 Jun 2024 11:36:07 -0700
Subject: [PATCH 2/5] feat(compartment-mapper): expose captureFromMap

---
 packages/compartment-mapper/capture-lite.js   |   1 +
 packages/compartment-mapper/package.json      |   1 +
 .../compartment-mapper/src/capture-lite.js    | 322 ++++++++++++++++++
 packages/compartment-mapper/src/types.js      |  26 ++
 4 files changed, 350 insertions(+)
 create mode 100644 packages/compartment-mapper/capture-lite.js
 create mode 100644 packages/compartment-mapper/src/capture-lite.js

diff --git a/packages/compartment-mapper/capture-lite.js b/packages/compartment-mapper/capture-lite.js
new file mode 100644
index 0000000000..df4b0bee94
--- /dev/null
+++ b/packages/compartment-mapper/capture-lite.js
@@ -0,0 +1 @@
+export { captureFromMap } from './src/capture-lite.js';
diff --git a/packages/compartment-mapper/package.json b/packages/compartment-mapper/package.json
index d4d7c74d88..db8ab57a07 100644
--- a/packages/compartment-mapper/package.json
+++ b/packages/compartment-mapper/package.json
@@ -33,6 +33,7 @@
     "./archive.js": "./archive.js",
     "./archive-lite.js": "./archive-lite.js",
     "./archive-parsers.js": "./archive-parsers.js",
+    "./capture-lite.js": "./capture-lite.js",
     "./import-archive.js": "./import-archive.js",
     "./import-archive-lite.js": "./import-archive-lite.js",
     "./import-archive-parsers.js": "./import-archive-parsers.js",
diff --git a/packages/compartment-mapper/src/capture-lite.js b/packages/compartment-mapper/src/capture-lite.js
new file mode 100644
index 0000000000..a4ae16710b
--- /dev/null
+++ b/packages/compartment-mapper/src/capture-lite.js
@@ -0,0 +1,322 @@
+/**
+ * This module provides {@link captureFromMap}, which only "captures" the
+ * compartment map descriptors and sources from a partially completed
+ * compartment map--_without_ creating an archive. The resulting compartment map
+ * represents a well-formed dependency graph, laden with useful metadata. This
+ * could be used for e.g., automatic policy generation.
+ *
+ * Note that the resulting data structure ({@link CaptureResult}) contains a
+ * mapping of filepaths to compartment map names.
+ *
+ * These functions do not have a bias for any particular mapping, so you will
+ * need to use `mapNodeModules` from `@endo/compartment-map/node-modules.js` or
+ * a similar device to construct one. The default `parserForLanguage` mapping is
+ * empty. You will need to provide the `defaultParserForLanguage` from
+ * `@endo/compartment-mapper/import-parsers.js` or
+ * `@endo/compartment-mapper/archive-parsers.js`.
+ *
+ * If you use `@endo/compartment-mapper/archive-parsers.js`, the archive will
+ * contain pre-compiled ESM and CJS modules wrapped in a JSON envelope, suitable
+ * for use with the SES shim in any environment including a web page, without a
+ * client-side dependency on Babel.
+ *
+ * If you use `@endo/compartment-mapper/import-parsers.js`, the archive will
+ * contain original sources, so to import the archive with
+ * `src/import-archive-lite.js`, you will need to provide the archive parsers
+ * and entrain a runtime dependency on Babel.
+ *
+ * @module
+ */
+
+// @ts-check
+/* eslint no-shadow: 0 */
+
+/** @import {ReadFn} from './types.js' */
+/** @import {ReadPowers} from './types.js' */
+/** @import {CompartmentMapDescriptor} from './types.js' */
+/** @import {CaptureOptions} from './types.js' */
+/** @import {Sources} from './types.js' */
+/** @import {CompartmentDescriptor} from './types.js' */
+/** @import {ModuleDescriptor} from './types.js' */
+/** @import {CaptureResult} from './types.js' */
+
+import {
+  assertCompartmentMap,
+  pathCompare,
+  stringCompare,
+} from './compartment-map.js';
+import {
+  exitModuleImportHookMaker,
+  makeImportHookMaker,
+} from './import-hook.js';
+import { link } from './link.js';
+import { resolve } from './node-module-specifier.js';
+import { detectAttenuators } from './policy.js';
+import { unpackReadPowers } from './powers.js';
+
+const { freeze, assign, create, fromEntries, entries, keys } = Object;
+
+/**
+ * We attempt to produce compartment maps that are consistent regardless of
+ * whether the packages were originally laid out on disk for development or
+ * production, and other trivia like the fully qualified path of a specific
+ * installation.
+ *
+ * Naming compartments for the self-ascribed name and version of each Node.js
+ * package is insufficient because they are not guaranteed to be unique.
+ * Dependencies do not necessarilly come from the npm registry and may be
+ * for example derived from fully qualified URL's or Github org and project
+ * names.
+ * Package managers are also not required to fully deduplicate the hard
+ * copy of each package even when they are identical resources.
+ * Duplication is undesirable, but we elect to defer that problem to solutions
+ * in the package managers, as the alternative would be to consistently hash
+ * the original sources of the packages themselves, which may not even be
+ * available much less pristine for us.
+ *
+ * So, instead, we use the lexically least path of dependency names, delimited
+ * by hashes.
+ * The compartment maps generated by the ./node-modules.js tooling pre-compute
+ * these traces for our use here.
+ * We sort the compartments lexically on their self-ascribed name and version,
+ * and use the lexically least dependency name path as a tie-breaker.
+ * The dependency path is logical and orthogonal to the package manager's
+ * actual installation location, so should be orthogonal to the vagaries of the
+ * package manager's deduplication algorithm.
+ *
+ * @param {Record<string, CompartmentDescriptor>} compartments
+ * @returns {Record<string, string>} map from old to new compartment names.
+ */
+const renameCompartments = compartments => {
+  /** @type {Record<string, string>} */
+  const compartmentRenames = create(null);
+  let index = 0;
+  let prev = '';
+
+  // The sort below combines two comparators to avoid depending on sort
+  // stability, which became standard as recently as 2019.
+  // If that date seems quaint, please accept my regards from the distant past.
+  // We are very proud of you.
+  const compartmentsByPath = Object.entries(compartments)
+    .map(([name, compartment]) => ({
+      name,
+      path: compartment.path,
+      label: compartment.label,
+    }))
+    .sort((a, b) => {
+      if (a.label === b.label) {
+        assert(a.path !== undefined && b.path !== undefined);
+        return pathCompare(a.path, b.path);
+      }
+      return stringCompare(a.label, b.label);
+    });
+
+  for (const { name, label } of compartmentsByPath) {
+    if (label === prev) {
+      compartmentRenames[name] = `${label}-n${index}`;
+      index += 1;
+    } else {
+      compartmentRenames[name] = label;
+      prev = label;
+      index = 1;
+    }
+  }
+  return compartmentRenames;
+};
+
+/**
+ * @param {Record<string, CompartmentDescriptor>} compartments
+ * @param {Sources} sources
+ * @param {Record<string, string>} compartmentRenames
+ */
+const translateCompartmentMap = (compartments, sources, compartmentRenames) => {
+  const result = create(null);
+  for (const compartmentName of keys(compartmentRenames)) {
+    const compartment = compartments[compartmentName];
+    const { name, label, retained, policy } = compartment;
+    if (retained) {
+      // rename module compartments
+      /** @type {Record<string, ModuleDescriptor>} */
+      const modules = create(null);
+      const compartmentModules = compartment.modules;
+      if (compartment.modules) {
+        for (const name of keys(compartmentModules).sort()) {
+          const module = compartmentModules[name];
+          if (module.compartment !== undefined) {
+            modules[name] = {
+              ...module,
+              compartment: compartmentRenames[module.compartment],
+            };
+          } else {
+            modules[name] = module;
+          }
+        }
+      }
+
+      // integrate sources into modules
+      const compartmentSources = sources[compartmentName];
+      if (compartmentSources) {
+        for (const name of keys(compartmentSources).sort()) {
+          const source = compartmentSources[name];
+          const { location, parser, exit, sha512, deferredError } = source;
+          if (location !== undefined) {
+            modules[name] = {
+              location,
+              parser,
+              sha512,
+            };
+          } else if (exit !== undefined) {
+            modules[name] = {
+              exit,
+            };
+          } else if (deferredError !== undefined) {
+            modules[name] = {
+              deferredError,
+            };
+          }
+        }
+      }
+
+      result[compartmentRenames[compartmentName]] = {
+        name,
+        label,
+        location: compartmentRenames[compartmentName],
+        modules,
+        policy,
+        // `scopes`, `types`, and `parsers` are not necessary since every
+        // loadable module is captured in `modules`.
+      };
+    }
+  }
+
+  return result;
+};
+
+/**
+ * @param {Sources} sources
+ * @param {Record<string, string>} compartmentRenames
+ * @returns {Sources}
+ */
+const renameSources = (sources, compartmentRenames) => {
+  return fromEntries(
+    entries(sources).map(([name, compartmentSources]) => [
+      compartmentRenames[name],
+      compartmentSources,
+    ]),
+  );
+};
+
+/**
+ * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {Sources} sources
+ * @returns {CaptureResult}
+ */
+const captureCompartmentMap = (compartmentMap, sources) => {
+  const {
+    compartments,
+    entry: { compartment: entryCompartmentName, module: entryModuleSpecifier },
+  } = compartmentMap;
+
+  const compartmentRenames = renameCompartments(compartments);
+  const captureCompartments = translateCompartmentMap(
+    compartments,
+    sources,
+    compartmentRenames,
+  );
+  const captureEntryCompartmentName = compartmentRenames[entryCompartmentName];
+  const captureSources = renameSources(sources, compartmentRenames);
+
+  const captureCompartmentMap = {
+    tags: [],
+    entry: {
+      compartment: captureEntryCompartmentName,
+      module: entryModuleSpecifier,
+    },
+    compartments: captureCompartments,
+  };
+
+  // Cross-check:
+  // We assert that we have constructed a valid compartment map, not because it
+  // might not be, but to ensure that the assertCompartmentMap function can
+  // accept all valid compartment maps.
+  assertCompartmentMap(captureCompartmentMap);
+
+  return {
+    captureCompartmentMap,
+    captureSources,
+    compartmentRenames,
+  };
+};
+
+/**
+ * @param {ReadFn | ReadPowers} powers
+ * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {CaptureOptions} [options]
+ * @returns {Promise<CaptureResult>}
+ */
+export const captureFromMap = async (powers, compartmentMap, options = {}) => {
+  const {
+    moduleTransforms,
+    modules: exitModules = {},
+    searchSuffixes = undefined,
+    importHook: exitModuleImportHook = undefined,
+    policy = undefined,
+    sourceMapHook = undefined,
+    parserForLanguage: parserForLanguageOption = {},
+    languageForExtension: languageForExtensionOption = {},
+  } = options;
+
+  const parserForLanguage = freeze(
+    assign(create(null), parserForLanguageOption),
+  );
+  const languageForExtension = freeze(
+    assign(create(null), languageForExtensionOption),
+  );
+
+  const { read, computeSha512 } = unpackReadPowers(powers);
+
+  const {
+    compartments,
+    entry: { module: entryModuleSpecifier, compartment: entryCompartmentName },
+  } = compartmentMap;
+
+  /** @type {Sources} */
+  const sources = Object.create(null);
+
+  const consolidatedExitModuleImportHook = exitModuleImportHookMaker({
+    modules: exitModules,
+    exitModuleImportHook,
+  });
+
+  const makeImportHook = makeImportHookMaker(read, entryCompartmentName, {
+    sources,
+    compartmentDescriptors: compartments,
+    archiveOnly: true,
+    computeSha512,
+    searchSuffixes,
+    entryCompartmentName,
+    entryModuleSpecifier,
+    exitModuleImportHook: consolidatedExitModuleImportHook,
+    sourceMapHook,
+  });
+  // Induce importHook to record all the necessary modules to import the given module specifier.
+  const { compartment, attenuatorsCompartment } = link(compartmentMap, {
+    resolve,
+    makeImportHook,
+    moduleTransforms,
+    parserForLanguage,
+    languageForExtension,
+    archiveOnly: true,
+  });
+  await compartment.load(entryModuleSpecifier);
+  if (policy) {
+    // retain all attenuators.
+    await Promise.all(
+      detectAttenuators(policy).map(attenuatorSpecifier =>
+        attenuatorsCompartment.load(attenuatorSpecifier),
+      ),
+    );
+  }
+
+  return captureCompartmentMap(compartmentMap, sources);
+};
diff --git a/packages/compartment-mapper/src/types.js b/packages/compartment-mapper/src/types.js
index 4b2c36a89c..64df74e2fe 100644
--- a/packages/compartment-mapper/src/types.js
+++ b/packages/compartment-mapper/src/types.js
@@ -619,3 +619,29 @@ export {};
  *
  * @typedef {ExecuteOptions & ArchiveOptions} ImportLocationOptions
  */
+
+/**
+ * Options for `captureFromMap()`
+ *
+ * @typedef CaptureOptions
+ * @property {ModuleTransforms} [moduleTransforms]
+ * @property {Record<string, any>} [modules]
+ * @property {boolean} [dev]
+ * @property {SomePolicy} [policy]
+ * @property {Set<string>} [tags]
+ * @property {ExitModuleImportHook} [importHook]
+ * @property {Array<string>} [searchSuffixes]
+ * @property {Record<string, string>} [commonDependencies]
+ * @property {SourceMapHook} [sourceMapHook]
+ * @property {Record<string, ParserImplementation>} [parserForLanguage]
+ * @property {LanguageForExtension} [languageForExtension]
+ */
+
+/**
+ * The result of `captureFromMap()`
+ *
+ * @typedef CaptureResult
+ * @property {CompartmentMapDescriptor} captureCompartmentMap
+ * @property {Sources} captureSources
+ * @property {Record<string, string>} compartmentRenames
+ */

From 54b9585c3c1e34995b5144ce5a9dbc83433a2f29 Mon Sep 17 00:00:00 2001
From: Christopher Hiller <boneskull@boneskull.com>
Date: Wed, 5 Jun 2024 14:38:03 -0700
Subject: [PATCH 3/5] chore(compartment-mapper): update module docstring

Co-authored-by: Kris Kowal <kris@agoric.com>
---
 packages/compartment-mapper/src/capture-lite.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/compartment-mapper/src/capture-lite.js b/packages/compartment-mapper/src/capture-lite.js
index a4ae16710b..a6d143afff 100644
--- a/packages/compartment-mapper/src/capture-lite.js
+++ b/packages/compartment-mapper/src/capture-lite.js
@@ -2,8 +2,8 @@
  * This module provides {@link captureFromMap}, which only "captures" the
  * compartment map descriptors and sources from a partially completed
  * compartment map--_without_ creating an archive. The resulting compartment map
- * represents a well-formed dependency graph, laden with useful metadata. This
- * could be used for e.g., automatic policy generation.
+ * represents a well-formed dependency graph, laden with useful metadata. This,
+ * for example, could be used for automatic policy generation.
  *
  * Note that the resulting data structure ({@link CaptureResult}) contains a
  * mapping of filepaths to compartment map names.

From ebf23836064cd84e40aa85330fc7a6a751e34830 Mon Sep 17 00:00:00 2001
From: Christopher Hiller <boneskull@boneskull.com>
Date: Wed, 5 Jun 2024 14:38:32 -0700
Subject: [PATCH 4/5] chore(compartment-mapper): update module docstring

Co-authored-by: Kris Kowal <kris@agoric.com>
---
 packages/compartment-mapper/src/capture-lite.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/compartment-mapper/src/capture-lite.js b/packages/compartment-mapper/src/capture-lite.js
index a6d143afff..0ba4efbc6e 100644
--- a/packages/compartment-mapper/src/capture-lite.js
+++ b/packages/compartment-mapper/src/capture-lite.js
@@ -5,7 +5,7 @@
  * represents a well-formed dependency graph, laden with useful metadata. This,
  * for example, could be used for automatic policy generation.
  *
- * Note that the resulting data structure ({@link CaptureResult}) contains a
+ * The resulting data structure ({@link CaptureResult}) contains a
  * mapping of filepaths to compartment map names.
  *
  * These functions do not have a bias for any particular mapping, so you will

From 311358bb447bc98a40dd40d9939e8114442e1883 Mon Sep 17 00:00:00 2001
From: Christopher Hiller <boneskull@boneskull.com>
Date: Wed, 5 Jun 2024 15:27:25 -0700
Subject: [PATCH 5/5] chore(compartment-mapper): add a couple tests for
 capture-lite

---
 .../test/capture-lite.test.js                 | 98 +++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100644 packages/compartment-mapper/test/capture-lite.test.js

diff --git a/packages/compartment-mapper/test/capture-lite.test.js b/packages/compartment-mapper/test/capture-lite.test.js
new file mode 100644
index 0000000000..e91c04955e
--- /dev/null
+++ b/packages/compartment-mapper/test/capture-lite.test.js
@@ -0,0 +1,98 @@
+// @ts-check
+
+import 'ses';
+import fs from 'node:fs';
+import url from 'node:url';
+import path from 'node:path';
+import test from 'ava';
+import { captureFromMap } from '../capture-lite.js';
+import { mapNodeModules } from '../src/node-modules.js';
+import { makeReadPowers } from '../src/node-powers.js';
+import { defaultParserForLanguage } from '../src/import-parsers.js';
+
+const { keys, entries, fromEntries } = Object;
+
+test('captureFromMap() should resolve with a CaptureResult', async t => {
+  t.plan(5);
+
+  const readPowers = makeReadPowers({ fs, url });
+  const moduleLocation = `${new URL(
+    'fixtures-0/node_modules/bundle/main.js',
+    import.meta.url,
+  )}`;
+
+  const nodeCompartmentMap = await mapNodeModules(readPowers, moduleLocation);
+
+  const { captureCompartmentMap, captureSources, compartmentRenames } =
+    await captureFromMap(readPowers, nodeCompartmentMap, {
+      parserForLanguage: defaultParserForLanguage,
+    });
+
+  const renames = fromEntries(
+    entries(compartmentRenames).map(([filepath, id]) => [id, filepath]),
+  );
+
+  t.deepEqual(
+    keys(captureSources).sort(),
+    ['bundle', 'bundle-dep-v0.0.0'],
+    'captureSources should contain sources for each compartment map descriptor',
+  );
+
+  t.deepEqual(
+    keys(renames).sort(),
+    ['bundle', 'bundle-dep-v0.0.0'],
+    'compartmentRenames must contain same compartment names as in captureCompartmentMap',
+  );
+
+  t.is(
+    keys(compartmentRenames).length,
+    keys(captureCompartmentMap.compartments).length,
+    'Every compartment descriptor must have a corresponding value in compartmentRenames',
+  );
+
+  t.deepEqual(
+    captureCompartmentMap.entry,
+    {
+      compartment: 'bundle',
+      module: './main.js',
+    },
+    'The entry compartment should point to the "bundle" compartment map',
+  );
+
+  t.deepEqual(
+    keys(captureCompartmentMap.compartments).sort(),
+    ['bundle', 'bundle-dep-v0.0.0'],
+    'The "bundle" and "bundle-dep-v0.0.0" compartments should be present',
+  );
+});
+
+test('captureFromMap() should round-trip sources based on parsers', async t => {
+  const readPowers = makeReadPowers({ fs, url });
+  const moduleLocation = `${new URL(
+    'fixtures-0/node_modules/bundle/main.js',
+    import.meta.url,
+  )}`;
+
+  const nodeCompartmentMap = await mapNodeModules(readPowers, moduleLocation);
+
+  const { captureSources, compartmentRenames } = await captureFromMap(
+    readPowers,
+    nodeCompartmentMap,
+    {
+      // we are NOT pre-compiling sources
+      parserForLanguage: defaultParserForLanguage,
+    },
+  );
+
+  const renames = fromEntries(
+    entries(compartmentRenames).map(([filepath, id]) => [id, filepath]),
+  );
+  const decoder = new TextDecoder();
+  // the actual source depends on the value of `parserForLanguage` above
+  const actual = decoder.decode(captureSources.bundle['./icando.cjs'].bytes);
+  const expected = await fs.promises.readFile(
+    path.join(url.fileURLToPath(renames.bundle), 'icando.cjs'),
+    'utf-8',
+  );
+  t.is(actual, expected, 'Source code should not be pre-compiled');
+});