feat(compartment-mapper): Custom parser support (#2304)

Closes: #2303

## Description

This PR adds custom parser support as discussed in #2303. To support the
feature:

- A new prop `options` was added to the `PackagePolicy` type.
- Parsers now optionally accept `compartmentDescriptor` objects, so that
parsers can enforce policy

### Security Considerations

See #2303

### Scaling Considerations

See #2303

### Documentation Considerations

- Add documentation for a new option `parsers` for e.g.,
`importLocation`
- Add documentation for a new prop `options` in `PackagePolicy`

### Testing Considerations

- [x] A round-trip test 
- [x] A round-trip test w/ policy enforcement 
- [x] Assert `options` is ignored by policy validator
- [x] Assert custom parsers cannot override builtin parsers

### Compatibility Considerations

None

### Upgrade Considerations

- Update `NEWS.md`

---------

Co-authored-by: naugtur <naugtur@gmail.com>
Co-authored-by: Kris Kowal <kris@agoric.com>

packages/compartment-mapper/NEWS.md

@@ -5,6 +5,13 @@ User-visible changes to the compartment mapper:
 - Fixes incompatible behavior with Node.js package conditional exports #2276.
   Previously, the last matching tag would override all prior matches, often
   causing a bundle to adopt the `default` instead of a more specific condition.
+- Adds `parserForLanguage` and `languageForExtension` options to all modes of
+  operation such that the compartment mapper can analyze and bundle languages
+  apart from the built-in languages, which include `esm` and `cjs`.
+  The `languageForExtension` option provides defaults for the entire
+  application and the `"parsers"` property in individual `package.json`
+  descriptors may extend or override using any of the configured or built-in
+  language parser names.
 
 # 0.9.0 (2023-08-07)
 

packages/compartment-mapper/src/archive.js

@@ -1,12 +1,11 @@
 // @ts-check
 /* eslint no-shadow: 0 */
 
-/** @import {ArchiveOptions} from './types.js' */
+/** @import {ArchiveOptions, ParserForLanguage} from './types.js' */
 /** @import {ArchiveWriter} from './types.js' */
 /** @import {CompartmentDescriptor} from './types.js' */
 /** @import {CompartmentMapDescriptor} from './types.js' */
 /** @import {ModuleDescriptor} from './types.js' */
-/** @import {ParserImplementation} from './types.js' */
 /** @import {ReadFn} from './types.js' */
 /** @import {CaptureSourceLocationHook} from './types.js' */
 /** @import {ReadPowers} from './types.js' */
@@ -39,16 +38,20 @@ import { detectAttenuators } from './policy.js';
 
 const textEncoder = new TextEncoder();
 
-/** @type {Record<string, ParserImplementation>} */
-const parserForLanguage = {
-  mjs: parserArchiveMjs,
-  'pre-mjs-json': parserArchiveMjs,
-  cjs: parserArchiveCjs,
-  'pre-cjs-json': parserArchiveCjs,
-  json: parserJson,
-  text: parserText,
-  bytes: parserBytes,
-};
+const { assign, create, freeze } = Object;
+
+/** @satisfies {Readonly<ParserForLanguage>} */
+const defaultParserForLanguage = freeze(
+  /** @type {const} */ ({
+    mjs: parserArchiveMjs,
+    'pre-mjs-json': parserArchiveMjs,
+    cjs: parserArchiveCjs,
+    'pre-cjs-json': parserArchiveCjs,
+    json: parserJson,
+    text: parserText,
+    bytes: parserBytes,
+  }),
+);
 
 /**
  * @param {string} rel - a relative URL
@@ -92,7 +95,7 @@ const { keys, entries, fromEntries } = Object;
  */
 const renameCompartments = compartments => {
   /** @type {Record<string, string>} */
-  const compartmentRenames = Object.create(null);
+  const compartmentRenames = create(null);
   let index = 0;
   let prev = '';
 
@@ -133,14 +136,14 @@ const renameCompartments = compartments => {
  * @param {Record<string, string>} compartmentRenames
  */
 const translateCompartmentMap = (compartments, sources, compartmentRenames) => {
-  const result = Object.create(null);
+  const result = create(null);
   for (const compartmentName of keys(compartmentRenames)) {
     const compartment = compartments[compartmentName];
     const { name, label, retained, policy } = compartment;
     if (retained) {
       // rename module compartments
       /** @type {Record<string, ModuleDescriptor>} */
-      const modules = Object.create(null);
+      const modules = create(null);
       const compartmentModules = compartment.modules;
       if (compartment.modules) {
         for (const name of keys(compartmentModules).sort()) {
@@ -291,7 +294,7 @@ export const makeArchiveCompartmentMap = (compartmentMap, sources) => {
  * @param {ArchiveOptions} [options]
  * @returns {Promise<{sources: Sources, compartmentMapBytes: Uint8Array, sha512?: string}>}
  */
-const digestLocation = async (powers, moduleLocation, options) => {
+const digestLocation = async (powers, moduleLocation, options = {}) => {
   const {
     moduleTransforms,
     modules: exitModules = {},
@@ -303,7 +306,17 @@ const digestLocation = async (powers, moduleLocation, options) => {
     importHook: exitModuleImportHook = undefined,
     policy = undefined,
     sourceMapHook = undefined,
-  } = options || {};
+    parserForLanguage: parserForLanguageOption = {},
+    languageForExtension: languageForExtensionOption = {},
+  } = options;
+
+  const parserForLanguage = freeze(
+    assign(create(null), defaultParserForLanguage, parserForLanguageOption),
+  );
+  const languageForExtension = freeze(
+    assign(create(null), languageForExtensionOption),
+  );
+
   const { read, computeSha512 } = unpackReadPowers(powers);
   const {
     packageLocation,
@@ -359,6 +372,7 @@ const digestLocation = async (powers, moduleLocation, options) => {
     makeImportHook,
     moduleTransforms,
     parserForLanguage,
+    languageForExtension,
     archiveOnly: true,
   });
   await compartment.load(entryModuleSpecifier);

packages/compartment-mapper/src/bundle.js

@@ -1,9 +1,8 @@
 // @ts-check
 /* eslint no-shadow: 0 */
 
-/** @import {ResolveHook} from 'ses' */
 /** @import {PrecompiledStaticModuleInterface} from 'ses' */
-/** @import {ParserImplementation} from './types.js' */
+/** @import {ParserForLanguage} from './types.js' */
 /** @import {CompartmentDescriptor} from './types.js' */
 /** @import {CompartmentSources} from './types.js' */
 /** @import {ReadFn} from './types.js' */
@@ -29,18 +28,23 @@ import cjsSupport from './bundle-cjs.js';
 
 const textEncoder = new TextEncoder();
 
+const { freeze } = Object;
 const { quote: q } = assert;
 
-/** @type {Record<string, ParserImplementation>} */
-const parserForLanguage = {
-  mjs: parserArchiveMjs,
-  'pre-mjs-json': parserArchiveMjs,
-  cjs: parserArchiveCjs,
-  'pre-cjs-json': parserArchiveCjs,
-  json: parserJson,
-  text: parserText,
-  bytes: parserBytes,
-};
+/**
+ * @satisfies {Readonly<ParserForLanguage>}
+ */
+const defaultParserForLanguage = freeze(
+  /** @type {const} */ ({
+    mjs: parserArchiveMjs,
+    'pre-mjs-json': parserArchiveMjs,
+    cjs: parserArchiveCjs,
+    'pre-cjs-json': parserArchiveCjs,
+    json: parserJson,
+    text: parserText,
+    bytes: parserBytes,
+  }),
+);
 
 /**
  * @param {Record<string, CompartmentDescriptor>} compartmentDescriptors
@@ -161,13 +165,7 @@ function getBundlerKitForModule(module) {
 /**
  * @param {ReadFn} read
  * @param {string} moduleLocation
- * @param {object} [options]
- * @param {ModuleTransforms} [options.moduleTransforms]
- * @param {boolean} [options.dev]
- * @param {Set<string>} [options.tags]
- * @param {object} [options.commonDependencies]
- * @param {Array<string>} [options.searchSuffixes]
- * @param {import('./types.js').SourceMapHook} [options.sourceMapHook]
+ * @param {ArchiveOptions} [options]
  * @returns {Promise<string>}
  */
 export const makeBundle = async (read, moduleLocation, options) => {
@@ -178,9 +176,22 @@ export const makeBundle = async (read, moduleLocation, options) => {
     searchSuffixes,
     commonDependencies,
     sourceMapHook = undefined,
+    parserForLanguage: parserForLanguageOption = {},
+    languageForExtension: languageForExtensionOption = {},
   } = options || {};
   const tags = new Set(tagsOption);
 
+  const parserForLanguage = Object.freeze(
+    Object.assign(
+      Object.create(null),
+      defaultParserForLanguage,
+      parserForLanguageOption,
+    ),
+  );
+  const languageForExtension = Object.freeze(
+    Object.assign(Object.create(null), languageForExtensionOption),
+  );
+
   const {
     packageLocation,
     packageDescriptorText,
@@ -223,6 +234,7 @@ export const makeBundle = async (read, moduleLocation, options) => {
     makeImportHook,
     moduleTransforms,
     parserForLanguage,
+    languageForExtension,
   });
   await compartment.load(entryModuleSpecifier);