feat(compartment-mapper): Add hooks for sourceURL (#932)

Adds hooks to archive production and consumption for reading and writing source locations, such that other tools yet to be written can use these hooks to provide fully qualified local debug source URLs. Archive creation functions now accept a `captureSourceLocation(compartmentName, moduleSpecifier, sourceLocation)` hook and archive parsing functions accept `computeSourceLocation(compartmentName, moduleSpecifier)`.
endojs · Nov 6, 2021 · a7b42ae · a7b42ae
1 parent 0dfb83e
commit a7b42ae
Show file tree

Hide file tree

Showing 12 changed files with 167 additions and 45 deletions.
diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md
@@ -5,6 +5,13 @@ User-visible changes to the compartment mapper:
 - Adds source URL suffixes to archives, such that the archive hash remains
   orthogonal to the local directory but has sufficient information that editors
   like VS Code can match the suffix to a file in the IDE workspace.
+- Adds hooks to archive production and consumption for reading and writing
+  source locations, such that other tools yet to be written can use these hooks
+  to provide fully qualified local debug source URLs.
+  Archive creation functions now accept a
+  `captureSourceLocation(compartmentName, moduleSpecifier, sourceLocation)`
+  hook and archive parsing functions accept
+  `computeSourceLocation(compartmentName, moduleSpecifier)`.
 
 # 0.5.1 (2021-08-12)
 

diff --git a/packages/compartment-mapper/src/archive.js b/packages/compartment-mapper/src/archive.js
@@ -7,6 +7,7 @@
 /** @typedef {import('./types.js').ModuleDescriptor} ModuleDescriptor */
 /** @typedef {import('./types.js').ParseFn} ParseFn */
 /** @typedef {import('./types.js').ReadFn} ReadFn */
+/** @typedef {import('./types.js').CaptureSourceLocationHook} CaptureSourceLocationHook */
 /** @typedef {import('./types.js').ReadPowers} ReadPowers */
 /** @typedef {import('./types.js').HashPowers} HashPowers */
 /** @typedef {import('./types.js').Sources} Sources */
@@ -65,9 +66,9 @@ const renameCompartments = compartments => {
  */
 const translateCompartmentMap = (compartments, sources, renames) => {
   const result = {};
-  for (const name of keys(compartments).sort()) {
-    const compartment = compartments[name];
-    const { label } = compartment;
+  for (const compartmentName of keys(compartments).sort()) {
+    const compartment = compartments[compartmentName];
+    const { name, label } = compartment;
 
     // rename module compartments
     /** @type {Record<string, ModuleDescriptor>} */
@@ -87,7 +88,7 @@ const translateCompartmentMap = (compartments, sources, renames) => {
     }
 
     // integrate sources into modules
-    const compartmentSources = sources[name];
+    const compartmentSources = sources[compartmentName];
     if (compartmentSources) {
       for (const name of keys(compartmentSources).sort()) {
         const source = compartmentSources[name];
@@ -101,9 +102,10 @@ const translateCompartmentMap = (compartments, sources, renames) => {
       }
     }
 
-    result[renames[name]] = {
+    result[renames[compartmentName]] = {
+      name,
       label,
-      location: renames[name],
+      location: renames[compartmentName],
       modules,
       // `scopes`, `types`, and `parsers` are not necessary since every
       // loadable module is captured in `modules`.
@@ -149,15 +151,34 @@ const addSourcesToArchive = async (archive, sources) => {
   }
 };
 
+/**
+ * @param {Sources} sources
+ * @param {CaptureSourceLocationHook} captureSourceLocation
+ */
+const captureSourceLocations = async (sources, captureSourceLocation) => {
+  for (const compartmentName of keys(sources).sort()) {
+    const modules = sources[compartmentName];
+    for (const moduleSpecifier of keys(modules).sort()) {
+      const { sourceLocation } = modules[moduleSpecifier];
+      if (sourceLocation !== undefined) {
+        captureSourceLocation(compartmentName, moduleSpecifier, sourceLocation);
+      }
+    }
+  }
+};
 /**
  * @param {ReadFn | ReadPowers} powers
  * @param {string} moduleLocation
  * @param {ArchiveOptions} [options]
  * @returns {Promise<{archiveSources: Sources, archiveCompartmentMapBytes: Uint8Array}>}
  */
 const digestLocation = async (powers, moduleLocation, options) => {
-  const { moduleTransforms, modules: exitModules = {}, dev = false } =
-    options || {};
+  const {
+    moduleTransforms,
+    modules: exitModules = {},
+    dev = false,
+    captureSourceLocation = undefined,
+  } = options || {};
   const { read, computeSha512 } = unpackReadPowers(powers);
   const {
     packageLocation,
@@ -235,6 +256,10 @@ const digestLocation = async (powers, moduleLocation, options) => {
     archiveCompartmentMapText,
   );
 
+  if (captureSourceLocation !== undefined) {
+    captureSourceLocations(archiveSources, captureSourceLocation);
+  }
+
   return {
     archiveCompartmentMapBytes,
     archiveSources,

diff --git a/packages/compartment-mapper/src/import-archive.js b/packages/compartment-mapper/src/import-archive.js
@@ -11,6 +11,7 @@
 /** @typedef {import('./types.js').ReadFn} ReadFn */
 /** @typedef {import('./types.js').ReadPowers} ReadPowers */
 /** @typedef {import('./types.js').HashFn} HashFn */
+/** @typedef {import('./types.js').ComputeSourceLocationHook} ComputeSourceLocationHook */
 /** @typedef {import('./types.js').LoadArchiveOptions} LoadArchiveOptions */
 /** @typedef {import('./types.js').ExecuteOptions} ExecuteOptions */
 
@@ -21,6 +22,7 @@ import { parseJson } from './parse-json.js';
 import { parsePreMjs } from './parse-pre-mjs.js';
 import { parseLocatedJson } from './json.js';
 import { unpackReadPowers } from './powers.js';
+import { join } from './node-module-specifier.js';
 
 // q as in quote for strings in error messages.
 const q = JSON.stringify;
@@ -46,13 +48,15 @@ const parserForLanguage = {
  * @param {Record<string, CompartmentDescriptor>} compartments
  * @param {string} archiveLocation
  * @param {HashFn} [computeSha512]
+ * @param {ComputeSourceLocationHook} [computeSourceLocation]
  * @returns {ArchiveImportHookMaker}
  */
 const makeArchiveImportHookMaker = (
   archive,
   compartments,
   archiveLocation,
-  computeSha512,
+  computeSha512 = undefined,
+  computeSourceLocation = undefined,
 ) => {
   // per-assembly:
   /** @type {ArchiveImportHookMaker} */
@@ -94,13 +98,26 @@ const makeArchiveImportHookMaker = (
         }
       }
 
+      let sourceLocation = `file:///${moduleLocation}`;
+      if (packageName !== undefined) {
+        const base = packageName
+          .split('/')
+          .slice(-1)
+          .join('/');
+        sourceLocation = `.../${join(base, moduleSpecifier)}`;
+      }
+      if (computeSourceLocation !== undefined) {
+        sourceLocation =
+          computeSourceLocation(packageLocation, moduleSpecifier) ||
+          sourceLocation;
+      }
+
       // eslint-disable-next-line no-await-in-loop
       const { record } = await parse(
         moduleBytes,
         moduleSpecifier,
-        `file:///${moduleLocation}`,
+        sourceLocation,
         packageLocation,
-        packageName,
       );
       return record;
     };
@@ -115,14 +132,19 @@ const makeArchiveImportHookMaker = (
  * @param {Object} [options]
  * @param {string} [options.expectedSha512]
  * @param {HashFn} [options.computeSha512]
+ * @param {ComputeSourceLocationHook} [options.computeSourceLocation]
  * @returns {Promise<Application>}
  */
 export const parseArchive = async (
   archiveBytes,
   archiveLocation = '<unknown>',
   options = {},
 ) => {
-  const { computeSha512 = undefined, expectedSha512 = undefined } = options;
+  const {
+    computeSha512 = undefined,
+    expectedSha512 = undefined,
+    computeSourceLocation = undefined,
+  } = options;
 
   const archive = await readZip(archiveBytes, archiveLocation);
   const compartmentMapBytes = await archive.read('compartment-map.json');
@@ -168,6 +190,7 @@ export const parseArchive = async (
       compartments,
       archiveLocation,
       computeSha512,
+      computeSourceLocation,
     );
     const { compartment } = link(compartmentMap, {
       makeImportHook,
@@ -197,11 +220,12 @@ export const loadArchive = async (
   options = {},
 ) => {
   const { read, computeSha512 } = unpackReadPowers(readPowers);
-  const { expectedSha512 } = options;
+  const { expectedSha512, computeSourceLocation } = options;
   const archiveBytes = await read(archiveLocation);
   return parseArchive(archiveBytes, archiveLocation, {
     computeSha512,
     expectedSha512,
+    computeSourceLocation,
   });
 };
 

diff --git a/packages/compartment-mapper/src/import-hook.js b/packages/compartment-mapper/src/import-hook.js
@@ -56,7 +56,7 @@ export const makeImportHookMaker = (
 ) => {
   // per-assembly:
   /** @type {ImportHookMaker} */
-  const makeImportHook = (packageLocation, packageName, parse) => {
+  const makeImportHook = (packageLocation, _packageName, parse) => {
     // per-compartment:
     packageLocation = resolveLocation(packageLocation, baseLocation);
     const packageSources = sources[packageLocation] || {};
@@ -121,7 +121,6 @@ export const makeImportHookMaker = (
             candidateSpecifier,
             moduleLocation,
             packageLocation,
-            packageName,
           );
           const {
             parser,
@@ -156,6 +155,7 @@ export const makeImportHookMaker = (
           );
           packageSources[candidateSpecifier] = {
             location: packageRelativeLocation,
+            sourceLocation: moduleLocation,
             parser,
             bytes: transformedBytes,
             record,

diff --git a/packages/compartment-mapper/src/link.js b/packages/compartment-mapper/src/link.js
@@ -76,7 +76,7 @@ const makeExtensionParser = (
   parserForLanguage,
   transforms,
 ) => {
-  return async (bytes, specifier, location, packageLocation, packageName) => {
+  return async (bytes, specifier, location, packageLocation) => {
     let language;
     if (has(languageForModuleSpecifier, specifier)) {
       language = languageForModuleSpecifier[specifier];
@@ -105,7 +105,7 @@ const makeExtensionParser = (
       );
     }
     const parse = parserForLanguage[language];
-    return parse(bytes, specifier, location, packageLocation, packageName);
+    return parse(bytes, specifier, location, packageLocation);
   };
 };
 

diff --git a/packages/compartment-mapper/src/parse-archive-cjs.js b/packages/compartment-mapper/src/parse-archive-cjs.js
@@ -3,7 +3,6 @@
 /** @typedef {import('ses').ThirdPartyStaticModuleInterface} ThirdPartyStaticModuleInterface */
 
 import { analyzeCommonJS } from '@endo/cjs-module-analyzer';
-import { join } from './node-module-specifier.js';
 
 const textEncoder = new TextEncoder();
 const textDecoder = new TextDecoder();
@@ -17,28 +16,22 @@ freeze(noopExecute);
 export const parseArchiveCjs = async (
   bytes,
   specifier,
-  _location,
+  location,
   _packageLocation,
-  packageName,
 ) => {
   const source = textDecoder.decode(bytes);
-  const base = packageName
-    .split('/')
-    .slice(-1)
-    .join('/');
-  const sourceLocation = `.../${join(base, specifier)}`;
 
   const { requires: imports, exports, reexports } = analyzeCommonJS(
     source,
-    sourceLocation,
+    location,
   );
 
   const pre = textEncoder.encode(
     JSON.stringify({
       imports,
       exports,
       reexports,
-      source: `(function (require, exports, module, __filename, __dirname) { ${source} //*/\n})\n//# sourceURL=${sourceLocation}`,
+      source: `(function (require, exports, module, __filename, __dirname) { ${source} //*/\n})\n`,
     }),
   );
 

diff --git a/packages/compartment-mapper/src/parse-archive-mjs.js b/packages/compartment-mapper/src/parse-archive-mjs.js
@@ -1,26 +1,19 @@
 // @ts-check
 
 import { StaticModuleRecord } from '@endo/static-module-record';
-import { join } from './node-module-specifier.js';
 
 const textEncoder = new TextEncoder();
 const textDecoder = new TextDecoder();
 
 /** @type {import('./types.js').ParseFn} */
 export const parseArchiveMjs = async (
   bytes,
-  specifier,
+  _specifier,
   _location,
   _packageLocation,
-  packageName,
 ) => {
   const source = textDecoder.decode(bytes);
-  const base = packageName
-    .split('/')
-    .slice(-1)
-    .join('/');
-  const sourceLocation = `.../${join(base, specifier)}`;
-  const record = new StaticModuleRecord(source, sourceLocation);
+  const record = new StaticModuleRecord(source);
   const pre = textEncoder.encode(JSON.stringify(record));
   return {
     parser: 'pre-mjs-json',

diff --git a/packages/compartment-mapper/src/parse-pre-cjs.js b/packages/compartment-mapper/src/parse-pre-cjs.js
@@ -6,15 +6,13 @@ const { freeze } = Object;
 
 const textDecoder = new TextDecoder();
 
-const urlParent = location => new URL('./', location).toString();
-const pathParent = path => {
-  const index = path.lastIndexOf('/');
-  if (index > 0) {
-    return path.slice(0, index);
+const locationParent = location => {
+  const index = location.lastIndexOf('/');
+  if (index >= 0) {
+    return location.slice(0, index);
   }
-  return '/';
+  return location;
 };
-const locationParent = typeof URL !== 'undefined' ? urlParent : pathParent;
 
 /** @type {import('./types.js').ParseFn} */
 export const parsePreCjs = async (

diff --git a/packages/compartment-mapper/src/parse-pre-mjs.js b/packages/compartment-mapper/src/parse-pre-mjs.js
@@ -13,6 +13,8 @@ export const parsePreMjs = async (
 ) => {
   const text = textDecoder.decode(bytes);
   const record = parseLocatedJson(text, location);
+  // eslint-disable-next-line no-underscore-dangle
+  record.__syncModuleProgram__ += `//# sourceURL=${location}\n`;
   return {
     parser: 'pre-mjs-json',
     bytes,