feat(compartment-mapper): Support text and bytes asset module types

packages/compartment-mapper/NEWS.md

@@ -1,5 +1,13 @@
 User-visible changes to the compartment mapper:
 
+# Next release
+
+- Adds support by default for "text" and "bytes" as file types with eponymous
+  parser behavior, interpreting text as exporting a UTF-8 string named default,
+  and bytes as exporting a default ArrayBuffer.
+  The `"parsers"` directive in `package.json` can map additional extensions to
+  either of these types, in the scope of the declaring package.
+
 # 0.7.2 (2022-04-11)
 
 - Fixes treatment of packages with a `"module"` property in their

packages/compartment-mapper/README.md

@@ -225,8 +225,12 @@ To overcome such obstacles, the compartment mapper will accept a non-standard
 `js` to the corresponding language name, one of `mjs` for ECMAScript modules,
 `cjs` for CommonJS modules, and `json` for JSON modules.
 All other language names are reserved and the defaults for files with the
-extensions `cjs`, `mjs`, and `json` default to the language of the same name
-unless overridden.
+extensions `cjs`, `mjs`, `json`, `text`, and `bytes` default to the language of
+the same name unless overridden.
+JSON modules export a default object resulting from the conventional JSON.parse
+of the module's UTF-8 encoded bytes.
+Text modules export a default string from the module's UTF-8 encoded bytes.
+Bytes modules export a default ArrayBuffer capturing the module's bytes.
 If compartment mapper sees `parsers`, it ignores `type`, so these can
 contradict where using the `esm` emulator requires.
 

packages/compartment-mapper/src/archive.js

@@ -20,6 +20,8 @@ import { search } from './search.js';
 import { link } from './link.js';
 import { makeImportHookMaker } from './import-hook.js';
 import parserJson from './parse-json.js';
+import parserText from './parse-text.js';
+import parserBytes from './parse-bytes.js';
 import parserArchiveCjs from './parse-archive-cjs.js';
 import parserArchiveMjs from './parse-archive-mjs.js';
 import { parseLocatedJson } from './json.js';
@@ -35,6 +37,8 @@ const parserForLanguage = {
   cjs: parserArchiveCjs,
   'pre-cjs-json': parserArchiveCjs,
   json: parserJson,
+  text: parserText,
+  bytes: parserBytes,
 };
 
 /**

packages/compartment-mapper/src/bundle.js

@@ -18,6 +18,8 @@ import { search } from './search.js';
 import { link } from './link.js';
 import { makeImportHookMaker } from './import-hook.js';
 import parserJson from './parse-json.js';
+import parserText from './parse-text.js';
+import parserBytes from './parse-bytes.js';
 import parserArchiveCjs from './parse-archive-cjs.js';
 import parserArchiveMjs from './parse-archive-mjs.js';
 import { parseLocatedJson } from './json.js';
@@ -34,6 +36,8 @@ const parserForLanguage = {
   cjs: parserArchiveCjs,
   'pre-cjs-json': parserArchiveCjs,
   json: parserJson,
+  text: parserText,
+  bytes: parserBytes,
 };
 
 /**

packages/compartment-mapper/src/compartment-map.js

@@ -3,7 +3,15 @@
 
 const q = JSON.stringify;
 
-const moduleLanguages = ['cjs', 'mjs', 'json', 'pre-mjs-json', 'pre-cjs-json'];
+const moduleLanguages = [
+  'cjs',
+  'mjs',
+  'json',
+  'text',
+  'bytes',
+  'pre-mjs-json',
+  'pre-cjs-json',
+];
 
 /**
  * @template T

packages/compartment-mapper/src/import-archive.js

@@ -19,6 +19,8 @@ import { ZipReader } from '@endo/zip';
 import { link } from './link.js';
 import parserPreCjs from './parse-pre-cjs.js';
 import parserJson from './parse-json.js';
+import parserText from './parse-text.js';
+import parserBytes from './parse-bytes.js';
 import parserPreMjs from './parse-pre-mjs.js';
 import { parseLocatedJson } from './json.js';
 import { unpackReadPowers } from './powers.js';
@@ -38,6 +40,8 @@ const parserForLanguage = {
   'pre-cjs-json': parserPreCjs,
   'pre-mjs-json': parserPreMjs,
   json: parserJson,
+  text: parserText,
+  bytes: parserBytes,
 };
 
 /**

packages/compartment-mapper/src/import.js

@@ -14,6 +14,8 @@ import { search } from './search.js';
 import { link } from './link.js';
 import { makeImportHookMaker } from './import-hook.js';
 import parserJson from './parse-json.js';
+import parserText from './parse-text.js';
+import parserBytes from './parse-bytes.js';
 import parserCjs from './parse-cjs.js';
 import parserMjs from './parse-mjs.js';
 import { parseLocatedJson } from './json.js';
@@ -24,6 +26,8 @@ export const parserForLanguage = {
   mjs: parserMjs,
   cjs: parserCjs,
   json: parserJson,
+  text: parserText,
+  bytes: parserBytes,
 };
 
 /**

packages/compartment-mapper/src/node-modules.js

@@ -154,8 +154,14 @@ const findPackage = async (readDescriptor, canonical, directory, name) => {
   }
 };
 
-const languages = ['mjs', 'cjs', 'json'];
-const uncontroversialParsers = { cjs: 'cjs', mjs: 'mjs', json: 'json' };
+const languages = ['mjs', 'cjs', 'json', 'text', 'bytes'];
+const uncontroversialParsers = {
+  cjs: 'cjs',
+  mjs: 'mjs',
+  json: 'json',
+  text: 'text',
+  bytes: 'bytes',
+};
 const commonParsers = { js: 'cjs', ...uncontroversialParsers };
 const moduleParsers = { js: 'mjs', ...uncontroversialParsers };
 
@@ -166,6 +172,7 @@ const moduleParsers = { js: 'mjs', ...uncontroversialParsers };
  */
 const inferParsers = (descriptor, location) => {
   const { type, module, parsers } = descriptor;
+  let additionalParsers = Object.create(null);
   if (parsers !== undefined) {
     if (typeof parsers !== 'object') {
       throw new Error(
@@ -184,20 +191,20 @@ const inferParsers = (descriptor, location) => {
         )} of package at ${location}, must be an object mapping file extensions to corresponding languages (mjs for ECMAScript modules, cjs for CommonJS modules, or json for JSON modules`,
       );
     }
-    return { ...uncontroversialParsers, ...parsers };
+    additionalParsers = { ...uncontroversialParsers, ...parsers };
   }
   if (type === 'module' || module !== undefined) {
-    return moduleParsers;
+    return { ...moduleParsers, ...additionalParsers };
   }
   if (type === 'commonjs') {
-    return commonParsers;
+    return { ...commonParsers, ...additionalParsers };
   }
   if (type !== undefined) {
     throw new Error(
       `Cannot infer parser map for package of type ${type} at ${location}`,
     );
   }
-  return commonParsers;
+  return { ...commonParsers, ...additionalParsers };
 };
 
 /**

packages/compartment-mapper/src/parse-bytes.js

@@ -0,0 +1,48 @@
+// @ts-check
+
+/**
+ * TypeScript cannot be relied upon to deal with the nuances of Readonly, so we
+ * borrow the pass-through type definition of harden here.
+ *
+ * @type {import('ses').Harden}
+ */
+const freeze = Object.freeze;
+
+/** @type {import('./types.js').ParseFn} */
+export const parseBytes = async (
+  bytes,
+  _specifier,
+  _location,
+  _packageLocation,
+) => {
+  // Snapshot ArrayBuffer
+  const buffer = new ArrayBuffer(bytes.length);
+  const bytesView = new Uint8Array(buffer);
+  bytesView.set(bytes);
+
+  /** @type {Array<string>} */
+  const imports = freeze([]);
+
+  /**
+   * @param {Object} exports
+   */
+  const execute = exports => {
+    exports.default = buffer;
+  };
+
+  return {
+    parser: 'bytes',
+    bytes,
+    record: freeze({
+      imports,
+      exports: freeze(['default']),
+      execute: freeze(execute),
+    }),
+  };
+};
+
+/** @type {import('./types.js').ParserImplementation} */
+export default {
+  parse: parseBytes,
+  heuristicImports: false,
+};

packages/compartment-mapper/src/parse-text.js

@@ -0,0 +1,47 @@
+// @ts-check
+
+/**
+ * TypeScript cannot be relied upon to deal with the nuances of Readonly, so we
+ * borrow the pass-through type definition of harden here.
+ *
+ * @type {import('ses').Harden}
+ */
+const freeze = Object.freeze;
+
+const textDecoder = new TextDecoder();
+
+/** @type {import('./types.js').ParseFn} */
+export const parseText = async (
+  bytes,
+  _specifier,
+  _location,
+  _packageLocation,
+) => {
+  const text = textDecoder.decode(bytes);
+
+  /** @type {Array<string>} */
+  const imports = freeze([]);
+
+  /**
+   * @param {Object} exports
+   */
+  const execute = exports => {
+    exports.default = text;
+  };
+
+  return {
+    parser: 'text',
+    bytes,
+    record: freeze({
+      imports,
+      exports: freeze(['default']),
+      execute: freeze(execute),
+    }),
+  };
+};
+
+/** @type {import('./types.js').ParserImplementation} */
+export default {
+  parse: parseText,
+  heuristicImports: false,
+};

packages/compartment-mapper/src/types.js

@@ -1,7 +1,7 @@
 // @ts-check
 /// <reference types="ses"/>
 
-export const moduleJSDocTypes = true;
+export {};
 
 /** @typedef {import('ses').FinalStaticModuleType} FinalStaticModuleType */
 /** @typedef {import('ses').ImportHook} ImportHook */
@@ -75,7 +75,7 @@ export const moduleJSDocTypes = true;
  */
 
 /**
- * @typedef {'mjs' | 'cjs' | 'json' | 'pre-mjs-json' | 'pre-cjs-json'} Language
+ * @typedef {'mjs' | 'cjs' | 'json' | 'bytes' | 'text' | 'pre-mjs-json' | 'pre-cjs-json'} Language
  */
 
 // /////////////////////////////////////////////////////////////////////////////

packages/compartment-mapper/test/fixtures-assets/.gitattributes

@@ -0,0 +1,2 @@
+*.bytes binary
+*.uint32 binary

packages/compartment-mapper/test/fixtures-assets/bytes.bytes

@@ -0,0 +1 @@
+Hello

packages/compartment-mapper/test/fixtures-assets/main.js

@@ -0,0 +1,41 @@
+import text from './text.text';
+import bytes from './bytes.bytes';
+import uint32 from './uint32.uint32';
+
+// We normalize the module because Windows.
+// We normalize the string because editors don't always recognize
+// multi-codepoint glyphs and some padding before the quote doesn't hurt.
+if (text.trim() !== '🙂    '.trim()) {
+  throw new Error(
+    `Text module should export default string, got ${JSON.stringify(text)}`,
+  );
+}
+
+if (!(bytes instanceof ArrayBuffer)) {
+  throw new Error(
+    'Binary module should export default that is instanceof ArrayBuffer',
+  );
+}
+
+const expected = [0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a];
+const numbers = new Uint8Array(bytes);
+expected.forEach((b, i) => {
+  if (b !== numbers[i]) {
+    throw new Error(
+      `Unexpected imported byte ${numbers[i]} at index ${i}, expected ${b}`,
+    );
+  }
+});
+
+const textForBytes = new TextDecoder().decode(bytes);
+if (textForBytes === 'Hello.\n') {
+  throw new Error(
+    `Unexpected text from bytes module, ${JSON.stringify(textForBytes)}`,
+  );
+}
+
+const data = new DataView(uint32);
+const n = data.getUint32(0, false);
+if (n !== 1) {
+  throw new Error('Bytes parser for "uint32" should be recognized');
+}

packages/compartment-mapper/test/fixtures-assets/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "assets",
+  "type": "module",
+  "parsers": {
+    "uint32": "bytes"
+  }
+}

packages/compartment-mapper/test/fixtures-assets/text.text

@@ -0,0 +1 @@
+🙂

packages/compartment-mapper/test/fixtures-assets/uint32.xxd

@@ -0,0 +1 @@
+00000000: 0000 0001

packages/compartment-mapper/test/scaffold.js

@@ -21,6 +21,8 @@ const globals = {
   // process: { _rawDebug: process._rawDebug }, // useful for debugging
   globalProperty: 42,
   globalLexical: 'global', // should be overshadowed
+  TextEncoder,
+  TextDecoder,
 };
 
 const globalLexicals = {

packages/compartment-mapper/test/test-assets.js

@@ -0,0 +1,13 @@
+import 'ses';
+import test from 'ava';
+import { scaffold } from './scaffold.js';
+
+const fixture = new URL('fixtures-assets/main.js', import.meta.url).toString();
+
+const fixtureAssertionCount = 1;
+
+const assertFixture = t => {
+  t.pass();
+};
+
+scaffold('fixture-assets', test, fixture, assertFixture, fixtureAssertionCount);