From 52d276ec897df27bbbbbb1acbfdbbce2651778ac Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Sat, 29 Apr 2023 12:41:58 +0300 Subject: [PATCH 01/12] module: change default resolver to not throw on unknown scheme Fixes https://github.com/nodejs/loaders/issues/138 --- doc/api/esm.md | 94 +++++++++---------- lib/internal/modules/esm/load.js | 32 +++++++ lib/internal/modules/esm/resolve.js | 34 ------- .../test-esm-import-meta-resolve.mjs | 2 + .../test-esm-loader-default-resolver.mjs | 30 ++++++ .../es-module-loaders/http-loader.mjs | 18 ---- .../uyyt-dummy-loader-main.mjs | 1 + .../uyyt-dummy-loader-main2.mjs | 1 + .../es-module-loaders/uyyt-dummy-loader.mjs | 24 +++++ 9 files changed, 137 insertions(+), 99 deletions(-) create mode 100644 test/es-module/test-esm-loader-default-resolver.mjs create mode 100644 test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs create mode 100644 test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs create mode 100644 test/fixtures/es-module-loaders/uyyt-dummy-loader.mjs diff --git a/doc/api/esm.md b/doc/api/esm.md index 02fed0990542b2..b8942f936dcdfe 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1029,28 +1029,6 @@ and there is no security. // https-loader.mjs import { get } from 'node:https'; -export function resolve(specifier, context, nextResolve) { - const { parentURL = null } = context; - - // Normally Node.js would error on specifiers starting with 'https://', so - // this hook intercepts them and converts them into absolute URLs to be - // passed along to the later hooks below. - if (specifier.startsWith('https://')) { - return { - shortCircuit: true, - url: specifier, - }; - } else if (parentURL && parentURL.startsWith('https://')) { - return { - shortCircuit: true, - url: new URL(specifier, parentURL).href, - }; - } - - // Let Node.js handle all other specifiers. - return nextResolve(specifier); -} - export function load(url, context, nextLoad) { // For JavaScript to be loaded over the network, we need to fetch and // return it. @@ -1091,9 +1069,7 @@ prints the current version of CoffeeScript per the module at the URL in #### Transpiler loader Sources that are in formats Node.js doesn't understand can be converted into -JavaScript using the [`load` hook][load hook]. Before that hook gets called, -however, a [`resolve` hook][resolve hook] needs to tell Node.js not to -throw an error on unknown file types. +JavaScript using the [`load` hook][load hook]. This is less performant than transpiling source files before running Node.js; a transpiler loader should only be used for development and testing @@ -1109,25 +1085,6 @@ import CoffeeScript from 'coffeescript'; const baseURL = pathToFileURL(`${cwd()}/`).href; -// CoffeeScript files end in .coffee, .litcoffee, or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, nextResolve) { - if (extensionsRegex.test(specifier)) { - const { parentURL = baseURL } = context; - - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - return { - shortCircuit: true, - url: new URL(specifier, parentURL).href, - }; - } - - // Let Node.js handle all other specifiers. - return nextResolve(specifier); -} - export async function load(url, context, nextLoad) { if (extensionsRegex.test(url)) { // Now that we patched resolve to let CoffeeScript URLs through, we need to @@ -1220,6 +1177,50 @@ loaded from disk but before Node.js executes it; and so on for any `.coffee`, `.litcoffee` or `.coffee.md` files referenced via `import` statements of any loaded file. +#### Overriding loader + +The above two loaders hooked into the "load" phase of the module loader. +This loader hooks into the "resolution" phase. This loader reads an +`overrides.json` file that specifies which specifiers to override to another +url. + +```js +// overriding-loader.js +import fs from 'node:fs/promises'; + +const overrides = JSON.parse(await fs.readFile('overrides.json')); + +export async function resolve(specifier, context, nextResolve) { + if (specifier in overrides) { + return nextResolve(overrides[specifier], context); + } + + return nextResolve(specifier, context); +} +``` + +Let's assume we have these files: + +```js +// main.js +import 'a-module-to-override'; +``` + +```json +// overrides.json +{ + "a-module-to-override": "./module-override.js" +} +``` + +```js +// module-override.js +console.log('module overridden!'); +``` + +If you run `node --experimental-loader ./overriding-loader.js main.js` +the output will be `module overriden!`. + ## Resolution algorithm ### Features @@ -1506,9 +1507,9 @@ _isImports_, _conditions_) > 7. If _pjson?.type_ exists and is _"module"_, then > 1. If _url_ ends in _".js"_, then > 1. Return _"module"_. -> 2. Throw an _Unsupported File Extension_ error. +> 2. return **undefined**. > 8. Otherwise, -> 1. Throw an _Unsupported File Extension_ error. +> 1. return **undefined**. **LOOKUP\_PACKAGE\_SCOPE**(_url_) @@ -1581,7 +1582,6 @@ for ESM specifiers is [commonjs-extension-resolution-loader][]. [custom https loader]: #https-loader [load hook]: #loadurl-context-nextload [percent-encoded]: url.md#percent-encoding-in-urls -[resolve hook]: #resolvespecifier-context-nextresolve [special scheme]: https://url.spec.whatwg.org/#special-scheme [status code]: process.md#exit-codes [the official standard format]: https://tc39.github.io/ecma262/#sec-modules diff --git a/lib/internal/modules/esm/load.js b/lib/internal/modules/esm/load.js index d2ab555c84b76e..476fe6b91de968 100644 --- a/lib/internal/modules/esm/load.js +++ b/lib/internal/modules/esm/load.js @@ -79,6 +79,8 @@ async function defaultLoad(url, context = kEmptyObject) { source, } = context; + throwIfUnsupportedURLScheme(new URL(url), experimentalNetworkImports); + if (format == null) { format = await defaultGetFormat(url, context); } @@ -102,6 +104,36 @@ async function defaultLoad(url, context = kEmptyObject) { }; } +/** + * throws an error if the protocol is not one of the protocols + * that can be loaded in the default loader + * + * @param {URL} parsed + * @param {boolean} experimentalNetworkImports + */ +function throwIfUnsupportedURLScheme(parsed, experimentalNetworkImports) { + // Avoid accessing the `protocol` property due to the lazy getters. + const protocol = parsed?.protocol; + if ( + protocol && + protocol !== 'file:' && + protocol !== 'data:' && + protocol !== 'node:' && + ( + !experimentalNetworkImports || + ( + protocol !== 'https:' && + protocol !== 'http:' + ) + ) + ) { + const schemes = ['file', 'data', 'node']; + if (experimentalNetworkImports) { + ArrayPrototypePush(schemes, 'https', 'http'); + } + throw new ERR_UNSUPPORTED_ESM_URL_SCHEME(parsed, schemes); + } +} /** * For a falsy `format` returned from `load`, throw an error. diff --git a/lib/internal/modules/esm/resolve.js b/lib/internal/modules/esm/resolve.js index 21703d63f6aa41..702b575bc1721e 100644 --- a/lib/internal/modules/esm/resolve.js +++ b/lib/internal/modules/esm/resolve.js @@ -941,37 +941,6 @@ function throwIfInvalidParentURL(parentURL) { } } -function throwIfUnsupportedURLProtocol(url) { - // Avoid accessing the `protocol` property due to the lazy getters. - const protocol = url.protocol; - if (protocol !== 'file:' && protocol !== 'data:' && - protocol !== 'node:') { - throw new ERR_UNSUPPORTED_ESM_URL_SCHEME(url); - } -} - -function throwIfUnsupportedURLScheme(parsed, experimentalNetworkImports) { - // Avoid accessing the `protocol` property due to the lazy getters. - const protocol = parsed?.protocol; - if ( - protocol && - protocol !== 'file:' && - protocol !== 'data:' && - ( - !experimentalNetworkImports || - ( - protocol !== 'https:' && - protocol !== 'http:' - ) - ) - ) { - const schemes = ['file', 'data']; - if (experimentalNetworkImports) { - ArrayPrototypePush(schemes, 'https', 'http'); - } - throw new ERR_UNSUPPORTED_ESM_URL_SCHEME(parsed, schemes); - } -} function defaultResolve(specifier, context = {}) { let { parentURL, conditions } = context; @@ -1048,7 +1017,6 @@ function defaultResolve(specifier, context = {}) { // This must come after checkIfDisallowedImport if (parsed && parsed.protocol === 'node:') return { __proto__: null, url: specifier }; - throwIfUnsupportedURLScheme(parsed, experimentalNetworkImports); const isMain = parentURL === undefined; if (isMain) { @@ -1095,8 +1063,6 @@ function defaultResolve(specifier, context = {}) { throw error; } - throwIfUnsupportedURLProtocol(url); - return { __proto__: null, // Do NOT cast `url` to a string: that will work even when there are real diff --git a/test/es-module/test-esm-import-meta-resolve.mjs b/test/es-module/test-esm-import-meta-resolve.mjs index 22139122eb505c..cf4774d38a277c 100644 --- a/test/es-module/test-esm-import-meta-resolve.mjs +++ b/test/es-module/test-esm-import-meta-resolve.mjs @@ -30,6 +30,8 @@ assert.strictEqual( code: 'ERR_INVALID_ARG_TYPE', }) ); +assert.equal(import.meta.resolve('http://some-absolute/url'), 'http://some-absolute/url') +assert.equal(import.meta.resolve('some://weird/protocol'), 'some://weird/protocol') assert.strictEqual(import.meta.resolve('baz/', fixtures), fixtures + 'node_modules/baz/'); diff --git a/test/es-module/test-esm-loader-default-resolver.mjs b/test/es-module/test-esm-loader-default-resolver.mjs new file mode 100644 index 00000000000000..f23025127093ab --- /dev/null +++ b/test/es-module/test-esm-loader-default-resolver.mjs @@ -0,0 +1,30 @@ +import { spawnPromisified } from '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import assert from 'node:assert'; +import { execPath } from 'node:process'; +import {describe, it} from 'node:test' + +describe('default resolver', () => { + it('should accept foreign schemas without exception (e.g. uyyt://something/or-other', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + fixtures.fileURL('/es-module-loaders/uyyt-dummy-loader.mjs'), + fixtures.path('/es-module-loaders/uyyt-dummy-loader-main.mjs'), + ]); + assert.strictEqual(code, 0); + assert.strictEqual(stdout.trim(), 'index.mjs!'); + assert.strictEqual(stderr, ''); + }) + it('should resolve foreign schemas by doing regular url absolutization', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + fixtures.fileURL('/es-module-loaders/uyyt-dummy-loader.mjs'), + fixtures.path('/es-module-loaders/uyyt-dummy-loader-main2.mjs'), + ]); + assert.strictEqual(code, 0); + assert.strictEqual(stdout.trim(), '42'); + assert.strictEqual(stderr, ''); + }) +}) \ No newline at end of file diff --git a/test/fixtures/es-module-loaders/http-loader.mjs b/test/fixtures/es-module-loaders/http-loader.mjs index 8096dd9bb73a4c..4fe8fcb4797436 100644 --- a/test/fixtures/es-module-loaders/http-loader.mjs +++ b/test/fixtures/es-module-loaders/http-loader.mjs @@ -1,23 +1,5 @@ import { get } from 'http'; -export function resolve(specifier, context, nextResolve) { - const { parentURL = null } = context; - - if (specifier.startsWith('http://')) { - return { - shortCircuit: true, - url: specifier, - }; - } else if (parentURL?.startsWith('http://')) { - return { - shortCircuit: true, - url: new URL(specifier, parentURL).href, - }; - } - - return nextResolve(specifier); -} - export function load(url, context, nextLoad) { if (url.startsWith('http://')) { return new Promise((resolve, reject) => { diff --git a/test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs b/test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs new file mode 100644 index 00000000000000..e9b293aab883b1 --- /dev/null +++ b/test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs @@ -0,0 +1 @@ +import 'uyyt://1/index.mjs'; diff --git a/test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs b/test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs new file mode 100644 index 00000000000000..71b0403af728a1 --- /dev/null +++ b/test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs @@ -0,0 +1 @@ +import 'uyyt://1/index2.mjs'; diff --git a/test/fixtures/es-module-loaders/uyyt-dummy-loader.mjs b/test/fixtures/es-module-loaders/uyyt-dummy-loader.mjs new file mode 100644 index 00000000000000..360a974b431b33 --- /dev/null +++ b/test/fixtures/es-module-loaders/uyyt-dummy-loader.mjs @@ -0,0 +1,24 @@ +export function load(url, context, nextLoad) { + switch (url) { + case 'uyyt://1/index.mjs': + return { + source: 'console.log("index.mjs!")', + format: 'module', + shortCircuit: true, + }; + case 'uyyt://1/index2.mjs': + return { + source: 'import c from "./sub.mjs"; console.log(c);', + format: 'module', + shortCircuit: true, + }; + case 'uyyt://1/sub.mjs': + return { + source: 'export default 42', + format: 'module', + shortCircuit: true, + }; + default: + return nextLoad(url, context); + } +} From 94afcaee459c1a422c2ad32bdc789499483d5e45 Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Tue, 2 May 2023 05:57:03 +0300 Subject: [PATCH 02/12] fix lint errors --- lib/internal/modules/esm/load.js | 1 - lib/internal/modules/esm/resolve.js | 2 -- test/es-module/test-esm-import-meta-resolve.mjs | 4 ++-- test/es-module/test-esm-loader-default-resolver.mjs | 13 +++++++------ 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/internal/modules/esm/load.js b/lib/internal/modules/esm/load.js index 476fe6b91de968..9ab6f18f3fdda9 100644 --- a/lib/internal/modules/esm/load.js +++ b/lib/internal/modules/esm/load.js @@ -107,7 +107,6 @@ async function defaultLoad(url, context = kEmptyObject) { /** * throws an error if the protocol is not one of the protocols * that can be loaded in the default loader - * * @param {URL} parsed * @param {boolean} experimentalNetworkImports */ diff --git a/lib/internal/modules/esm/resolve.js b/lib/internal/modules/esm/resolve.js index 702b575bc1721e..927b118f8ede2b 100644 --- a/lib/internal/modules/esm/resolve.js +++ b/lib/internal/modules/esm/resolve.js @@ -3,7 +3,6 @@ const { ArrayIsArray, ArrayPrototypeJoin, - ArrayPrototypePush, ArrayPrototypeShift, JSONStringify, ObjectGetOwnPropertyNames, @@ -51,7 +50,6 @@ const { ERR_PACKAGE_PATH_NOT_EXPORTED, ERR_UNSUPPORTED_DIR_IMPORT, ERR_NETWORK_IMPORT_DISALLOWED, - ERR_UNSUPPORTED_ESM_URL_SCHEME, } = require('internal/errors').codes; const { Module: CJSModule } = require('internal/modules/cjs/loader'); diff --git a/test/es-module/test-esm-import-meta-resolve.mjs b/test/es-module/test-esm-import-meta-resolve.mjs index cf4774d38a277c..ec6cd37ab01e10 100644 --- a/test/es-module/test-esm-import-meta-resolve.mjs +++ b/test/es-module/test-esm-import-meta-resolve.mjs @@ -30,8 +30,8 @@ assert.strictEqual( code: 'ERR_INVALID_ARG_TYPE', }) ); -assert.equal(import.meta.resolve('http://some-absolute/url'), 'http://some-absolute/url') -assert.equal(import.meta.resolve('some://weird/protocol'), 'some://weird/protocol') +assert.strictEqual(import.meta.resolve('http://some-absolute/url'), 'http://some-absolute/url'); +assert.strictEqual(import.meta.resolve('some://weird/protocol'), 'some://weird/protocol'); assert.strictEqual(import.meta.resolve('baz/', fixtures), fixtures + 'node_modules/baz/'); diff --git a/test/es-module/test-esm-loader-default-resolver.mjs b/test/es-module/test-esm-loader-default-resolver.mjs index f23025127093ab..0ca936d977e6f1 100644 --- a/test/es-module/test-esm-loader-default-resolver.mjs +++ b/test/es-module/test-esm-loader-default-resolver.mjs @@ -2,11 +2,11 @@ import { spawnPromisified } from '../common/index.mjs'; import * as fixtures from '../common/fixtures.mjs'; import assert from 'node:assert'; import { execPath } from 'node:process'; -import {describe, it} from 'node:test' +import { describe, it } from 'node:test'; describe('default resolver', () => { it('should accept foreign schemas without exception (e.g. uyyt://something/or-other', async () => { - const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + const { code, stdout, stderr } = await spawnPromisified(execPath, [ '--no-warnings', '--experimental-loader', fixtures.fileURL('/es-module-loaders/uyyt-dummy-loader.mjs'), @@ -15,9 +15,10 @@ describe('default resolver', () => { assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), 'index.mjs!'); assert.strictEqual(stderr, ''); - }) + }); + it('should resolve foreign schemas by doing regular url absolutization', async () => { - const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + const { code, stdout, stderr } = await spawnPromisified(execPath, [ '--no-warnings', '--experimental-loader', fixtures.fileURL('/es-module-loaders/uyyt-dummy-loader.mjs'), @@ -26,5 +27,5 @@ describe('default resolver', () => { assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), '42'); assert.strictEqual(stderr, ''); - }) -}) \ No newline at end of file + }); +}); From c87c78c16c7404d692e837ed1eeea487938f56bf Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Tue, 2 May 2023 21:20:40 +0300 Subject: [PATCH 03/12] Update doc/api/esm.md Co-authored-by: Geoffrey Booth --- doc/api/esm.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index b8942f936dcdfe..cb66d4d8582c73 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1179,10 +1179,10 @@ loaded file. #### Overriding loader -The above two loaders hooked into the "load" phase of the module loader. -This loader hooks into the "resolution" phase. This loader reads an +The previous two loaders defined `load` hooks. This is an example of a loader +that does its work via the `resolve` hook. This loader reads an `overrides.json` file that specifies which specifiers to override to another -url. +URL. ```js // overriding-loader.js From 3df4003705dd7e424e7815cc78ade0bcea605ad4 Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Wed, 3 May 2023 17:06:41 +0300 Subject: [PATCH 04/12] added test for extensions, and simplified protocol test --- .../test-esm-loader-default-resolver.mjs | 25 +++++++++++++++---- .../byop-dummy-loader-main.mjs | 1 + .../byop-dummy-loader-main2.mjs | 1 + .../byop-dummy-loader-main3.mjs | 1 + ...dummy-loader.mjs => byop-dummy-loader.mjs} | 12 ++++++--- .../uyyt-dummy-loader-main.mjs | 1 - .../uyyt-dummy-loader-main2.mjs | 1 - 7 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs create mode 100644 test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs create mode 100644 test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs rename test/fixtures/es-module-loaders/{uyyt-dummy-loader.mjs => byop-dummy-loader.mjs} (66%) delete mode 100644 test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs delete mode 100644 test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs diff --git a/test/es-module/test-esm-loader-default-resolver.mjs b/test/es-module/test-esm-loader-default-resolver.mjs index 0ca936d977e6f1..07a5bafcbd3305 100644 --- a/test/es-module/test-esm-loader-default-resolver.mjs +++ b/test/es-module/test-esm-loader-default-resolver.mjs @@ -5,12 +5,14 @@ import { execPath } from 'node:process'; import { describe, it } from 'node:test'; describe('default resolver', () => { - it('should accept foreign schemas without exception (e.g. uyyt://something/or-other', async () => { + // In these tests `byop` is an acronym for "bring your own protocol", and is the + // protocol our byop-dummy-loader.mjs can load + it('should accept foreign schemas without exception (e.g. byop://something/or-other)', async () => { const { code, stdout, stderr } = await spawnPromisified(execPath, [ '--no-warnings', '--experimental-loader', - fixtures.fileURL('/es-module-loaders/uyyt-dummy-loader.mjs'), - fixtures.path('/es-module-loaders/uyyt-dummy-loader-main.mjs'), + fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), + fixtures.path('/es-module-loaders/byop-dummy-loader-main.mjs'), ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), 'index.mjs!'); @@ -21,11 +23,24 @@ describe('default resolver', () => { const { code, stdout, stderr } = await spawnPromisified(execPath, [ '--no-warnings', '--experimental-loader', - fixtures.fileURL('/es-module-loaders/uyyt-dummy-loader.mjs'), - fixtures.path('/es-module-loaders/uyyt-dummy-loader-main2.mjs'), + fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), + fixtures.path('/es-module-loaders/byop-dummy-loader-main2.mjs'), ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), '42'); assert.strictEqual(stderr, ''); }); + + // In this test, `byoe` is an acronym for "bring your own extension" + it('should accept foreign extensions without exception (e.g. ..//something.byoe)', async () => { + const { code, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), + fixtures.path('/es-module-loaders/byop-dummy-loader-main3.mjs'), + ]); + assert.strictEqual(code, 0); + assert.strictEqual(stdout.trim(), 'index.byoe!'); + assert.strictEqual(stderr, ''); + }); }); diff --git a/test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs b/test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs new file mode 100644 index 00000000000000..4b0156bf3af824 --- /dev/null +++ b/test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs @@ -0,0 +1 @@ +import 'byop://1/index.mjs'; diff --git a/test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs b/test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs new file mode 100644 index 00000000000000..5d406447c09079 --- /dev/null +++ b/test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs @@ -0,0 +1 @@ +import 'byop://1/index2.mjs'; diff --git a/test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs b/test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs new file mode 100644 index 00000000000000..ae5bae546c5c3a --- /dev/null +++ b/test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs @@ -0,0 +1 @@ +import 'byop://1/index.byoe'; diff --git a/test/fixtures/es-module-loaders/uyyt-dummy-loader.mjs b/test/fixtures/es-module-loaders/byop-dummy-loader.mjs similarity index 66% rename from test/fixtures/es-module-loaders/uyyt-dummy-loader.mjs rename to test/fixtures/es-module-loaders/byop-dummy-loader.mjs index 360a974b431b33..17a3430b44e3f6 100644 --- a/test/fixtures/es-module-loaders/uyyt-dummy-loader.mjs +++ b/test/fixtures/es-module-loaders/byop-dummy-loader.mjs @@ -1,23 +1,29 @@ export function load(url, context, nextLoad) { switch (url) { - case 'uyyt://1/index.mjs': + case 'byop://1/index.mjs': return { source: 'console.log("index.mjs!")', format: 'module', shortCircuit: true, }; - case 'uyyt://1/index2.mjs': + case 'byop://1/index2.mjs': return { source: 'import c from "./sub.mjs"; console.log(c);', format: 'module', shortCircuit: true, }; - case 'uyyt://1/sub.mjs': + case 'byop://1/sub.mjs': return { source: 'export default 42', format: 'module', shortCircuit: true, }; + case 'byop://1/index.byoe': + return { + source: 'console.log("index.byoe!")', + format: 'module', + shortCircuit: true, + }; default: return nextLoad(url, context); } diff --git a/test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs b/test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs deleted file mode 100644 index e9b293aab883b1..00000000000000 --- a/test/fixtures/es-module-loaders/uyyt-dummy-loader-main.mjs +++ /dev/null @@ -1 +0,0 @@ -import 'uyyt://1/index.mjs'; diff --git a/test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs b/test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs deleted file mode 100644 index 71b0403af728a1..00000000000000 --- a/test/fixtures/es-module-loaders/uyyt-dummy-loader-main2.mjs +++ /dev/null @@ -1 +0,0 @@ -import 'uyyt://1/index2.mjs'; From f615248b81cfd6cd9222ab35ad89a34a326a4491 Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Wed, 3 May 2023 17:48:28 +0300 Subject: [PATCH 05/12] remove fixtures by inlining them in the test --- test/es-module/test-esm-loader-default-resolver.mjs | 12 +++++++++--- .../es-module-loaders/byop-dummy-loader-main.mjs | 1 - .../es-module-loaders/byop-dummy-loader-main2.mjs | 1 - .../es-module-loaders/byop-dummy-loader-main3.mjs | 1 - 4 files changed, 9 insertions(+), 6 deletions(-) delete mode 100644 test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs delete mode 100644 test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs delete mode 100644 test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs diff --git a/test/es-module/test-esm-loader-default-resolver.mjs b/test/es-module/test-esm-loader-default-resolver.mjs index 07a5bafcbd3305..e733d053dd1b3d 100644 --- a/test/es-module/test-esm-loader-default-resolver.mjs +++ b/test/es-module/test-esm-loader-default-resolver.mjs @@ -12,7 +12,9 @@ describe('default resolver', () => { '--no-warnings', '--experimental-loader', fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), - fixtures.path('/es-module-loaders/byop-dummy-loader-main.mjs'), + '--input-type=module', + '--eval', + "import 'byop://1/index.mjs'" ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), 'index.mjs!'); @@ -24,7 +26,9 @@ describe('default resolver', () => { '--no-warnings', '--experimental-loader', fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), - fixtures.path('/es-module-loaders/byop-dummy-loader-main2.mjs'), + '--input-type=module', + '--eval', + "import 'byop://1/index2.mjs'" ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), '42'); @@ -37,7 +41,9 @@ describe('default resolver', () => { '--no-warnings', '--experimental-loader', fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), - fixtures.path('/es-module-loaders/byop-dummy-loader-main3.mjs'), + '--input-type=module', + '--eval', + "import 'byop://1/index.byoe'" ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), 'index.byoe!'); diff --git a/test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs b/test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs deleted file mode 100644 index 4b0156bf3af824..00000000000000 --- a/test/fixtures/es-module-loaders/byop-dummy-loader-main.mjs +++ /dev/null @@ -1 +0,0 @@ -import 'byop://1/index.mjs'; diff --git a/test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs b/test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs deleted file mode 100644 index 5d406447c09079..00000000000000 --- a/test/fixtures/es-module-loaders/byop-dummy-loader-main2.mjs +++ /dev/null @@ -1 +0,0 @@ -import 'byop://1/index2.mjs'; diff --git a/test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs b/test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs deleted file mode 100644 index ae5bae546c5c3a..00000000000000 --- a/test/fixtures/es-module-loaders/byop-dummy-loader-main3.mjs +++ /dev/null @@ -1 +0,0 @@ -import 'byop://1/index.byoe'; From e4b4cdfe12979e0aa589bca5ac7339bb7cdc7853 Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Wed, 3 May 2023 17:48:46 +0300 Subject: [PATCH 06/12] change load example to "import map" semantics --- doc/api/esm.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index cb66d4d8582c73..9267c56bab2495 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1177,22 +1177,23 @@ loaded from disk but before Node.js executes it; and so on for any `.coffee`, `.litcoffee` or `.coffee.md` files referenced via `import` statements of any loaded file. -#### Overriding loader +#### "import map" loader The previous two loaders defined `load` hooks. This is an example of a loader that does its work via the `resolve` hook. This loader reads an -`overrides.json` file that specifies which specifiers to override to another -URL. +`import-map.json` file that specifies which specifiers to override to another +URL (this is a very simplistic implemenation of a small subset of the +"import maps" specification). ```js -// overriding-loader.js +// import-map-loader.js import fs from 'node:fs/promises'; -const overrides = JSON.parse(await fs.readFile('overrides.json')); +const {imports} = JSON.parse(await fs.readFile('import-map.json')); export async function resolve(specifier, context, nextResolve) { - if (specifier in overrides) { - return nextResolve(overrides[specifier], context); + if (specifier in imports) { + return nextResolve(imports[specifier], context); } return nextResolve(specifier, context); @@ -1203,23 +1204,25 @@ Let's assume we have these files: ```js // main.js -import 'a-module-to-override'; +import 'a-module'; ``` ```json -// overrides.json +// import-map.json { - "a-module-to-override": "./module-override.js" + "imports": { + "a-module": "./some-module.js" + } } ``` ```js -// module-override.js -console.log('module overridden!'); +// some-module.js +console.log('some module!'); ``` -If you run `node --experimental-loader ./overriding-loader.js main.js` -the output will be `module overriden!`. +If you run `node --experimental-loader ./import-map-loader.js main.js` +the output will be `some module!`. ## Resolution algorithm From a387b9c7aeb7a73710478d6f4090fabc7013479b Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Wed, 3 May 2023 17:56:52 +0300 Subject: [PATCH 07/12] fix lint errors --- doc/api/esm.md | 2 +- test/es-module/test-esm-loader-default-resolver.mjs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index 9267c56bab2495..4ec9f6cf43be48 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1189,7 +1189,7 @@ URL (this is a very simplistic implemenation of a small subset of the // import-map-loader.js import fs from 'node:fs/promises'; -const {imports} = JSON.parse(await fs.readFile('import-map.json')); +const { imports } = JSON.parse(await fs.readFile('import-map.json')); export async function resolve(specifier, context, nextResolve) { if (specifier in imports) { diff --git a/test/es-module/test-esm-loader-default-resolver.mjs b/test/es-module/test-esm-loader-default-resolver.mjs index e733d053dd1b3d..27320fcfcfe862 100644 --- a/test/es-module/test-esm-loader-default-resolver.mjs +++ b/test/es-module/test-esm-loader-default-resolver.mjs @@ -14,7 +14,7 @@ describe('default resolver', () => { fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), '--input-type=module', '--eval', - "import 'byop://1/index.mjs'" + "import 'byop://1/index.mjs'", ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), 'index.mjs!'); @@ -28,7 +28,7 @@ describe('default resolver', () => { fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), '--input-type=module', '--eval', - "import 'byop://1/index2.mjs'" + "import 'byop://1/index2.mjs'", ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), '42'); @@ -43,7 +43,7 @@ describe('default resolver', () => { fixtures.fileURL('/es-module-loaders/byop-dummy-loader.mjs'), '--input-type=module', '--eval', - "import 'byop://1/index.byoe'" + "import 'byop://1/index.byoe'", ]); assert.strictEqual(code, 0); assert.strictEqual(stdout.trim(), 'index.byoe!'); From 88a4ce9bc531db7c924085d38c82886fc3313240 Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Thu, 4 May 2023 07:42:49 +0300 Subject: [PATCH 08/12] Update doc/api/esm.md to improve "import map" loader Co-authored-by: Antoine du Hamel --- doc/api/esm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index 4ec9f6cf43be48..63f7784e89493e 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1192,7 +1192,7 @@ import fs from 'node:fs/promises'; const { imports } = JSON.parse(await fs.readFile('import-map.json')); export async function resolve(specifier, context, nextResolve) { - if (specifier in imports) { + if (Object.hasOwn(imports, specifier)) { return nextResolve(imports[specifier], context); } From b0eb1c0676fa51a6f9c0895a34d2da4751edbfe6 Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Fri, 5 May 2023 10:42:07 +0300 Subject: [PATCH 09/12] Document resolution not failing on url protocol --- doc/api/esm.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/api/esm.md b/doc/api/esm.md index 63f7784e89493e..4dfa94b8db1d6c 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1236,6 +1236,7 @@ The resolver has the following properties: * No default extensions * No folder mains * Bare specifier package resolution lookup through node\_modules +* Does not fail on unknown extensions or protocols ### Resolver algorithm @@ -1243,6 +1244,11 @@ The algorithm to load an ES module specifier is given through the **ESM\_RESOLVE** method below. It returns the resolved URL for a module specifier relative to a parentURL. +The algorithm does not determine whether the resolved URL protocol can be +loaded by Node.js, because other loaders may be able to load it. +The default loader _does_ fail if it was asked to load a URL +that has an unsuppported protocol. + The algorithm to determine the module format of a resolved URL is provided by **ESM\_FORMAT**, which returns the unique module format for any file. The _"module"_ format is returned for an ECMAScript From 98924ddcfa059c4c0fbde50409d68dd77eb958ee Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Sun, 7 May 2023 06:05:33 +0300 Subject: [PATCH 10/12] add some loader info to esm docs and change to "default resolver" --- doc/api/esm.md | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index 4dfa94b8db1d6c..b443ac526194b3 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -143,8 +143,9 @@ There are three types of specifiers: * _Absolute specifiers_ like `'file:///opt/nodejs/config.js'`. They refer directly and explicitly to a full path. -Bare specifier resolutions are handled by the [Node.js module resolution -algorithm][]. All other specifier resolutions are always only resolved with +Bare specifier resolutions are handled by the [Node.js default module +resolution and loading algorithm][]. +All other specifier resolutions are always only resolved with the standard relative [URL][] resolution semantics. Like in CommonJS, module files within packages can be accessed by appending a @@ -1224,21 +1225,29 @@ console.log('some module!'); If you run `node --experimental-loader ./import-map-loader.js main.js` the output will be `some module!`. -## Resolution algorithm +## Default resolution and loading algorithm ### Features -The resolver has the following properties: +The default resolver has the following properties: * FileURL-based resolution as is used by ES modules -* Support for builtin module loading * Relative and absolute URL resolution * No default extensions * No folder mains * Bare specifier package resolution lookup through node\_modules * Does not fail on unknown extensions or protocols -### Resolver algorithm +The default loader has the following properties + +* Support for builtin module loading via `node:` URLs +* Support for "inline" module loading via `data:` URLs +* Support for `file:` module loading +* Fails on any other URL protocol +* Fails on unknown extensions for `file:` loading + (supports only `.cjs`, `.js`, and `.mjs`) + +### Default resolver algorithm The algorithm to load an ES module specifier is given through the **ESM\_RESOLVE** method below. It returns the resolved URL for a @@ -1247,10 +1256,16 @@ module specifier relative to a parentURL. The algorithm does not determine whether the resolved URL protocol can be loaded by Node.js, because other loaders may be able to load it. The default loader _does_ fail if it was asked to load a URL -that has an unsuppported protocol. +that has an unsuppported protocol (not `file:`, `data:`, or `node:`). + +The algorithm also tries to determine the format of the file based +on the extension (see `ESM_FILE_FORMAT` algorithm below), but does +not fail if it does not recognize the extension. The default loader +_does_ fail if it was asked to load a file with an unsupported +extension (not `.mjs`, `.cjs`, or `.json`). The algorithm to determine the module format of a resolved URL is -provided by **ESM\_FORMAT**, which returns the unique module +provided by **ESM\_FILE\_FORMAT**, which returns the unique module format for any file. The _"module"_ format is returned for an ECMAScript Module, while the _"commonjs"_ format is used to indicate loading through the legacy CommonJS loader. Additional formats such as _"addon"_ can be extended in @@ -1277,7 +1292,7 @@ The resolver can throw the following errors: * _Unsupported Directory Import_: The resolved path corresponds to a directory, which is not a supported target for module imports. -### Resolver Algorithm Specification +### Default resolver Algorithm Specification **ESM\_RESOLVE**(_specifier_, _parentURL_) @@ -1311,7 +1326,7 @@ The resolver can throw the following errors: > 8. Otherwise, > 1. Set _format_ the module format of the content type associated with the > URL _resolved_. -> 9. Load _resolved_ as module format, _format_. +> 9. Return _format_ and _resolved_ **PACKAGE\_RESOLVE**(_packageSpecifier_, _parentURL_) @@ -1516,9 +1531,9 @@ _isImports_, _conditions_) > 7. If _pjson?.type_ exists and is _"module"_, then > 1. If _url_ ends in _".js"_, then > 1. Return _"module"_. -> 2. return **undefined**. +> 2. Return **undefined**. > 8. Otherwise, -> 1. return **undefined**. +> 1. Return **undefined**. **LOOKUP\_PACKAGE\_SCOPE**(_url_) From 357e8db7e948e7580f688301b3e4e6a10995c34a Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Tue, 9 May 2023 17:38:21 +0300 Subject: [PATCH 11/12] esm.md fixed based on comments --- doc/api/esm.md | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index b443ac526194b3..df50f93dddf1c9 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -143,7 +143,7 @@ There are three types of specifiers: * _Absolute specifiers_ like `'file:///opt/nodejs/config.js'`. They refer directly and explicitly to a full path. -Bare specifier resolutions are handled by the [Node.js default module +Bare specifier resolutions are handled by the [Node.js module resolution and loading algorithm][]. All other specifier resolutions are always only resolved with the standard relative [URL][] resolution semantics. @@ -1225,7 +1225,7 @@ console.log('some module!'); If you run `node --experimental-loader ./import-map-loader.js main.js` the output will be `some module!`. -## Default resolution and loading algorithm +## Resolution and loading algorithm ### Features @@ -1237,6 +1237,7 @@ The default resolver has the following properties: * No folder mains * Bare specifier package resolution lookup through node\_modules * Does not fail on unknown extensions or protocols +* Can optionally provide a hint of the format to the loading phase The default loader has the following properties @@ -1247,22 +1248,26 @@ The default loader has the following properties * Fails on unknown extensions for `file:` loading (supports only `.cjs`, `.js`, and `.mjs`) -### Default resolver algorithm +### Resolution algorithm The algorithm to load an ES module specifier is given through the **ESM\_RESOLVE** method below. It returns the resolved URL for a module specifier relative to a parentURL. -The algorithm does not determine whether the resolved URL protocol can be -loaded by Node.js, because other loaders may be able to load it. -The default loader _does_ fail if it was asked to load a URL -that has an unsuppported protocol (not `file:`, `data:`, or `node:`). +The resolution algorithm determines the full resolved URL for a module +load, along with its suggested module format. The resolution algorithm +does not determine whether the resolved URL protocol can be loaded, +or whether the file extensions are permitted, instead these validations +are applied by Node.js during the load phase +(for example, if it was asked to load a URL that has a protocol that is +not `file:`, `data:`, `node:`, or if `--experimental-network-imports` +is enabled, `https:`). The algorithm also tries to determine the format of the file based -on the extension (see `ESM_FILE_FORMAT` algorithm below), but does -not fail if it does not recognize the extension. The default loader -_does_ fail if it was asked to load a file with an unsupported -extension (not `.mjs`, `.cjs`, or `.json`). +on the extension (see `ESM_FILE_FORMAT` algorithm below). If it does +not recognize the file extension (eg if it is not `.mjs`, `.cjs`, or +`.json`), then a format of `undefined` is returned, +which will throw during the load phase. The algorithm to determine the module format of a resolved URL is provided by **ESM\_FILE\_FORMAT**, which returns the unique module @@ -1292,7 +1297,7 @@ The resolver can throw the following errors: * _Unsupported Directory Import_: The resolved path corresponds to a directory, which is not a supported target for module imports. -### Default resolver Algorithm Specification +### Resolution Algorithm Specification **ESM\_RESOLVE**(_specifier_, _parentURL_) @@ -1326,7 +1331,7 @@ The resolver can throw the following errors: > 8. Otherwise, > 1. Set _format_ the module format of the content type associated with the > URL _resolved_. -> 9. Return _format_ and _resolved_ +> 9. Return _format_ and _resolved_ to the loading phase **PACKAGE\_RESOLVE**(_packageSpecifier_, _parentURL_) @@ -1577,7 +1582,7 @@ for ESM specifiers is [commonjs-extension-resolution-loader][]. [Import Assertions proposal]: https://github.com/tc39/proposal-import-assertions [JSON modules]: #json-modules [Loaders API]: #loaders -[Node.js Module Resolution Algorithm]: #resolver-algorithm-specification +[Node.js Module Resolution And Loading Algorithm]: #resolution-algorithm-specification [Terminology]: #terminology [URL]: https://url.spec.whatwg.org/ [`"exports"`]: packages.md#exports From 2c7e695c9dd36ff7a4db34216190c71dbb3fc03a Mon Sep 17 00:00:00 2001 From: Gil Tayar Date: Fri, 19 May 2023 02:51:16 +0300 Subject: [PATCH 12/12] fix expected error message --- test/es-module/test-esm-dynamic-import.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/es-module/test-esm-dynamic-import.js b/test/es-module/test-esm-dynamic-import.js index 95d34244357e7c..ac6b35ebc1bc15 100644 --- a/test/es-module/test-esm-dynamic-import.js +++ b/test/es-module/test-esm-dynamic-import.js @@ -59,7 +59,7 @@ function expectFsNamespace(result) { 'ERR_UNSUPPORTED_ESM_URL_SCHEME'); if (common.isWindows) { const msg = - 'Only URLs with a scheme in: file and data are supported by the default ' + + 'Only URLs with a scheme in: file, data, and node are supported by the default ' + 'ESM loader. On Windows, absolute paths must be valid file:// URLs. ' + "Received protocol 'c:'"; expectModuleError(import('C:\\example\\foo.mjs'),