Implement the nodejs_compat_v2 compat flag

The `nodejs_compat_v2` flag supersedes the 'nodejs_compat' flag.
When enabled...

1. Node.js built-ins are available for import/require
2. Unlike the original `nodejs_compat` flag, Node.js imports do not
   require the 'node:' specifier prefix. Internally, the implementation
   will detect a Node.js raw specifier and convert it into the appropriate
   prefixed specifier, e.g. 'fs' becomes 'node:fs'
3. The `Buffer` and `process` global objects are exposed on the global
4. A user worker bundle can still provide it's own implementations of
   all `node:` modules which will take precendence over the built-ins
5. The new `process.getBuiltinModule(...)` API is implemented.
   See nodejs/node#52762

A worker can replace the implementation of `node:process` if they
choose, which may mean that `getBuiltinModule(...)` is not available.

src/node/internal/process.ts

@@ -16,6 +16,8 @@ import {
   ERR_INVALID_ARG_VALUE,
 } from 'node-internal:internal_errors'
 
+import { default as utilImpl } from 'node-internal:util';
+
 export function nextTick(cb: Function, ...args: unknown[]) {
   queueMicrotask(() => { cb(...args); });
 };
@@ -62,7 +64,12 @@ export const env = new Proxy({}, {
   }
 });
 
+export function getBuiltinModule(id: string) : any {
+  return utilImpl.getBuiltinModule(id);
+}
+
 export default {
   nextTick,
   env,
+  getBuiltinModule,
 };

src/node/internal/util.d.ts

@@ -107,3 +107,5 @@ export function isWeakMap(value: unknown): value is WeakMap<any, unknown>;
 export function isWeakSet(value: unknown): value is WeakSet<any>;
 export function isAnyArrayBuffer(value: unknown): value is ArrayBuffer | SharedArrayBuffer;
 export function isBoxedPrimitive(value: unknown): value is Number | String | Boolean | BigInt | Symbol;
+
+export function getBuiltinModule(id: string): any;

src/workerd/api/global-scope.c++

@@ -777,6 +777,56 @@ void ServiceWorkerGlobalScope::reportError(jsg::Lock& js, jsg::JsValue error) {
   }
 }
 
+namespace {
+jsg::JsValue resolveFromRegistry(jsg::Lock& js, kj::StringPtr specifier) {
+  auto moduleRegistry = jsg::ModuleRegistry::from(js);
+  if (moduleRegistry == nullptr) {
+    // TODO: Return the known built-in instead? This gets a bit tricky as we currently
+    // have no mechanism defined for accessing and caching the built-in module without
+    // the module registry. Should we even support this case at all? Without the module
+    // registry the user can't access the other importable modules anyway. For now, just
+    // returning undefined in this case seems the most appropriate.
+    return js.undefined();
+  }
+
+  auto spec = kj::Path::parse(specifier);
+  auto& info = JSG_REQUIRE_NONNULL(moduleRegistry->resolve(js, spec), Error,
+                                   kj::str("No such module: ", specifier));
+  auto module = info.module.getHandle(js);
+  jsg::instantiateModule(js, module);
+
+  auto handle = jsg::check(module->Evaluate(js.v8Context()));
+  KJ_ASSERT(handle->IsPromise());
+  auto prom = handle.As<v8::Promise>();
+  KJ_ASSERT(prom->State() != v8::Promise::PromiseState::kPending);
+  if (module->GetStatus() == v8::Module::kErrored) {
+    jsg::throwTunneledException(js.v8Isolate, module->GetException());
+  }
+  return jsg::JsValue(js.v8Get(module->GetModuleNamespace().As<v8::Object>(), "default"_kj));
+}
+}  // namespace
+
+jsg::JsValue ServiceWorkerGlobalScope::getBuffer(jsg::Lock& js) {
+  KJ_ASSERT(FeatureFlags::get(js).getNodeJsCompatV2());
+  auto value = resolveFromRegistry(js, "node:buffer"_kj);
+  auto obj = JSG_REQUIRE_NONNULL(value.tryCast<jsg::JsObject>(), TypeError,
+                                 "Invalid node:buffer implementation");
+  // Unlike the getProcess() case below, this getter is returning an object that
+  // is exported by the node:buffer module and not the module itself, so we need
+  // this additional get to the grab the reference to the thing we're actually
+  // returning.
+  auto buffer = obj.get(js, "Buffer"_kj);
+  JSG_REQUIRE(buffer.isFunction(), TypeError, "Invalid node:buffer implementation");
+  return buffer;
+}
+
+jsg::JsValue ServiceWorkerGlobalScope::getProcess(jsg::Lock& js) {
+  KJ_ASSERT(FeatureFlags::get(js).getNodeJsCompatV2());
+  auto value = resolveFromRegistry(js, "node:process"_kj);
+  JSG_REQUIRE(value.isObject(), TypeError, "Invalid node:process implementation");
+  return value;
+}
+
 double Performance::now() {
   // We define performance.now() for compatibility purposes, but due to spectre concerns it
   // returns exactly what Date.now() returns.

src/workerd/api/global-scope.h

@@ -454,6 +454,12 @@ class ServiceWorkerGlobalScope: public WorkerGlobalScope {
 
   void reportError(jsg::Lock& js, jsg::JsValue error);
 
+  // When the nodejs_compat_v2 compatibility flag is enabled, we expose the Node.js
+  // compat Buffer and process at the global scope in all modules as lazy instance
+  // properties.
+  jsg::JsValue getBuffer(jsg::Lock& js);
+  jsg::JsValue getProcess(jsg::Lock& js);
+
   JSG_RESOURCE_TYPE(ServiceWorkerGlobalScope, CompatibilityFlags::Reader flags) {
     JSG_INHERIT(WorkerGlobalScope);
 
@@ -527,6 +533,12 @@ class ServiceWorkerGlobalScope: public WorkerGlobalScope {
       JSG_NESTED_TYPE(TransformStreamDefaultController);
     }
 
+    if (flags.getNodeJsCompatV2()) {
+      JSG_LAZY_INSTANCE_PROPERTY(Buffer, getBuffer);
+      JSG_LAZY_INSTANCE_PROPERTY(process, getProcess);
+      JSG_LAZY_INSTANCE_PROPERTY(global, getSelf);
+    }
+
     JSG_NESTED_TYPE(CompressionStream);
     JSG_NESTED_TYPE(DecompressionStream);
     JSG_NESTED_TYPE(TextEncoderStream);

src/workerd/api/node/node-compat-v2-test.js

@@ -0,0 +1,64 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+//
+
+// Imports of Node.js built-ins should work both with and without
+// the 'node:' prefix.
+import { default as assert } from 'node:assert';
+import { default as assert2 } from 'assert';
+const assert3 = (await import('node:assert')).default;
+const assert4 = (await import('assert')).default;
+
+assert.strictEqual(assert, assert2);
+assert.strictEqual(assert, assert3);
+assert.strictEqual(assert, assert4);
+
+export const nodeJsExpectedGlobals = {
+  async test() {
+    // Expected Node.js globals Buffer, process, and global should be present.
+    const { Buffer } = await import('node:buffer');
+    assert.strictEqual(Buffer, globalThis.Buffer);
+
+    const { default: process } = await import('node:process');
+    assert.strictEqual(process, globalThis.process);
+
+    assert.strictEqual(global, globalThis);
+  }
+};
+
+export const nodeJsGetBuiltins = {
+  async test() {
+    // node:* modules in the worker bundle should override the built-in modules...
+    const { default: fs } = await import('node:fs');
+    const { default: path } = await import('node:path');
+
+    await import ('node:path');
+
+    // But process.getBuiltinModule should always return the built-in module.
+    const builtInPath = process.getBuiltinModule('node:path');
+    const builtInFs = process.getBuiltinModule('node:fs');
+
+    // These are from the worker bundle....
+    assert.strictEqual(fs, 1);
+    assert.strictEqual(path, 2);
+
+    // But these are from the built-ins...
+    // node:fs is not implemented currently so it should be undefined here.
+    assert.strictEqual(builtInFs, undefined);
+
+    // node:path is implemented tho...
+    assert.notStrictEqual(path, builtInPath);
+    assert.strictEqual(typeof builtInPath, 'object');
+    assert.strictEqual(typeof builtInPath.join, 'function');
+
+    // While process.getBuiltinModule(...) in Node.js only returns Node.js
+    // built-ins, our impl will return cloudflare: and workerd: built-ins
+    // also, for completeness. A key difference, however, is that for non-Node.js
+    // built-ins, the return value is the module namespace rather than the default
+    // export.
+
+    const socket = await import('cloudflare:sockets');
+    assert.strictEqual(process.getBuiltinModule('cloudflare:sockets'), socket);
+  }
+};

src/workerd/api/node/node-compat-v2-test.wd-test

@@ -0,0 +1,17 @@
+using Workerd = import "/workerd/workerd.capnp";
+
+const unitTests :Workerd.Config = (
+  services = [
+    ( name = "node-compat-v2-test",
+      worker = (
+        modules = [
+          (name = "worker", esModule = embed "node-compat-v2-test.js"),
+          (name = "node:fs", esModule = "export default 1"),
+          (name = "node:path", esModule = "export default 2"),
+        ],
+        compatibilityDate = "2024-05-01",
+        compatibilityFlags = ["nodejs_compat_v2", "experimental"]
+      )
+    ),
+  ],
+);

src/workerd/api/node/node.h

@@ -60,16 +60,19 @@ void registerNodeJsCompatModules(
 #undef V
 #undef NODEJS_MODULES
 
+  bool nodeJsCompatEnabled = featureFlags.getNodeJsCompat() ||
+                             featureFlags.getNodeJsCompatV2();
+
   // If the `nodejs_compat` flag isn't enabled, only register internal modules.
   // We need these for `console.log()`ing when running `workerd` locally.
   kj::Maybe<jsg::ModuleType> maybeFilter;
-  if (!featureFlags.getNodeJsCompat()) maybeFilter = jsg::ModuleType::INTERNAL;
+  if (!nodeJsCompatEnabled) maybeFilter = jsg::ModuleType::INTERNAL;
 
   registry.addBuiltinBundle(NODE_BUNDLE, maybeFilter);
 
   // If the `nodejs_compat` flag is off, but the `nodejs_als` flag is on, we
   // need to register the `node:async_hooks` module from the bundle.
-  if (!featureFlags.getNodeJsCompat() && featureFlags.getNodeJsAls()) {
+  if (!nodeJsCompatEnabled && featureFlags.getNodeJsAls()) {
     jsg::Bundle::Reader reader = NODE_BUNDLE;
     for (auto module : reader.getModules()) {
       auto specifier = module.getName();

src/workerd/api/node/util.c++

@@ -3,6 +3,7 @@
 //     https://opensource.org/licenses/Apache-2.0
 #include "util.h"
 #include <kj/vector.h>
+#include <workerd/jsg/modules.h>
 
 namespace workerd::api::node {
 
@@ -181,5 +182,46 @@ jsg::Name UtilModule::getResourceTypeInspect(jsg::Lock& js) {
   return js.newApiSymbol("kResourceTypeInspect"_kj);
 }
 
+jsg::JsValue UtilModule::getBuiltinModule(jsg::Lock& js, kj::String specifier) {
+  auto rawSpecifier = kj::str(specifier);
+  bool isNode = false;
+  KJ_IF_SOME(spec, jsg::checkNodeSpecifier(specifier)) {
+    isNode = true;
+    specifier = kj::mv(spec);
+  }
+
+  auto registry = jsg::ModuleRegistry::from(js);
+  if (registry == nullptr) return js.undefined();
+  auto path = kj::Path::parse(specifier);
+
+  KJ_IF_SOME(info, registry->resolve(js, path, kj::none,
+      jsg::ModuleRegistry::ResolveOption::BUILTIN_ONLY,
+      jsg::ModuleRegistry::ResolveMethod::IMPORT,
+      rawSpecifier.asPtr())) {
+    auto module = info.module.getHandle(js);
+    jsg::instantiateModule(js, module);
+    auto handle = jsg::check(module->Evaluate(js.v8Context()));
+    KJ_ASSERT(handle->IsPromise());
+    auto prom = handle.As<v8::Promise>();
+    KJ_ASSERT(prom->State() != v8::Promise::PromiseState::kPending);
+    if (module->GetStatus() == v8::Module::kErrored) {
+      jsg::throwTunneledException(js.v8Isolate, module->GetException());
+    }
+
+    // For Node.js modules, we want to grab the default export and return that.
+    // For other built-ins, we'll return the module namespace instead. Can be
+    // a bit confusing but it's a side effect of Node.js modules originally
+    // being commonjs and the official getBuiltinModule returning what is
+    // expected to be the default export, while the behavior of other built-ins
+    // is not really defined by Node.js' implementation.
+    if (isNode) {
+      return jsg::JsValue(js.v8Get(module->GetModuleNamespace().As<v8::Object>(), "default"_kj));
+    } else {
+      return jsg::JsValue(module->GetModuleNamespace());
+    }
+  }
+
+  return js.undefined();
+}
 
 }  // namespace workerd::api::node

src/workerd/api/node/util.h

@@ -196,6 +196,8 @@ class UtilModule final: public jsg::Object {
   bool isAnyArrayBuffer(jsg::JsValue value);
   bool isBoxedPrimitive(jsg::JsValue value);
 
+  jsg::JsValue getBuiltinModule(jsg::Lock& js, kj::String specifier);
+
   JSG_RESOURCE_TYPE(UtilModule) {
     JSG_NESTED_TYPE(MIMEType);
     JSG_NESTED_TYPE(MIMEParams);
@@ -220,6 +222,8 @@ class UtilModule final: public jsg::Object {
   #undef V
     JSG_METHOD(isAnyArrayBuffer);
     JSG_METHOD(isBoxedPrimitive);
+
+    JSG_METHOD(getBuiltinModule);
   }
 };
 

src/workerd/io/compatibility-date.capnp

@@ -441,4 +441,12 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
     $compatDisableFlag("fetch_legacy_url")
     $compatEnableDate("2024-06-03");
   # Ensures that WHATWG standard URL parsing is used in the fetch API implementation.
+
+  nodeJsCompatV2 @50 :Bool
+      $compatEnableFlag("nodejs_compat_v2")
+      $compatDisableFlag("no_nodejs_compat_v2")
+      $experimental;
+  # Implies nodeJSCompat with the following additional modifications:
+  # * Node.js Compat built-ins may be imported/required with or without the node: prefix
+  # * Node.js Compat the globals Buffer and process are available everywhere
 }

src/workerd/io/worker.c++

@@ -1018,6 +1018,9 @@ Worker::Isolate::Isolate(kj::Own<Api> apiParam,
 
     lock->setCaptureThrowsAsRejections(features.getCaptureThrowsAsRejections());
     lock->setCommonJsExportDefault(features.getExportCommonJsDefaultNamespace());
+    if (features.getNodeJsCompatV2()) {
+      lock->setNodeJsCompatEnabled();
+    }
 
     if (impl->inspector != kj::none || ::kj::_::Debug::shouldLog(::kj::LogSeverity::INFO)) {
       lock->setLoggerCallback([this](jsg::Lock& js, kj::StringPtr message) {

src/workerd/jsg/jsg.c++

@@ -192,6 +192,10 @@ void Lock::setCaptureThrowsAsRejections(bool capture) {
   IsolateBase::from(v8Isolate).setCaptureThrowsAsRejections({}, capture);
 }
 
+void Lock::setNodeJsCompatEnabled() {
+  IsolateBase::from(v8Isolate).setNodeJsCompatEnabled({}, true);
+}
+
 void Lock::setCommonJsExportDefault(bool exportDefault) {
   IsolateBase::from(v8Isolate).setCommonJsExportDefault({}, exportDefault);
 }

src/workerd/jsg/jsg.h

@@ -2296,6 +2296,8 @@ class Lock {
   void setCaptureThrowsAsRejections(bool capture);
   void setCommonJsExportDefault(bool exportDefault);
 
+  void setNodeJsCompatEnabled();
+
   using Logger = void(Lock&, kj::StringPtr);
   void setLoggerCallback(kj::Function<Logger>&& logger);