[Flight] Add bundler-less version of RSC using plain ESM (facebook#26889

)

This isn't really meant to be actually used, there are many issues with
this approach, but it shows the capabilities as a proof-of-concept.

It's a new reference implementation package `react-server-dom-esm` as
well as a fixture in `fixtures/flight-esm` (fork of `fixtures/flight`).
This works pretty much the same as pieces we already have in the Webpack
implementation but instead of loading modules using Webpack on the
client it uses native browser ESM.

To really show it off, I don't use any JSX in the fixture and so it also
doesn't use Babel or any compilation of the files.

This works because we don't actually bundle the server in the reference
implementation in the first place. We instead use [Node.js
Loaders](https://nodejs.org/api/esm.html#loaders) to intercept files
that contain `"use client"` and `"use server"` and replace them. There's
a simple check for those exact bytes, and no parsing, so this is very
fast.

Since the client isn't actually bundled, there's no module map needed.
We can just send the file path to the file we want to load in the RSC
payload for client references.

Since the existing reference implementation for Node.js already used ESM
to load modules on the server, that all works the same, including Server
Actions. No bundling.

There is one case that isn't implemented here. Importing a `"use
server"` file from a Client Component. We don't have that implemented in
the Webpack reference implementation neither - only in Next.js atm. In
Webpack it would be implemented as a Webpack loader.

There are a few ways this can be implemented without a bundler:

- We can intercept the request from the browser importing this file in
the HTTP server, and do a quick scan for `"use server"` in the file and
replace it just like we do with loaders in Node.js. This is effectively
how Vite works and likely how anyone using this technique would have to
support JSX anyway.
- We can use native browser "loaders" once that's eventually available
in the same way as in Node.js.
- We can generate import maps for each file and replace it with a
pointer to a placeholder file. This requires scanning these ahead of
time which defeats the purposes.

Another case that's not implemented is the inline `"use server"` closure
in a Server Component. That would require the existing loader to be a
bit smarter but would still only "compile" files that contains those
bytes in the fast path check. This would also happen in the loader that
already exists so wouldn't do anything substantially different than what
we currently have here.

.eslintrc.js

@@ -324,6 +324,7 @@ module.exports = {
         'packages/react-devtools-shared/**/*.js',
         'packages/react-noop-renderer/**/*.js',
         'packages/react-refresh/**/*.js',
+        'packages/react-server-dom-esm/**/*.js',
         'packages/react-server-dom-webpack/**/*.js',
         'packages/react-test-renderer/**/*.js',
         'packages/react-debug-tools/**/*.js',

fixtures/flight-esm/.gitignore

@@ -0,0 +1,20 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

fixtures/flight-esm/loader/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

fixtures/flight-esm/loader/region.js

@@ -0,0 +1,52 @@
+import {
+  resolve,
+  load as reactLoad,
+  getSource as getSourceImpl,
+  transformSource as reactTransformSource,
+} from 'react-server-dom-esm/node-loader';
+
+export {resolve};
+
+async function textLoad(url, context, defaultLoad) {
+  const {format} = context;
+  const result = await defaultLoad(url, context, defaultLoad);
+  if (result.format === 'module') {
+    if (typeof result.source === 'string') {
+      return result;
+    }
+    return {
+      source: Buffer.from(result.source).toString('utf8'),
+      format: 'module',
+    };
+  }
+  return result;
+}
+
+export async function load(url, context, defaultLoad) {
+  return await reactLoad(url, context, (u, c) => {
+    return textLoad(u, c, defaultLoad);
+  });
+}
+
+async function textTransformSource(source, context, defaultTransformSource) {
+  const {format} = context;
+  if (format === 'module') {
+    if (typeof source === 'string') {
+      return {source};
+    }
+    return {
+      source: Buffer.from(source).toString('utf8'),
+    };
+  }
+  return defaultTransformSource(source, context, defaultTransformSource);
+}
+
+async function transformSourceImpl(source, context, defaultTransformSource) {
+  return await reactTransformSource(source, context, (s, c) => {
+    return textTransformSource(s, c, defaultTransformSource);
+  });
+}
+
+export const transformSource =
+  process.version < 'v16' ? transformSourceImpl : undefined;
+export const getSource = process.version < 'v16' ? getSourceImpl : undefined;

fixtures/flight-esm/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "flight-esm",
+  "type": "module",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "body-parser": "^1.20.1",
+    "browserslist": "^4.18.1",
+    "busboy": "^1.6.0",
+    "compression": "^1.7.4",
+    "concurrently": "^7.3.0",
+    "nodemon": "^2.0.19",
+    "prompts": "^2.4.2",
+    "react": "experimental",
+    "react-dom": "experimental",
+    "undici": "^5.20.0"
+  },
+  "scripts": {
+    "predev": "cp -r ../../build/oss-experimental/* ./node_modules/",
+    "prestart": "cp -r ../../build/oss-experimental/* ./node_modules/",
+    "dev": "concurrently \"npm run dev:region\" \"npm run dev:global\"",
+    "dev:global": "NODE_ENV=development BUILD_PATH=dist node server/global",
+    "dev:region": "NODE_ENV=development BUILD_PATH=dist nodemon --watch src --watch dist -- --experimental-loader ./loader/region.js --conditions=react-server server/region",
+    "start": "concurrently \"npm run start:region\" \"npm run start:global\"",
+    "start:global": "NODE_ENV=production node server/global",
+    "start:region": "NODE_ENV=production node --experimental-loader ./loader/region.js --conditions=react-server server/region"
+  }
+}

fixtures/flight-esm/server/global.js

@@ -0,0 +1,137 @@
+'use strict';
+
+// This is a server to host CDN distributed resources like module source files and SSR
+
+const path = require('path');
+const url = require('url');
+
+const fs = require('fs').promises;
+const compress = require('compression');
+const chalk = require('chalk');
+const express = require('express');
+const http = require('http');
+
+const {renderToPipeableStream} = require('react-dom/server');
+const {createFromNodeStream} = require('react-server-dom-esm/client');
+
+const moduleBasePath = new URL('../src', url.pathToFileURL(__filename)).href;
+
+const app = express();
+
+app.use(compress());
+
+function request(options, body) {
+  return new Promise((resolve, reject) => {
+    const req = http.request(options, res => {
+      resolve(res);
+    });
+    req.on('error', e => {
+      reject(e);
+    });
+    body.pipe(req);
+  });
+}
+
+app.all('/', async function (req, res, next) {
+  // Proxy the request to the regional server.
+  const proxiedHeaders = {
+    'X-Forwarded-Host': req.hostname,
+    'X-Forwarded-For': req.ips,
+    'X-Forwarded-Port': 3000,
+    'X-Forwarded-Proto': req.protocol,
+  };
+  // Proxy other headers as desired.
+  if (req.get('rsc-action')) {
+    proxiedHeaders['Content-type'] = req.get('Content-type');
+    proxiedHeaders['rsc-action'] = req.get('rsc-action');
+  } else if (req.get('Content-type')) {
+    proxiedHeaders['Content-type'] = req.get('Content-type');
+  }
+
+  const promiseForData = request(
+    {
+      host: '127.0.0.1',
+      port: 3001,
+      method: req.method,
+      path: '/',
+      headers: proxiedHeaders,
+    },
+    req
+  );
+
+  if (req.accepts('text/html')) {
+    try {
+      const rscResponse = await promiseForData;
+
+      const moduleBaseURL = '/src';
+
+      // For HTML, we're a "client" emulator that runs the client code,
+      // so we start by consuming the RSC payload. This needs the local file path
+      // to load the source files from as well as the URL path for preloads.
+      const root = await createFromNodeStream(
+        rscResponse,
+        moduleBasePath,
+        moduleBaseURL
+      );
+      // Render it into HTML by resolving the client components
+      res.set('Content-type', 'text/html');
+      const {pipe} = renderToPipeableStream(root, {
+        // TODO: bootstrapModules inserts a preload before the importmap which causes
+        // the import map to be invalid. We need to fix that in Float somehow.
+        // bootstrapModules: ['/src/index.js'],
+      });
+      pipe(res);
+    } catch (e) {
+      console.error(`Failed to SSR: ${e.stack}`);
+      res.statusCode = 500;
+      res.end();
+    }
+  } else {
+    try {
+      const rscResponse = await promiseForData;
+      // For other request, we pass-through the RSC payload.
+      res.set('Content-type', 'text/x-component');
+      rscResponse.on('data', data => {
+        res.write(data);
+        res.flush();
+      });
+      rscResponse.on('end', data => {
+        res.end();
+      });
+    } catch (e) {
+      console.error(`Failed to proxy request: ${e.stack}`);
+      res.statusCode = 500;
+      res.end();
+    }
+  }
+});
+
+app.use(express.static('public'));
+app.use('/src', express.static('src'));
+app.use(
+  '/node_modules/react-server-dom-esm/esm',
+  express.static('node_modules/react-server-dom-esm/esm')
+);
+
+app.listen(3000, () => {
+  console.log('Global Fizz/Webpack Server listening on port 3000...');
+});
+
+app.on('error', function (error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  switch (error.code) {
+    case 'EACCES':
+      console.error('port 3000 requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error('Port 3000 is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+});

fixtures/flight-esm/server/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

fixtures/flight-esm/server/region.js

@@ -0,0 +1,140 @@
+'use strict';
+
+// This is a server to host data-local resources like databases and RSC
+
+const path = require('path');
+const url = require('url');
+
+if (typeof fetch === 'undefined') {
+  // Patch fetch for earlier Node versions.
+  global.fetch = require('undici').fetch;
+}
+
+const express = require('express');
+const bodyParser = require('body-parser');
+const busboy = require('busboy');
+const app = express();
+const compress = require('compression');
+const {Readable} = require('node:stream');
+
+app.use(compress());
+
+// Application
+
+const {readFile} = require('fs').promises;
+
+const React = require('react');
+
+const moduleBasePath = new URL('../src', url.pathToFileURL(__filename)).href;
+
+async function renderApp(res, returnValue) {
+  const {renderToPipeableStream} = await import('react-server-dom-esm/server');
+  const m = await import('../src/App.js');
+
+  const App = m.default;
+  const root = React.createElement(App);
+  // For client-invoked server actions we refresh the tree and return a return value.
+  const payload = returnValue ? {returnValue, root} : root;
+  const {pipe} = renderToPipeableStream(payload, moduleBasePath);
+  pipe(res);
+}
+
+app.get('/', async function (req, res) {
+  await renderApp(res, null);
+});
+
+app.post('/', bodyParser.text(), async function (req, res) {
+  const {
+    renderToPipeableStream,
+    decodeReply,
+    decodeReplyFromBusboy,
+    decodeAction,
+  } = await import('react-server-dom-esm/server');
+  const serverReference = req.get('rsc-action');
+  if (serverReference) {
+    // This is the client-side case
+    const [filepath, name] = serverReference.split('#');
+    const action = (await import(filepath))[name];
+    // Validate that this is actually a function we intended to expose and
+    // not the client trying to invoke arbitrary functions. In a real app,
+    // you'd have a manifest verifying this before even importing it.
+    if (action.$$typeof !== Symbol.for('react.server.reference')) {
+      throw new Error('Invalid action');
+    }
+
+    let args;
+    if (req.is('multipart/form-data')) {
+      // Use busboy to streamingly parse the reply from form-data.
+      const bb = busboy({headers: req.headers});
+      const reply = decodeReplyFromBusboy(bb, moduleBasePath);
+      req.pipe(bb);
+      args = await reply;
+    } else {
+      args = await decodeReply(req.body, moduleBasePath);
+    }
+    const result = action.apply(null, args);
+    try {
+      // Wait for any mutations
+      await result;
+    } catch (x) {
+      // We handle the error on the client
+    }
+    // Refresh the client and return the value
+    renderApp(res, result);
+  } else {
+    // This is the progressive enhancement case
+    const UndiciRequest = require('undici').Request;
+    const fakeRequest = new UndiciRequest('http://localhost', {
+      method: 'POST',
+      headers: {'Content-Type': req.headers['content-type']},
+      body: Readable.toWeb(req),
+      duplex: 'half',
+    });
+    const formData = await fakeRequest.formData();
+    const action = await decodeAction(formData, moduleBasePath);
+    try {
+      // Wait for any mutations
+      await action();
+    } catch (x) {
+      const {setServerState} = await import('../src/ServerState.js');
+      setServerState('Error: ' + x.message);
+    }
+    renderApp(res, null);
+  }
+});
+
+app.get('/todos', function (req, res) {
+  res.json([
+    {
+      id: 1,
+      text: 'Shave yaks',
+    },
+    {
+      id: 2,
+      text: 'Eat kale',
+    },
+  ]);
+});
+
+app.listen(3001, () => {
+  console.log('Regional Flight Server listening on port 3001...');
+});
+
+app.on('error', function (error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  switch (error.code) {
+    case 'EACCES':
+      console.error('port 3001 requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error('Port 3001 is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+});