feat(metro-resolver-symlinks): experimental option to retry resolving from disk

.changeset/wet-cheetahs-explain.md

@@ -0,0 +1,5 @@
+---
+"@rnx-kit/metro-resolver-symlinks": patch
+---
+
+Add an experimental option for retrying module resolution from disk if not found in Haste map

packages/metro-resolver-symlinks/README.md

@@ -41,9 +41,10 @@ Import and assign the resolver to `resolver.resolveRequest` in your
 
 ## Options
 
-| Option      | Type                           | Description                                         |
-| :---------- | :----------------------------- | :-------------------------------------------------- |
-| remapModule | `(moduleId: string) => string` | A custom function for remapping additional modules. |
+| Option                                | Type                           | Description                                                                                                                                                                                                                                                                                          |
+| :------------------------------------ | :----------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `remapModule`                         | `(moduleId: string) => string` | A custom function for remapping additional modules.                                                                                                                                                                                                                                                  |
+| `experimental_retryResolvingFromDisk` | boolean                        | [Experimental] Whether to retry module resolution on disk if not found in Haste map. This option is useful for scenarios where you want to reduce the number of watched files (and thus the initial time spent on crawling). Note that enabling this will likely be slower than having a warm cache. |
 
 ### `remapModule`
 

packages/metro-resolver-symlinks/src/symlinkResolver.ts

@@ -1,17 +1,47 @@
 import { normalizePath } from "@rnx-kit/tools-node";
-import type { CustomResolver, ResolutionContext } from "metro-resolver";
+import type {
+  CustomResolver,
+  Resolution,
+  ResolutionContext,
+} from "metro-resolver";
 import { requireModuleFromMetro } from "./helper";
 import { remapReactNativeModule, resolveModulePath } from "./resolver";
 import type { MetroResolver, Options } from "./types";
+import { applyEnhancedResolver } from "./utils/enhancedResolve";
+import {
+  patchMetro,
+  shouldEnableRetryResolvingFromDisk,
+} from "./utils/patchMetro";
 import { remapImportPath } from "./utils/remapImportPath";
 
+function applyMetroResolver(
+  resolve: CustomResolver,
+  context: ResolutionContext,
+  moduleName: string,
+  platform: string
+): Resolution {
+  const modifiedModuleName = resolveModulePath(context, moduleName, platform);
+  return resolve(context, normalizePath(modifiedModuleName), platform, null);
+}
+
 export function makeResolver(options: Options = {}): MetroResolver {
   const { resolve: metroResolver } =
     requireModuleFromMetro<typeof import("metro-resolver")>("metro-resolver");
 
   const { remapModule = (_, moduleName, __) => moduleName } = options;
 
-  const remappers = [remapModule, remapReactNativeModule, resolveModulePath];
+  const enableRetryResolvingFromDisk =
+    shouldEnableRetryResolvingFromDisk(options);
+
+  const applyResolver = enableRetryResolvingFromDisk
+    ? applyEnhancedResolver
+    : applyMetroResolver;
+
+  if (enableRetryResolvingFromDisk) {
+    patchMetro(options);
+  }
+
+  const remappers = [remapModule, remapReactNativeModule];
   const symlinkResolver = (
     context: ResolutionContext,
     moduleName: string,
@@ -56,12 +86,7 @@ export function makeResolver(options: Options = {}): MetroResolver {
         moduleName
       );
 
-      return resolve(
-        context,
-        normalizePath(modifiedModuleName),
-        platform,
-        null
-      );
+      return applyResolver(resolve, context, modifiedModuleName, platform);
     } finally {
       if (!context.resolveRequest) {
         // Restoring `resolveRequest` must happen last

packages/metro-resolver-symlinks/src/types.ts

@@ -1,5 +1,9 @@
 import type { ResolutionContext as MetroResolutionContext } from "metro-resolver";
 
+type ExperimentalOptions = {
+  experimental_retryResolvingFromDisk?: boolean | "force";
+};
+
 export type MetroResolver = typeof import("metro-resolver").resolve;
 
 export type ResolutionContext = Pick<
@@ -13,6 +17,6 @@ export type ModuleResolver = (
   platform: string
 ) => string;
 
-export type Options = {
+export type Options = ExperimentalOptions & {
   remapModule?: ModuleResolver;
 };

packages/metro-resolver-symlinks/src/utils/enhancedResolve.ts

@@ -0,0 +1,68 @@
+import { isPackageModuleRef, parseModuleRef } from "@rnx-kit/tools-node";
+import { expandPlatformExtensions } from "@rnx-kit/tools-react-native/platform";
+import type {
+  CustomResolver,
+  Resolution,
+  ResolutionContext,
+} from "metro-resolver";
+import * as path from "path";
+
+type Resolver = (fromDir: string, moduleId: string) => string | false;
+
+const getEnhancedResolver = (() => {
+  const resolvers: Record<string, Resolver> = {};
+  return (context: ResolutionContext, platform = "common") => {
+    if (!resolvers[platform]) {
+      // @ts-expect-error Property 'mainFields' does not exist on type 'ResolutionContext'
+      const { mainFields, sourceExts } = context;
+      const extensions = sourceExts.map((ext) => `.${ext}`);
+      resolvers[platform] = require("enhanced-resolve").create.sync({
+        aliasFields: ["browser"],
+        extensions:
+          platform === "common"
+            ? extensions
+            : expandPlatformExtensions(platform, extensions),
+        mainFields,
+      });
+    }
+    return resolvers[platform];
+  };
+})();
+
+function getFromDir(context: ResolutionContext, moduleName: string): string {
+  const { extraNodeModules, originModulePath } = context;
+  if (extraNodeModules) {
+    const ref = parseModuleRef(moduleName);
+    if (isPackageModuleRef(ref)) {
+      const pkgName = ref.scope ? `${ref.scope}/${ref.name}` : ref.name;
+      const dir = extraNodeModules[pkgName];
+      if (dir) {
+        return dir;
+      }
+    }
+  }
+
+  return originModulePath ? path.dirname(originModulePath) : process.cwd();
+}
+
+export function applyEnhancedResolver(
+  _resolve: CustomResolver,
+  context: ResolutionContext,
+  moduleName: string,
+  platform: string
+): Resolution {
+  if (!platform) {
+    return { type: "empty" };
+  }
+
+  const enhancedResolve = getEnhancedResolver(context, platform);
+  const filePath = enhancedResolve(getFromDir(context, moduleName), moduleName);
+  if (filePath === false) {
+    return { type: "empty" };
+  }
+
+  return {
+    type: "sourceFile",
+    filePath,
+  };
+}

packages/metro-resolver-symlinks/src/utils/patchMetro.ts

@@ -0,0 +1,136 @@
+import * as fs from "fs";
+import * as path from "path";
+import { ensureResolveFrom, getMetroSearchPath } from "../helper";
+import type { Options } from "../types";
+
+function fileExists(path: string): boolean {
+  const stat = fs.statSync(path, { throwIfNoEntry: false });
+  return Boolean(stat && stat.isFile());
+}
+
+function importMetroModule(path: string) {
+  const metroPath = ensureResolveFrom("metro", getMetroSearchPath());
+  const modulePath = metroPath + path;
+  try {
+    return require(modulePath);
+  } catch (_) {
+    throw new Error(
+      `Cannot find '${modulePath}'. This probably means that ` +
+        "'experimental_retryResolvingFromDisk' is not compatible with the " +
+        "version of 'metro' that you are currently using. Please update to " +
+        "the latest version and try again. If the issue still persists after " +
+        "the update, please file a bug at " +
+        "https://github.com/microsoft/rnx-kit/issues."
+    );
+  }
+}
+
+function getDependencyGraph() {
+  return importMetroModule("/src/node-haste/DependencyGraph");
+}
+
+function supportsRetryResolvingFromDisk(): boolean {
+  const { version } = importMetroModule("/package.json");
+  const [major, minor] = version.split(".");
+  const v = major * 1000 + minor;
+  return v >= 64 && v <= 73;
+}
+
+export function shouldEnableRetryResolvingFromDisk({
+  experimental_retryResolvingFromDisk,
+}: Options): boolean {
+  if (
+    !supportsRetryResolvingFromDisk() &&
+    experimental_retryResolvingFromDisk !== "force"
+  ) {
+    console.warn(
+      "The version of Metro you're using has not been tested with " +
+        "`experimental_retryResolvingFromDisk`. If you still want to enable " +
+        "it, please set it to 'force'."
+    );
+    return false;
+  }
+
+  return Boolean(experimental_retryResolvingFromDisk);
+}
+
+/**
+ * Monkey-patches Metro to not use HasteFS as the only source for module
+ * resolution.
+ *
+ * Practically every file system operation in Metro must go through HasteFS,
+ * most notably watching for file changes and resolving node modules. If Metro
+ * cannot find a file in the Haste map, it does not exist. This means that for
+ * Metro to find a file, all folders must be declared in `watchFolders`,
+ * including `node_modules` and any dependency storage folders (e.g. pnpm)
+ * regardless of whether we need to watch them. In big monorepos, this can
+ * easily overwhelm file watchers, even with Watchman installed.
+ *
+ * There's no way to avoid the initial crawling of the file system. However, we
+ * can drastically reduce the number of files that needs to be crawled/watched
+ * by not relying solely on Haste for module resolution. This requires patching
+ * Metro to use `fs.existsSync` instead of `HasteFS.exists`. With this change,
+ * we can list only the folders that we care about in `watchFolders`. In some
+ * cases, like on CI, we can even set `watchFolders` to an empty array to limit
+ * watched files to the current package only.
+ *
+ * Why didn't we use `hasteImplModulePath`? Contrary to the name, it doesn't
+ * let you replace HasteFS. As of 0.73, it is only used to retrieve the path of
+ * a module. The default implementation returns
+ * `path.relative(projectRoot, filePath)` if the entry is not found in the map.
+ *
+ * @param options Options passed to Metro
+ */
+export function patchMetro(options: Options): void {
+  if (!shouldEnableRetryResolvingFromDisk(options)) {
+    return;
+  }
+
+  const DependencyGraph = getDependencyGraph();
+
+  // Patch `_createModuleResolver` and `_doesFileExist` to use `fs.existsSync`.
+  DependencyGraph.prototype.orig__createModuleResolver =
+    DependencyGraph.prototype._createModuleResolver;
+  DependencyGraph.prototype._createModuleResolver = function (): void {
+    this._doesFileExist = (filePath: string): boolean => {
+      return this._hasteFS.exists(filePath) || fileExists(filePath);
+    };
+
+    this.orig__createModuleResolver();
+    if (typeof this._moduleResolver._options.resolveAsset !== "function") {
+      throw new Error("Could not find `resolveAsset` in `ModuleResolver`");
+    }
+
+    this._moduleResolver._options.resolveAsset = (
+      dirPath: string,
+      assetName: string,
+      extension: string
+    ) => {
+      const basePath = dirPath + path.sep + assetName;
+      const assets = [
+        basePath + extension,
+        ...this._config.resolver.assetResolutions.map(
+          (resolution: string) => basePath + "@" + resolution + "x" + extension
+        ),
+      ].filter(this._doesFileExist);
+      return assets.length ? assets : null;
+    };
+  };
+
+  // Since we will be resolving files outside of `watchFolders`, their hashes
+  // will not be found. We'll return the `filePath` as they should be unique.
+  DependencyGraph.prototype.orig_getSha1 = DependencyGraph.prototype.getSha1;
+  DependencyGraph.prototype.getSha1 = function (filePath: string): string {
+    try {
+      return this.orig_getSha1(filePath);
+    } catch (e) {
+      // `ReferenceError` will always be thrown when Metro encounters a file
+      // that does not exist in the Haste map.
+      if (e instanceof ReferenceError) {
+        return filePath;
+      }
+
+      throw e;
+    }
+  };
+}