feat: add support for npm modules (#454)

* feat: simplify `ImportMap`

* refactor: tweak `filterScopes`

* chore: fix test

* feat: add support for npm modules

* refactor: tidy up

* fix: only process import statements

* feat: support unprefixed Node.js built-ins

* feat: add `try/catch`

* refactor: convert stub path to file URL

* refactor: simplify code

* refactor: rename variable

* refactor: rename variable

* refactor: use `builtinModules` from `node:module`

* refactor: add try/catch

* chore: update lock file

.eslintrc.cjs

@@ -12,6 +12,7 @@ module.exports = {
     complexity: 'off',
     'import/extensions': 'off',
     'max-lines': 'off',
+    'max-lines-per-function': 'off',
     'max-statements': 'off',
     'node/no-missing-import': 'off',
     'no-magic-numbers': 'off',

.gitignore

@@ -2,6 +2,7 @@
 *.swp
 npm-debug.log
 node_modules
+!test/fixtures/**/node_modules
 /core
 .eslintcache
 .npmrc

deno/bundle.ts

@@ -1,10 +1,10 @@
 import { writeStage2 } from './lib/stage2.ts'
 
 const [payload] = Deno.args
-const { basePath, destPath, externals, functions, importMapData } = JSON.parse(payload)
+const { basePath, destPath, externals, functions, importMapData, vendorDirectory } = JSON.parse(payload)
 
 try {
-  await writeStage2({ basePath, destPath, externals, functions, importMapData })
+  await writeStage2({ basePath, destPath, externals, functions, importMapData, vendorDirectory })
 } catch (error) {
   if (error instanceof Error && error.message.includes("The module's source code could not be parsed")) {
     delete error.stack

deno/lib/stage2.ts

@@ -3,7 +3,7 @@ import { build, LoadResponse } from 'https://deno.land/x/eszip@v0.40.0/mod.ts'
 import * as path from 'https://deno.land/std@0.177.0/path/mod.ts'
 
 import type { InputFunction, WriteStage2Options } from '../../shared/stage2.ts'
-import { importMapSpecifier, virtualRoot } from '../../shared/consts.ts'
+import { importMapSpecifier, virtualRoot, virtualVendorRoot } from '../../shared/consts.ts'
 import { LEGACY_PUBLIC_SPECIFIER, PUBLIC_SPECIFIER, STAGE2_SPECIFIER } from './consts.ts'
 import { inlineModule, loadFromVirtualRoot, loadWithRetry } from './common.ts'
 
@@ -63,7 +63,13 @@ const getVirtualPath = (basePath: string, filePath: string) => {
   return url
 }
 
-const stage2Loader = (basePath: string, functions: InputFunction[], externals: Set<string>, importMapData?: string) => {
+const stage2Loader = (
+  basePath: string,
+  functions: InputFunction[],
+  externals: Set<string>,
+  importMapData: string | undefined,
+  vendorDirectory?: string,
+) => {
   return async (specifier: string): Promise<LoadResponse | undefined> => {
     if (specifier === STAGE2_SPECIFIER) {
       const stage2Entry = getStage2Entry(basePath, functions)
@@ -91,13 +97,24 @@ const stage2Loader = (basePath: string, functions: InputFunction[], externals: S
       return loadFromVirtualRoot(specifier, virtualRoot, basePath)
     }
 
+    if (vendorDirectory !== undefined && specifier.startsWith(virtualVendorRoot)) {
+      return loadFromVirtualRoot(specifier, virtualVendorRoot, vendorDirectory)
+    }
+
     return await loadWithRetry(specifier)
   }
 }
 
-const writeStage2 = async ({ basePath, destPath, externals, functions, importMapData }: WriteStage2Options) => {
+const writeStage2 = async ({
+  basePath,
+  destPath,
+  externals,
+  functions,
+  importMapData,
+  vendorDirectory,
+}: WriteStage2Options) => {
   const importMapURL = importMapData ? importMapSpecifier : undefined
-  const loader = stage2Loader(basePath, functions, new Set(externals), importMapData)
+  const loader = stage2Loader(basePath, functions, new Set(externals), importMapData, vendorDirectory)
   const bytes = await build([STAGE2_SPECIFIER], loader, importMapURL)
   const directory = path.dirname(destPath)
 

node/bundler.test.ts

@@ -142,7 +142,7 @@ test('Prints a nice error message when user tries importing NPM module', async (
   } catch (error) {
     expect(error).toBeInstanceOf(BundleError)
     expect((error as BundleError).message).toEqual(
-      `It seems like you're trying to import an npm module. This is only supported via CDNs like esm.sh. Have you tried 'import mod from "https://esm.sh/p-retry"'?`,
+      `It seems like you're trying to import an npm module. This is only supported via CDNs like esm.sh. Have you tried 'import mod from "https://esm.sh/parent-1"'?`,
     )
   } finally {
     await cleanup()
@@ -457,3 +457,32 @@ test('Handles imports with the `node:` prefix', async () => {
 
   await cleanup()
 })
+
+test('Loads npm modules from bare specifiers with and without the `npm:` prefix', async () => {
+  const { basePath, cleanup, distPath } = await useFixture('imports_npm_module')
+  const sourceDirectory = join(basePath, 'functions')
+  const declarations: Declaration[] = [
+    {
+      function: 'func1',
+      path: '/func1',
+    },
+  ]
+  const vendorDirectory = await tmp.dir()
+
+  await bundle([sourceDirectory], distPath, declarations, {
+    basePath,
+    featureFlags: { edge_functions_npm_modules: true },
+    vendorDirectory: vendorDirectory.path,
+  })
+  const manifestFile = await readFile(resolve(distPath, 'manifest.json'), 'utf8')
+  const manifest = JSON.parse(manifestFile)
+  const bundlePath = join(distPath, manifest.bundles[0].asset)
+  const { func1 } = await runESZIP(bundlePath, vendorDirectory.path)
+
+  expect(func1).toBe(
+    `<parent-1><child-1>JavaScript</child-1></parent-1>, <parent-2><child-2><grandchild-1>APIs<cwd>${process.cwd()}</cwd></grandchild-1></child-2></parent-2>, <parent-3><child-2><grandchild-1>Markup<cwd>${process.cwd()}</cwd></grandchild-1></child-2></parent-3>`,
+  )
+
+  await cleanup()
+  await rm(vendorDirectory.path, { force: true, recursive: true })
+})

node/bundler.ts

@@ -11,30 +11,33 @@ import type { Bundle } from './bundle.js'
 import { FunctionConfig, getFunctionConfig } from './config.js'
 import { Declaration, mergeDeclarations } from './declaration.js'
 import { load as loadDeployConfig } from './deploy_config.js'
+import { EdgeFunction } from './edge_function.js'
 import { FeatureFlags, getFlags } from './feature_flags.js'
 import { findFunctions } from './finder.js'
 import { bundle as bundleESZIP } from './formats/eszip.js'
 import { ImportMap } from './import_map.js'
-import { getLogger, LogFunction } from './logger.js'
+import { getLogger, LogFunction, Logger } from './logger.js'
 import { writeManifest } from './manifest.js'
+import { vendorNPMSpecifiers } from './npm_dependencies.js'
 import { ensureLatestTypes } from './types.js'
 
-interface BundleOptions {
+export interface BundleOptions {
   basePath?: string
+  bootstrapURL?: string
   cacheDirectory?: string
   configPath?: string
   debug?: boolean
   distImportMapPath?: string
   featureFlags?: FeatureFlags
   importMapPaths?: (string | undefined)[]
+  internalSrcFolder?: string
   onAfterDownload?: OnAfterDownloadHook
   onBeforeDownload?: OnBeforeDownloadHook
   systemLogger?: LogFunction
-  internalSrcFolder?: string
-  bootstrapURL?: string
+  vendorDirectory?: string
 }
 
-const bundle = async (
+export const bundle = async (
   sourceDirectories: string[],
   distDirectory: string,
   tomlDeclarations: Declaration[] = [],
@@ -46,10 +49,11 @@ const bundle = async (
     distImportMapPath,
     featureFlags: inputFeatureFlags,
     importMapPaths = [],
+    internalSrcFolder,
     onAfterDownload,
     onBeforeDownload,
     systemLogger,
-    internalSrcFolder,
+    vendorDirectory,
   }: BundleOptions = {},
 ) => {
   const logger = getLogger(systemLogger, debug)
@@ -93,6 +97,11 @@ const bundle = async (
   const userFunctions = userSourceDirectories.length === 0 ? [] : await findFunctions(userSourceDirectories)
   const internalFunctions = internalSrcFolder ? await findFunctions([internalSrcFolder]) : []
   const functions = [...internalFunctions, ...userFunctions]
+  const vendor = await safelyVendorNPMSpecifiers({ basePath, featureFlags, functions, logger, vendorDirectory })
+
+  if (vendor) {
+    importMap.add(vendor.importMap)
+  }
 
   const functionBundle = await bundleESZIP({
     basePath,
@@ -104,6 +113,7 @@ const bundle = async (
     functions,
     featureFlags,
     importMap,
+    vendorDirectory: vendor?.directory,
   })
 
   // The final file name of the bundles contains a SHA256 hash of the contents,
@@ -116,7 +126,6 @@ const bundle = async (
   const internalConfigPromises = internalFunctions.map(
     async (func) => [func.name, await getFunctionConfig({ func, importMap, deno, log: logger })] as const,
   )
-
   const userConfigPromises = userFunctions.map(
     async (func) => [func.name, await getFunctionConfig({ func, importMap, deno, log: logger })] as const,
   )
@@ -152,6 +161,8 @@ const bundle = async (
     layers: deployConfig.layers,
   })
 
+  await vendor?.cleanup()
+
   if (distImportMapPath) {
     await importMap.writeToFile(distImportMapPath)
   }
@@ -224,5 +235,32 @@ const createFunctionConfig = ({ internalFunctionsWithConfig, declarations }: Cre
     }
   }, {} as Record<string, FunctionConfig>)
 
-export { bundle }
-export type { BundleOptions }
+interface VendorNPMOptions {
+  basePath: string
+  featureFlags: FeatureFlags
+  functions: EdgeFunction[]
+  logger: Logger
+  vendorDirectory: string | undefined
+}
+
+const safelyVendorNPMSpecifiers = async ({
+  basePath,
+  featureFlags,
+  functions,
+  logger,
+  vendorDirectory,
+}: VendorNPMOptions) => {
+  if (!featureFlags.edge_functions_npm_modules) {
+    return
+  }
+
+  try {
+    return await vendorNPMSpecifiers(
+      basePath,
+      functions.map(({ path }) => path),
+      vendorDirectory,
+    )
+  } catch (error) {
+    logger.system(error)
+  }
+}

node/feature_flags.ts

@@ -1,5 +1,6 @@
 const defaultFlags = {
   edge_functions_fail_unsupported_regex: false,
+  edge_functions_npm_modules: false,
 }
 
 type FeatureFlag = keyof typeof defaultFlags

node/formats/eszip.ts

@@ -1,7 +1,7 @@
 import { join } from 'path'
 import { pathToFileURL } from 'url'
 
-import { virtualRoot } from '../../shared/consts.js'
+import { virtualRoot, virtualVendorRoot } from '../../shared/consts.js'
 import type { WriteStage2Options } from '../../shared/stage2.js'
 import { DenoBridge } from '../bridge.js'
 import { Bundle, BundleFormat } from '../bundle.js'
@@ -23,6 +23,7 @@ interface BundleESZIPOptions {
   featureFlags: FeatureFlags
   functions: EdgeFunction[]
   importMap: ImportMap
+  vendorDirectory?: string
 }
 
 const bundleESZIP = async ({
@@ -34,23 +35,27 @@ const bundleESZIP = async ({
   externals,
   functions,
   importMap,
+  vendorDirectory,
 }: BundleESZIPOptions): Promise<Bundle> => {
   const extension = '.eszip'
   const destPath = join(distDirectory, `${buildID}${extension}`)
-  const { bundler, importMap: bundlerImportMap } = getESZIPPaths()
-
-  // Transforming all paths under `basePath` to use the virtual root prefix.
   const importMapPrefixes: Record<string, string> = {
     [`${pathToFileURL(basePath)}/`]: virtualRoot,
   }
-  const importMapData = importMap.getContents(importMapPrefixes)
 
+  if (vendorDirectory !== undefined) {
+    importMapPrefixes[`${pathToFileURL(vendorDirectory)}/`] = virtualVendorRoot
+  }
+
+  const { bundler, importMap: bundlerImportMap } = getESZIPPaths()
+  const importMapData = JSON.stringify(importMap.getContents(importMapPrefixes))
   const payload: WriteStage2Options = {
     basePath,
     destPath,
     externals,
     functions,
-    importMapData: JSON.stringify(importMapData),
+    importMapData,
+    vendorDirectory,
   }
   const flags = ['--allow-all', '--no-config', `--import-map=${bundlerImportMap}`]