refactor: support glob import under import.meta.glob

BREAKING CHANGE: Glob import syntax has changed. The feature is now
exposed under `import.meta.glob` (lazy, exposes dynamic import functions)
and `import.meta.globEager` (eager, exposes already imported modules).

docs/guide/features.md

@@ -181,39 +181,55 @@ import { field } from './example.json'
 
 ## Glob Import
 
-> Requires ^2.0.0-beta.13
+> Requires ^2.0.0-beta.17
 
-Vite supports importing multiple modules from the file system using a [glob pattern](https://github.com/mrmlnc/fast-glob#pattern-syntax):
+Vite supports importing multiple modules from the file system via the special `import.meta.glob` function:
 
 ```js
-import modules from 'glob:./dir/**'
+const modules = import.meta.glob('./dir/*.js')
 ```
 
-Will be transformed to the following:
+The above will be transformed into the following:
 
 ```js
 // code produced by vite
-import * as __glob__0_0 from './dir/foo.js'
-import * as __glob__0_1 from './dir/bar.js'
 const modules = {
-  './dir/foo.js': __glob__0_0,
-  './dir/bar.js': __glob__0_1
+  './dir/foo.js': () => import('./dir/foo.js'),
+  './dir/bar.js': () => import('./dir/bar.js')
 }
 ```
 
 You can then iterate over the keys of the `modules` object to access the corresponding modules:
 
 ```js
 for (const path in modules) {
-  console.log(modules[path])
+  modules[path]().then((mod) => {
+    console.log(path, mod)
+  })
 }
 ```
 
-Some notes on the glob import syntax:
+Matched files are by default lazy loaded via dynamic import and will be split into separate chunks during build. If you'd rather import all the modules directly (e.g. relying on side-effects in these modules to be applied first), you can use `import.meta.globEager` instead:
 
-- Glob imports must start with the `glob:` prefix and followed by a [glob pattern](https://github.com/mrmlnc/fast-glob#pattern-syntax).
-- Glob patterns must be relative and start with `.`
-- Glob imports can only use the default import specifier (no named imports, no `import * as ...`).
+```js
+const modules = import.meta.glob('./dir/*.js')
+```
+
+The above will be transformed into the following:
+
+```js
+// code produced by vite
+const modules = {
+  './dir/foo.js': () => import('./dir/foo.js'),
+  './dir/bar.js': () => import('./dir/bar.js')
+}
+```
+
+Note that:
+
+- This is a Vite-only feature and is not a web or ES standard.
+- The glob patterns must be relative and start with `.`.
+- The glob matching is done via `fast-glob` - check out its documentation for [supported glob patterns](https://github.com/mrmlnc/fast-glob#pattern-syntax).
 
 ## Web Assembly
 

packages/playground/glob-import/__tests__/glob-import.spec.ts

@@ -31,19 +31,12 @@ const allResult = {
   './dir/index.js': {
     modules: filteredResult
   },
-  './dir/nested/bar.js': isBuild
-    ? {
-        msg: 'bar',
-        modules: {
-          '../baz.json': json
-        }
-      }
-    : {
-        modules: {
-          '../baz.json': json
-        },
-        msg: 'bar'
-      }
+  './dir/nested/bar.js': {
+    modules: {
+      '../baz.json': json
+    },
+    msg: 'bar'
+  }
 }
 
 test('should work', async () => {

packages/playground/glob-import/dir/index.js

@@ -1,3 +1,3 @@
-import modules from 'glob:./*.js'
+const modules = import.meta.globEager('./*.js')
 
 export { modules }

packages/playground/glob-import/dir/nested/bar.js

@@ -1,4 +1,4 @@
-import modules from 'glob:../*.json'
+const modules = import.meta.globEager('../*.json')
 
 export const msg = 'bar'
 export { modules }

packages/playground/glob-import/index.html

@@ -2,7 +2,20 @@
 
 <script type="module" src="./dir/index.js"></script>
 <script type="module">
-  import modules from 'glob:./dir/**'
+  const modules = import.meta.glob('./dir/**')
 
-  document.querySelector('.result').textContent = JSON.stringify(modules, null, 2)
+  for (const path in modules) {
+    modules[path]().then((mod) => {
+      console.log(path, mod)
+    })
+  }
+
+  const keys = Object.keys(modules)
+  Promise.all(keys.map((key) => modules[key]())).then((mods) => {
+    const res = {}
+    mods.forEach((m, i) => {
+      res[keys[i]] = m
+    })
+    document.querySelector('.result').textContent = JSON.stringify(res, null, 2)
+  })
 </script>

packages/vite/client.d.ts

@@ -24,8 +24,25 @@ interface ImportMeta {
   }
 
   readonly env: ImportMetaEnv
-}
 
+  glob(
+    pattern: string
+  ): Record<
+    string,
+    () => Promise<{
+      [key: string]: any
+    }>
+  >
+
+  globEager(
+    pattern: string
+  ): Record<
+    string,
+    {
+      [key: string]: any
+    }
+  >
+}
 interface ImportMetaEnv {
   [key: string]: string | boolean | undefined
   BASE_URL: string
@@ -187,9 +204,3 @@ declare module '*?worker&inline' {
   }
   export default workerConstructor
 }
-
-// glob import
-declare module 'glob:*' {
-  const modules: Record<string, Record<string, any>>
-  export default modules
-}

packages/vite/src/node/plugins/importAnalysis.ts

@@ -254,25 +254,22 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin {
             }
           } else if (prop === '.env') {
             hasEnv = true
+          } else if (prop === '.glo' && source[end + 4] === 'b') {
+            // transform import.meta.glob()
+            // e.g. `import.meta.glob('glob:./dir/*.js')`
+            const { imports, exp, endIndex } = await transformImportGlob(
+              source,
+              start,
+              importer,
+              index,
+              normalizeUrl
+            )
+            str().prepend(imports)
+            str().overwrite(expStart, endIndex, exp)
           }
           continue
         }
 
-        // transform import context
-        // e.g. `import modules from 'glob:./dir/*.js'`
-        if (url.startsWith('glob:')) {
-          const result = await transformImportGlob(
-            source.slice(expStart, expEnd),
-            url,
-            importer,
-            index,
-            start,
-            normalizeUrl
-          )
-          str().overwrite(expStart, expEnd, result)
-          continue
-        }
-
         // For dynamic id, check if it's a literal that we can resolve
         let hasViteIgnore = false
         let isLiteralDynamicId = false

packages/vite/src/node/plugins/importGlob.ts

@@ -2,11 +2,10 @@ import path from 'path'
 import glob from 'fast-glob'
 import { ResolvedConfig } from '../config'
 import { Plugin } from '../plugin'
-import { parse as parseJS } from 'acorn'
 import { cleanUrl } from '../utils'
-import type { Node } from 'estree'
 import MagicString from 'magic-string'
 import { ImportSpecifier, init, parse as parseImports } from 'es-module-lexer'
+import { RollupError } from 'rollup'
 
 /**
  * Build only. During serve this is performed as part of ./importAnalysis.
@@ -20,7 +19,7 @@ export function importGlobPlugin(config: ResolvedConfig): Plugin {
         // skip deps
         importer.includes('node_modules') ||
         // fast check for presence of glob keyword
-        source.indexOf('glob:.') < 0
+        source.indexOf('import.meta.glob') < 0
       ) {
         return
       }
@@ -42,17 +41,17 @@ export function importGlobPlugin(config: ResolvedConfig): Plugin {
       const str = () => s || (s = new MagicString(source))
 
       for (let index = 0; index < imports.length; index++) {
-        const { s: start, e: end, ss: expStart, se: expEnd } = imports[index]
+        const { s: start, e: end, ss: expStart } = imports[index]
         const url = source.slice(start, end)
-        if (url.startsWith('glob:')) {
-          const result = await transformImportGlob(
-            source.slice(expStart, expEnd),
-            url,
+        if (url === 'import.meta' && source.slice(end, end + 5) === '.glob') {
+          const { imports, exp, endIndex } = await transformImportGlob(
+            source,
+            start,
             importer,
-            index,
-            start
+            index
           )
-          str().overwrite(expStart, expEnd, result)
+          str().prepend(imports)
+          str().overwrite(expStart, endIndex, exp)
         }
       }
 
@@ -67,44 +66,22 @@ export function importGlobPlugin(config: ResolvedConfig): Plugin {
 }
 
 export async function transformImportGlob(
-  exp: string,
-  url: string,
+  source: string,
+  pos: number,
   importer: string,
   importIndex: number,
-  pos: number,
   normalizeUrl?: (url: string, pos: number) => Promise<[string, string]>
-): Promise<string> {
+): Promise<{ imports: string; exp: string; endIndex: number }> {
   const err = (msg: string) => {
     const e = new Error(`Invalid glob import syntax: ${msg}`)
     ;(e as any).pos = pos
     return e
   }
 
-  const node = (parseJS(exp, {
-    ecmaVersion: 2020,
-    sourceType: 'module'
-  }) as any).body[0] as Node
-
-  if (node.type !== 'ImportDeclaration') {
-    throw err(`statement must be an import declaration.`)
-  }
-
-  let localName: string | undefined
-  for (const spec of node.specifiers) {
-    if (spec.type !== 'ImportDefaultSpecifier') {
-      throw err(`can only use the default import.`)
-    }
-    localName = spec.local.name
-    break
-  }
-  if (!localName) {
-    throw err(`missing default import.`)
-  }
-
   importer = cleanUrl(importer)
   const importerBasename = path.basename(importer)
 
-  let pattern = url.slice(5)
+  let [pattern, endIndex] = lexGlobPattern(source, pos)
   if (!pattern.startsWith('.')) {
     throw err(`pattern must start with "."`)
   }
@@ -133,9 +110,84 @@ export async function transformImportGlob(
       ;[importee] = await normalizeUrl(file, pos)
     }
     const identifier = `__glob_${importIndex}_${i}`
-    imports += `import * as ${identifier} from "${importee}";\n`
-    entries += `\n  ${JSON.stringify(file)}: ${identifier},`
+    const isEager = source.slice(pos, pos + 21) === 'import.meta.globEager'
+    if (isEager) {
+      imports += `import * as ${identifier} from ${JSON.stringify(importee)};`
+      entries += ` ${JSON.stringify(file)}: ${identifier},`
+    } else {
+      entries += ` ${JSON.stringify(file)}: () => import(${JSON.stringify(
+        importee
+      )}),`
+    }
+  }
+
+  return {
+    imports,
+    exp: `{${entries}}`,
+    endIndex
+  }
+}
+
+const enum LexerState {
+  inCall,
+  inSingleQuoteString,
+  inDoubleQuoteString,
+  inTemplateString
+}
+
+function lexGlobPattern(code: string, pos: number): [string, number] {
+  let state = LexerState.inCall
+  let pattern = ''
+
+  let i = code.indexOf(`(`, pos) + 1
+  outer: for (; i < code.length; i++) {
+    const char = code.charAt(i)
+    switch (state) {
+      case LexerState.inCall:
+        if (char === `'`) {
+          state = LexerState.inSingleQuoteString
+        } else if (char === `"`) {
+          state = LexerState.inDoubleQuoteString
+        } else if (char === '`') {
+          state = LexerState.inTemplateString
+        } else if (/\s/.test(char)) {
+          continue
+        } else {
+          error(i)
+        }
+        break
+      case LexerState.inSingleQuoteString:
+        if (char === `'`) {
+          break outer
+        } else {
+          pattern += char
+        }
+        break
+      case LexerState.inDoubleQuoteString:
+        if (char === `"`) {
+          break outer
+        } else {
+          pattern += char
+        }
+        break
+      case LexerState.inTemplateString:
+        if (char === '`') {
+          break outer
+        } else {
+          pattern += char
+        }
+        break
+      default:
+        throw new Error('unknown import.meta.glob lexer state')
+    }
   }
+  return [pattern, code.indexOf(`)`, i) + 1]
+}
 
-  return `${imports}const ${localName} = {${entries}\n}`
+function error(pos: number) {
+  const err = new Error(
+    `import.meta.glob() can only accept string literals.`
+  ) as RollupError
+  err.pos = pos
+  throw err
 }

packages/vite/src/node/server/hmr.ts

@@ -328,7 +328,7 @@ export function lexAcceptedHmrDeps(
         }
         break
       default:
-        throw new Error('unknown lexer state')
+        throw new Error('unknown import.meta.hot lexer state')
     }
   }
   return false