scanImports() function reads all files at once, can cause EMFILE in l…

…arger projects #1245 (#1256)

* Recovers webpack magic comments and adds to import

Webpack's magic comments must be extracted from dynamic imports before the spec is passed to `matchDynamicImportValue`. If a match is found for these types of comments, they are appended to the rewritten import.

* scanImports() function reads all files at once, can cause EMFILE in larger projects #1245

See #1245
Adds 'p-queue' dependency
Creates `getLoadedFiles` mapper function to improve type legibility

* Bumps CONCURRENT_FILE_READS to 1000 and removes onIdle call #1256

snowpack/src/scan-imports.ts

@@ -14,6 +14,9 @@ import {
   readFile,
   SVELTE_VUE_REGEX,
 } from './util';
+import PQueue from 'p-queue';
+
+const CONCURRENT_FILE_READS = 1000;
 
 // [@\w] - Match a word-character or @ (valid package name)
 // (?!.*(:\/\/)) - Ignore if previous match was a protocol (ex: http://)
@@ -248,16 +251,19 @@ export async function scanImports(
   }
 
   // Scan every matched JS file for web dependency imports
-  const loadedFiles: (SnowpackSourceFile | null)[] = await Promise.all(
-    includeFiles.map(async (filePath) => {
+  const loadFileQueue = new PQueue({concurrency: CONCURRENT_FILE_READS});
+  const getLoadedFiles = async (filePath: string): Promise<SnowpackSourceFile | null> =>
+    loadFileQueue.add(async () => {
       const {baseExt, expandedExt} = getExt(filePath);
       return {
         baseExt,
         expandedExt,
         locOnDisk: filePath,
         contents: await readFile(filePath),
       };
-    }),
+    });
+  const loadedFiles: (SnowpackSourceFile | null)[] = await Promise.all(
+    includeFiles.map(getLoadedFiles),
   );
 
   return scanImportsFromFiles(loadedFiles.filter(isTruthy), config);