🏗 Replace globby with fast-glob (ampproject#35846)

.eslintignore

@@ -29,6 +29,7 @@ validator/**/dist
 validator/export
 
 # Files and directories explicitly ignored by eslint
+**/node_modules/**
 build-system/babel-plugins/**/fixtures/**/*.js
 build-system/babel-plugins/**/fixtures/**/*.mjs
 build-system/tasks/make-extension/template/**/*

build-system/common/utils.js

@@ -1,7 +1,7 @@
 const argv = require('minimist')(process.argv.slice(2));
 const experimentsConfig = require('../global-configs/experiments-config.json');
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const {clean} = require('../tasks/clean');
 const {cyan, green, red, yellow} = require('./colors');
 const {default: ignore} = require('ignore');
@@ -49,7 +49,7 @@ function getExperimentConfig(experiment) {
  * @return {!Array<string>}
  */
 function getFilesChanged(globs, options) {
-  const allFiles = globby.sync(globs, options).map(String);
+  const allFiles = fastGlob.sync(globs, options).map(String);
   return gitDiffNameOnlyMain().filter((changedFile) => {
     return fs.existsSync(changedFile) && allFiles.includes(changedFile);
   });
@@ -79,12 +79,11 @@ function getFilesFromArgv() {
   if (!argv.files) {
     return [];
   }
-  // TODO(#30223): globby only takes posix globs. Find a Windows alternative.
   const toPosix = (str) => str.replace(/\\\\?/g, '/');
   const globs = Array.isArray(argv.files) ? argv.files : argv.files.split(',');
   const allFiles = [];
   for (const glob of globs) {
-    const files = globby.sync(toPosix(glob.trim()));
+    const files = fastGlob.sync(toPosix(glob.trim()));
     if (files.length == 0) {
       log(red('ERROR:'), 'Argument', cyan(glob), 'matched zero files.');
       throw new Error('Argument matched zero files.');
@@ -133,7 +132,7 @@ function getFilesToCheck(globs, options = {}, ignoreFile = undefined) {
     }
     return logFiles(filesChanged);
   }
-  return ignored.filter(globby.sync(globs, options).map(String));
+  return ignored.filter(fastGlob.sync(globs, options).map(String));
 }
 
 /**

build-system/compile/compile.js

@@ -1,8 +1,8 @@
 'use strict';
 const argv = require('minimist')(process.argv.slice(2));
 const del = require('del');
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const path = require('path');
 const {checkForUnknownDeps} = require('./check-for-unknown-deps');
 const {CLOSURE_SRC_GLOBS} = require('./sources');
@@ -195,8 +195,8 @@ function generateCompilerOptions(outputFilename, options) {
       'third_party/web-animations-externs/web_animations.js',
       'third_party/react-externs/externs.js',
       'third_party/moment/moment.extern.js',
-      ...globby.sync('src/core{,/**}/*.extern.js'),
-      ...globby.sync('build-system/externs/*.extern.js'),
+      ...fastGlob.sync('src/core{,/**}/*.extern.js'),
+      ...fastGlob.sync('build-system/externs/*.extern.js'),
       ...externs,
     ];
   }
@@ -405,7 +405,7 @@ async function compile(
     ? entryModuleFilenames.concat(options.extraGlobs || [])
     : getSrcs(entryModuleFilenames, options);
   const transformedSrcFiles = await Promise.all(
-    globby
+    fastGlob
       .sync(srcs)
       .map((src) => preClosureBabel(src, outputFilename, options))
   );

build-system/compile/pre-closure-babel.js

@@ -1,8 +1,8 @@
 'use strict';
 
 const babel = require('@babel/core');
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const path = require('path');
 const tempy = require('tempy');
 const {BABEL_SRC_GLOBS} = require('./sources');
@@ -47,7 +47,11 @@ function getBabelOutputDir() {
  * @return {!Array<string>}
  */
 function getFilesToTransform() {
-  return globby.sync([...BABEL_SRC_GLOBS, '!node_modules/', '!third_party/']);
+  return fastGlob.sync([
+    ...BABEL_SRC_GLOBS,
+    '!node_modules/**',
+    '!third_party/**',
+  ]);
 }
 
 /**

build-system/pr-check/build-targets.js

@@ -6,7 +6,7 @@
  * determine which tasks are required to run for pull request builds.
  */
 const config = require('../test-configs/config');
-const globby = require('globby');
+const fastGlob = require('fast-glob');
 const minimatch = require('minimatch');
 const path = require('path');
 const {cyan} = require('../common/colors');
@@ -393,7 +393,7 @@ function expandFileLists() {
   ];
   for (const globName of globNames) {
     const fileListName = globName.replace('Globs', 'Files');
-    fileLists[fileListName] = globby.sync(config[globName], {dot: true});
+    fileLists[fileListName] = fastGlob.sync(config[globName], {dot: true});
   }
 }
 

build-system/pr-check/utils.js

@@ -1,7 +1,7 @@
 'use strict';
 
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const path = require('path');
 const {
   ciPullRequestSha,
@@ -245,7 +245,7 @@ function storeBuildToWorkspace_(containerDirectory) {
       }
     }
     // Bento components are compiled inside the extension source file.
-    for (const componentFile of globby.sync('extensions/*/?.?/dist/*.js')) {
+    for (const componentFile of fastGlob.sync('extensions/*/?.?/dist/*.js')) {
       fs.ensureDirSync(
         `/tmp/workspace/builds/${containerDirectory}/${path.dirname(
           componentFile

build-system/server/typescript-compile.js

@@ -1,5 +1,5 @@
 const esbuild = require('esbuild');
-const globby = require('globby');
+const fastGlob = require('fast-glob');
 const path = require('path');
 const {accessSync} = require('fs-extra');
 const {cyan, green} = require('../common/colors');
@@ -30,7 +30,7 @@ async function buildNewServer() {
     green('at'),
     cyan(outdir) + green('...')
   );
-  const entryPoints = await globby(`${SERVER_TRANSFORM_PATH}/**/*.ts`);
+  const entryPoints = await fastGlob(`${SERVER_TRANSFORM_PATH}/**/*.ts`);
   const startTime = Date.now();
   await esbuild.build({
     ...esbuildOptions,

build-system/tasks/3p-vendor-helpers.js

@@ -1,5 +1,5 @@
 const debounce = require('../common/debounce');
-const globby = require('globby');
+const fastGlob = require('fast-glob');
 const {cyan, red} = require('../common/colors');
 const {endBuildStep} = require('./helpers');
 const {esbuildCompile} = require('./helpers');
@@ -104,7 +104,7 @@ function generateBundles() {
  * @return {!Array<string>}
  */
 function listVendors() {
-  const filesToBuild = globby.sync(SRCPATH);
+  const filesToBuild = fastGlob.sync(SRCPATH);
   const srcMatcher = /^3p\/vendors\/(.*)\.js/;
   const results = [];
 

build-system/tasks/analytics-vendor-configs.js

@@ -1,8 +1,8 @@
 const minimist = require('minimist');
 const argv = minimist(process.argv.slice(2));
 const debounce = require('../common/debounce');
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const jsonminify = require('jsonminify');
 const {basename, dirname, extname, join} = require('path');
 const {endBuildStep} = require('./helpers');
@@ -37,7 +37,7 @@ async function analyticsVendorConfigs(opt_options) {
 
   const startTime = Date.now();
 
-  const srcFiles = await globby(srcPath);
+  const srcFiles = await fastGlob(srcPath);
   await fs.ensureDir(destPath);
   for (const srcFile of srcFiles) {
     let destFile = join(destPath, basename(srcFile));
@@ -63,7 +63,7 @@ async function analyticsVendorConfigs(opt_options) {
     }
     await fs.writeFile(destFile, contents, 'utf-8');
   }
-  if ((await globby(srcPath)).length > 0) {
+  if ((await fastGlob(srcPath)).length > 0) {
     endBuildStep(
       'Compiled all analytics vendor configs into',
       destPath,

build-system/tasks/bundle-size/index.js

@@ -1,8 +1,8 @@
 'use strict';
 
 const argv = require('minimist')(process.argv.slice(2));
+const fastGlob = require('fast-glob');
 const fetch = require('node-fetch');
-const globby = require('globby');
 const path = require('path');
 const url = require('url');
 const {
@@ -226,7 +226,7 @@ async function reportBundleSize() {
  * @return {Promise<void>}
  */
 async function getLocalBundleSize() {
-  if ((await globby(fileGlobs)).length === 0) {
+  if ((await fastGlob(fileGlobs)).length === 0) {
     log('Could not find runtime files.');
     log('Run', cyan('amp dist --noextensions'), 'and re-run this task.');
     process.exitCode = 1;

build-system/tasks/check-analytics-vendors-list.js

@@ -1,4 +1,4 @@
-const globby = require('globby');
+const fastGlob = require('fast-glob');
 const {basename} = require('path');
 const {readFile} = require('fs-extra');
 const {writeDiffOrFail} = require('../common/diff');
@@ -32,7 +32,7 @@ const blockRegExp = (name) =>
  * @return {Promise<void>}
  */
 async function checkAnalyticsVendorsList() {
-  const vendors = globby
+  const vendors = fastGlob
     .sync(vendorsGlob)
     .map((path) => basename(path, '.json'))
     .sort();

build-system/tasks/check-build-system.js

@@ -1,4 +1,4 @@
-const globby = require('globby');
+const fastGlob = require('fast-glob');
 const path = require('path');
 const {cyan, green} = require('../common/colors');
 const {execOrThrow} = require('../common/exec');
@@ -10,7 +10,7 @@ const {updateSubpackages} = require('../common/update-packages');
  * Skips npm checks during CI (already done while running each task).
  */
 function updateBuildSystemSubpackages() {
-  const packageFiles = globby.sync('build-system/tasks/*/package.json');
+  const packageFiles = fastGlob.sync('build-system/tasks/*/package.json');
   for (const packageFile of packageFiles) {
     const packageDir = path.dirname(packageFile);
     updateSubpackages(packageDir, /* skipNpmChecks */ true);

build-system/tasks/check-exact-versions.js

@@ -1,7 +1,7 @@
 'use strict';
 
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const semver = require('semver');
 const {cyan, green, red} = require('../common/colors');
 const {gitDiffFileMain} = require('../common/git');
@@ -35,9 +35,9 @@ function check(file) {
  * @return {!Promise}
  */
 async function checkExactVersions() {
-  const packageJsonFiles = await globby([
+  const packageJsonFiles = await fastGlob([
     '**/package.json',
-    '!**/node_modules',
+    '!**/node_modules/**',
   ]);
   packageJsonFiles.forEach((file) => {
     if (check(file)) {

build-system/tasks/check-types.js

@@ -1,5 +1,5 @@
 const argv = require('minimist')(process.argv.slice(2));
-const globby = require('globby');
+const fastGlob = require('fast-glob');
 const {
   createCtrlcHandler,
   exitCtrlcHandler,
@@ -135,12 +135,12 @@ const TYPE_CHECK_TARGETS = {
       'ads/inabox/inabox-host.js',
       'src/web-worker/web-worker.js',
     ],
-    extraGlobs: ['src/inabox/*.js', '!node_modules/preact'],
+    extraGlobs: ['src/inabox/*.js', '!node_modules/preact/**'],
     warningLevel: 'QUIET',
   },
   'extensions': () => ({
     entryPoints: getExtensionSrcPaths(),
-    extraGlobs: ['src/inabox/*.js', '!node_modules/preact'],
+    extraGlobs: ['src/inabox/*.js', '!node_modules/preact/**'],
     warningLevel: 'QUIET',
   }),
   'integration': {
@@ -204,7 +204,7 @@ async function typeCheck(targetName) {
 
   // If srcGlobs and externGlobs are defined, determine the externs/extraGlobs
   if (srcGlobs.length || externGlobs.length) {
-    opts.externs = externGlobs.flatMap(globby.sync);
+    opts.externs = externGlobs.flatMap(fastGlob.sync);
 
     // Included globs should explicitly exclude any externs
     const excludedExterns = externGlobs.map((glob) => `!${glob}`);

build-system/tasks/check-video-interface-list.js

@@ -1,4 +1,4 @@
-const globby = require('globby');
+const fastGlob = require('fast-glob');
 const {getStdout} = require('../common/process');
 const {readFile} = require('fs-extra');
 const {writeDiffOrFail} = require('../common/diff');
@@ -29,7 +29,7 @@ const entry = (name) =>
  */
 const generateList = () =>
   getStdout(
-    ['grep -lr', `"${grepJsContent}"`, ...globby.sync(grepJsFiles)].join(' ')
+    ['grep -lr', `"${grepJsContent}"`, ...fastGlob.sync(grepJsFiles)].join(' ')
   )
     .trim()
     .split('\n')

build-system/tasks/compile-jison.js

@@ -1,7 +1,7 @@
 'use strict';
 
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const jison = require('jison');
 const path = require('path');
 const {jisonPath} = require('../test-configs/config');
@@ -28,7 +28,7 @@ const imports = new Map([
  * @return {!Promise}
  */
 async function compileJison(searchDir = jisonPath) {
-  const jisonFiles = await globby(searchDir);
+  const jisonFiles = await fastGlob(searchDir);
   await Promise.all(
     jisonFiles.map((jisonFile) => {
       const jsFile = path.basename(jisonFile, '.jison');

build-system/tasks/css/index.js

@@ -1,6 +1,6 @@
 const debounce = require('../../common/debounce');
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const path = require('path');
 const {buildExtensions} = require('../extension-helpers');
 const {endBuildStep, watchDebounceDelay} = require('../helpers');
@@ -77,7 +77,7 @@ async function copyCss() {
   for (const {outCss} of cssEntryPoints) {
     await fs.copy(`build/css/${outCss}`, `dist/${outCss}`);
   }
-  const cssFiles = await globby('build/css/amp-*.css');
+  const cssFiles = await fastGlob('build/css/amp-*.css');
   await Promise.all(
     cssFiles.map((cssFile) => {
       return fs.copy(cssFile, `dist/v0/${path.basename(cssFile)}`);

build-system/tasks/dist.js

@@ -1,6 +1,6 @@
 const colors = require('../common/colors');
+const fastGlob = require('fast-glob');
 const fs = require('fs-extra');
-const globby = require('globby');
 const path = require('path');
 const {
   bootstrapThirdPartyFrames,
@@ -241,7 +241,7 @@ async function preBuildWebPushPublisherFiles() {
       const js = await fs.readFile(`${srcPath}/${fileName}.js`, 'utf8');
       const builtName = `${fileName}.js`;
       await fs.outputFile(`${destPath}/${builtName}`, js);
-      const jsFiles = await globby(`${srcPath}/*.js`);
+      const jsFiles = await fastGlob(`${srcPath}/*.js`);
       await Promise.all(
         jsFiles.map((jsFile) => {
           return fs.copy(jsFile, `${destPath}/${path.basename(jsFile)}`);
@@ -304,7 +304,7 @@ async function preBuildExperiments() {
   const js = await fs.readFile(jsSrcPath, 'utf8');
   const builtName = 'experiments.max.js';
   await fs.outputFile(`${jsDir}/${builtName}`, js);
-  const jsFiles = await globby(`${expDir}/*.js`);
+  const jsFiles = await fastGlob(`${expDir}/*.js`);
   await Promise.all(
     jsFiles.map((jsFile) => {
       return fs.copy(jsFile, `${jsDir}/${path.basename(jsFile)}`);
@@ -346,7 +346,7 @@ async function preBuildLoginDoneVersion(version) {
   const js = await fs.readFile(jsPath, 'utf8');
   const builtName = `amp-login-done-${version}.max.js`;
   await fs.outputFile(`${buildDir}/${builtName}`, js);
-  const jsFiles = await globby(`${srcDir}/*.js`);
+  const jsFiles = await fastGlob(`${srcDir}/*.js`);
   await Promise.all(
     jsFiles.map((jsFile) => {
       return fs.copy(jsFile, `${buildDir}/${path.basename(jsFile)}`);