fix(gatsby): add retry mechanism for gatsby-node/config.ts compilation

packages/gatsby-cli/src/structured-errors/error-map.ts

@@ -383,7 +383,7 @@ const errors = {
   },
   "10127": {
     text: (context): string =>
-      `Your "${context.configName}.ts" file failed to compile to "${context.configName}.js. Please run "gatsby clean" and try again.\n\nIf the issue persists, please open an issue with a reproduction at https://github.com/gatsbyjs/gatsby/issues/new for more help."`,
+      `Your "${context.configName}.ts" file failed to compile to "${context.configName}.js". Please run "gatsby clean" and try again.\n\nIf the issue persists, please open an issue with a reproduction at https://github.com/gatsbyjs/gatsby/issues/new for more help."`,
     type: Type.CONFIG,
     level: Level.ERROR,
     category: ErrorCategory.USER,
@@ -731,6 +731,21 @@ const errors = {
     type: Type.COMPILATION,
     category: ErrorCategory.USER,
   },
+  "11904": {
+    text: (context): string =>
+      `Expected compiled files not found after compilation for ${
+        context.siteRoot
+      } after ${context.retries} retries.\nFile expected to be valid: ${
+        context.compiledFileLocation
+      }${
+        context.sourceFileLocation
+          ? `\nCompiled from: ${context.sourceFileLocation}`
+          : ``
+      }\n\nPlease run "gatsby clean" and try again. If the issue persists, please open an issue with a reproduction at https://github.com/gatsbyjs/gatsby/issues/new for more help.`,
+    level: Level.ERROR,
+    type: Type.COMPILATION,
+    category: ErrorCategory.SYSTEM,
+  },
   "12100": {
     text: (
       context

packages/gatsby/src/utils/parcel/compile-gatsby-files.ts

@@ -1,12 +1,25 @@
 import { Parcel } from "@parcel/core"
 import type { Diagnostic } from "@parcel/diagnostic"
 import reporter from "gatsby-cli/lib/reporter"
-import { ensureDir, emptyDir, existsSync } from "fs-extra"
+import { ensureDir, emptyDir, existsSync, remove } from "fs-extra"
 import telemetry from "gatsby-telemetry"
 
 export const COMPILED_CACHE_DIR = `.cache/compiled`
 export const PARCEL_CACHE_DIR = `.cache/.parcel-cache`
 export const gatsbyFileRegex = `gatsby-+(node|config).ts`
+const RETRY_COUNT = 5
+
+function getCacheDir(siteRoot: string): string {
+  return `${siteRoot}/${PARCEL_CACHE_DIR}`
+}
+
+function exponentialBackoff(retry: number): Promise<void> {
+  if (retry === 0) {
+    return Promise.resolve()
+  }
+  const timeout = 50 * Math.pow(2, retry)
+  return new Promise(resolve => setTimeout(resolve, timeout))
+}
 
 /**
  * Construct Parcel with config.
@@ -31,33 +44,80 @@ export function constructParcel(siteRoot: string): Parcel {
         distDir: `${siteRoot}/${COMPILED_CACHE_DIR}`,
       },
     },
-    cacheDir: `${siteRoot}/${PARCEL_CACHE_DIR}`,
+    cacheDir: getCacheDir(siteRoot),
   })
 }
 
 /**
  * Compile known gatsby-* files (e.g. `gatsby-config`, `gatsby-node`)
  * and output in `<SITE_ROOT>/.cache/compiled`.
  */
-export async function compileGatsbyFiles(siteRoot: string): Promise<void> {
+export async function compileGatsbyFiles(
+  siteRoot: string,
+  retry: number = 0
+): Promise<void> {
   try {
-    const parcel = constructParcel(siteRoot)
     const distDir = `${siteRoot}/${COMPILED_CACHE_DIR}`
     await ensureDir(distDir)
     await emptyDir(distDir)
+
+    await exponentialBackoff(retry)
+
+    const parcel = constructParcel(siteRoot)
     const { bundleGraph } = await parcel.run()
 
-    if (telemetry.isTrackingEnabled()) {
-      const bundles = bundleGraph.getBundles()
+    await exponentialBackoff(retry)
 
-      if (bundles.length === 0) return
+    const bundles = bundleGraph.getBundles()
 
-      let compiledTSFilesCount = 0
-      for (const bundle of bundles) {
-        if (bundle?.getMainEntry()?.filePath?.endsWith(`.ts`)) {
+    if (bundles.length === 0) return
+
+    let compiledTSFilesCount = 0
+    for (const bundle of bundles) {
+      // validate that output exists and is valid
+      try {
+        delete require.cache[bundle.filePath]
+        require(bundle.filePath)
+      } catch (e) {
+        if (retry >= RETRY_COUNT) {
+          reporter.panic({
+            id: `11904`,
+            context: {
+              siteRoot,
+              retries: RETRY_COUNT,
+              compiledFileLocation: bundle.filePath,
+              sourceFileLocation: bundle.getMainEntry()?.filePath,
+            },
+          })
+        } else if (retry > 0) {
+          // first retry is most flaky and it seems it always get in good state
+          // after that - most likely cache clearing is the trick that fixes the problem
+          reporter.verbose(
+            `Failed to import compiled file "${
+              bundle.filePath
+            }" after retry, attempting another retry (#${
+              retry + 1
+            } of ${RETRY_COUNT}) - "${e.message}"`
+          )
+        }
+
+        // sometimes parcel cache gets in weird state
+        await remove(getCacheDir(siteRoot))
+
+        await compileGatsbyFiles(siteRoot, retry + 1)
+        return
+      }
+
+      const mainEntry = bundle.getMainEntry()?.filePath
+      // mainEntry won't exist for shared chunks
+      if (mainEntry) {
+        if (mainEntry.endsWith(`.ts`)) {
           compiledTSFilesCount = compiledTSFilesCount + 1
         }
       }
+    }
+
+    if (telemetry.isTrackingEnabled()) {
       telemetry.trackCli(`PARCEL_COMPILATION_END`, {
         valueInteger: compiledTSFilesCount,
         name: `count of compiled ts files`,