fix(gatsby): Show meaningful error when directory names are too long (#…

…21518)

* Truncate long paths automatically

* Write out truncated paths

* Map to correct disk paths

* Serve page-data

* Update snapshots

* Remove console.log

* Make opt-in into truncateLongPaths optional

* Empty line

* Fix bugs

* Add comments and tests

* Add tests and update snapshots

* Change 200 to 255

* Remove string-hash dependency

* Yarn.lock

* Move function to utils

* Look at segments instead of whole path

* Detect problematic paths and panic in production

* Reuse isWindows variable

* Improve error messages, add development handling:

* Change variable name

* Remove truncatePaths from config scheme

* Remove unneeded export

* update config snapshot

* Add comments

* move most of logic to utils/path, same error for build/develop with slightly different message

* no chalk in error messages

* test: add basic test for tooLongSegmentsInPath

* fixup error message

Co-authored-by: Kirill Vasiltsov <v_kirill@yumemi.co.jp>
Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>

packages/gatsby-cli/src/structured-errors/error-map.js

@@ -377,6 +377,26 @@ const errorMap = {
     type: `PLUGIN`,
     level: `ERROR`,
   },
+  // Directory/file name exceeds OS character limit
+  "11331": {
+    text: context =>
+      [
+        `One or more path segments are too long - they exceed OS filename length limit.\n`,
+        `Page path: "${context.path}"`,
+        `Invalid segments:\n${context.invalidPathSegments
+          .map(segment => ` - "${segment}"`)
+          .join(`\n`)}`,
+        ...(!context.isProduction
+          ? [
+              `\nThis will fail production builds, please adjust your paths.`,
+              `\nIn development mode gatsby truncated to: "${context.truncatedPath}"`,
+            ]
+          : []),
+      ]
+        .filter(Boolean)
+        .join(`\n`),
+    level: `ERROR`,
+  },
   // node object didn't pass validation
   "11467": {
     text: context =>

packages/gatsby/src/redux/actions/public.js

@@ -16,7 +16,11 @@ const { store } = require(`..`)
 const fileExistsSync = require(`fs-exists-cached`).sync
 const joiSchemas = require(`../../joi-schemas/joi`)
 const { generateComponentChunkName } = require(`../../utils/js-chunk-names`)
-const { getCommonDir } = require(`../../utils/path`)
+const {
+  getCommonDir,
+  truncatePath,
+  tooLongSegmentsInPath,
+} = require(`../../utils/path`)
 const apiRunnerNode = require(`../../utils/api-runner-node`)
 const { trackCli } = require(`gatsby-telemetry`)
 const { getNonGatsbyCodeFrame } = require(`../../utils/stack-trace-utils`)
@@ -373,6 +377,24 @@ ${reservedFields.map(f => `  * "${f}"`).join(`\n`)}
     internalComponentName = `Component${pascalCase(page.path)}`
   }
 
+  const invalidPathSegments = tooLongSegmentsInPath(page.path)
+
+  if (invalidPathSegments.length > 0) {
+    const truncatedPath = truncatePath(page.path)
+    report.panicOnBuild({
+      id: `11331`,
+      context: {
+        path: page.path,
+        invalidPathSegments,
+
+        // we will only show truncatedPath in non-production scenario
+        isProduction: process.env.NODE_ENV === `production`,
+        truncatedPath,
+      },
+    })
+    page.path = truncatedPath
+  }
+
   const internalPage: Page = {
     internalComponentName,
     path: page.path,

packages/gatsby/src/utils/__tests__/path.ts

@@ -1,5 +1,10 @@
 import { joinPath } from "gatsby-core-utils"
-import { withBasePath, getCommonDir } from "../path"
+import {
+  withBasePath,
+  getCommonDir,
+  truncatePath,
+  tooLongSegmentsInPath,
+} from "../path"
 import os from "os"
 
 describe(`paths`, () => {
@@ -108,4 +113,59 @@ describe(`paths`, () => {
       expect(getCommonDir(path1, path2)).toBe(expected)
     })
   })
+
+  describe(`tooLongSegmentsInPath`, () => {
+    it.each<[string, { input: string; expected: Array<string> }]>([
+      [
+        `doesn't touch short paths`,
+        {
+          input: `/short/path/`,
+          expected: [],
+        },
+      ],
+      [
+        `finds long segments`,
+        {
+          input: `/lo${`o`.repeat(500)}ng/path/`,
+          expected: [`lo${`o`.repeat(500)}ng`],
+        },
+      ],
+    ])(`%s`, (_label, { input, expected }) => {
+      expect(tooLongSegmentsInPath(input)).toEqual(expected)
+    })
+  })
+
+  describe(`truncatePath`, () => {
+    const SHORT_PATH = `/short/path/without/trailing/slash`
+    const SHORT_PATH_TRAILING = `/short/path/with/trailing/slash/`
+    const VERY_LONG_PATH = `/` + `x`.repeat(256) + `/`
+    const VERY_LONG_PATH_NON_LATIN = `/` + `あ`.repeat(255) + `/`
+
+    it(`Truncates long paths correctly`, () => {
+      const truncatedPathLatin = truncatePath(VERY_LONG_PATH)
+      const truncatedPathNonLatin = truncatePath(VERY_LONG_PATH_NON_LATIN)
+      for (const segment of truncatedPathLatin) {
+        expect(segment.length).toBeLessThanOrEqual(255)
+      }
+      for (const segment of truncatedPathNonLatin) {
+        expect(segment.length).toBeLessThanOrEqual(255)
+      }
+    })
+
+    it(`Preserves trailing slash`, () => {
+      const truncatedPathLong = truncatePath(VERY_LONG_PATH)
+      const truncatedPathShort = truncatePath(SHORT_PATH_TRAILING)
+      expect(truncatedPathLong.substring(truncatedPathLong.length - 1)).toEqual(
+        `/`
+      )
+      expect(
+        truncatedPathShort.substring(truncatedPathShort.length - 1)
+      ).toEqual(`/`)
+    })
+
+    it(`Does not truncate short paths`, () => {
+      const truncatedPath = truncatePath(SHORT_PATH)
+      expect(truncatedPath).toEqual(SHORT_PATH)
+    })
+  })
 })

packages/gatsby/src/utils/path.ts

@@ -1,5 +1,5 @@
 import path from "path"
-import { joinPath } from "gatsby-core-utils"
+import { joinPath, createContentDigest } from "gatsby-core-utils"
 
 export const withBasePath = (basePath: string) => (
   ...paths: string[]
@@ -31,3 +31,38 @@ export const getCommonDir = (path1: string, path2: string): string => {
 
   return posixJoinWithLeadingSlash(path1Segments)
 }
+
+// MacOS (APFS) and Windows (NTFS) filename length limit = 255 chars, Others = 255 bytes
+const MAX_PATH_SEGMENT_CHARS = 255
+const MAX_PATH_SEGMENT_BYTES = 255
+const SLICING_INDEX = 50
+const pathSegmentRe = /[^/]+/g
+
+const isMacOs = process.platform === `darwin`
+const isWindows = process.platform === `win32`
+
+const isNameTooLong = (segment: string): boolean =>
+  isMacOs || isWindows
+    ? segment.length > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars)
+    : Buffer.from(segment).length > MAX_PATH_SEGMENT_BYTES // Other (255 bytes)
+
+export const tooLongSegmentsInPath = (path: string): Array<string> => {
+  const invalidFilenames: Array<string> = []
+  for (const segment of path.split(`/`)) {
+    if (isNameTooLong(segment)) {
+      invalidFilenames.push(segment)
+    }
+  }
+  return invalidFilenames
+}
+
+export const truncatePath = (path: string): string =>
+  path.replace(pathSegmentRe, match => {
+    if (isNameTooLong(match)) {
+      return (
+        match.slice(0, SLICING_INDEX) +
+        createContentDigest(match.slice(SLICING_INDEX))
+      )
+    }
+    return match
+  })