Fix broken HTML inlining of non UTF-8 decodable binary data from Flight payload

packages/next/src/client/app-index.tsx

@@ -43,7 +43,7 @@ const appElement: HTMLElement | Document | null = document
 
 const encoder = new TextEncoder()
 
-let initialServerDataBuffer: string[] | undefined = undefined
+let initialServerDataBuffer: (string | Uint8Array)[] | undefined = undefined
 let initialServerDataWriter: ReadableStreamDefaultController | undefined =
   undefined
 let initialServerDataLoaded = false
@@ -56,6 +56,7 @@ function nextServerDataCallback(
     | [isBootStrap: 0]
     | [isNotBootstrap: 1, responsePartial: string]
     | [isFormState: 2, formState: any]
+    | [isBinary: 3, responseBase64Partial: string]
 ): void {
   if (seg[0] === 0) {
     initialServerDataBuffer = []
@@ -70,6 +71,22 @@ function nextServerDataCallback(
     }
   } else if (seg[0] === 2) {
     initialFormStateData = seg[1]
+  } else if (seg[0] === 3) {
+    if (!initialServerDataBuffer)
+      throw new Error('Unexpected server data: missing bootstrap script.')
+
+    // Decode the base64 string back to binary data.
+    const binaryString = atob(seg[1])
+    const decodedChunk = new Uint8Array(binaryString.length)
+    for (var i = 0; i < binaryString.length; i++) {
+      decodedChunk[i] = binaryString.charCodeAt(i)
+    }
+
+    if (initialServerDataWriter) {
+      initialServerDataWriter.enqueue(decodedChunk)
+    } else {
+      initialServerDataBuffer.push(decodedChunk)
+    }
   }
 }
 
@@ -84,7 +101,7 @@ function nextServerDataCallback(
 function nextServerDataRegisterWriter(ctr: ReadableStreamDefaultController) {
   if (initialServerDataBuffer) {
     initialServerDataBuffer.forEach((val) => {
-      ctr.enqueue(encoder.encode(val))
+      ctr.enqueue(typeof val === 'string' ? encoder.encode(val) : val)
     })
     if (initialServerDataLoaded && !initialServerDataFlushed) {
       ctr.close()

packages/next/src/server/app-render/app-render.tsx

@@ -395,7 +395,7 @@ function createFlightDataResolver(ctx: AppRenderContext) {
   // Generate the flight data and as soon as it can, convert it into a string.
   const promise = generateFlight(ctx)
     .then(async (result) => ({
-      flightData: await result.toUnchunkedString(true),
+      flightData: await result.toUnchunkedBuffer(true),
     }))
     // Otherwise if it errored, return the error.
     .catch((err) => ({ err }))

packages/next/src/server/app-render/use-flight-response.tsx

@@ -9,6 +9,7 @@ const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge'
 const INLINE_FLIGHT_PAYLOAD_BOOTSTRAP = 0
 const INLINE_FLIGHT_PAYLOAD_DATA = 1
 const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2
+const INLINE_FLIGHT_PAYLOAD_BINARY = 3
 
 const flightResponses = new WeakMap<BinaryStreamOf<any>, Promise<any>>()
 const encoder = new TextEncoder()
@@ -96,10 +97,8 @@ export function createInlinedDataReadableStream(
     ? `<script nonce=${JSON.stringify(nonce)}>`
     : '<script>'
 
-  const decoder = new TextDecoder('utf-8', { fatal: true })
-  const decoderOptions = { stream: true }
-
   const flightReader = flightStream.getReader()
+  const decoder = new TextDecoder('utf-8', { fatal: true })
 
   const readable = new ReadableStream({
     type: 'bytes',
@@ -114,15 +113,26 @@ export function createInlinedDataReadableStream(
     async pull(controller) {
       try {
         const { done, value } = await flightReader.read()
-        if (done) {
-          const tail = decoder.decode(value, { stream: false })
-          if (tail.length) {
-            writeFlightDataInstruction(controller, startScriptTag, tail)
+
+        if (value) {
+          try {
+            const decodedString = decoder.decode(value, { stream: !done })
+
+            // The chunk cannot be decoded as valid UTF-8 string as it might
+            // have arbitrary binary data.
+            writeFlightDataInstruction(
+              controller,
+              startScriptTag,
+              decodedString
+            )
+          } catch {
+            // The chunk cannot be decoded as valid UTF-8 string.
+            writeFlightDataInstruction(controller, startScriptTag, value)
           }
+        }
+
+        if (done) {
           controller.close()
-        } else {
-          const chunkAsString = decoder.decode(value, decoderOptions)
-          writeFlightDataInstruction(controller, startScriptTag, chunkAsString)
         }
       } catch (error) {
         // There was a problem in the upstream reader or during decoding or enqueuing
@@ -154,13 +164,28 @@ function writeInitialInstructions(
 function writeFlightDataInstruction(
   controller: ReadableStreamDefaultController,
   scriptStart: string,
-  chunkAsString: string
+  chunk: string | Uint8Array
 ) {
+  let htmlInlinedData: string
+
+  if (typeof chunk === 'string') {
+    htmlInlinedData = htmlEscapeJsonString(
+      JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunk])
+    )
+  } else {
+    // The chunk cannot be embedded as a UTF-8 string in the script tag.
+    // Instead let's inline it in base64.
+    // Credits to Devon Govett (devongovett) for the technique.
+    // https://github.com/devongovett/rsc-html-stream
+    const base64 = btoa(String.fromCodePoint(...chunk))
+    htmlInlinedData = htmlEscapeJsonString(
+      JSON.stringify([INLINE_FLIGHT_PAYLOAD_BINARY, base64])
+    )
+  }
+
   controller.enqueue(
     encoder.encode(
-      `${scriptStart}self.__next_f.push(${htmlEscapeJsonString(
-        JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunkAsString])
-      )})</script>`
+      `${scriptStart}self.__next_f.push(${htmlInlinedData})</script>`
     )
   )
 }

packages/next/src/server/base-server.ts

@@ -13,6 +13,8 @@ import type { ParsedUrlQuery } from 'querystring'
 import type { RenderOptsPartial as PagesRenderOptsPartial } from './render'
 import type { RenderOptsPartial as AppRenderOptsPartial } from './app-render/types'
 import type {
+  CachedAppPageValue,
+  CachedPageValue,
   ResponseCacheBase,
   ResponseCacheEntry,
   ResponseGenerator,
@@ -2528,16 +2530,29 @@ export default abstract class Server<
         return null
       }
 
+      if (isAppPath) {
+        return {
+          value: {
+            kind: 'APP_PAGE',
+            html: result,
+            headers,
+            rscData: metadata.flightData,
+            postponed: metadata.postponed,
+            status: res.statusCode,
+          } satisfies CachedAppPageValue,
+          revalidate: metadata.revalidate,
+        }
+      }
+
       // We now have a valid HTML result that we can return to the user.
       return {
         value: {
           kind: 'PAGE',
           html: result,
-          pageData: metadata.pageData ?? metadata.flightData,
-          postponed: metadata.postponed,
+          pageData: metadata.pageData,
           headers,
-          status: isAppPath ? res.statusCode : undefined,
-        },
+          status: res.statusCode,
+        } satisfies CachedPageValue,
         revalidate: metadata.revalidate,
       }
     }
@@ -2650,7 +2665,6 @@ export default abstract class Server<
               value: {
                 kind: 'PAGE',
                 html: RenderResult.fromStatic(html),
-                postponed: undefined,
                 status: undefined,
                 headers: undefined,
                 pageData: {},
@@ -2719,7 +2733,7 @@ export default abstract class Server<
     }
 
     const didPostpone =
-      cacheEntry.value?.kind === 'PAGE' &&
+      cacheEntry.value?.kind === 'APP_PAGE' &&
       typeof cacheEntry.value.postponed === 'string'
 
     if (
@@ -2886,7 +2900,11 @@ export default abstract class Server<
     } else if (isAppPath) {
       // If the request has a postponed state and it's a resume request we
       // should error.
-      if (cachedData.postponed && minimalPostponed) {
+      if (
+        cachedData.kind === 'APP_PAGE' &&
+        cachedData.postponed &&
+        minimalPostponed
+      ) {
         throw new Error(
           'Invariant: postponed state should not be present on a resume request'
         )
@@ -2934,7 +2952,11 @@ export default abstract class Server<
       }
 
       // Mark that the request did postpone if this is a data request.
-      if (cachedData.postponed && isRSCRequest) {
+      if (
+        cachedData.kind === 'APP_PAGE' &&
+        cachedData.postponed &&
+        isRSCRequest
+      ) {
         res.setHeader(NEXT_DID_POSTPONE_HEADER, '1')
       }
 
@@ -2945,8 +2967,15 @@ export default abstract class Server<
       if (isDataReq && !isPreviewMode) {
         // If this is a dynamic RSC request, then stream the response.
         if (isDynamicRSCRequest) {
-          if (cachedData.pageData) {
-            throw new Error('Invariant: Expected pageData to be undefined')
+          if (cachedData.kind !== 'APP_PAGE') {
+            console.error({ url: req.url, pathname }, cachedData)
+            throw new Error(
+              `Invariant: expected cache data kind of APP_PAGE got ${cachedData.kind}`
+            )
+          }
+
+          if (cachedData.rscData) {
+            throw new Error('Invariant: Expected rscData to be undefined')
           }
 
           if (cachedData.postponed) {
@@ -2965,17 +2994,23 @@ export default abstract class Server<
           }
         }
 
-        if (typeof cachedData.pageData !== 'string') {
+        if (cachedData.kind !== 'APP_PAGE') {
+          throw new Error(
+            `Invariant: expected cached data to be APP_PAGE got ${cachedData.kind}`
+          )
+        }
+
+        if (!Buffer.isBuffer(cachedData.rscData)) {
           throw new Error(
-            `Invariant: expected pageData to be a string, got ${typeof cachedData.pageData}`
+            `Invariant: expected rscData to be a Buffer, got ${typeof cachedData.rscData}`
           )
         }
 
         // As this isn't a prefetch request, we should serve the static flight
         // data.
         return {
           type: 'rsc',
-          body: RenderResult.fromStatic(cachedData.pageData),
+          body: RenderResult.fromStatic(cachedData.rscData),
           revalidate: cacheEntry.revalidate,
         }
       }
@@ -2986,7 +3021,10 @@ export default abstract class Server<
       // If there's no postponed state, we should just serve the HTML. This
       // should also be the case for a resume request because it's completed
       // as a server render (rather than a static render).
-      if (!cachedData.postponed || this.minimalMode) {
+      if (
+        !(cachedData.kind === 'APP_PAGE' && cachedData.postponed) ||
+        this.minimalMode
+      ) {
         return {
           type: 'html',
           body,
@@ -3015,7 +3053,7 @@ export default abstract class Server<
             throw new Error('Invariant: expected a result to be returned')
           }
 
-          if (result.value?.kind !== 'PAGE') {
+          if (result.value?.kind !== 'APP_PAGE') {
             throw new Error(
               `Invariant: expected a page response, got ${result.value?.kind}`
             )
@@ -3041,6 +3079,11 @@ export default abstract class Server<
         revalidate: 0,
       }
     } else if (isDataReq) {
+      if (cachedData.kind !== 'PAGE') {
+        throw new Error(
+          `Invariant: expected cached data to be PAGE got ${cachedData.kind}`
+        )
+      }
       return {
         type: 'json',
         body: RenderResult.fromStatic(JSON.stringify(cachedData.pageData)),

packages/next/src/server/lib/incremental-cache/fetch-cache.ts

@@ -109,7 +109,10 @@ export default class FetchCache implements CacheHandler {
             }
             // rough estimate of size of cache value
             return (
-              value.html.length + (JSON.stringify(value.pageData)?.length || 0)
+              value.html.length +
+              (JSON.stringify(
+                value.kind === 'APP_PAGE' ? value.rscData : value.pageData
+              )?.length || 0)
             )
           },
         })