feat: waiting for all content to load for spiders in streaming ssr (#…

…5186)

.changeset/new-games-tell.md

@@ -0,0 +1,8 @@
+---
+'@modern-js/runtime': patch
+'@modern-js/prod-server': patch
+'@modern-js/types': patch
+---
+
+feat: Waiting for all content to load for spiders in streaming ssr
+feat: 在 streaming ssr 时，为爬虫等待所有内容加载完毕

packages/document/main-doc/docs/en/guides/advanced-features/ssr/stream.mdx

@@ -228,6 +228,14 @@ function ErrorElement() {
 }
 ```
 
+## Waiting for all content to load for spiders
+
+Streaming offers a better user experience because the user can see the content as it becomes available.
+
+However, when a spider visits your page, you might want to let all of the content load first and then produce the final HTML output instead of revealing it progressively.
+
+Modern.js uses [isbot](https://www.npmjs.com/package/isbot) to examine the user-agent of requests, determining whether they come from a crawler.
+
 :::info More
 
 1. [Deferred Data](https://reactrouter.com/en/main/guides/deferred)

packages/document/main-doc/docs/zh/guides/advanced-features/ssr/stream.mdx

@@ -232,6 +232,14 @@ function ErrorElement() {
 }
 ```
 
+## 为爬虫等待所有内容加载完毕
+
+流式传输可以提高用户体验，因为当页面内容可用时，用户可以及时感知到它们。
+
+然而，当一个爬虫访问该页面时，它可能需要先加载所有内容，直接输出整个 HTML，而不是渐进式地加载它。
+
+Modern.js 使用 [isbot](https://www.npmjs.com/package/isbot) 对请求的 `uesr-agent`, 以判断请求是否来自爬虫。
+
 :::info 补充信息
 
 1. [Deferred Data](https://reactrouter.com/en/main/guides/deferred)

packages/runtime/plugin-runtime/src/ssr/serverRender/renderToStream/renderToPipe.ts

@@ -19,6 +19,9 @@ function renderToPipe(
 ) {
   let shellChunkStatus = ShellChunkStatus.START;
 
+  // When a crawler visit the page, we should waiting for entrie content of page
+  const onReady = context.ssrContext?.isSpider ? 'onAllReady' : 'onShellReady';
+
   const { ssrContext } = context;
   const chunkVec: string[] = [];
   const forUserPipe: Pipe<Writable> = stream => {
@@ -32,7 +35,7 @@ function renderToPipe(
       const { pipe } = renderToPipeableStream(rootElement, {
         ...options,
         nonce: ssrContext?.nonce,
-        onShellReady() {
+        [onReady]() {
           getTemplates(context, RenderLevel.SERVER_RENDER, pluginConfig).then(
             ({ shellAfter, shellBefore }) => {
               options?.onShellReady?.();

packages/runtime/plugin-runtime/src/ssr/serverRender/renderToStream/renderToPipe.worker.ts

@@ -45,7 +45,15 @@ function renderToPipe(
             options?.onError?.(error);
           },
         });
+
+      if (context.ssrContext?.isSpider) {
+        // However, when a crawler visits your page, or if you’re generating the pages at the build time,
+        // you might want to let all of the content load first and then produce the final HTML output instead of revealing it progressively.
+        // from: https://react.dev/reference/react-dom/server/renderToReadableStream#handling-different-errors-in-different-ways
+        await readableOriginal.allReady;
+      }
       const reader: ReadableStreamDefaultReader = readableOriginal.getReader();
+
       const injectableStream = new ReadableStream({
         start(controller) {
           async function push() {

packages/server/prod-server/package.json

@@ -74,6 +74,7 @@
     "@modern-js/server-core": "workspace:*",
     "@modern-js/utils": "workspace:*",
     "@modern-js/runtime-utils": "workspace:*",
+    "isbot": "^4.2.0",
     "@swc/helpers": "0.5.3",
     "cookie": "0.5.0",
     "etag": "^1.8.1",

packages/server/prod-server/src/libs/render/ssr.ts

@@ -7,6 +7,7 @@ import {
   SERVER_RENDER_FUNCTION_NAME,
 } from '@modern-js/utils';
 import type { ModernServerContext } from '@modern-js/types';
+import { isbot } from 'isbot';
 import { RenderResult, ServerHookRunner } from '../../type';
 import { createAfterStreamingRenderContext } from '../hook-api';
 import { afterRenderInjectableStream } from '../hook-api/afterRenderForStream';
@@ -47,6 +48,8 @@ export const render = async (
     ? require(routesManifestUri)
     : undefined;
 
+  const isSpider = isbot(ctx.headers['user-agent'] || null);
+
   const context: SSRServerContext = {
     request: {
       baseUrl: urlPath,
@@ -79,6 +82,7 @@ export const render = async (
     req: ctx.req,
     res: ctx.res,
     enableUnsafeCtx,
+    isSpider,
     nonce,
   };
   context.logger = createLogger(context, ctx.logger);

packages/toolkit/types/server/context.d.ts

@@ -109,6 +109,8 @@ export type BaseSSRServerContext<T extends 'node' | 'worker' = 'node'> = {
   res: T extends 'worker' ? BaseResponseLike : ModernServerContext['res'];
 
   mode?: SSRMode; // ssr type
+
+  isSpider?: boolean; // Check if it's spider request
 };
 
 export interface ServerInitHookContext {