feat: defaults to noindex nofollow (#11623)

jacobsfletch · web-flow · commit 5285518562f5 · 2025-03-11T13:29:49.000-04:00
We now have the ability to define all page metadata for the admin panel via the Payload Config as a result of #11593. This means we can now set sensible defaults for additional properties, e.g. `noindex` and `nofollow` on the `robots` property. Setting this will prevent these pages from being indexed and appearing in search results. Note that setting this property prevents _indexing_ these pages, but does not prevent them from being _crawled_. To prevent crawling as well, you must add a standalone `robots.txt` file to your root directory.
diff --git a/docs/admin/metadata.mdx b/docs/admin/metadata.mdx
@@ -53,7 +53,6 @@ The following options are available for Root Metadata:
 | Key | Type | Description |
 | --- | --- | --- |
 | `defaultOGImageType` | `dynamic` (default), `static`, or `off` | The type of default OG image to use. If set to `dynamic`, Payload will use Next.js image generation to create an image with the title of the page. If set to `static`, Payload will use the `defaultOGImage` URL. If set to `off`, Payload will not generate an OG image. |
-| `icons` | `IconConfig[]` | An array of icon objects. [More details](#icons). |
 | `titleSuffix` | `string` | A suffix to append to the end of the title of every page. Defaults to "- Payload". |
 | `[keyof Metadata]` | `unknown` | Any other properties that Next.js supports within the `generateMetadata` function. [More details](https://nextjs.org/docs/app/api-reference/functions/generate-metadata). |
 
@@ -68,7 +67,7 @@ The Icons Config corresponds to the `<link>` tags that are used to specify icons
 
 The most common icon type is the favicon, which is displayed in the browser tab. This is specified by the `rel` attribute `icon`. Other common icon types include `apple-touch-icon`, which is used by Apple devices when the Admin Panel is saved to the home screen, and `mask-icon`, which is used by Safari to mask the Admin Panel icon.
 
-To customize icons, use the `icons` key within the `admin.meta` object in your Payload Config:
+To customize icons, use the `admin.meta.icons` property in your Payload Config:
 
 ```ts
 {
@@ -100,7 +99,7 @@ For a full list of all available Icon options, see the [Next.js documentation](h
 
 Open Graph metadata is a set of tags that are used to control how URLs are displayed when shared on social media platforms. Open Graph metadata is automatically generated by Payload, but can be customized at the Root level.
 
-To customize Open Graph metadata, use the `openGraph` key within the `admin.meta` object in your Payload Config:
+To customize Open Graph metadata, use the `admin.meta.openGraph` property in your Payload Config:
 
 ```ts
 {
@@ -128,6 +127,45 @@ To customize Open Graph metadata, use the `openGraph` key within the `admin.meta
 
 For a full list of all available Open Graph options, see the [Next.js documentation](https://nextjs.org/docs/app/api-reference/functions/generate-metadata#opengraph).
 
+### Robots
+
+Setting the `robots` property will allow you to control the `robots` meta tag that is rendered within the `<head>` of the Admin Panel. This can be used to control how search engines index pages and displays them in search results.
+
+By default, the Admin Panel is set to prevent search engines from indexing pages within the Admin Panel.
+
+To customize the Robots Config, use the `admin.meta.robots` property in your Payload Config:
+
+```ts
+{
+  // ...
+  admin: {
+    meta: {
+      // highlight-start
+      robots: 'noindex, nofollow',
+      // highlight-end
+    },
+  },
+}
+```
+
+For a full list of all available Robots options, see the [Next.js documentation](https://nextjs.org/docs/app/api-reference/functions/generate-metadata#robots).
+
+##### Prevent Crawling 
+
+While setting meta tags via `admin.meta.robots` can prevent search engines from _indexing_ web pages, it does not prevent them from being _crawled_.
+
+To prevent your pages from being crawled altogether, add a `robots.txt` file to your root directory.
+
+```txt
+User-agent: *
+Disallow: /admin/
+```
+
+<Banner type="info">
+  **Note:**
+  If you've customized the path to your Admin Panel via `config.routes`, be sure to update the `Disallow` directive to match your custom path.
+</Banner>
+
 ## Collection Metadata
 
 Collection Metadata is the metadata that is applied to all pages within any given Collection within the Admin Panel. This metadata is used to customize the title and description of all views within any given Collection, unless overridden by the view itself.
diff --git a/packages/payload/src/config/defaults.ts b/packages/payload/src/config/defaults.ts
@@ -18,6 +18,7 @@ export const defaults: Omit<Config, 'db' | 'editor' | 'secret'> = {
     },
     meta: {
       defaultOGImageType: 'dynamic',
+      robots: 'noindex, nofollow',
       titleSuffix: '- Payload',
     },
     routes: {
@@ -91,6 +92,7 @@ export const addDefaultsToConfig = (config: Config): Config => {
     },
     meta: {
       defaultOGImageType: 'dynamic',
+      robots: 'noindex, nofollow',
       titleSuffix: '- Payload',
       ...(config?.admin?.meta || {}),
     },
diff --git a/test/admin/config.ts b/test/admin/config.ts
@@ -129,7 +129,6 @@ export default buildConfigWithDefaults({
         description: 'This is a custom OG description',
         title: 'This is a custom OG title',
       },
-      robots: 'nofollow noindex',
       titleSuffix: '- Custom Title Suffix',
     },
     routes: customAdminRoutes,
diff --git a/test/admin/e2e/general/e2e.spec.ts b/test/admin/e2e/general/e2e.spec.ts
@@ -158,6 +158,16 @@ describe('General', () => {
       })
     })
 
+    describe('robots', () => {
+      test('should apply default robots meta tag', async () => {
+        await page.goto(`${serverURL}/admin`)
+        await expect(page.locator('meta[name="robots"]')).toHaveAttribute(
+          'content',
+          /noindex, nofollow/,
+        )
+      })
+    })
+
     describe('favicons', () => {
       test('should render custom favicons', async () => {
         await page.goto(postsUrl.admin)
