Fix frontend robots.txt (#4186)

frontend/.gitignore

@@ -27,3 +27,7 @@ src/locales/scripts/wp-locales.json
 /.npmrc
 /.pnpmfile.cjs
 /pnpm-lock.yaml
+
+# To prevent accidentally adding a hardcoded robots.txt, see
+# /src/server-middleware/robots.js for the robots.txt file.
+src/static/robots.txt

frontend/src/server-middleware/robots.js

@@ -1,22 +1,44 @@
 const { LOCAL, PRODUCTION } = require("../constants/deploy-env")
 
+const deniedUserAgents = [
+  "GPTBot",
+  "CCBot",
+  "ChatGPT-User",
+  "Google-Extended",
+  "anthropic-ai",
+  "Omgilibot",
+  "Omgili",
+  "FacebookBot",
+  "Diffbot",
+  "Bytespider",
+  "ImagesiftBot",
+  "cohere-ai",
+]
+
+const aiDisallowRules = deniedUserAgents
+  .map((ua) => `User-agent: ${ua}\nDisallow: /\n`)
+  .join("\n")
+
 /**
  * Send the correct robots.txt information per-environment.
  */
 export default function robots(_, res) {
   const deployEnv = process.env.DEPLOYMENT_ENV ?? LOCAL
 
   const contents =
-    deployEnv === PRODUCTION
-      ? `# Block search result pages
+    deployEnv !== PRODUCTION
+      ? `# Block search result pages and single result pages
 User-agent: *
+Crawl-delay: 10
 Disallow: /search/audio/
 Disallow: /search/image/
 Disallow: /search/
 Disallow: /image/
 Disallow: /audio/
+
+${aiDisallowRules}
       `
-      : `# Block crawlers from the staging site
+      : `# Block everyone from the staging site
 User-agent: *
 Disallow: /
 `