autogenerate robots.txt

this automatically generates our robots.txt file, grabbing an updated list of ai scrapers to block each time
nyancrimew · Nov 10, 2024 · 500a00f · 500a00f
1 parent 395fb36
commit 500a00f
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 5 deletions.
diff --git a/src/_data/aibots.js b/src/_data/aibots.js
@@ -0,0 +1,4 @@
+module.exports = async function() {
+    response = await fetch("https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/refs/heads/main/robots.txt");
+    return response.text()
+}
diff --git a/src/robots.njk b/src/robots.njk
@@ -0,0 +1,16 @@
+---
+permalink: /robots.txt
+---
+# omg haiiiii robots ^-^
+# i love robots :3
+
+# AI scrapers
+{{ aibots }}
+
+# everyone else
+User-agent: *
+Allow: /
+
+# sitemaps
+Sitemap: {{ site.url }}/sitemap.xml
+Sitemap: {{ site.url }}/sitemap-news.xml
diff --git a/src/static/robots.txt b/src/static/robots.txt