robots.txt

freedit-org · Oct 23, 2023 · b2b637b · b2b637b
1 parent 80c7e13
commit b2b637b
Show file tree

Hide file tree

Showing 4 changed files with 110 additions and 46 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/controller/inn.rs b/src/controller/inn.rs
@@ -207,7 +207,7 @@ pub(crate) async fn mod_inn_post(
         // remove the old inn name
         if input.inn_name != inn.inn_name {
             let old_inn_name_key = inn.inn_name.replace(' ', "_").to_lowercase();
-            inn_names_tree.remove(&old_inn_name_key)?;
+            inn_names_tree.remove(old_inn_name_key)?;
         }
 
         // remove the old inn topics

diff --git a/src/controller/meta_handler.rs b/src/controller/meta_handler.rs
@@ -156,7 +156,7 @@ pub(crate) async fn style() -> (HeaderMap, &'static str) {
 }
 
 pub(crate) async fn robots() -> &'static str {
-    "User-agent: *\nDisallow: /search"
+    include_str!("../../static/robots.txt")
 }
 
 pub async fn shutdown_signal() {

diff --git a/static/robots.txt b/static/robots.txt
@@ -0,0 +1,25 @@
+User-agent: *
+Disallow: /admin
+Disallow: /mod
+Disallow: /preview
+
+User-agent: MJ12Bot
+Disallow: /
+
+User-agent: PetalBot
+Disallow: /
+
+User-agent: AspiegelBot
+Disallow: /
+
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: SEMrushBot
+Disallow: /
+
+User-agent: DotBot
+Disallow: /
+
+User-agent: MauiBot
+Disallow: /