AP users: update robots.txt for new URL prefixes

for #512
snarfed · May 30, 2023 · 852bd5d · 852bd5d
1 parent 29f1a1d
commit 852bd5d
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/static/robots.txt b/static/robots.txt
@@ -1,5 +1,11 @@
 User-agent: *
 Disallow: /*?*
-Disallow: /user/*
+Disallow: /ap/*
+Disallow: /atp/*
+Disallow: /bluesky/*
+Disallow: /convert/*
+Disallow: /nostr/*
 Disallow: /recent*
 Disallow: /r/*
+Disallow: /user/*
+Disallow: /web/*