Added robots.txt to Stupid to prevent crawling

CalebQ42 · Aug 7, 2023 · 96d1011 · 96d1011
1 parent de4477b
commit 96d1011
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/embed/robots.txt b/embed/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/stupid.go b/stupid.go
@@ -2,6 +2,7 @@ package stupid
 
 import (
 	"crypto/ed25519"
+	_ "embed"
 	"log"
 	"net/http"
 	"path"
@@ -12,6 +13,9 @@ import (
 	"github.com/CalebQ42/stupid-backend/pkg/db"
 )
 
+//go:embed embed/robots.txt
+var robotsTxt []byte
+
 // An instance of the stupid backend. Implements http.Handler
 type Stupid struct {
 	keys            db.Table
@@ -86,6 +90,10 @@ func (s *Stupid) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		Method: r.Method,
 		Resp:   w,
 	}
+	if len(req.Path) == 1 && req.Path[0] == "robots.txt" {
+		w.Write(robotsTxt)
+		return
+	}
 	if s.headerValues != nil {
 		for k, v := range s.headerValues {
 			w.Header().Set(k, v)