tooling: add robots txt (#470)

Dockerfile

@@ -9,6 +9,7 @@ COPY migrations /app/migrations
 COPY views /app/views
 COPY dist /app/dist
 COPY LICENSE /app
+COPY robots.txt /app
 COPY fider /app
 
 EXPOSE 3000

app/cmd/routes.go

@@ -16,6 +16,11 @@ func routes(r *web.Engine) *web.Engine {
 	r.Use(middlewares.Secure())
 	r.Use(middlewares.Compress())
 
+	robots := r.Group()
+	{
+		robots.Get("/robots.txt", handlers.RobotsTXT())
+	}
+
 	assets := r.Group()
 	{
 		assets.Use(middlewares.CORS())

app/handlers/common.go

@@ -36,6 +36,17 @@ func LegalPage(title, file string) web.HandlerFunc {
 	}
 }
 
+//RobotsTXT return content of robots.txt file
+func RobotsTXT() web.HandlerFunc {
+	return func(c web.Context) error {
+		bytes, err := ioutil.ReadFile(env.Path("./robots.txt"))
+		if err != nil {
+			return c.NotFound()
+		}
+		return c.String(http.StatusOK, string(bytes))
+	}
+}
+
 //Page returns a page without properties
 func Page(title, description string) web.HandlerFunc {
 	return func(c web.Context) error {

app/handlers/common_test.go

@@ -48,3 +48,11 @@ func TestLegalPageHandler_Invalid(t *testing.T) {
 
 	Expect(code).Equals(http.StatusNotFound)
 }
+
+func TestRobotsTXT(t *testing.T) {
+	RegisterT(t)
+
+	server, _ := mock.NewServer()
+	code, _ := server.Execute(handlers.RobotsTXT())
+	Expect(code).Equals(http.StatusOK)
+}

robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /api/
+Disallow: /admin/
+Disallow: /oauth/

scripts/git-prune-local.sh

@@ -1,2 +1,3 @@
 # https://medium.com/@kcmueller/delete-local-git-branches-that-were-deleted-on-remote-repository-b596b71b530c
+git fetch -p
 git branch -vv | grep ' gone]' | awk '{print $1}' | xargs git branch -D