From 20598daf0e2806e0d67ce0f747ba3a97201172b2 Mon Sep 17 00:00:00 2001
From: Jeremy Prevost <jprevost@mit.edu>
Date: Mon, 24 Aug 2015 13:10:49 -0400
Subject: [PATCH] Adds robots.txt

closes #333
---
 conf/routes       |  1 +
 public/robots.txt | 11 +++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 public/robots.txt

diff --git a/conf/routes b/conf/routes
index 74d92f2..a88becd 100644
--- a/conf/routes
+++ b/conf/routes
@@ -196,3 +196,4 @@ POST    /submodel                    controllers.Application.addSubscriberModel
 
 # Map static resources from the /public folder to the /assets URL path
 GET     /assets/*file               controllers.Assets.at(path="/public", file)
+GET     /robots.txt                 controllers.Assets.at(path="/public", file="robots.txt")
diff --git a/public/robots.txt b/public/robots.txt
new file mode 100644
index 0000000..9ca367a
--- /dev/null
+++ b/public/robots.txt
@@ -0,0 +1,11 @@
+User-agent: *
+Disallow: /item
+Disallow: /item/
+Disallow: /items
+Disallow: /items/
+Disallow: /topic
+Disallow: /topic/
+Disallow: /topics
+Disallow: /topics/
+Disallow: /search
+Disallow: /search/