Merge pull request #813 from pulibrary/811-disallow-more

Try Disallow: /*?*, as used in orangelight
pulibrary · Jul 20, 2020 · b485cba · b485cba
2 parents b244104 + 1aaa2a0
commit b485cba
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/app/views/pages/robots.text.erb b/app/views/pages/robots.text.erb
@@ -1,7 +1,7 @@
 <% if Rails.env.production? %>
   User-agent: *
   Crawl-delay: 10
-  Disallow: /*?
+  Disallow: /*?*
 <% else %>
   User-Agent: *
   Disallow: /

diff --git a/spec/views/pages/robots.text.erb_spec.rb b/spec/views/pages/robots.text.erb_spec.rb
@@ -21,7 +21,7 @@
     end
 
     it "slows down spiders and disallows indexing anything with a query-string" do
-      expect(response).to include "Disallow: /*?"
+      expect(response).to include "Disallow: /*?*"
       expect(response).to include "Crawl-delay: 10"
     end
   end