Skip to content
Browse files

Don't check to fetch /robots.txt when trying to fetch /robots.txt

  • Loading branch information...
1 parent 7f1d85c commit 3dd7725965a89f247716c88596015ddaf100d09e @drbrain committed
Showing with 29 additions and 0 deletions.
  1. +1 −0 lib/webrobots.rb
  2. +28 −0 test/test_webrobots.rb
View
1 lib/webrobots.rb
@@ -117,6 +117,7 @@ def split_uri(url)
def evaluate(url)
site, request_uri = split_uri(url)
+ return nil, request_uri if request_uri == '/robots.txt'
return get_robots_txt(site), request_uri
end
View
28 test/test_webrobots.rb
@@ -517,4 +517,32 @@ class TestWebRobots < Test::Unit::TestCase
assert @doc.meta_robots('googlebot').include?('noarchive')
end
end
+
+ class Agent
+ def initialize
+ @robots = WebRobots.new 'agent', :http_get => method(:get)
+ end
+
+ def get uri
+ @robots.allowed? uri
+
+ if uri.request_uri == '/robots.txt' then
+ ''
+ else
+ 'content'
+ end
+ end
+ end
+
+ context "embedded in a user-agent" do
+ setup do
+ @agent = Agent.new
+ end
+
+ should "fetch robots.txt" do
+ body = @agent.get URI.parse 'http://example/robots.html'
+
+ assert_equal 'content', body
+ end
+ end
end

0 comments on commit 3dd7725

Please sign in to comment.
Something went wrong with that request. Please try again.