Allow robots.txt with no useragent

rapportive-oss · Sep 12, 2011 · 1f55490 · 1f55490
1 parent 411cbb7
commit 1f55490
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 11 deletions.
diff --git a/lib/robotstxt/parser.rb b/lib/robotstxt/parser.rb
@@ -253,21 +253,20 @@ def parse(body)
 
             when "Disallow"
 
-              if @rules.any?
-                parser_mode = :rules
-                if value == ""
-                  @rules.last[1] << ["*", true]
-                else
-                  @rules.last[1] << [value, false]
-                end
+              parser_mode = :rules
+              @rules << ["*", []] if @rules.empty?
+
+              if value == ""
+                @rules.last[1] << ["*", true]
+              else
+                @rules.last[1] << [value, false]
               end
 
             when "Allow"
 
-              if @rules.any?
-                parser_mode = :rules
-                @rules.last[1] << [value, true]
-              end
+              parser_mode = :rules
+              @rules << ["*", []] if @rules.empty?
+              @rules.last[1] << [value, true]
 
             when "Sitemap"
               @sitemaps << value

diff --git a/test/parser_test.rb b/test/parser_test.rb
@@ -87,4 +87,12 @@ def test_useragents
     assert false == Robotstxt::Parser.new("Bing", robotstxt).allowed?("/hello")
   end
 
+  def test_missing_useragent
+    robotstxt = <<-ROBOTS
+Disallow: /index
+ROBOTS
+    assert true === Robotstxt::Parser.new("Google", robotstxt).allowed?("/hello")
+    assert false === Robotstxt::Parser.new("Google", robotstxt).allowed?("/index/wold")
+  end
+
 end