Fix http://github.com/fizx/robots/issues#issue/1; Tests don't rely on…

… network.

CHANGELOG

@@ -1,3 +1,6 @@
+0.9.0
+  - Fix http://github.com/fizx/robots/issues#issue/1
+  - Tests don't rely on network.
 0.8.0
   - Add multiple values from robots.txt (via joost)
 0.7.3

lib/robots.rb

@@ -12,15 +12,7 @@ class ParsedRobots
     def initialize(uri, user_agent)
       @last_accessed = Time.at(1)
 
-      io = nil
-      begin
-        Timeout::timeout(Robots.timeout) do
-          io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
-        end 
-      rescue Timeout::Error
-        STDERR.puts "robots.txt request timed out"
-      end
-
+      io = Robots.get_robots_txt(uri)
 
       if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
         io = StringIO.new("User-agent: *\nAllow: /\n")
@@ -99,12 +91,23 @@ def other_values
   protected
 
     def to_regex(pattern)
+      return /should-not-match-anything-123456789/ if pattern.strip.empty?
       pattern = Regexp.escape(pattern)
       pattern.gsub!(Regexp.escape("*"), ".*")
       Regexp.compile("^#{pattern}")
     end
   end
 
+  def self.get_robots_txt(uri)
+    begin
+      Timeout::timeout(Robots.timeout) do
+        io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
+      end 
+    rescue Timeout::Error
+      STDERR.puts "robots.txt request timed out"
+    end
+  end
+
   def self.timeout=(t)
     @timeout = t
   end

test/fixtures/emptyish.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: