@@ -1,14 +1,27 @@ require "open-uri" require "uri" require "rubygems" +require "timeout" class Robots + DEFAULT_TIMEOUT = 3 + class ParsedRobots - def initialize(uri) + def initialize(uri, user_agent) @last_accessed = Time.at(1) - io = open(URI.join(uri.to_s, "/robots.txt")) rescue nil + + io = nil + begin + Timeout::timeout(Robots.timeout) do + io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil + end + rescue Timeout::Error + STDERR.puts "robots.txt request timed out" + end + + if !io || io.content_type != "text/plain" || io.status != ["200", "OK"] io = StringIO.new("User-agent: *\nAllow: /\n") end @@ -91,6 +104,14 @@ class Robots end end + def self.timeout=(t) + @timeout = t + end + + def self.timeout + @timeout || DEFAULT_TIMEOUT + end + def initialize(user_agent) @user_agent = user_agent @parsed = {} @@ -99,14 +120,14 @@ class Robots def allowed?(uri) uri = URI.parse(uri.to_s) unless uri.is_a?(URI) host = uri.host - @parsed[host] ||= ParsedRobots.new(uri) + @parsed[host] ||= ParsedRobots.new(uri, @user_agent) @parsed[host].allowed?(uri, @user_agent) end def other_values(uri) uri = URI.parse(uri.to_s) unless uri.is_a?(URI) host = uri.host - @parsed[host] ||= ParsedRobots.new(uri) + @parsed[host] ||= ParsedRobots.new(uri, @user_agent) @parsed[host].other_values end end lib/robots.rb @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "robots" - s.version = "0.6.0" + s.version = "0.7.1" s.date = "2008-12-10" s.summary = "Simple robots.txt parser" s.email = "kyle@kylemaxwell.com" robots.gemspec d45462fd2f26e9e1a14986e04da28cb498af80e1 Kyle Maxwell kyle@111.flood.pivotallabs.com http://github.com/fizx/robots/commit/f3cdfc5020785e4e34654a29b2e75b5b9ca70b5b f3cdfc5020785e4e34654a29b2e75b5b9ca70b5b 2009-03-24T15:11:13-07:00 2009-03-24T15:11:13-07:00 better 6c9d0f1da7c5953f46545e91b66b12097298100c Kyle Maxwell kyle@111.flood.pivotallabs.com