@@ -36,47 +36,47 @@ class Server # You can pass in multiple paths, and it will try each one until it gets a 200 def fetch(*paths) paths.each do |path| - begin - Merb.logger.debug("loading... " + base_url + path) + result = nil - result = fetch_from_web(path) + Merb.logger.debug("loading... " + base_url + path) - compressed_size = result.downloaded_content_length + result = fetch_from_web(path) + - Merb.logger.debug("loaded #{result.downloaded_content_length} compressed characters") + compressed_size = result.downloaded_content_length - if result.response_code == 200 - - if result.header_str.include?("Cache-Lookup: HIT") - Merb.logger.debug("Spider HIT") - else - Merb.logger.debug("Spider MISS") - end + Merb.logger.debug("loaded #{result.downloaded_content_length} compressed characters") - body = nil - + if result.response_code == 200 + + if result.header_str.include?("Cache-Lookup: HIT") + Merb.logger.debug("Spider HIT") + else + Merb.logger.debug("Spider MISS") + end + + body = nil + + + time_to "decompress" do - time_to "decompress" do - - begin - gz = Zlib::GzipReader.new( StringIO.new( result.body_str ) ) - body = gz.read - rescue Zlib::GzipFile::Error - # If its not looking gzipped, just display it - body = result.body_str - end - - body = body.force_encoding("UTF-8") + begin + gz = Zlib::GzipReader.new( StringIO.new( result.body_str ) ) + body = gz.read + rescue Zlib::GzipFile::Error + # If its not looking gzipped, just display it + body = result.body_str end - Merb.logger.debug("Decompressed to #{body.size} characters") - - return {:url => result.last_effective_url, :body => body} + body = body.force_encoding("UTF-8") end - - rescue Curl::Err::HostResolutionError, Curl::Err::GotNothingError, Curl::Err::ConnectionFailedError, Curl::Err::PartialFileError - Merb.logger.error("Could not connect to " + base_url + path) + + Merb.logger.debug("Decompressed to #{body.size} characters") + + return {:url => result.last_effective_url, :body => body} end + + end return {} @@ -85,11 +85,21 @@ class Server # :api: private def fetch_from_web(path) time_to "fetch" do - Curl::Easy.perform(base_url + path) do |curl| - # This configures Curl::Easy to follow redirects - curl.follow_location = true - curl.headers = {"Accept-Encoding" => "compress, gzip"} - end + if defined?(Curl) + begin + Curl::Easy.perform(base_url + path) do |curl| + # This configures Curl::Easy to follow redirects + curl.follow_location = true + curl.headers = {"Accept-Encoding" => "compress, gzip"} + end + rescue Curl::Err::HostResolutionError, Curl::Err::GotNothingError, Curl::Err::ConnectionFailedError, Curl::Err::PartialFileError + Merb.logger.error("Could not connect to " + base_url + path) + end + else + # This is for if we are using a non-curl supported version of Ruby + require 'open-uri' + open(base_url + path).read + end end end end \ No newline at end of file app/models/server.rb @@ -8,11 +8,16 @@ end require 'sass' require 'cgi' -require 'curb' require 'merb-haml' require 'nokogiri' require 'mime/types' +begin + require 'curb' +rescue + puts "no curb installed.. using open-uri" +end + if is19? Encoding.default_internal = Encoding.default_external = "UTF-8" require Merb.root / 'merb' / 'monkey' / 'haml_fix' config/init.rb @@ -4,7 +4,12 @@ describe Article do before(:all) do # Stub out the device - @device = Device.new(nil) + request = mock("request") + request.stub!("preferred_format").and_return nil + request.stub!("accept").and_return "xhtml" + + + @device = Device.new(request) @device.instance_eval("@user_agent = 'Webkit'") # Stubs out networking spec/models/article_spec.rb 14b1f34ad408d144eb2bf65104b1e0fc5e12cc69 Hampton Catlin hcatlin@gmail.com http://github.com/hcatlin/wikimedia-mobile/commit/2bba31f7ffdd7a1740955c97ec470b6f85837e25 2bba31f7ffdd7a1740955c97ec470b6f85837e25 2009-10-25T04:59:55-07:00 2009-10-25T04:59:55-07:00 refactoring the server object to not require curb, but use it if its installed... should help with non-MRI-based interpreters 1ae30c1b213740db92cfc03b3f866d3e4439369a Hampton Catlin hcatlin@gmail.com