@@ -20,6 +20,7 @@ require "rubygems" require "hpricot" require "open-uri" +require "net/http" module Feedbag @@ -32,23 +33,32 @@ module Feedbag 'application/rdf+xml', ] + @debug = 1 + $feeds = [] + $base_uri = nil def self.find(url) $feeds = [] begin html = open(url) do |f| if @content_types.include?(f.content_type.downcase) - return self.add_feed(url) + return self.add_feed(url, nil) end doc = Hpricot(f.read) + if doc.at("base") and doc.at("base")["href"] + $base_uri = doc.at("base")["href"] + else + $base_uri = nil + end + # first with links (doc/"link").each do |l| next unless l["rel"] if l["type"] and @content_types.include?(l["type"].downcase.strip) and (l["rel"].downcase =~ /alternate/i or l["rel"] == "service.feed") - self.add_feed(l["href"], url) + self.add_feed(l["href"], url, $base_uri) end end @@ -59,7 +69,7 @@ module Feedbag a["href"] =~ /feed=(rss2|atom)/i or a["href"] =~ /(atom|feed)\/$/i) - self.add_feed(a["href"], url) + self.add_feed(a["href"], url, $base_uri) end end @@ -71,9 +81,17 @@ module Feedbag $feeds end - def self.add_feed(feed_url, orig_url) + def self.add_feed(feed_url, orig_url, base_uri = nil) + puts "#{feed_url} - #{orig_url}" url = feed_url.sub(/^feed:/, '').strip + if base_uri + url = base_uri + feed_url + puts "base_uri: #{base_uri}" if @debug + puts "orig_url: #{orig_url}" if @debug + puts "feed_url: #{feed_url}" if @debug + end + begin uri = URI.parse(url) rescue @@ -84,7 +102,21 @@ module Feedbag orig = URI.parse(orig_url) url = orig.merge(url).to_s end - $feeds.push(url) + + # verify url is really valid + $feeds.push(url) if self._is_http_valid(URI.parse(url), url) + end + + def self._is_http_valid(uri, url) + req = Net::HTTP.get_response(uri) + case req + when Net::HTTPSuccess then + return true + else + guess_url = "#{uri.scheme}://#{uri.host}#{uri.path}" + return false if guess_url == url + return true if self._is_http_valid(URI.parse(guess_url), guess_url) + end end end feedbag.rb @@ -1,6 +1,6 @@ #!/usr/bin/ruby -require "../feedbag" +require "#{File.dirname(__FILE__)}/../feedbag" require "test/unit" require "open-uri" require "hpricot" @@ -9,35 +9,36 @@ require "pp" class AtomAutoDiscoveryTest < Test::Unit::TestCase def test_autodisc base_url = "http://diveintomark.org/tests/client/autodiscovery/" - url = "html4-001.html" + url = base_url + "html4-001.html" i = 1 puts "trying now with #{url}" while(i) + puts i = 0 # unless otherwise found - f = Feedbag.find base_url + url + f = Feedbag.find url assert_instance_of Array, f - assert f.size == 1, "Feedbag didn't find a feed on #{base_url + url} or found more than one" + assert f.size == 1, "Feedbag didn't find a feed on #{url} or found more than one" puts " found #{f[0]}" feed = Hpricot(open(f[0])) (feed/"link").each do |l| next unless l["rel"] == "alternate" - assert_equal l["href"], base_url + url + assert_equal l["href"], url end # ahora me voy al siguiente - html = Hpricot(open(base_url + url)) + html = Hpricot(open(url)) (html/"link").each do |l| next unless l["rel"] == "next" - url = l["href"] + url = URI.parse(base_url).merge(l["href"]).to_s puts "trying now with #{url}" i = 1 end - + end end test/atom_autodiscovery_test.rb 6bda23094a9639cff53fc27cb2ba320cceca02c2 David Moreno david@axiombox.com http://github.com/damog/feedbag/commit/98c2023b540b8d6d7931bd8df1ddfe1edd81ec3c 98c2023b540b8d6d7931bd8df1ddfe1edd81ec3c 2008-12-25T09:18:04-08:00 2008-12-25T09:18:04-08:00 adding _is_http_valid for unsuccessful testing: undone 784d2c25262846f71c27fdb0f87b6be6488e05ce David Moreno david@axiombox.com