lib/rfeedparser/expat_parser.rb lib/rfeedparser/libxml_parser.rb lib/rfeedparser/loose_feed_parser.rb tests/to_skip.yml @@ -1,5 +1,6 @@ === Testing rFeedParser === - Simply run `ruby tests/rfeedparsertest.rb` to run all of the FeedParser tests. +Simply run rake to run all of the FeedParser tests. + Optionally, you can start up rfeedparserserver.rb and, in another shell, run rfeedparser.rb against "http://localhost:8097/tests/path/to/testcase.xml" if you want to try a test individually. You can, of course, run rfeedparser.rb with @@ -9,7 +10,7 @@ the HTTP server path. I'll probably merge feedparserserver.rb into rfeedparsertest.rb soon. === Last Count 20070321 === - By my last count, rfeedparsertest.rb says that there are 45 assertions +By my last count, rfeedparsertest.rb says that there are 45 assertions that fail, and 4 that error out. I've included here a few tests that "Failed, Sort Of". By that I mean, the behaviors the tests are meant to check are correct, but the test fails because of some other superficial RUBY-TESTING @@ -1,11 +1,16 @@ require 'rubygems' -Gem::manage_gems require 'rake/testtask' require 'rake/gempackagetask' +begin + require 'lib/rfeedparser' +rescue LoadError + module FeedParser; VERSION = '0.0.0'; end + puts "Problem loading rfeedparser; try rake setup" +end spec = Gem::Specification.new do |s| s.name = "rfeedparser" - s.version = "0.9.940" # Don't forget the Version in rfeedparser.rb + s.version = FeedParser::VERSION s.author = "Jeff Hodges" s.email = "jeff at somethingsimilar dot com" s.homepage = "http://rfeedparser.rubyforge.org/" @@ -15,7 +20,7 @@ spec = Gem::Specification.new do |s| s.require_path = "lib" # s.autorequire = "feedparser" # tHe 3vil according to Why. s.test_file = "tests/rfeedparsertest.rb" - s.has_rdoc = false + s.has_rdoc = false # TODO: fix s.extra_rdoc_files = ['README','LICENSE', 'RUBY-TESTING'] s.rubyforge_project = 'rfeedparser' @@ -27,11 +32,10 @@ spec = Gem::Specification.new do |s| s.add_dependency('htmltools', '>= 1.10') s.add_dependency('htmlentities', '4.0.0') s.add_dependency('mongrel', '>=1.0.1') - s.requirements << "Yoshida Masato's Ruby bindings to the Expat XML parser" + s.add_dependency('addressable', '>= 1.0.4') + s.requirements << "Ruby bindings to the Expat XML parser or libxml2 (version 0.5.2 or greater)" end - - Rake::GemPackageTask.new(spec) do |pkg| pkg.need_zip = true end @@ -45,6 +49,7 @@ end task :default => [:test] # Taken liberally from http://blog.labnotes.org/2008/02/28/svn-checkout-rake-setup/ +# TODO: update to work with rubygems 1.0+ desc "If you're building from source, run this task first to setup the necessary dependencies." task :setup do puts "\nOn top of these gems, you'll also need #{spec.requirements.join(',')}." Rakefile @@ -1,104 +1,109 @@ #!/usr/bin/env ruby -"""Universal feed parser in Ruby +# Universal feed parser in Ruby +# +# Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds +# +# Visit http://feedparser.org/ for the latest version in Python +# Visit http://feedparser.org/docs/ for the latest documentation +# Email Jeff Hodges at jeff@obquo.com for questions +# +# Required: Ruby 1.8 -Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds - -Visit http://feedparser.org/ for the latest version in Python -Visit http://feedparser.org/docs/ for the latest documentation -Email Jeff Hodges at jeff@obquo.com for questions - -Required: Ruby 1.8 -""" $KCODE = 'UTF8' require 'stringio' require 'uri' +require 'open-uri' require 'cgi' # escaping html require 'time' require 'pp' -require 'rubygems' require 'base64' require 'iconv' require 'zlib' +require 'rubygems' + + +# If available, Nikolai's UTF-8 library will ease use of utf-8 documents. +# See http://git.bitwi.se/ruby-character-encodings.git/. +begin + gem 'character-encodings', ">=0.2.0" + require 'encoding/character/utf-8' +rescue LoadError +end + +# TODO: require these in the files that need them, not in the toplevel gem 'hpricot', "=0.6" require 'hpricot' -gem 'character-encodings', ">=0.2.0" + gem 'htmltools', ">=1.10" +require 'html/sgml-parser' + gem 'htmlentities', ">=4.0.0" +require 'htmlentities' + gem 'activesupport', ">=1.4.1" -gem 'rchardet', ">=1.0" +require 'active_support' -require 'xml/saxdriver' # calling expat through the xmlparser gem +gem 'addressable', ">= 1.0.4" +require 'addressable/uri' +gem 'rchardet', ">=1.0" require 'rchardet' $chardet = true -require 'encoding/character/utf-8' -require 'html/sgml-parser' -require 'htmlentities' -require 'active_support' -require 'open-uri' -include OpenURI - $debug = false $compatible = true -$LOAD_PATH << File.expand_path(File.dirname(__FILE__)) +$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) require 'rfeedparser/utilities' -require 'rfeedparser/forgiving_uri' require 'rfeedparser/better_sgmlparser' require 'rfeedparser/better_attributelist' require 'rfeedparser/feedparserdict' require 'rfeedparser/parser_mixin' -require 'rfeedparser/parsers' -require 'rfeedparser/monkey_patches' + +require 'rfeedparser/loose_feed_parser' + +begin + require 'rfeedparser/expat_parser' + StrictFeedParser = FeedParser::Expat::StrictFeedParser + +rescue LoadError, NameError + STDERR.puts "Could not load expat; trying libxml." + + begin + require 'rfeedparser/libxml_parser' + StrictFeedParser = FeedParser::LibXml::StrictFeedParser + rescue LoadError, NameError + STDERR.puts "Could not load libxml either; will use loose parser." + end +end +require 'rfeedparser/monkey_patches' module FeedParser extend FeedParserUtilities - Version = "0.9.940" - - License = """Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE.""" - - Translator_From_Python_To_Ruby = "Jeff Hodges <http://somethingsimilar.com>" - Author = "Mark Pilgrim <http://diveintomark.org/>" - Contributors = [ "Jason Diamond <http://injektilo.org/>", - "John Beimler <http://john.beimler.org/>", - "Fazal Majid <http://www.majid.info/mylos/weblog/>", - "Aaron Swartz <http://aaronsw.com/>", - "Kevin Marks <http://epeus.blogspot.com/>", - "Jesse Newland <http://jnewland.com/>" - ] + VERSION = "0.9.940" + + AUTHOR = "Mark Pilgrim <http://diveintomark.org/>" + PORTER = "Jeff Hodges <http://somethingsimilar.com>" + CONTRIBUTERS = ["Jason Diamond <http://injektilo.org/>", + "John Beimler <http://john.beimler.org/>", + "Fazal Majid <http://www.majid.info/mylos/weblog/>", + "Aaron Swartz <http://aaronsw.com/>", + "Kevin Marks <http://epeus.blogspot.com/>", + "Jesse Newland <http://jnewland.com/>", + "Charlie Savage <http://cfis.savagexi.com/>", + "Phil Hagelberg <http://technomancy.us>"] + # HTTP "User-Agent" header to send to servers when downloading feeds. # If you are embedding feedparser in a larger application, you should # change this to your application name and URL. - USER_AGENT = "rFeedParser/#{Version} +http://rfeedparser.rubyforge.org/" + USER_AGENT = "rFeedParser/#{VERSION} +http://rfeedparser.rubyforge.org/" # HTTP "Accept" header to send to servers when downloading feeds. If you don't - # want to send an Accept header, set this to None. + # want to send an Accept header, set this to nil. ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1" @@ -159,14 +164,14 @@ module FeedParser url_file_stream_or_string.strip! - furi = ForgivingURI.parse(url_file_stream_or_string) - if furi && ['http','https','ftp'].include?(furi.scheme) + uri = Addressable::URI.parse(url_file_stream_or_string) + if uri && ['http','https','ftp'].include?(uri.scheme) auth = nil - if furi.host && furi.password - auth = Base64::encode64("#{furi.user}:#{furi.password}").strip - furi.password = nil - url_file_stream_or_string = furi.to_s + if uri.host && uri.password + auth = Base64::encode64("#{uri.user}:#{uri.password}").strip + uri.password = nil + url_file_stream_or_string = uri.to_s end req_headers = {} @@ -213,8 +218,8 @@ module FeedParser # Use the default compatibility if compatible is nil $compatible = options[:compatible].nil? ? $compatible : options[:compatible] - - strictklass = options[:strict] || StrictFeedParser + + strictklass = options[:strict] || StrictFeedParser rescue nil looseklass = options[:loose] || LooseFeedParser options[:handlers] = options[:handlers] || [] @@ -392,26 +397,15 @@ module FeedParser result['encoding'] = proposed_encoding end + use_strict_parser = false if !defined? StrictFeedParser + if use_strict_parser - # initialize the SAX parser - saxparser = XML::SAX::Helpers::ParserFactory.makeParser("XML::Parser::SAXDriver") - feedparser = strictklass.new(baseuri, baselang, 'utf-8') - saxparser.setDocumentHandler(feedparser) - saxparser.setDTDHandler(feedparser) - saxparser.setEntityResolver(feedparser) - saxparser.setErrorHandler(feedparser) - - inputdata = XML::SAX::InputSource.new('parsedfeed') - inputdata.setByteStream(StringIO.new(data)) begin - saxparser.parse(inputdata) - - rescue StandardError, XML::SAX::SAXParseException => parseerr # resparse - - if $debug - $stderr << "xml parsing failed\n" - $stderr << parseerr.to_s+"\n" # Hrmph. - end + parser = StrictFeedParser.new(baseuri, baselang) + feedparser = parser.handler + parser.parse(data) + rescue => err + $stderr << "xml parsing failed: #{err.message}\n#{err.backtrace.join("\n")}" if $debug result['bozo'] = true result['bozo_exception'] = feedparser.exc || e use_strict_parser = false lib/rfeedparser.rb @@ -58,7 +58,7 @@ module FeedParserUtilities ename, eattr = l h.search(ename).each do |elem| euri = elem.attributes[eattr] - uri = ForgivingURI.parse(URI.encode(euri)) rescue nil + uri = Addressable::URI.parse(Addressable::URI.encode(euri)) rescue nil if euri and not euri.empty? and uri and uri.relative? elem.raw_attributes[eattr] = urljoin(baseURI, euri) end lib/rfeedparser/markup_helpers.rb @@ -1,4 +1,5 @@ #!/usr/bin/env ruby + module FeedParser module FeedParserMixin include FeedParserUtilities @@ -386,7 +387,7 @@ module FeedParserMixin end # resolve relative URIs - if @can_be_relative_uri.include?element and output and not output.empty? + if @can_be_relative_uri.include?(element) and !output.blank? output = resolveURI(output) end @@ -1240,4 +1241,18 @@ module FeedParserMixin end end # End FeedParserMixin -end \ No newline at end of file +end + +def urljoin(base, uri) + urifixer = /^([A-Za-z][A-Za-z0-9+-.]*:\/\/)(\/*)(.*?)/u + uri = uri.sub(urifixer, '\1\3') + pbase = Addressable::URI.parse(base) rescue nil + if pbase && pbase.absolute? + puri = Addressable::URI.parse(uri) rescue nil + if puri && puri.relative? + # ForgivingURI.join does the wrong thing. What the hell. + return Addressable::URI.join(base, uri).to_s.gsub(/[^:]\/{2,}/, '') + end + end + return uri +end lib/rfeedparser/parser_mixin.rb @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + require 'test/unit' require File.join(File.dirname(__FILE__),'../lib/rfeedparser') @@ -5,10 +7,11 @@ begin require 'rubygems' gem 'mongrel' require 'mongrel' -rescue => details +rescue LoadError STDERR.puts "Whoops, had an error with loading mongrel as a gem. Trying just 'require'. Mongrel is required for testing." require 'mongrel' end + Mongrel::HTTP_STATUS_CODES[220] = "Unspecified success" def uconvert(one, two, three); FeedParser::uconvert(one, two, three); end @@ -251,12 +254,3 @@ class FeedParserTestRequestHandler < Mongrel::DirHandler end end end - - -class XMLTests < Test::Unit::TestCase - # Empty, but here for clarity - def setup - end - def teardown - end -end \ No newline at end of file tests/rfeedparser_test_helper.rb @@ -1,34 +1,46 @@ #!/usr/bin/env ruby + # This is based off of Sam Ruby's xml_filetest.rb # I've adapted it for rfeedparser # http://intertwingly.net/blog/2005/10/30/Testing-FeedTools-Dynamically/ +require 'yaml' require File.join(File.dirname(__FILE__),'rfeedparser_test_helper') -# default methods to be public -XMLTests.send(:public) -# add one unit test for each file -Dir["#{File.dirname(__FILE__)}/**/*.xml"].each do |xmlfile| - methname = "tests_"+xmlfile.gsub('./', '').gsub('/','_').sub('.xml','') - XMLTests.send(:define_method, methname) { - - options = {} - options[:compatible] = true - # This keeps compatibility with 4.1 feedparser tests (i.e. no - # smart stripping of styles). This is not (yet) required, as - # rfeedparser is compatible by default. +class XMLTests < Test::Unit::TestCase + # Some tests are known to fail because we're copying the Python + # version's feed suite verbatim, and we have minor implementation + # details that don't constitute brokenness but are still + # different. Running `rake test skip=y' will skip these. + # + # Additionally if you want to run a single test, run: + # rake test n=test_tests_wellformed_encoding_x80macroman + # + def self.skip?(name) + return true if ENV['n'] and ENV['n'] != name + if ENV['skip'] + @to_skip ||= YAML.load(File.open(File.dirname(__FILE__) + '/to_skip.yml')) + @to_skip.include? name + end + end - # Evaluate feed + Dir["#{File.dirname(__FILE__)}/**/*.xml"].each do |xmlfile| + name = "test_#{xmlfile.gsub('./', '').gsub('/','_').sub('.xml','')}" + next if skip?(name) - fp = FeedParser.parse("http://127.0.0.1:#{$PORT}/#{xmlfile}", options) - # I should point out that the 'compatible' arg is not necessary, - # but probably will be in the future if we decide to change the default. + define_method(name) do + fp = FeedParser.parse("http://127.0.0.1:#{$PORT}/#{xmlfile}", :compatible => true) + # I should point out that the 'compatible' arg is not necessary, + # but probably will be in the future if we decide to change the default. - description, evalString = scrape_assertion_strings(xmlfile) - assert fp.instance_eval(evalString), description.inspect - } + description, evalString = scrape_assertion_strings(xmlfile) + assert fp.instance_eval(evalString), description + end + end end + +# TODO: don't fail if the rfeedparserserver.rb is already running # Start up the mongrel server and tell it how to send the tests server = Mongrel::HttpServer.new("0.0.0.0",$PORT) Mongrel::DirHandler::add_mime_type('.xml','application/xml') tests/rfeedparsertest.rb lib/rfeedparser/forgiving_uri.rb lib/rfeedparser/parsers.rb 8dcbfc037aa91d71e2a82090af2da4a278b1ca68 617224694da87fa9edd165fc92c9f942427a4f87 Jeff Hodges jeff@somethingsimilar.com http://github.com/jmhodges/rfeedparser/commit/3a125f29c5be176be2e36b26697dcc97b7b92145 3a125f29c5be176be2e36b26697dcc97b7b92145 2008-05-08T15:39:16-07:00 2008-05-08T15:39:16-07:00 Merge branch 'technomancy_master' e2faaad95aad9f7a62978beecd5596071ba943c5 Jeff Hodges jeff@somethingsimilar.com