Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

finished initial version of the fetching API. Need to mock out feed g…

…etting (since it's actually hitting the net right now) and need to refactor since the code is currently ugly.
  • Loading branch information...
commit 13d1dface24928faa3e9b2a8fe6021ca35d83737 1 parent f7cf9fc
@pauldix pauldix authored
Showing with 94 additions and 11 deletions.
  1. +31 −1 lib/feedzirra/feed.rb
  2. +63 −10 spec/feedzirra/feed_spec.rb
View
32 lib/feedzirra/feed.rb
@@ -30,7 +30,9 @@ def self.fetch_raw(urls, options = {})
responses = {}
urls.each do |url|
easy = Curl::Easy.new(url) do |curl|
- curl.headers["User-Agent"] = "feedzirra"
+ curl.headers["User-Agent"] = (options[:user_agent] || "feedzirra http://github.com/pauldix/feedzirra/tree/master")
+ curl.headers["If-Modified-Since"] = options[:if_modified_since] if options.has_key?(:if_modified_since)
+ curl.headers["If-None-Match"] = options[:if_none_match] if options.has_key?(:if_none_match)
curl.follow_location = true
curl.on_success do |c|
responses[url] = c.body_str
@@ -46,6 +48,34 @@ def self.fetch_raw(urls, options = {})
return responses.size == 1 ? responses.values.first : responses
end
+ def self.fetch_and_parse(urls, options = {})
+ urls = [*urls]
+ multi = Curl::Multi.new
+ responses = {}
+ urls.each do |url|
+ easy = Curl::Easy.new(url) do |curl|
+ curl.headers["User-Agent"] = (options[:user_agent] || "feedzirra http://github.com/pauldix/feedzirra/tree/master")
+ curl.headers["If-Modified-Since"] = options[:if_modified_since] if options.has_key?(:if_modified_since)
+ curl.headers["If-None-Match"] = options[:if_none_match] if options.has_key?(:if_none_match)
+ curl.follow_location = true
+ curl.on_success do |c|
+ feed = Feed.parse(c.body_str)
+ feed.feed_url ||= c.last_effective_url
+ responses[url] = feed
+ options[:on_success].call(url, feed) if options.has_key?(:on_success)
+ end
+ curl.on_failure do |c|
+ responses[url] = c.response_code
+ options[:on_failure].call(url, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+ end
+ end
+ multi.add(easy)
+ end
+
+ multi.perform
+ return responses.size == 1 ? responses.values.first : responses
+ end
+
def self.etag_from_header(header)
header =~ /.*ETag:\s(.*)\r/
$1
View
73 spec/feedzirra/feed_spec.rb
@@ -73,25 +73,78 @@
describe "fetching feeds" do
before(:each) do
- @feed_url = "http://feeds.feedburner.com/PaulDixExplainsNothing"
- @feed_url2 = "http://feeds.feedburner.com/trottercashion"
+ @paul_feed_url = "http://feeds.feedburner.com/PaulDixExplainsNothing"
+ @trotter_feed_url = "http://feeds.feedburner.com/trottercashion"
end
describe "#fetch_raw" do
- it "should return raw xml when not passed a block" do
- Feedzirra::Feed.fetch_raw(@feed_url).should =~ /^#{Regexp.escape('<?xml version="1.0" encoding="UTF-8"?>')}/
+ it "should take :user_agent as an option"
+ it "should take :if_modified_since as an option"
+ it "should take :if_none_match as an option"
+ it "should take an optional on_success lambda"
+ it "should take an optional on_failure lambda"
+
+ it "should return raw xml" do
+ Feedzirra::Feed.fetch_raw(@paul_feed_url).should =~ /^#{Regexp.escape('<?xml version="1.0" encoding="UTF-8"?>')}/
end
- it "should take multiple feed urls and return a hash of urls and responses" do
- results = Feedzirra::Feed.fetch_raw([@feed_url, @feed_url2])
- results.keys.should include(@feed_url)
- results.keys.should include(@feed_url2)
- results[@feed_url].should =~ /Paul Dix/
- results[@feed_url2].should =~ /Trotter Cashion/
+ it "should take multiple feed urls and return a hash of urls and response xml" do
+ results = Feedzirra::Feed.fetch_raw([@paul_feed_url, @trotter_feed_url])
+ results.keys.should include(@paul_feed_url)
+ results.keys.should include(@trotter_feed_url)
+ results[@paul_feed_url].should =~ /Paul Dix/
+ results[@trotter_feed_url].should =~ /Trotter Cashion/
end
end
describe "#fetch_and_parse" do
+ it "should return a feed object for a single url" do
+ feed = Feedzirra::Feed.fetch_and_parse(@paul_feed_url)
+ feed.title.should == "Paul Dix Explains Nothing"
+ end
+
+ it "should set the feed_url to the new url if redirected" do
+ feed = Feedzirra::Feed.fetch_and_parse("http://tinyurl.com/tenderlovemaking")
+ feed.feed_url.should == "http://tenderlovemaking.com/feed/"
+ end
+
+ it "should set the feed_url for an rdf feed" do
+ feed = Feedzirra::Feed.fetch_and_parse("http://www.avibryant.com/rss.xml")
+ feed.feed_url.should == "http://www.avibryant.com/rss.xml"
+ end
+
+ it "should set the feed_url for an rss feed" do
+ feed = Feedzirra::Feed.fetch_and_parse("http://tenderlovemaking.com/feed/")
+ feed.feed_url.should == "http://tenderlovemaking.com/feed/"
+ end
+
+ it "should return a hash of feed objects with the passed in feed_url for the key and parsed feed for the value for multiple feeds" do
+ feeds = Feedzirra::Feed.fetch_and_parse([@paul_feed_url, @trotter_feed_url])
+ feeds.size.should == 2
+ feeds[@paul_feed_url].feed_url.should == @paul_feed_url
+ feeds[@trotter_feed_url].feed_url.should == @trotter_feed_url
+ end
+
+ it "should yeild the url and feed object to a :on_success lambda" do
+ successful_call_mock = mock("successful_call_mock")
+ successful_call_mock.should_receive(:call)
+ Feedzirra::Feed.fetch_and_parse(@paul_feed_url, :on_success => lambda { |feed_url, feed|
+ feed_url.should == @paul_feed_url
+ feed.class.should == Feedzirra::AtomFeedBurner
+ successful_call_mock.call})
+ end
+
+ it "should yield the url, response_code, response_header, and response_body to a :on_failure lambda" do
+ failure_call_mock = mock("failure_call_mock")
+ failure_call_mock.should_receive(:call)
+ fail_url = "http://localhost"
+ Feedzirra::Feed.fetch_and_parse(fail_url, :on_failure => lambda {|feed_url, response_code, response_header, response_body|
+ feed_url.should == fail_url
+ response_code.should == 0
+ response_header.should == ""
+ response_body.should == ""
+ failure_call_mock.call})
+ end
end
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.