@@ -27,7 +27,8 @@ class NewsArticleFeed < ActiveRecord::Base # Parse the feed and create any new NewsArticles def create_news_articles(rssdata = nil) rss = get_rss_entries(rssdata) - articles = rss.entries.collect do |e| + entries = NewsArticleFeedFilter.filter(rss.entries) + articles = entries.collect do |e| url = e[:link] page = WebPageParser::ParserFactory.parser_for(:url => url, :page => nil) next nil if page.nil? @@ -43,7 +44,7 @@ class NewsArticleFeed < ActiveRecord::Base a.parser = page.class.to_s.split('::').last begin a.save! - logger.info "NewsArticleFeed #{id}, NewsArticle #{a.id} discovered" + logger.debug "NewsArticleFeed #{id}, NewsArticle #{a.id} discovered" next a rescue ActiveRecord::RecordInvalid logger.error "NewsArticleFeed #{id}, NewsArticle '#{a.guid}' not created: #{a.errors.full_messages}" app/models/news_article_feed.rb @@ -21,7 +21,7 @@ class NewsArticleFeedFilter < ActiveRecord::Base # Takes a hash of an rss feed entry and checks it against the filters def allows?(entry) - url_filter_allows?(entry[:url]) + url_filter_allows?(entry[:url] || entry[:link]) end def url_filter_allows?(url) app/models/news_article_feed_filter.rb @@ -23,6 +23,10 @@ describe NewsArticleFeedFilter do @naff.allows?(:url => @good_url).should == true end + it "should match links against the url filter" do + @naff.allows?(:link => @bad_url).should == false + end + it "should not match urls when there is no url filter defined" do NewsArticleFeedFilter.new.allows?(:url => @bad_url).should == true end @@ -42,7 +46,6 @@ describe NewsArticleFeedFilter do it "should validate url_filter as a regular expression" do naff = NewsArticleFeedFilter.new(@valid_attributes.merge(:url_filter => '.*\1')) naff.valid?.should == false - end it "should match titles against the title filter" spec/models/news_article_feed_filter_spec.rb @@ -75,6 +75,13 @@ describe NewsArticleFeed do articles.first.source.should == @valid_attributes[:source] end + it "should not create NewsArticles for entries that match NewsArticleFeedFilters" do + NewsArticleFeedFilter.create!(:name => "Test", :url_filter => 'bbc') + f = NewsArticleFeed.create!(@valid_attributes) + articles = f.create_news_articles(some_rss_feed_xml) + articles.size.should == 0 + end + end spec/models/news_article_feed_spec.rb ea689291de0471622c427745731325ad7a8db4be John Leach john@johnleach.co.uk http://github.com/johnl/news-sniffer/commit/62d84b11b954a097c88f173214cb050014dcc3a3 62d84b11b954a097c88f173214cb050014dcc3a3 2009-05-25T13:27:51-07:00 2009-05-25T13:27:51-07:00 spec: NewsArticleFeedFilter should match links against the url filter spec: NewsArticleFeed should not create NewsArticles for entries that match NewsArticleFeedFilters 12f3eaadabffa92aa4179e77baf926427b4b0f66 John Leach john@johnleach.co.uk