@@ -2,7 +2,7 @@ # David Moreno <david@axiombox.com> # -require "#{File.expand_path(File.dirname(__FILE__))}/rfeed.rb" +require "#{File.expand_path(File.dirname(__FILE__))}/lib/rfeed.rb" task :'db:migrate' => :environment do ActiveRecord::Migrator.migrate("#{File.dirname(__FILE__)}/db/migrate", ENV["VERSION"] ? ENV["VERSION"].to_i : nil) Rakefile @@ -10,14 +10,18 @@ raise ArgumentError, "Sin comando" unless ARGV[0] cmd = ARGV.shift case cmd - when "update" - fs = Feed.find :all - fs.each do |f| - f.update + when "fetch" + if ARGV[0] + fs = Feed.find ARGV[0] + fs.fetch + else + fs = Feed.find :all + fs.each do |f| + f.fetch + end end when "post" - when "add" when "new" raise ArgumentError, "No feed specified" unless ARGV[0] f = Feed.new :feed => ARGV[0] @@ -32,6 +36,30 @@ case cmd end end + when /(del|delete|remove)/i + raise ArgumentError, "No feed id specified to remove" unless ARGV[0] + f = Feed.find ARGV[0] + if f.destroy + puts "#{f.name} successfully deleted." + else + $stderr.puts "ERROR:" + f.errors.full_messages.each do |e| + $stderr.puts " #{e}" + end + end + + + when "list" + fs = Feed.find :all + fs.each do |f| + puts "#{ f.id}:#{f.feed_url}" + pp f + end + + when /^\d+$/ + f = Feed.find cmd + pp f + else raise ArgumentError, "No comand found for `#{cmd}'" end bin/rfeed @@ -8,7 +8,7 @@ class CreatePosts < ActiveRecord::Migration p.text :description p.timestamp :date p.timestamp :updated_at - p.timestamp :last_processed_at + p.timestamp :created_at end end db/migrate/002_create_posts.rb @@ -1,6 +1,7 @@ require "rfeedparser" require "feedbag" require "goodies/goodies/lwr-simple" +require 'digest/md5' class Feed < ActiveRecord::Base has_many :posts @@ -9,9 +10,13 @@ class Feed < ActiveRecord::Base validates_presence_of :link, :message => "wasn't found on `feed_url'" validates_uniqueness_of :feed_url, :message => "already exists on database" - attr_accessor :fp + attr_accessor :fp, :changed - def parse(url) + def name + return "#{self.id}:#{self.feed_url}" + end + + def parse(url, params = {}) url = LWR::Simple.normalize(url).to_s # checking that the url is a feed: @@ -26,13 +31,18 @@ class Feed < ActiveRecord::Base $stderr.puts "URL provided not a feed but using `#{url}'" end - fp = FeedParser.parse url + fp = FeedParser.parse url, { :etag => params[:etag], :modified => params[:modified] } if fp.status.nil? or fp.status >= 400.to_s $stderr.puts "Errors with #{url}!" return nil end + if fp.status == "304" and fp.feed.empty? and fp.entries.empty? # not changed! + self.changed = false + return self + end + # there's no real easy way to do this # so we'll only do if the feed_validator gem is available validate = false # assuming it's not there @@ -56,22 +66,44 @@ class Feed < ActiveRecord::Base self.link = fp.feed.link self.title = fp.feed.title - self.etag = fp.etag - self.last_modified = fp.modified_time + if params[:new] + self.etag = nil + self.last_modified = nil + else + puts "ahh!" + self.etag = fp.etag + self.last_modified = fp.modified_time + end self.feed_url = url + self.changed = true self.fp = fp + self end - def update - p = self.parse(self.feed_url) + def fetch + parse(self.feed_url, :etag => self.etag, :modified => self.last_modified) + + unless self.changed + puts "#{self.name}: Feed unchanged" + return + end + + self.fp.entries.each do |e| + post = Post.entry(e, self) + end + + self.save - p.entries.each do |e| -# Post.new !!!! + if feed.save + puts "#{feed.name} successfully fetched!" + else + puts feed.errors.full_messages + pp feed end end def feed=(url) - p = self.parse(url) + p = self.parse(url, :new => true) if p.nil? return nil lib/rfeed/models/Feed.rb @@ -2,4 +2,34 @@ class Post < ActiveRecord::Base belongs_to :feed validates_presence_of :entry_id, :feed_id, :title, :link + validates_uniqueness_of :entry_id + + def self.entry(e, feed = nil) + entry_id = nil + if not e["id"].empty? # e.id will try to trigger Object.id + puts "using id" + entry_id = e["id"] + elsif not e.link.empty? + puts "using link" + entry_id = e.link + elsif not e.title.empty? + puts "using title" + entry_id = MD5::Digest.hexdigest(e.title) + elsif not e.summary.empty? + puts "using summary" + entry_id = MD5::Digest.hexdigest(e.summary) + else + $stderr.puts "No entry_id found for #{pp e}" + return nil + end + + post = Post.find_or_create_by_entry_id(entry_id) + post.feed_id = feed["id"] + post.link = e.link || e.links[0]["href"] || feed.link || "#" + post.title = e.title || e.link + post.description = e.description || e.content || e.summary || "no content" + post.date = e.created_time || e.updated_time || e.published_time || feed.fp.updated_time || feed.fp.modified_time || Time.now + return post + end + end lib/rfeed/models/Post.rb @@ -42,3 +42,15 @@ Migrating to CreatePosts (2) *[4;36;1mSQL (0.2ms)*[0m *[0;1mSELECT version FROM schema_migrations*[0m Migrating to CreateFeeds (1) Migrating to CreatePosts (2) + *[4;36;1mSQL (0.1ms)*[0m *[0;1mSET SQL_AUTO_IS_NULL=0*[0m + *[4;35;1mSQL (0.2ms)*[0m *[0mSHOW TABLES*[0m + *[4;36;1mSQL (9.3ms)*[0m *[0;1mCREATE TABLE `schema_migrations` (`version` varchar(255) NOT NULL) ENGINE=InnoDB*[0m + *[4;35;1mSQL (3.9ms)*[0m *[0mCREATE UNIQUE INDEX `unique_schema_migrations` ON `schema_migrations` (`version`)*[0m + *[4;36;1mSQL (0.2ms)*[0m *[0;1mSHOW TABLES*[0m + *[4;35;1mSQL (0.2ms)*[0m *[0mSELECT version FROM schema_migrations*[0m +Migrating to CreateFeeds (1) + *[4;36;1mSQL (5.6ms)*[0m *[0;1mCREATE TABLE `feeds` (`id` int(11) DEFAULT NULL auto_increment PRIMARY KEY, `link` varchar(255), `feed_url` varchar(255), `title` varchar(255), `etag` varchar(255), `last_modified` datetime, `created_at` datetime, `updated_at` datetime) ENGINE=InnoDB*[0m + *[4;35;1mSQL (0.4ms)*[0m *[0mINSERT INTO schema_migrations (version) VALUES ('1')*[0m +Migrating to CreatePosts (2) + *[4;36;1mSQL (20.4ms)*[0m *[0;1mCREATE TABLE `posts` (`id` int(11) DEFAULT NULL auto_increment PRIMARY KEY, `entry_id` varchar(255), `feed_id` int(11), `title` varchar(255), `link` varchar(255), `description` text, `date` datetime, `updated_at` datetime, `created_at` datetime) ENGINE=InnoDB*[0m + *[4;35;1mSQL (0.6ms)*[0m *[0mINSERT INTO schema_migrations (version) VALUES ('2')*[0m log/database.log 471905fe43779d13a62da0d4847e6ad7acf97715 David Moreno david@axiombox.com http://github.com/damog/rfeed/commit/5b4f68620de7ef53f115c399d6aa2ecfb71a2293 5b4f68620de7ef53f115c399d6aa2ecfb71a2293 2009-01-11T18:53:19-08:00 2009-01-11T18:53:19-08:00 i cant stand this is broken b835397174f307bad1ca97f0b600e810f07e18a2 David Moreno david@axiombox.com