0
@@ -1,13 +1,15 @@
0
 class FeedsController < ApplicationController
0
   
0
   def index
0
+    @feeds = Feed.find(:all)
0
   end
0
   
0
   def new
0
     pp params
0
     
0
-    feed = Feed.new(:title => params[:url], :link => params[:url], :link_regexp => params[:link], 
0
-            :title_regexp => params[:title], :content_regexp => params[:content])
0
+    feed = Feed.new(:title => params[:url].gsub('[', '[@'), :link => params[:url].gsub('[', '[@'), 
0
+            :link_regexp => params[:link].gsub('[', '[@'), :title_regexp => params[:title].gsub('[', '[@'), 
0
+            :content_regexp => params[:content].gsub('[', '[@'))
0
             
0
     feed.save!
0
     
0
0
@@ -17,9 +19,28 @@
0
   def show
0
     @feed = Feed.find_by_id(params[:id])
0
     
0
-    @feed.execute
0
+    parsed_items = @feed.execute
0
+
0
+    existing_items = Item.find(:all, 
0
+      :conditions => ["feed_id =? and guid in (?)", @feed.id, parsed_items.collect(&:guid)])
0
+
0
+
0
+    existing_items.collect!(&:guid) unless existing_items.empty?
0
+
0
+    new_items = parsed_items.reject {|item| existing_items.include?(item.guid)}
0
+
0
+    @feed.items << new_items unless new_items.empty?
0
+
0
+    @feed.last_published = Time.now
0
+
0
+    @feed.save unless new_items.empty?
0
     
0
-    pp @feed
0
+    @items = Item.find(:all, :conditions => ["feed_id = ?", @feed.id], 
0
+      :order => "created_at DESC")
0
+    
0
+    respond_to do |format|
0
+      format.xml
0
+    end
0
   end
0
   
0
   

0
@@ -1,88 +1,58 @@
0
 require "net/http"
0
 require "uri"
0
-require "scrapi"
0
-Tidy.path = "/usr/lib/libtidy.dylib"
0
+require "hpricot"
0
 
0
-# class ScrapePosts < Scraper::Base
0
-#   #cattr_reader :title_matcher
0
-#   
0
-#   @@title_matcher = ""
0
-#   
0
-#   def initialize(title, source, options=nil)
0
-#     @@title_matcher = title
0
-#     super source, options
0
-#   end
0
-#   
0
-#   
0
-#   array :titles
0
-#   
0
-#   process @@title_matcher, :titles => :text
0
-# 
0
-#   
0
-# end
0
-
0
 class Feed < ActiveRecord::Base
0
   has_many :items
0
-  
0
-  #t.string :title, :description, :link, :link_regexp, :title_regexp, :content_regexp, :more_regexp
0
 
0
-  
0
   def execute
0
-    #grab the contents of the page to scrape
0
-    html = Net::HTTP.get(URI.parse(self.link))
0
+    items = parse_page(link)
0
+  end
0
+  
0
+  def parse_page(link)
0
+    doc = Hpricot(fetch_page(link))
0
     
0
-    #scrape the data we want from the page
0
-    result = page_scraper_for(self.title_regexp, self.link_regexp, self.content_regexp).scrape(html)
0
+    links = doc.search(link_regexp).collect do |link|
0
+      fix_relative_url(link.search("a[@href]").first.attributes["href"])
0
+    end
0
     
0
+    titles = doc.search(title_regexp).collect do |title|
0
+      clean(title.to_s)
0
+    end
0
+    
0
+    contents = doc.search(content_regexp).collect do |content|
0
+      absolutize_links(content).to_s #make all links absolute
0
+    end
0
+    
0
     items = []
0
-    result.link_array.each_index do |i|
0
-      link = fix_relative_url(result.link_array[i])
0
-      content = absolutize_links(result.content_array[i]).to_s #make all links absolute
0
-      title = clean(result.title_array[i]) #remove html, newlines and extra spaces
0
-      items << {:link => link, :title => title, :content => content}
0
+    links.each_index do |i|
0
+      items << Item.new({:link=>links[i], :title=>titles[i], :content=>contents[i], :guid=>links[i].hash})
0
     end
0
     
0
     items
0
-
0
   end
0
   
0
   private
0
-  # returns the scraper for the patterns passed in
0
-  def page_scraper_for(title_matcher, link_matcher, content_matcher)
0
-    Scraper.define do
0
-      selector :select_link, "a"
0
-      
0
-      array :title_array
0
-      process title_matcher, :title_array => :text
0
-      
0
-      array :link_array
0
-      process link_matcher, 
0
-        :link_array => Scraper.define {process "a[href]",:link =>"@href";result :link}
0
-      
0
-      process content_matcher do |element|
0
-        @content_array ||= []
0
-        @content_array << element
0
-      end
0
-      attr_accessor :content_array
0
-    end
0
+  def fetch_page(link)
0
+    uri = URI.parse(link)
0
+    path_with_query = uri.query.blank? ? uri.path : (uri.path + "?" + uri.query)
0
+    
0
+    response, html = Net::HTTP.new(uri.host, uri.port).get2(path_with_query, HEADERS)
0
+    return html
0
   end
0
   
0
   # parses through the html nodes, finds the links and fixes any that are relative
0
   def absolutize_links(node)
0
-    sibling = node
0
-    while(sibling = sibling.next_sibling)
0
-      absolutize_links(sibling) if sibling.tag? && sibling.name!="a"
0
+    node.search("a") do |item|
0
+      item.attributes["href"] = fix_relative_url(item.attributes["href"])
0
     end
0
-    
0
-    node.children.each do |node| 
0
-      node.attributes["href"] = fix_relative_url(node.attributes["href"]) if node.tag? && node.name=="a"
0
-      absolutize_links(node) if node.tag? && node.name!="a"
0
-    end
0
   end
0
   
0
   # pull out html tags, get rid of new lines, remove extra spaces
0
   def clean(string)
0
-    string.gsub(/<\/?[^>]*>/, "").gsub("\n", " ").squeeze(" ").strip
0
+    return "NO TITLE FOUND" if string == nil
0
+    string = string.gsub(/<\/?[^>]*>/, "").gsub("\n", " ").gsub("\t", " ").squeeze(" ").strip
0
+    return string.blank? ? "NO TITLE FOUND" : string
0
   end
0
   
0
   # some sites use relative urls, these wont work when clicking on them from the rss feed so
0
@@ -94,5 +64,11 @@
0
     end
0
     return url
0
   end
0
+  
0
+  HEADERS = {
0
+    'Accept' => '*/*',
0
+    'Accept-Language' => 'en-ca',
0
+    'User-Agent' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322)'
0
+  }
0
 end