@@ -10,8 +10,8 @@ Scraper element :title elements 'div.hentry' => :articles do - element 'h2' => :title - element 'a/@href' => :url + element :title => h2' + element :url => 'a/@href' end end README.md @@ -9,24 +9,24 @@ require 'date' # extracts data from a single bookmark class Bookmark < Scraper - element 'h4 a' => :title - element '.description' => :description + element :title => 'h4 a' + element :description => '.description' # extract attribute with xpath - element './/h4/a/@href' => :url + element :url => './/h4/a/@href' # tags are plural - elements 'ul.tag-chain li span' => :tags + elements :tags => 'ul.tag-chain li span' # dates are in form "22 OCT 09" - element '.dateGroup span' => :date, :with => lambda { |span| + element :date => '.dateGroup span', :with => lambda { |span| Date.strptime(span.inner_text.strip, '%d %b %y') } end # finds all bookmarks on the page class Delicious < Scraper - elements '#bookmarklist div.bookmark' => :bookmarks, :with => Bookmark + elements :bookmarks => '#bookmarklist div.bookmark', :with => Bookmark end mislav = Delicious.parse open('http://delicious.com/mislav/ruby') examples/delicious.rb @@ -21,7 +21,7 @@ class Scraper # Specify a new singular scraping rule def self.element(*args, &block) - selector, name, delegate = parse_rule_declaration(*args, &block) + name, selector, delegate = parse_rule_declaration(*args, &block) rules[name] = [selector, delegate] attr_accessor name name @@ -52,7 +52,7 @@ class Scraper # Defines accessors on the fly from lambda # # class LinkScraper < Scraper - # element 'a/@href' => :link, :with => lambda { |link| + # element :link => 'a/@href', :with => lambda { |link| # link.inner_text =~ /http:\/\/(.*?)\.(.*)/ # self.domain = $2 # self.subdomain = $1 @@ -60,7 +60,7 @@ class Scraper # } # end # - # scraper = LinkScraper.scrape(HTML) + # scraper = LinkScraper.parse(HTML) # scraper.domain, scraper.subdomain, scraper.link #=> ... class Proxy def initialize(target) @@ -94,19 +94,19 @@ class Scraper # Rule declaration is in Hash or single argument form: # - # { '//some/selector' => :name, :with => MyClass } - # #=> ['//some/selector', :name, MyClass] + # { :name => '//some/selector', :with => MyClass } + # #=> [:name, '//some/selector', MyClass] # # :title - # #=> ['title', :title, nil] + # #=> [:title, 'title', nil] def self.parse_rule_declaration(*args, &block) options, name = Hash === args.last ? args.pop : {}, args.first delegate = options.delete(:with) - selector, property = name ? [name.to_s, name.to_sym] : options.to_a.flatten + property, selector = name ? [name.to_sym, name.to_s] : options.to_a.flatten raise ArgumentError, "invalid rule declaration: #{args.inspect}" unless property # eval block in context of a new scraper subclass delegate = Class.new(delegate || Scraper, &block) if block_given? - return selector, property, delegate + return property, selector, delegate end def self.rules @@ -141,12 +141,12 @@ if __FILE__ == $0 HTML = DATA.read class Article < Scraper - element 'h1' => :title - element 'a/@href' => :link + element :title => 'h1' + element :link => 'a/@href' end class TimestampedArticle < Article - element 'p.pubdate' => :published, :with => lambda { |node| + element :published => 'p.pubdate', :with => lambda { |node| node.inner_text.sub('Published on ', '') } @@ -161,25 +161,25 @@ if __FILE__ == $0 class BlogScraper < Scraper element :title - elements '#nav li' => :navigation_items + elements :navigation_items => '#nav li' end class OverrideBlogScraper < BlogScraper elements :title - element '#nav li' => :navigation_items + element :navigation_items => '#nav li' end class BlogWithArticles < BlogScraper - elements 'div.hentry' => :articles, :with => Article + elements :articles => 'div.hentry', :with => Article end class BlogWithTimestampedArticles < BlogScraper - elements 'div.hentry' => :articles, :with => TimestampedArticle + elements :articles => 'div.hentry', :with => TimestampedArticle end class BlogWithArticlesBlock < BlogScraper - elements 'div.hentry' => :articles do - element 'h1' => :title + elements :articles => 'div.hentry' do + element :title => 'h1' end end @@ -313,7 +313,7 @@ if __FILE__ == $0 describe "setting instance variables on the fly from lambda" do it "should be able to set scraper instance variables dinamically" do scraper = Scraper.define do - element 'span/a/@href' => :link, :with => lambda { |link| + element :link => 'span/a/@href', :with => lambda { |link| href = link.inner_text href =~ /http:\/\/(.*?)\.(.*)/ self.domain = $2 scraper.rb fb64c07e8ca3df0a190dc691b5bbf0046135d59a David Krmpotic david.krmpotic@gmail.com http://github.com/davidhq/scraper/commit/3baf43c16d3bece015205a3f7673cf9cd9a14d95 3baf43c16d3bece015205a3f7673cf9cd9a14d95 2009-10-24T17:18:02-07:00 2009-10-24T17:02:43-07:00 make API more sensible, so it's :name => "selector" now ed19174d7cbc0be5faf5d1d307f8a8649834b15a David Krmpotic david.krmpotic@gmail.com