@@ -17,7 +17,7 @@ task "cleanup_readme" => ["rdoc"] gem_spec = Gem::Specification.new do |s| s.name = 'scrubyt' - s.version = '0.4.26' + s.version = '0.4.30' s.summary = 'A powerful Web-scraping framework built on Mechanize and Hpricot (and FireWatir)' s.description = %{scRUBYt! is an easy to learn and use, yet powerful and effective web scraping framework. It's most interesting part is a Web-scraping DSL built on HPricot and WWW::Mechanize, which allows to navigate to the page of interest, then extract and query data records with a few lines of code. It is hard to describe scRUBYt! in a few sentences - you have to see it for yourself!} # Files containing Test::Unit test cases. Rakefile @@ -13,8 +13,8 @@ module Scrubyt module Firewatir def self.included(base) - base.module_eval do - @@agent = FireWatir::Firefox.new + base.module_eval do + @@agent = FireWatir::Firefox.new unless defined? @@agent @@current_doc_url = nil @@current_doc_protocol = nil @@base_dir = nil @@ -60,6 +60,11 @@ module Scrubyt store_host_name(@@agent.url) # in case we're on a new host end + def self.use_current_page + @@mechanize_doc = "<html>#{@@agent.html}</html>" + @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc)) + end + def self.frame(attribute, value) if @@current_frame @@current_frame.frame(attribute, value) @@ -111,7 +116,24 @@ module Scrubyt @@mechanize_doc = "<html>#{@@agent.html}</html>" @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc)) Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}" - end + end + + def self.click_by_xpath_if_exists(xpath, wait_secs=0) + begin + result_page = @@agent.element_by_xpath(xpath).click + sleep(wait_secs) if wait_secs > 0 + @@agent.wait + + extractor.evaluate_extractor + + @@current_doc_url = @@agent.url + @@mechanize_doc = "<html>#{@@agent.html}</html>" + @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc)) + Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}" + rescue Watir::Exception::UnknownObjectException + Scrubyt.log :INFO, "XPath #{xpath} doesn't exist in this document" + end + end def self.click_by_xpath(xpath, wait_secs=0) Scrubyt.log :ACTION, "Clicking by XPath : %p" % xpath lib/scrubyt/core/navigation/agents/firewatir.rb @@ -56,6 +56,10 @@ module Scrubyt def fetch(*args) FetchAction.fetch(*args) end + + def use_current_page + FetchAction.use_current_page + end ## #Submit the current form def submit(index=nil, type=nil) @@ -76,6 +80,10 @@ module Scrubyt FetchAction.click_link(link_spec, 0, sleep_secs) end + def click_by_xpath_if_exists(xpath, sleep_secs=0) + FetchAction.click_by_xpath_if_exists(xpath, sleep_secs) + end + def click_by_xpath(xpath) FetchAction.click_by_xpath(xpath) end lib/scrubyt/core/navigation/navigation_actions.rb @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = %q{scrubyt} s.summary = "A powerful Web-scraping framework built on Mechanize and Hpricot (and FireWatir)" - s.version = "0.4.28" + s.version = "0.4.30" s.authors = ["Peter Szinek", "Glenn Gillen"] s.date = %q{2009-01-31} s.description = %q{scRUBYt! is an easy to learn and use, yet powerful and effective web scraping framework. It's most interesting part is a Web-scraping DSL built on HPricot and WWW::Mechanize, which allows to navigate to the page of interest, then extract and query data records with a few lines of code. It is hard to describe scRUBYt! in a few sentences - you have to see it for yourself!} scrubyt.gemspec e47fe28ded618acafe5f9c86d34ac0cb4a008c96 Peter Szinek peter@rubyrailways.com http://github.com/scrubber/scrubyt/commit/3eef63564a8838fc9266658baf0823976b224eaf 3eef63564a8838fc9266658baf0823976b224eaf

2009-05-25T10:07:26-07:00

Added some new navigation actions (FireWatir only) a81efb082802fb3563012f08353c4916bce974f8 Peter Szinek peter@rubyrailways.com