examples/misc/on_click_next/next_page_link.rb examples/misc/on_click_next/page_1.html examples/misc/on_click_next/page_2.html examples/misc/on_click_next/page_3.html @@ -103,6 +103,10 @@ module Scrubyt end sleep(wait_secs) if wait_secs > 0 @@agent.wait + + # evaluate the results + extractor.evaluate_extractor + @@current_doc_url = @@agent.url @@mechanize_doc = "<html>#{@@agent.html}</html>" @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc)) lib/scrubyt/core/navigation/agents/firewatir.rb @@ -14,6 +14,15 @@ module Scrubyt @@host_name = nil @@history = [] @@current_form = nil + @@extractor = nil + + def self.extractor=(extractor) + @@extractor = extractor + end + + def self.extractor + return @@extractor + end ## # At any given point, the current document can be queried with this method; Typically used lib/scrubyt/core/navigation/fetch_action.rb @@ -71,6 +71,7 @@ module Scrubyt root_pattern end end + FetchAction.extractor = self context.extractor = self context.instance_eval(&extractor_definition) @evaluating_extractor_definition = false @@ -82,10 +83,10 @@ module Scrubyt end #Once all is set up, evaluate the extractor from the root pattern! - root_results = evaluate_extractor + evaluate_extractor @result = ScrubytResult.new('root') - @result.push(*root_results) + @result.push(*@root_results) @result.root_patterns = @root_patterns @result.source_file = source_file @result.source_proc = extractor_definition @@ -127,14 +128,14 @@ module Scrubyt end def evaluate_extractor - root_results = [] + @root_results ||= [] current_page_count = 1 catch :quit_next_page_loop do loop do url = get_current_doc_url #TODO need absolute address here 2/4 @processed_pages << url @root_patterns.each do |root_pattern| - root_results.push(*root_pattern.evaluate(get_hpricot_doc, nil)) + @root_results.push(*root_pattern.evaluate(get_hpricot_doc, nil)) end while @processed_pages.include? url #TODO need absolute address here 3/4 @@ -160,7 +161,8 @@ module Scrubyt current_page_count += 1 end end - root_results + @root_patterns = [] + @root_results end end lib/scrubyt/core/shared/extractor.rb @@ -20,6 +20,7 @@ module Scrubyt end def to_s + return "" if result.nil? text = (@result.is_a? String) ? @result : @result.inner_html.gsub(/<.*?>/, '') text = SharedUtils.unescape_entities(text) text.strip! @@ -29,6 +30,10 @@ module Scrubyt text end end + + def inspect + to_s + end def to_libxml libxml_node = XML::Node.new(name) lib/scrubyt/output/result_node.rb 17de69ffd9f00f61b2e5f5d2c07d03e864068d9b Mikkel Garcia mikkel@mikkel-garcias-macbook-pro-15.local http://github.com/scrubber/scrubyt/commit/2a8253673fea7c6e7d9f4b062ed58215c2e114fb 2a8253673fea7c6e7d9f4b062ed58215c2e114fb

2009-04-06T13:37:26-07:00

Added the ability to scrape, click_link, then scrape again. (Only for firefox agent) 8c14c1a5b358c626100c4bba61d97b4437075134 Mikkel Garcia mikkel@mikkel-garcias-macbook-pro-15.local