@@ -3,8 +3,10 @@ class Google BASE_SEARCH_URL = "http://www.google.com/search?q=" - def self.search_url query - BASE_SEARCH_URL + URI.encode(query) + def self.search_url query, options = {} + url = BASE_SEARCH_URL + URI.encode(query) + url += "&start=#{ (options[:page] - 1) * 10 }" if options[:page] and options[:page] != 1 + url end def self.fetch_url url @@ -15,8 +17,16 @@ class Google ( Hpricot(html) / 'div.g' ).map { |result| Result.new(result) } end - def self.search query - parse_results( fetch_url(search_url(query)) ) + def self.search query, options = {} + if options[:pages] and options[:pages] > 1 + results = [] + options[:pages].times do |i| + results += parse_results( fetch_url(search_url(query, :page => i+1)) ) + end + results + else + parse_results( fetch_url(search_url(query)) ) + end end # Simple class representing a single Google result lib/domain-finder/google.rb @@ -12,6 +12,12 @@ describe Google do Google.search_url(@example_query).should == @example_url end + it '.search_url should support pagination' do + Google.search_url(@example_query, :page => 1 ).should == @example_url + Google.search_url(@example_query, :page => 2 ).should == @example_url + '&start=10' + Google.search_url(@example_query, :page => 10).should == @example_url + '&start=90' + end + it '.fetch_url should fetch the results of a full url' do Google.should_receive(:open).with(@example_url).and_return(StringIO.new(@example_html)) Google.fetch_url(@example_url).should == @example_html @@ -30,7 +36,19 @@ describe Google do it '.search should return GoogleResult objects for a query' do Google.should_receive(:open).with(@example_url).and_return(StringIO.new(@example_html)) - Google.search(@example_query).first.should be_a_kind_of(Google::Result) + results = Google.search(@example_query) + results.length.should == 10 + results.first.should be_a_kind_of(Google::Result) + end + + it '.search should support pagination' do + Google.should_receive(:open).with(@example_url).and_return(StringIO.new(@example_html)) + Google.should_receive(:open).with(@example_url + '&start=10').and_return(StringIO.new(@example_html)) + Google.should_receive(:open).with(@example_url + '&start=20').and_return(StringIO.new(@example_html)) + + results = Google.search(@example_query, :pages => 3) + results.length.should == 30 + results.first.should be_a_kind_of(Google::Result) end end spec/google_spec.rb 03488e1c162988c9904b559e64b8e8ca068374a9 remi remi@remitaylor.com http://github.com/remi/domain-finder/commit/a6e706532cb74e08d94662c2d35ef96f8b1a4256 a6e706532cb74e08d94662c2d35ef96f8b1a4256 2008-06-14T15:34:10-07:00 2008-06-14T15:34:10-07:00 added Google pagination ... Google.search 'query', :pages => 3 (should give 30 results) 1512ca8426593e476dcbbb88cc8a87f5b79fabe2 remi remi@remitaylor.com