Add a batched ActiveRecord extension.

* Implements a batched class method for models to select records for a model
  in batches without putting too much strain on the database server.

Rakefile

@@ -5,3 +5,14 @@ require File.expand_path('../config/application', __FILE__)
 require 'rake'
 
 Appsterdam::Application.load_tasks
+
+namespace :test do
+  Rake::TestTask.new('lib') do |t|
+    t.test_files = FileList['test/lib/**/*_test.rb']
+    t.verbose = true
+  end
+end
+
+task :test do
+  Rake::Task['test:lib'].invoke
+end

lib/batched.rb

@@ -0,0 +1,24 @@
+module Batched
+  DEFAULT_BATCH_SIZE = 512
+
+  def _max_id
+    connection.execute("SELECT MAX(id) FROM #{table_name}").first[0]
+  end
+
+  def _ids_in_batch(batch_size, index)
+    (index*batch_size...(index+1)*batch_size).to_a
+  end
+
+  # Yields collections for the specified size until all records
+  # have been returned in the most database efficient way. Note
+  # that this doesn't guarantee a specific batch size.
+  def batched(options={})
+    max_batch_size = DEFAULT_BATCH_SIZE
+    batch_count = (_max_id / max_batch_size.to_f).ceil
+    (0...batch_count).each do |index|
+      yield scoped(:conditions => { :id => _ids_in_batch(max_batch_size, index) }).all(options)
+    end
+  end
+end
+
+ActiveRecord::Base.send(:extend, Batched)

test/lib/batched_test.rb

@@ -0,0 +1,40 @@
+require File.expand_path('../../test_helper', __FILE__)
+
+describe Batched do
+  it "finds the largest ID in the table" do
+    Member._max_id.should == Member.all.map(&:id).max
+  end
+
+  it "returns a list of ID's in a batch" do
+    Member._ids_in_batch(0, 0).should == []
+
+    Member._ids_in_batch(3, 0).should == [0, 1, 2]
+    Member._ids_in_batch(3, 1).should == [3, 4, 5]
+
+    Member._ids_in_batch(4, 0).should == [0, 1, 2, 3]
+    Member._ids_in_batch(4, 1).should == [4, 5, 6, 7]
+  end
+
+  it "selects batches of a maximum size" do
+    collection = stub('Collection')
+    scope = stub('Scope')
+    scope.stubs(:all).returns(collection)
+
+    Member.stubs(:_max_id).returns(3653)
+    (0..7).each do |index|
+      Member.expects(:scoped).with(:conditions => { :id => Member._ids_in_batch(512, index) }).returns(scope)
+    end
+
+    Member.batched do |collection|
+      collection.should == collection
+    end
+  end
+
+  it "passes additional options to the finder method" do
+    Member.stubs(:_max_id).returns(12)
+    scope = stub('Scope')
+    Member.expects(:scoped).with(:conditions => { :id => Member._ids_in_batch(512, 0) }).returns(scope)
+    scope.expects(:all).with(:include => :account).returns(nil)
+    Member.batched(:include => :account) {}
+  end
+end