+ slight db source rewrite

server/lib/picky/sources/db.rb

@@ -107,22 +107,23 @@ def snapshot_table_name type # :nodoc:
 
     # Harvests the data to index in chunks.
     #
-    def harvest type, category
+    def harvest type, category, &block
       connect_backend
 
       (0..count(type)).step(chunksize) do |offset|
-        get_data(type, category, offset).each do |indexed_id, text|
-          next unless text
-          text.force_encoding 'utf-8' # TODO Still needed?
-          yield indexed_id, text
-        end
+        get_data type, category, offset, &block
       end
     end
 
     # Gets the data from the backend.
     #
-    def get_data type, category, offset # :nodoc:
-      database.connection.execute harvest_statement_with_offset(type, category, offset)
+    def get_data type, category, offset, &block # :nodoc:
+      select_statement = harvest_statement_with_offset(type, category, offset)
+      database.connection.execute(select_statement).each do |indexed_id, text|
+        next unless text
+        text.force_encoding 'utf-8' # TODO Still needed? Or move to backend?
+        yield indexed_id, text
+      end
     end
 
     # Builds a harvest statement for getting data to index.

server/spec/lib/sources/db_spec.rb

@@ -17,13 +17,33 @@
   end
 
   describe "get_data" do
-    it "delegates" do
-      type     = stub :type,     :name => :some_type
-      category = stub :category, :from => :some_category
-
-      @connection.should_receive(:execute).once.with 'SELECT indexed_id, some_category FROM some_type_type_index st WHERE st.id > some_offset LIMIT 25000'
-
-      @source.get_data type, category, :some_offset
+    before(:each) do
+      @type     = stub :type,     :name => :some_type
+      @category = stub :category, :from => :some_category
+    end
+    context 'no data' do
+      it "delegates to the connection" do
+
+        @connection.should_receive(:execute).
+                    once.
+                    with('SELECT indexed_id, some_category FROM some_type_type_index st WHERE st.id > some_offset LIMIT 25000').
+                    and_return []
+
+        @source.get_data @type, @category, :some_offset
+      end
+    end
+    context 'with data' do
+      it 'yields to the caller' do
+        @connection.should_receive(:execute).
+                    any_number_of_times.
+                    with('SELECT indexed_id, some_category FROM some_type_type_index st WHERE st.id > some_offset LIMIT 25000').
+                    and_return [[1, 'text']]
+
+        @source.get_data @type, @category, :some_offset do |id, text|
+          id.should == 1
+          text.should == 'text'
+        end
+      end
     end
   end
 
@@ -67,48 +87,22 @@
     end
   end
 
-  # TODO Redo.
-  #
-  # describe "harvest" do
-  #   before(:each) do
-  #     @source.stub! :harvest_statement_with_offset
-  #   end
-  #   context 'expectations' do
-  #     before(:each) do
-  #       @connection.stub! :execute => []
-  #       @connection.stub! :select_value
-  #     end
-  #     after(:each) do
-  #       @source.harvest :type_name, :some_field
-  #     end
-  #     context "with WHERE" do
-  #       before(:each) do
-  #         @source.stub! :select_statement => 'bla WHERE blu'
-  #       end
-  #       it "should connect" do
-  #         @source.should_receive(:connect_backend).once.with
-  #       end
-  #       it "should call the harvest statement with an offset" do
-  #         @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset
-  #       end
-  #     end
-  #     context "without WHERE" do
-  #       it "should connect" do
-  #         @adapter.should_receive(:connect).once.with
-  #       end
-  #       it "should call the harvest statement with an offset" do
-  #         @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset
-  #       end
-  #     end
-  #   end
-  #   context 'returns' do
-  #     it "should return whatever the execute statement returns" do
-  #       @connection.stub! :execute => :some_result
-  #       
-  #       @source.harvest(:some_type, :some_field).should == :some_result
-  #     end
-  #   end
-  # end
+  describe 'harvest' do
+    before(:each) do
+      @type     = stub :type,     :name => :some_type
+      @category = stub :category, :name => :some_category
+
+      @source.should_receive(:get_data).any_number_of_times.and_return [[:some_id, 'some_text']].cycle
+      @source.stub! :count => 17
+    end
+    it 'calls connect_backend' do
+      @source.should_receive(:connect_backend).once.with()
+
+      @source.harvest @type, @category do |id, text|
+        p [id, text]
+      end
+    end
+  end
 
   describe "harvest_statement_with_offset" do
     before(:each) do