modified delta functionality to allow for more detailed delta'ing.

you can now pass a datetime to the delta option allowing for time-based deltas instead
of the basic boolean type.
the basic boolean delta are still available and are the default option

lib/thinking_sphinx.rb

@@ -2,6 +2,7 @@
 require 'riddle'
 
 require 'thinking_sphinx/active_record'
+require 'thinking_sphinx/collection'
 require 'thinking_sphinx/association'
 require 'thinking_sphinx/attribute'
 require 'thinking_sphinx/configuration'

lib/thinking_sphinx/active_record.rb

@@ -73,7 +73,7 @@ def define_index(&block)
             end
 
             if index.delta?
-              before_save   :toggle_delta
+              before_save   :toggle_delta if index.simple_delta?
               after_commit  :index_delta
             end
 

lib/thinking_sphinx/index.rb

@@ -49,7 +49,7 @@ def empty?(part = :core)
     def to_config(index, database_conf, charset_type)
       # Set up associations and joins
       link!
-      
+
       attr_sources = attributes.collect { |attrib|
         attrib.to_sphinx_clause
       }.join("\n  ")
@@ -84,22 +84,40 @@ def to_config(index, database_conf, charset_type)
       SOURCE
 
       if delta?
-        config += <<-SOURCE
-
-source #{model.indexes.first.name}_#{index}_delta : #{model.indexes.first.name}_#{index}_core
-{
-sql_query_pre    = 
-sql_query_pre    = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""}
-#{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
-sql_query        = #{to_sql(:delta => true).gsub(/\n/, ' ')}
-sql_query_range  = #{to_sql_query_range :delta => true}
-}
-        SOURCE
+        config += delta_config(index, adapter, charset_type)
       end
 
       config
     end
 
+    def delta_config(index, adapter, charset_type)
+      return '' unless delta?
+
+      sql = <<-SOURCE
+      
+      source #{model.indexes.first.name}_#{index}_delta : #{model.indexes.first.name}_#{index}_core
+      {
+      sql_query_pre    = 
+      sql_query_pre    = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""}
+      #{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
+      SOURCE
+
+      if !complex_delta?
+        sql += <<-SOURCE
+        sql_query        = #{to_sql(true, :delta => true).gsub(/\n/, ' ')}
+        sql_query_range  = #{to_sql_query_range(true, :delta => true)}
+        }
+        SOURCE
+      else        
+        sql += <<-SOURCE
+        sql_query        = #{to_sql(true, @delta).gsub(/\n/, ' ')}
+        sql_query_range  = #{to_sql_query_range(true, @delta)}
+        }
+        SOURCE
+      end
+
+      sql
+    end
     # Link all the fields and associations to their corresponding
     # associations and joins. This _must_ be called before interrogating
     # the index's fields and associations for anything that may reference
@@ -138,12 +156,16 @@ def link!
     #   index.to_sql
     #   index.to_sql(:delta => true)
     #
-    def to_sql(options={})
+    def to_sql(for_delta=false, options={})
       assocs = all_associations
 
       where_clause = ""
-      if self.delta?
-        where_clause << " AND #{@model.quoted_table_name}.#{quote_column('delta')}" +" = #{options[:delta] ? db_boolean(true) : db_boolean(false)}"
+      if for_delta && self.delta?
+        if !self.complex_delta?
+          where_clause << " AND #{@model.quoted_table_name}.#{quote_column('delta')}" +" = #{options[:delta] ? db_boolean(true) : db_boolean(false)}"
+        else
+          where_clause << " AND #{@model.quoted_table_name}.#{quote_column(options[:field])} > DATE_SUB(NOW(), INTERVAL #{options[:threshold]} SECOND)"
+        end
       end
       unless @conditions.empty?
         where_clause << " AND " << @conditions.join(" AND ")
@@ -186,7 +208,7 @@ def to_sql_query_info
     # returns minimum and maximum id values. These can be filtered by delta -
     # so pass in :delta => true to get the delta version of the SQL.
     # 
-    def to_sql_query_range(options={})
+    def to_sql_query_range(for_delta=false, options={})
       min_statement = "MIN(#{quote_column(@model.primary_key)})"
       max_statement = "MAX(#{quote_column(@model.primary_key)})"
 
@@ -198,8 +220,16 @@ def to_sql_query_range(options={})
 
       sql = "SELECT #{min_statement}, #{max_statement} " +
             "FROM #{@model.quoted_table_name} "
-      sql << "WHERE #{@model.quoted_table_name}.#{quote_column('delta')} " + 
-            "= #{options[:delta] ? db_boolean(true) : db_boolean(false)}" if self.delta?
+
+      if for_delta && self.delta?
+        if !self.complex_delta?
+          sql << "WHERE #{@model.quoted_table_name}.#{quote_column('delta')} " + 
+            "= #{options[:delta] ? db_boolean(true) : db_boolean(false)}" 
+        else
+          sql << "WHERE #{@model.quoted_table_name}.#{quote_column(options[:field])} > DATE_SUB(NOW(), INTERVAL #{options[:threshold]} SECOND)"
+        end
+      end
+
       sql
     end
 
@@ -208,7 +238,12 @@ def to_sql_query_range(options={})
     # back to 0.
     #
     def to_sql_query_pre
-      self.delta? ? "UPDATE #{@model.quoted_table_name} SET #{quote_column('delta')} = #{db_boolean(false)}" : ""
+      if self.simple_delta? 
+        "UPDATE #{@model.quoted_table_name} SET #{quote_column('delta')} = #{db_boolean(false)}"
+      else
+        ""
+      end
+
     end
 
     # Flag to indicate whether this index has a corresponding delta index.
@@ -217,6 +252,14 @@ def delta?
       @delta
     end
 
+    def complex_delta?
+      delta? && @delta.is_a?(Hash) && @delta.has_key?(:field) && @delta.has_key?(:threshold)
+    end
+
+    def simple_delta?
+      delta? && !complex_delta?      
+    end
+
     def adapter
       @adapter ||= case @model.connection.class.name
       when "ActiveRecord::ConnectionAdapters::MysqlAdapter"

lib/thinking_sphinx/search.rb

@@ -18,15 +18,9 @@ def search_for_ids(*args)
         options = args.extract_options!
         page    = options[:page] ? options[:page].to_i : 1
 
-        begin
-          pager = WillPaginate::Collection.create(page,
-            client.limit, results[:total_found] || 0) do |collection|
-            collection.replace results[:matches].collect { |match| match[:doc] }
-            collection.instance_variable_set :@total_entries, results[:total_found]
-          end
-        rescue
-          results[:matches].collect { |match| match[:doc] }
-        end
+        pager = ThinkingSphinx::Collection.new(page, client.limit, results[:total] || 0, results[:total_found] || 0)
+        pager.replace results[:matches].collect { |match| match[:doc] }
+        [pager, results]        
       end
 
       # Searches through the Sphinx indexes for relevant matches. There's
@@ -192,16 +186,10 @@ def search(*args)
         options = args.extract_options!
         klass   = options[:class]
         page    = options[:page] ? options[:page].to_i : 1
-
-        begin
-          pager = WillPaginate::Collection.create(page,
-            client.limit, results[:total] || 0) do |collection|
-            collection.replace instances_from_results(results[:matches], options, klass)
-            collection.instance_variable_set :@total_entries, results[:total_found]
-          end
-        rescue StandardError => err
-          instances_from_results(results[:matches], options, klass)
-        end
+
+        pager = ThinkingSphinx::Collection.new(page, client.limit, results[:total] || 0, results[:total_found] || 0)
+        pager.replace instances_from_results(results[:matches], options, klass)
+        (options[:include_raw] ? [pager, results] : pager)
       end
 
       # Checks if a document with the given id exists within a specific index.

tasks/thinking_sphinx_tasks.rb

@@ -54,6 +54,34 @@
     puts cmd
     system cmd
   end
+
+  desc "Reindex the passed delta"
+  task :index_delta => [:app_env, :configure] do
+    config = ThinkingSphinx::Configuration.new
+
+    index_name = get_index_name(ENV['MODEL'])
+
+    cmd = "indexer --config '#{config.config_file}'"
+    cmd << " --rotate" if sphinx_running?
+    cmd << " #{index_name}_delta"
+
+    system cmd
+
+  end
+
+  desc "Merge the passed indexes delta into the core"
+  task :index_merge => [:app_env, :configure] do
+    config = ThinkingSphinx::Configuration.new
+
+    index_name = get_index_name(ENV['MODEL'])
+
+    cmd = "indexer --config '#{config.config_file}'"
+    cmd << " --rotate" if sphinx_running?
+    cmd << " --merge #{index_name}_core #{index_name}_delta"
+
+    system cmd
+
+  end
 end
 
 namespace :ts do
@@ -83,4 +111,10 @@ def sphinx_pid
 
 def sphinx_running?
   sphinx_pid && `ps -p #{sphinx_pid} | wc -l`.to_i > 1
+end
+
+def get_index_name(str)
+  klass = str.to_s.strip.classify.constantize
+  raise "The class '#{klass}' has no Thinking Sphinx indexes defined" if !klass.indexes || klass.indexes.empty?
+  klass.indexes.first.name    
 end