+ typed backends now responsible for intersecting ids, + this means t…

…hat Picky now has exchangeable backends! Hooray!

server/lib/picky/backends/memory.rb

@@ -12,6 +12,35 @@ def configure_with bundle
         @configuration = File::JSON.new    bundle.index_path(:configuration)
       end
 
+      # Returns the result ids for the allocation.
+      #
+      # Sorts the ids by size and & through them in the following order (sizes):
+      # 0. [100_000, 400, 30, 2]
+      # 1. [2, 30, 400, 100_000]
+      # 2. (100_000 & (400 & (30 & 2))) # => result
+      #
+      # Note: Uses a C-optimized intersection routine (in performant.c)
+      #       for speed and memory efficiency.
+      #
+      # Note: In the memory based version we ignore the amount and offset hints.
+      #       We cannot use the information to speed up the algorithm, unfortunately.
+      #
+      def ids combinations, _, _
+        return [] if combinations.empty?
+
+        # Get the ids for each combination.
+        #
+        id_arrays = combinations.inject([]) do |total, combination|
+          total << combination.ids
+        end
+
+        # Call the optimized C algorithm.
+        #
+        # Note: It orders the passed arrays by size.
+        #
+        Performant::Array.memory_efficient_intersect id_arrays
+      end
+
     end
 
   end

server/lib/picky/backends/redis.rb

@@ -20,6 +20,61 @@ def configure_with bundle
         @configuration = Redis::StringHash.new "#{bundle.identifier}:configuration", actual_backend
       end
 
+      # Returns the result ids for the allocation.
+      #
+      # Developers wanting to program fast intersection
+      # routines, can do so analogue to this in their own
+      # backend implementations.
+      #
+      # Note: We use the amount and offset hints to speed Redis up.
+      #
+      def ids combinations, amount, offset
+        return [] if combinations.empty?
+
+        identifiers = combinations.inject([]) do |identifiers, combination|
+          identifiers << "#{combination.identifier}"
+        end
+
+        result_id = generate_intermediate_result_id
+
+        # Intersect and store.
+        #
+        actual_backend.zinterstore result_id, identifiers
+
+        # Get the stored result.
+        #
+        results = actual_backend.zrange result_id, offset, (offset + amount)
+
+        # Delete the stored result as it was only for temporary purposes.
+        #
+        # Note: I could also not delete it, but that would not be clean at all.
+        #
+        actual_backend.del result_id
+
+        results
+      end
+
+      # Generate a multiple host/process safe result id.
+      #
+      # Note: Generated when this class loads.
+      #
+      require 'socket'
+      def self.extract_host
+        @host ||= Socket.gethostname
+      end
+      def host
+        self.class.extract_host
+      end
+      extract_host
+      def pid
+        @pid ||= Process.pid
+      end
+      # Use the host and pid (generated lazily in child processes) for the result.
+      #
+      def generate_intermediate_result_id
+        :"#{host}:#{pid}:picky:result"
+      end
+
     end
 
   end

server/lib/picky/loader.rb

@@ -154,9 +154,7 @@ def self.load_framework_internals
       # Query combinations, qualifiers, weigher.
       #
       load_relative 'query/combination'
-      load_relative 'query/combinations/base'
-      load_relative 'query/combinations/memory'
-      load_relative 'query/combinations/redis'
+      load_relative 'query/combinations'
 
       load_relative 'query/allocation'
       load_relative 'query/allocations'
@@ -166,10 +164,7 @@ def self.load_framework_internals
       load_relative 'query/weights'
 
       load_relative 'query/indexes'
-
-      # Configuration.
-      #
-      # load_internals 'configuration/index'
+      load_relative 'query/indexes_check'
 
       # Adapters.
       #
@@ -230,7 +225,6 @@ def self.load_user_interface
       # Search.
       #
       load_relative 'search'
-      load_relative 'search/index_combinations_type'
 
       # Sources.
       #

server/lib/picky/query/allocation.rb

@@ -7,13 +7,18 @@ module Query
     #
     class Allocation # :nodoc:all
 
-      attr_reader :count, :ids, :score, :combinations, :result_identifier
+      attr_reader :count,
+                  :ids,
+                  :score,
+                  :combinations,
+                  :result_identifier
 
       #
       #
-      def initialize combinations, result_identifier
+      def initialize index, combinations
         @combinations      = combinations
-        @result_identifier = result_identifier
+        @result_identifier = index.result_identifier # TODO Make cleverer.
+        @backend           = index.backend           # TODO Make cleverer.
       end
 
       def hash
@@ -30,10 +35,10 @@ def calculate_score weights
         @score ||= @combinations.calculate_score(weights)
       end
 
-      # Asks the combinations for the (intersected) ids.
+      # Asks the backend for the (intersected) ids.
       #
       def calculate_ids amount, offset
-        @combinations.ids amount, offset # Calculate as many ids as are necessary.
+        @backend.ids combinations, amount, offset
       end
 
       # This starts the searching process.

server/lib/picky/query/combinations.rb

@@ -0,0 +1,70 @@
+module Picky
+
+  module Query
+
+    # Combinations are a number of Combination-s.
+    #
+    # They are the core of an allocation.
+    # An allocation consists of a number of combinations.
+    #
+    # Base Combinations contain methods for calculating score and ids.
+    #
+    class Combinations # :nodoc:all
+
+      attr_reader :combinations
+
+      delegate :empty?, :inject, :to => :@combinations
+
+      def initialize combinations = []
+        @combinations = combinations
+      end
+
+      def hash
+        @combinations.hash
+      end
+
+      # Uses user specific weights to calculate a score for the combinations.
+      #
+      def calculate_score weights
+        total_score + weighted_score(weights)
+      end
+      def total_score
+        @combinations.sum &:weight
+      end
+      def weighted_score weights
+        weights.score_for @combinations
+      end
+
+      # Filters the tokens and identifiers such that only identifiers
+      # that are passed in, remain, including their tokens.
+      #
+      # Note: This method is not totally independent of the calculate_ids one.
+      #       Since identifiers are only nullified, we need to not include the
+      #       ids that have an associated identifier that is nil.
+      #
+      def keep identifiers = []
+        @combinations.reject! { |combination| !combination.in?(identifiers) }
+      end
+
+      # Filters the tokens and identifiers such that identifiers
+      # that are passed in, are removed, including their tokens.
+      #
+      # Note: This method is not totally independent of the calculate_ids one.
+      #       Since identifiers are only nullified, we need to not include the
+      #       ids that have an associated identifier that is nil.
+      #
+      def remove identifiers = []
+        @combinations.reject! { |combination| combination.in?(identifiers) }
+      end
+
+      #
+      #
+      def to_result
+        @combinations.map &:to_result
+      end
+
+    end
+
+  end
+
+end

server/lib/picky/query/indexes.rb

@@ -20,10 +20,11 @@ class Indexes
       # Its job is to generate all possible combinations.
       # Note: We cannot mix memory and redis indexes just yet.
       #
-      def initialize *indexes, combinations_type
-        @indexes           = indexes
-        @combinations_type = combinations_type
-        @mapper            = Query::QualifierCategoryMapper.new
+      def initialize *indexes
+        IndexesCheck.check_backend_types indexes
+
+        @indexes = indexes
+        @mapper  = Query::QualifierCategoryMapper.new
         map_categories
       end
       def map_categories
@@ -81,12 +82,12 @@ def allocation_for tokens, index
 
         # Generate all possible combinations.
         #
-        expanded_combinations = expand_combinations_from possible_combinations
+        all_possible_combinations = expand_combinations_from possible_combinations
 
         # Add the wrapped possible allocations to the ones we already have.
         #
-        expanded_combinations.map! do |expanded_combination|
-          Allocation.new @combinations_type.new(expanded_combination), index.result_identifier # TODO Do not extract result_identifier.
+        all_possible_combinations.map! do |expanded_combinations|
+          Allocation.new index, Query::Combinations.new(expanded_combinations)
         end
       end
 

server/lib/picky/query/indexes_check.rb

@@ -0,0 +1,43 @@
+module Picky
+
+  # TODO Remove.
+  #
+  class IndexesCheck
+
+    class << self
+
+      # Returns the right combinations strategy for
+      # a number of query indexes.
+      #
+      # Currently it isn't possible using Memory and Redis etc.
+      # indexes in the same query index group.
+      #
+      # Picky will raise a Query::Indexes::DifferentTypesError.
+      #
+      def check_backend_types index_definitions_ary # :nodoc:
+        backend_types = index_definitions_ary.map(&:backend).map(&:class)
+        backend_types.uniq!
+        raise_different backend_types if backend_types.size > 1
+        backend_types
+      end
+      def raise_different backend_types # :nodoc:
+        raise DifferentTypesError.new(backend_types)
+      end
+
+      # Currently it isn't possible using Memory and Redis etc.
+      # indexes in the same query index group.
+      #
+      class DifferentTypesError < StandardError # :nodoc:all
+        def initialize types
+          @types = types
+        end
+        def to_s
+          "Currently it isn't possible to mix Indexes with backends #{@types.join(" and ")} in the same Search instance."
+        end
+      end
+
+    end
+
+  end
+
+end

server/lib/picky/search.rb

@@ -36,7 +36,7 @@ class Search
     #   end
     #
     def initialize *index_definitions
-      @indexes = Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
+      @indexes = Query::Indexes.new *index_definitions
 
       instance_eval(&Proc.new) if block_given?