Permalink
Browse files

+ typed backends now responsible for intersecting ids, + this means t…

…hat Picky now has exchangeable backends! Hooray!
  • Loading branch information...
1 parent bda840e commit c10bdf5f49041ccfde516e6f10eeec15b7fb513d @floere committed Aug 25, 2011
@@ -12,6 +12,35 @@ def configure_with bundle
@configuration = File::JSON.new bundle.index_path(:configuration)
end
+ # Returns the result ids for the allocation.
+ #
+ # Sorts the ids by size and & through them in the following order (sizes):
+ # 0. [100_000, 400, 30, 2]
+ # 1. [2, 30, 400, 100_000]
+ # 2. (100_000 & (400 & (30 & 2))) # => result
+ #
+ # Note: Uses a C-optimized intersection routine (in performant.c)
+ # for speed and memory efficiency.
+ #
+ # Note: In the memory based version we ignore the amount and offset hints.
+ # We cannot use the information to speed up the algorithm, unfortunately.
+ #
+ def ids combinations, _, _
+ return [] if combinations.empty?
+
+ # Get the ids for each combination.
+ #
+ id_arrays = combinations.inject([]) do |total, combination|
+ total << combination.ids
+ end
+
+ # Call the optimized C algorithm.
+ #
+ # Note: It orders the passed arrays by size.
+ #
+ Performant::Array.memory_efficient_intersect id_arrays
+ end
+
end
end
@@ -20,6 +20,61 @@ def configure_with bundle
@configuration = Redis::StringHash.new "#{bundle.identifier}:configuration", actual_backend
end
+ # Returns the result ids for the allocation.
+ #
+ # Developers wanting to program fast intersection
+ # routines, can do so analogue to this in their own
+ # backend implementations.
+ #
+ # Note: We use the amount and offset hints to speed Redis up.
+ #
+ def ids combinations, amount, offset
+ return [] if combinations.empty?
+
+ identifiers = combinations.inject([]) do |identifiers, combination|
+ identifiers << "#{combination.identifier}"
+ end
+
+ result_id = generate_intermediate_result_id
+
+ # Intersect and store.
+ #
+ actual_backend.zinterstore result_id, identifiers
+
+ # Get the stored result.
+ #
+ results = actual_backend.zrange result_id, offset, (offset + amount)
+
+ # Delete the stored result as it was only for temporary purposes.
+ #
+ # Note: I could also not delete it, but that would not be clean at all.
+ #
+ actual_backend.del result_id
+
+ results
+ end
+
+ # Generate a multiple host/process safe result id.
+ #
+ # Note: Generated when this class loads.
+ #
+ require 'socket'
+ def self.extract_host
+ @host ||= Socket.gethostname
+ end
+ def host
+ self.class.extract_host
+ end
+ extract_host
+ def pid
+ @pid ||= Process.pid
+ end
+ # Use the host and pid (generated lazily in child processes) for the result.
+ #
+ def generate_intermediate_result_id
+ :"#{host}:#{pid}:picky:result"
+ end
+
end
end
View
@@ -154,9 +154,7 @@ def self.load_framework_internals
# Query combinations, qualifiers, weigher.
#
load_relative 'query/combination'
- load_relative 'query/combinations/base'
- load_relative 'query/combinations/memory'
- load_relative 'query/combinations/redis'
+ load_relative 'query/combinations'
load_relative 'query/allocation'
load_relative 'query/allocations'
@@ -166,10 +164,7 @@ def self.load_framework_internals
load_relative 'query/weights'
load_relative 'query/indexes'
-
- # Configuration.
- #
- # load_internals 'configuration/index'
+ load_relative 'query/indexes_check'
# Adapters.
#
@@ -230,7 +225,6 @@ def self.load_user_interface
# Search.
#
load_relative 'search'
- load_relative 'search/index_combinations_type'
# Sources.
#
@@ -7,13 +7,18 @@ module Query
#
class Allocation # :nodoc:all
- attr_reader :count, :ids, :score, :combinations, :result_identifier
+ attr_reader :count,
+ :ids,
+ :score,
+ :combinations,
+ :result_identifier
#
#
- def initialize combinations, result_identifier
+ def initialize index, combinations
@combinations = combinations
- @result_identifier = result_identifier
+ @result_identifier = index.result_identifier # TODO Make cleverer.
+ @backend = index.backend # TODO Make cleverer.
end
def hash
@@ -30,10 +35,10 @@ def calculate_score weights
@score ||= @combinations.calculate_score(weights)
end
- # Asks the combinations for the (intersected) ids.
+ # Asks the backend for the (intersected) ids.
#
def calculate_ids amount, offset
- @combinations.ids amount, offset # Calculate as many ids as are necessary.
+ @backend.ids combinations, amount, offset
end
# This starts the searching process.
@@ -0,0 +1,70 @@
+module Picky
+
+ module Query
+
+ # Combinations are a number of Combination-s.
+ #
+ # They are the core of an allocation.
+ # An allocation consists of a number of combinations.
+ #
+ # Base Combinations contain methods for calculating score and ids.
+ #
+ class Combinations # :nodoc:all
+
+ attr_reader :combinations
+
+ delegate :empty?, :inject, :to => :@combinations
+
+ def initialize combinations = []
+ @combinations = combinations
+ end
+
+ def hash
+ @combinations.hash
+ end
+
+ # Uses user specific weights to calculate a score for the combinations.
+ #
+ def calculate_score weights
+ total_score + weighted_score(weights)
+ end
+ def total_score
+ @combinations.sum &:weight
+ end
+ def weighted_score weights
+ weights.score_for @combinations
+ end
+
+ # Filters the tokens and identifiers such that only identifiers
+ # that are passed in, remain, including their tokens.
+ #
+ # Note: This method is not totally independent of the calculate_ids one.
+ # Since identifiers are only nullified, we need to not include the
+ # ids that have an associated identifier that is nil.
+ #
+ def keep identifiers = []
+ @combinations.reject! { |combination| !combination.in?(identifiers) }
+ end
+
+ # Filters the tokens and identifiers such that identifiers
+ # that are passed in, are removed, including their tokens.
+ #
+ # Note: This method is not totally independent of the calculate_ids one.
+ # Since identifiers are only nullified, we need to not include the
+ # ids that have an associated identifier that is nil.
+ #
+ def remove identifiers = []
+ @combinations.reject! { |combination| combination.in?(identifiers) }
+ end
+
+ #
+ #
+ def to_result
+ @combinations.map &:to_result
+ end
+
+ end
+
+ end
+
+end
@@ -20,10 +20,11 @@ class Indexes
# Its job is to generate all possible combinations.
# Note: We cannot mix memory and redis indexes just yet.
#
- def initialize *indexes, combinations_type
- @indexes = indexes
- @combinations_type = combinations_type
- @mapper = Query::QualifierCategoryMapper.new
+ def initialize *indexes
+ IndexesCheck.check_backend_types indexes
+
+ @indexes = indexes
+ @mapper = Query::QualifierCategoryMapper.new
map_categories
end
def map_categories
@@ -81,12 +82,12 @@ def allocation_for tokens, index
# Generate all possible combinations.
#
- expanded_combinations = expand_combinations_from possible_combinations
+ all_possible_combinations = expand_combinations_from possible_combinations
# Add the wrapped possible allocations to the ones we already have.
#
- expanded_combinations.map! do |expanded_combination|
- Allocation.new @combinations_type.new(expanded_combination), index.result_identifier # TODO Do not extract result_identifier.
+ all_possible_combinations.map! do |expanded_combinations|
+ Allocation.new index, Query::Combinations.new(expanded_combinations)
end
end
@@ -0,0 +1,43 @@
+module Picky
+
+ # TODO Remove.
+ #
+ class IndexesCheck
+
+ class << self
+
+ # Returns the right combinations strategy for
+ # a number of query indexes.
+ #
+ # Currently it isn't possible using Memory and Redis etc.
+ # indexes in the same query index group.
+ #
+ # Picky will raise a Query::Indexes::DifferentTypesError.
+ #
+ def check_backend_types index_definitions_ary # :nodoc:
+ backend_types = index_definitions_ary.map(&:backend).map(&:class)
+ backend_types.uniq!
+ raise_different backend_types if backend_types.size > 1
+ backend_types
+ end
+ def raise_different backend_types # :nodoc:
+ raise DifferentTypesError.new(backend_types)
+ end
+
+ # Currently it isn't possible using Memory and Redis etc.
+ # indexes in the same query index group.
+ #
+ class DifferentTypesError < StandardError # :nodoc:all
+ def initialize types
+ @types = types
+ end
+ def to_s
+ "Currently it isn't possible to mix Indexes with backends #{@types.join(" and ")} in the same Search instance."
+ end
+ end
+
+ end
+
+ end
+
+end
@@ -36,7 +36,7 @@ class Search
# end
#
def initialize *index_definitions
- @indexes = Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
+ @indexes = Query::Indexes.new *index_definitions
instance_eval(&Proc.new) if block_given?
@@ -1,47 +0,0 @@
-module Picky
-
- class Search
-
- # Returns the right combinations strategy for
- # a number of query indexes.
- #
- # Currently it isn't possible using Memory and Redis etc.
- # indexes in the same query index group.
- #
- # Picky will raise a Query::Indexes::DifferentTypesError.
- #
- @@mapping = {
- Backends::Memory => Query::Combinations::Memory,
- Backends::Redis => Query::Combinations::Redis
- }
- def combinations_type_for index_definitions_ary # :nodoc:
- backend_types = extract_backend_types index_definitions_ary
- !backend_types.empty? && @@mapping[*backend_types] || Query::Combinations::Memory
- end
- def extract_backend_types index_definitions_ary # :nodoc:
- backend_types = index_definitions_ary.map(&:backend).map(&:class)
- backend_types.uniq!
- check_backend_types backend_types
- backend_types
- end
- def check_backend_types backend_types # :nodoc:
- raise_different backend_types if backend_types.size > 1
- end
- # Currently it isn't possible using Memory and Redis etc.
- # indexes in the same query index group.
- #
- class DifferentTypesError < StandardError # :nodoc:all
- def initialize types
- @types = types
- end
- def to_s
- "Currently it isn't possible to mix Indexes with backends #{@types.join(" and ")} in the same Search instance."
- end
- end
- def raise_different backend_types # :nodoc:
- raise DifferentTypesError.new(backend_types)
- end
-
- end
-
-end

0 comments on commit c10bdf5

Please sign in to comment.