From 536a852100f7a9ec7b16d132b69e5969c617f35b Mon Sep 17 00:00:00 2001 From: Florian Hanke Date: Thu, 25 Aug 2011 15:42:05 +1000 Subject: [PATCH] + backend rewrite, towards making it more exchangeable --- .../lib/picky/{backend => backends}/base.rb | 2 +- .../picky/{backend => backends}/file/basic.rb | 2 +- .../picky/{backend => backends}/file/json.rb | 2 +- .../{backend => backends}/file/marshal.rb | 2 +- .../picky/{backend => backends}/file/text.rb | 2 +- .../{backend/files.rb => backends/memory.rb} | 4 +- .../lib/picky/{backend => backends}/redis.rb | 2 +- .../{backend => backends}/redis/basic.rb | 2 +- .../{backend => backends}/redis/float_hash.rb | 2 +- .../{backend => backends}/redis/list_hash.rb | 2 +- .../redis/string_hash.rb | 2 +- server/lib/picky/bundle.rb | 12 +- server/lib/picky/category.rb | 10 +- server/lib/picky/indexed/bundle.rb | 112 +++++++ server/lib/picky/indexed/bundle/base.rb | 116 ------- server/lib/picky/indexed/bundle/memory.rb | 31 -- server/lib/picky/indexed/bundle/redis.rb | 29 -- .../lib/picky/indexed/wrappers/exact_first.rb | 2 +- server/lib/picky/indexes/index.rb | 9 +- server/lib/picky/indexes/memory.rb | 7 +- server/lib/picky/indexes/redis.rb | 7 +- server/lib/picky/indexing/bundle.rb | 239 +++++++++++++++ server/lib/picky/indexing/bundle/base.rb | 242 --------------- server/lib/picky/indexing/bundle/memory.rb | 28 -- server/lib/picky/indexing/bundle/redis.rb | 28 -- server/lib/picky/loader.rb | 33 +- server/spec/lib/backend/file/basic_spec.rb | 4 +- server/spec/lib/backend/file/json_spec.rb | 4 +- server/spec/lib/backend/file/marshal_spec.rb | 4 +- server/spec/lib/backend/file/text_spec.rb | 2 +- .../backend/{files_spec.rb => memory_spec.rb} | 8 +- server/spec/lib/backend/redis/basic_spec.rb | 4 +- .../spec/lib/backend/redis/float_hash_spec.rb | 4 +- .../spec/lib/backend/redis/list_hash_spec.rb | 4 +- .../lib/backend/redis/string_hash_spec.rb | 4 +- server/spec/lib/backend/redis_spec.rb | 2 +- server/spec/lib/category_indexed_spec.rb | 8 +- server/spec/lib/indexed/bundle/redis_spec.rb | 17 -- .../lib/indexed/{bundle => }/memory_spec.rb | 8 +- server/spec/lib/indexing/bundle/base_spec.rb | 38 --- server/spec/lib/indexing/bundle/redis_spec.rb | 283 ------------------ ...> bundle_partial_generation_speed_spec.rb} | 4 +- .../{bundle/memory_spec.rb => bundle_spec.rb} | 36 ++- 43 files changed, 458 insertions(+), 905 deletions(-) rename server/lib/picky/{backend => backends}/base.rb (99%) rename server/lib/picky/{backend => backends}/file/basic.rb (99%) rename server/lib/picky/{backend => backends}/file/json.rb (97%) rename server/lib/picky/{backend => backends}/file/marshal.rb (97%) rename server/lib/picky/{backend => backends}/file/text.rb (98%) rename server/lib/picky/{backend/files.rb => backends/memory.rb} (93%) rename server/lib/picky/{backend => backends}/redis.rb (96%) rename server/lib/picky/{backend => backends}/redis/basic.rb (98%) rename server/lib/picky/{backend => backends}/redis/float_hash.rb (94%) rename server/lib/picky/{backend => backends}/redis/list_hash.rb (97%) rename server/lib/picky/{backend => backends}/redis/string_hash.rb (96%) create mode 100644 server/lib/picky/indexed/bundle.rb delete mode 100644 server/lib/picky/indexed/bundle/base.rb delete mode 100644 server/lib/picky/indexed/bundle/memory.rb delete mode 100644 server/lib/picky/indexed/bundle/redis.rb create mode 100644 server/lib/picky/indexing/bundle.rb delete mode 100644 server/lib/picky/indexing/bundle/base.rb delete mode 100644 server/lib/picky/indexing/bundle/memory.rb delete mode 100644 server/lib/picky/indexing/bundle/redis.rb rename server/spec/lib/backend/{files_spec.rb => memory_spec.rb} (93%) delete mode 100644 server/spec/lib/indexed/bundle/redis_spec.rb rename server/spec/lib/indexed/{bundle => }/memory_spec.rb (93%) delete mode 100644 server/spec/lib/indexing/bundle/base_spec.rb delete mode 100644 server/spec/lib/indexing/bundle/redis_spec.rb rename server/spec/lib/indexing/{bundle/memory_partial_generation_speed_spec.rb => bundle_partial_generation_speed_spec.rb} (91%) rename server/spec/lib/indexing/{bundle/memory_spec.rb => bundle_spec.rb} (88%) diff --git a/server/lib/picky/backend/base.rb b/server/lib/picky/backends/base.rb similarity index 99% rename from server/lib/picky/backend/base.rb rename to server/lib/picky/backends/base.rb index 0adb045b..a6d4415b 100644 --- a/server/lib/picky/backend/base.rb +++ b/server/lib/picky/backends/base.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends class Base diff --git a/server/lib/picky/backend/file/basic.rb b/server/lib/picky/backends/file/basic.rb similarity index 99% rename from server/lib/picky/backend/file/basic.rb rename to server/lib/picky/backends/file/basic.rb index 475702c3..5026205e 100644 --- a/server/lib/picky/backend/file/basic.rb +++ b/server/lib/picky/backends/file/basic.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends # Handles all aspects of index files, such as dumping/loading. # diff --git a/server/lib/picky/backend/file/json.rb b/server/lib/picky/backends/file/json.rb similarity index 97% rename from server/lib/picky/backend/file/json.rb rename to server/lib/picky/backends/file/json.rb index 45f95095..86f9b0ec 100644 --- a/server/lib/picky/backend/file/json.rb +++ b/server/lib/picky/backends/file/json.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends module File diff --git a/server/lib/picky/backend/file/marshal.rb b/server/lib/picky/backends/file/marshal.rb similarity index 97% rename from server/lib/picky/backend/file/marshal.rb rename to server/lib/picky/backends/file/marshal.rb index 473c2870..dc5691e3 100644 --- a/server/lib/picky/backend/file/marshal.rb +++ b/server/lib/picky/backends/file/marshal.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends module File diff --git a/server/lib/picky/backend/file/text.rb b/server/lib/picky/backends/file/text.rb similarity index 98% rename from server/lib/picky/backend/file/text.rb rename to server/lib/picky/backends/file/text.rb index 419a84ba..1792425b 100644 --- a/server/lib/picky/backend/file/text.rb +++ b/server/lib/picky/backends/file/text.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends module File diff --git a/server/lib/picky/backend/files.rb b/server/lib/picky/backends/memory.rb similarity index 93% rename from server/lib/picky/backend/files.rb rename to server/lib/picky/backends/memory.rb index b471c5b5..a39f7ed2 100644 --- a/server/lib/picky/backend/files.rb +++ b/server/lib/picky/backends/memory.rb @@ -1,8 +1,8 @@ module Picky - module Backend + module Backends - class Files < Base + class Memory < Base def initialize bundle super bundle diff --git a/server/lib/picky/backend/redis.rb b/server/lib/picky/backends/redis.rb similarity index 96% rename from server/lib/picky/backend/redis.rb rename to server/lib/picky/backends/redis.rb index 312408b4..d58295b6 100644 --- a/server/lib/picky/backend/redis.rb +++ b/server/lib/picky/backends/redis.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends # # diff --git a/server/lib/picky/backend/redis/basic.rb b/server/lib/picky/backends/redis/basic.rb similarity index 98% rename from server/lib/picky/backend/redis/basic.rb rename to server/lib/picky/backends/redis/basic.rb index 52979696..012ede92 100644 --- a/server/lib/picky/backend/redis/basic.rb +++ b/server/lib/picky/backends/redis/basic.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends class Redis diff --git a/server/lib/picky/backend/redis/float_hash.rb b/server/lib/picky/backends/redis/float_hash.rb similarity index 94% rename from server/lib/picky/backend/redis/float_hash.rb rename to server/lib/picky/backends/redis/float_hash.rb index 9f803cd8..22578f80 100644 --- a/server/lib/picky/backend/redis/float_hash.rb +++ b/server/lib/picky/backends/redis/float_hash.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends class Redis diff --git a/server/lib/picky/backend/redis/list_hash.rb b/server/lib/picky/backends/redis/list_hash.rb similarity index 97% rename from server/lib/picky/backend/redis/list_hash.rb rename to server/lib/picky/backends/redis/list_hash.rb index 75a481ba..2e22dd07 100644 --- a/server/lib/picky/backend/redis/list_hash.rb +++ b/server/lib/picky/backends/redis/list_hash.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends class Redis diff --git a/server/lib/picky/backend/redis/string_hash.rb b/server/lib/picky/backends/redis/string_hash.rb similarity index 96% rename from server/lib/picky/backend/redis/string_hash.rb rename to server/lib/picky/backends/redis/string_hash.rb index 8ea15884..198e30e9 100644 --- a/server/lib/picky/backend/redis/string_hash.rb +++ b/server/lib/picky/backends/redis/string_hash.rb @@ -1,6 +1,6 @@ module Picky - module Backend + module Backends class Redis diff --git a/server/lib/picky/bundle.rb b/server/lib/picky/bundle.rb index d20405a3..d216b17d 100644 --- a/server/lib/picky/bundle.rb +++ b/server/lib/picky/bundle.rb @@ -35,16 +35,22 @@ class Bundle delegate :[], :[]=, :to => :configuration delegate :index_directory, :to => :category - def initialize name, category, similarity_strategy, options = {} + def initialize name, category, backend_class, similarity_strategy, options = {} @name = name @category = category - # TODO Still needed? + # TODO Refactor further. + # + @backend = backend_class.new self + + # Default backend values. + # + # TODO Use a default (memory) backend and load instantly. # @inverted = {} @weights = {} @similarity = {} - @configuration = {} # A hash with config options. + @configuration = {} @similarity_strategy = similarity_strategy end diff --git a/server/lib/picky/category.rb b/server/lib/picky/category.rb index 8223e0f3..af6761cf 100644 --- a/server/lib/picky/category.rb +++ b/server/lib/picky/category.rb @@ -35,16 +35,16 @@ def initialize name, index, options = {} partial = options[:partial] || Generators::Partial::Default similarity = options[:similarity] || Generators::Similarity::Default - @indexing_exact = index.indexing_bundle_class.new :exact, self, weights, Generators::Partial::None.new, similarity, options - @indexing_partial = index.indexing_bundle_class.new :partial, self, weights, partial, Generators::Similarity::None.new, options + @indexing_exact = Indexing::Bundle.new :exact, self, index.backend_class, weights, Generators::Partial::None.new, similarity, options + @indexing_partial = Indexing::Bundle.new :partial, self, index.backend_class, weights, partial, Generators::Similarity::None.new, options # Indexed. # - @indexed_exact = index.indexed_bundle_class.new :exact, self, similarity + @indexed_exact = Indexed::Bundle.new :exact, self, index.backend_class, similarity if partial.use_exact_for_partial? @indexed_partial = @indexed_exact else - @indexed_partial = index.indexed_bundle_class.new :partial, self, similarity + @indexed_partial = Indexed::Bundle.new :partial, self, index.backend_class, similarity end # @exact = exact_lambda.call(@exact, @partial) if exact_lambda = options[:exact_lambda] @@ -92,7 +92,7 @@ def prepared_index_path # Note: If you don't use it with the block, do not forget to close it. # def prepared_index_file &block - @prepared_index_file ||= Backend::File::Text.new prepared_index_path + @prepared_index_file ||= Backends::File::Text.new prepared_index_path @prepared_index_file.open &block end # Creates the index directory including all necessary paths above it. diff --git a/server/lib/picky/indexed/bundle.rb b/server/lib/picky/indexed/bundle.rb new file mode 100644 index 00000000..b9f70d99 --- /dev/null +++ b/server/lib/picky/indexed/bundle.rb @@ -0,0 +1,112 @@ +module Picky + + module Indexed # :nodoc:all + + # An indexed bundle is a number of memory/redis + # indexes that compose the indexes for a single category: + # * core (inverted) index + # * weights index + # * similarity index + # * index configuration + # + # Indexed refers to them being indexed. + # This class notably offers the methods: + # * load + # * clear + # + # To (re)load or clear the current indexes. + # + class Bundle < Picky::Bundle + + # Get the ids for the given symbol. + # + # Returns a (potentially empty) array of ids. + # + def ids sym + @inverted[sym] || [] + end + + # Get a weight for the given symbol. + # + # Returns a number, or nil. + # + def weight sym + @weights[sym] + end + + # Get settings for this bundle. + # + # Returns an object. + # + def [] sym + @configuration[sym] + end + + # Loads all indexes. + # + # Loading loads index objects from the backend. + # They should each respond to []. + # + def load + load_inverted + load_weights + load_similarity + load_configuration + end + + # Loads the core index. + # + def load_inverted + self.inverted = @backend.load_inverted + end + # Loads the weights index. + # + def load_weights + self.weights = @backend.load_weights + end + # Loads the similarity index. + # + def load_similarity + self.similarity = @backend.load_similarity + end + # Loads the configuration. + # + def load_configuration + self.configuration = @backend.load_configuration + end + + # Clears all indexes. + # + def clear + clear_inverted + clear_weights + clear_similarity + clear_configuration + end + + # Clears the core index. + # + def clear_inverted + inverted.clear + end + # Clears the weights index. + # + def clear_weights + weights.clear + end + # Clears the similarity index. + # + def clear_similarity + similarity.clear + end + # Clears the configuration. + # + def clear_configuration + configuration.clear + end + + end + + end + +end \ No newline at end of file diff --git a/server/lib/picky/indexed/bundle/base.rb b/server/lib/picky/indexed/bundle/base.rb deleted file mode 100644 index b4881d9b..00000000 --- a/server/lib/picky/indexed/bundle/base.rb +++ /dev/null @@ -1,116 +0,0 @@ -module Picky - - module Indexed # :nodoc:all - - # An indexed bundle is a number of memory/redis - # indexes that compose the indexes for a single category: - # * core (inverted) index - # * weights index - # * similarity index - # * index configuration - # - # Indexed refers to them being indexed. - # This class notably offers the methods: - # * load - # * clear - # - # To (re)load or clear the current indexes. - # - module Bundle - - class Base < Picky::Bundle - - # Get the ids for the given symbol. - # - # Returns a (potentially empty) array of ids. - # - def ids sym - @inverted[sym] || [] - end - - # Get a weight for the given symbol. - # - # Returns a number, or nil. - # - def weight sym - @weights[sym] - end - - # Get settings for this bundle. - # - # Returns an object. - # - def [] sym - @configuration[sym] - end - - # Loads all indexes. - # - # Loading loads index objects from the backend. - # They should each respond to []. - # - def load - load_inverted - load_weights - load_similarity - load_configuration - end - - # Loads the core index. - # - def load_inverted - self.inverted = @backend.load_inverted - end - # Loads the weights index. - # - def load_weights - self.weights = @backend.load_weights - end - # Loads the similarity index. - # - def load_similarity - self.similarity = @backend.load_similarity - end - # Loads the configuration. - # - def load_configuration - self.configuration = @backend.load_configuration - end - - # Clears all indexes. - # - def clear - clear_inverted - clear_weights - clear_similarity - clear_configuration - end - - # Clears the core index. - # - def clear_inverted - inverted.clear - end - # Clears the weights index. - # - def clear_weights - weights.clear - end - # Clears the similarity index. - # - def clear_similarity - similarity.clear - end - # Clears the configuration. - # - def clear_configuration - configuration.clear - end - - end - - end - - end - -end \ No newline at end of file diff --git a/server/lib/picky/indexed/bundle/memory.rb b/server/lib/picky/indexed/bundle/memory.rb deleted file mode 100644 index 202ba871..00000000 --- a/server/lib/picky/indexed/bundle/memory.rb +++ /dev/null @@ -1,31 +0,0 @@ -# encoding: utf-8 -# -module Picky - - module Indexed # :nodoc:all - - # TODO Replace inheritance by passing in the backend. Before that, unify the backend interface. - # - module Bundle - - # This is the _actual_ index (based on memory). - # - # Handles exact/partial index, weights index, and similarity index. - # - # Delegates file handling and checking to an *Indexed*::*Files* object. - # - class Memory < Base - - def initialize name, category, *args - super name, category, *args - - @backend = Backend::Files.new self # TODO Rename to Memory and push functionality there. - end - - end - - end - - end - -end \ No newline at end of file diff --git a/server/lib/picky/indexed/bundle/redis.rb b/server/lib/picky/indexed/bundle/redis.rb deleted file mode 100644 index dafad506..00000000 --- a/server/lib/picky/indexed/bundle/redis.rb +++ /dev/null @@ -1,29 +0,0 @@ -# encoding: utf-8 -# -module Picky - - module Indexed # :nodoc:all - - # TODO Replace inheritance by passing in the backend. Before that, unify the backend interface. - # - module Bundle - - # This is the _actual_ index (based on Redis). - # - # Handles exact/partial index, weights index, and similarity index. - # - class Redis < Base - - def initialize name, category, *args - super name, category, *args - - @backend = Backend::Redis.new self - end - - end - - end - - end - -end \ No newline at end of file diff --git a/server/lib/picky/indexed/wrappers/exact_first.rb b/server/lib/picky/indexed/wrappers/exact_first.rb index ee70cc49..dac29bb3 100644 --- a/server/lib/picky/indexed/wrappers/exact_first.rb +++ b/server/lib/picky/indexed/wrappers/exact_first.rb @@ -9,7 +9,7 @@ module Wrappers # This index combines an exact and partial index. # It serves to order the results such that exact hits are found first. # - class ExactFirst < Indexed::Bundle::Base + class ExactFirst < Indexed::Bundle delegate :similar, :identifier, diff --git a/server/lib/picky/indexes/index.rb b/server/lib/picky/indexes/index.rb index 57157930..a8389714 100644 --- a/server/lib/picky/indexes/index.rb +++ b/server/lib/picky/indexes/index.rb @@ -128,13 +128,10 @@ def initialize name, options = {} instance_eval(&Proc.new) if block_given? end - # Default bundles. + # Default backend. # - def indexing_bundle_class - Indexing::Bundle::Memory - end - def indexed_bundle_class - Indexed::Bundle::Memory + def backend_class + Backends::Memory end # Defines a searchable category on the index. diff --git a/server/lib/picky/indexes/memory.rb b/server/lib/picky/indexes/memory.rb index 31bd1843..e81f2000 100644 --- a/server/lib/picky/indexes/memory.rb +++ b/server/lib/picky/indexes/memory.rb @@ -6,11 +6,8 @@ class Indexes # class Memory < Index - def indexing_bundle_class - Indexing::Bundle::Memory - end - def indexed_bundle_class - Indexed::Bundle::Memory + def backend_class + Backends::Memory end end diff --git a/server/lib/picky/indexes/redis.rb b/server/lib/picky/indexes/redis.rb index 56ac7262..dff2af28 100644 --- a/server/lib/picky/indexes/redis.rb +++ b/server/lib/picky/indexes/redis.rb @@ -6,11 +6,8 @@ class Indexes # class Redis < Index - def indexing_bundle_class - Indexing::Bundle::Redis - end - def indexed_bundle_class - Indexed::Bundle::Redis + def backend_class + Backends::Redis end end diff --git a/server/lib/picky/indexing/bundle.rb b/server/lib/picky/indexing/bundle.rb new file mode 100644 index 00000000..35a8ea00 --- /dev/null +++ b/server/lib/picky/indexing/bundle.rb @@ -0,0 +1,239 @@ +module Picky + + module Indexing # :nodoc:all + + # A Bundle is a number of indexes + # per [index, category] combination. + # + # At most, there are three indexes: + # * *core* index (always used) + # * *weights* index (always used) + # * *similarity* index (used with similarity) + # + # In Picky, indexing is separated from the index + # handling itself through a parallel structure. + # + # Both use methods provided by this base class, but + # have very different goals: + # + # * *Indexing*::*Bundle* is just concerned with creating index files + # and providing helper functions to e.g. check the indexes. + # + # * *Index*::*Bundle* is concerned with loading these index files into + # memory and looking up search data as fast as possible. + # + # This is the indexing bundle. + # + # It does all menial tasks that have nothing to do + # with the actual index running etc. + # (Find these in Indexed::Bundle) + # + class Bundle < Picky::Bundle + + attr_reader :backend, + :prepared + + attr_accessor :partial_strategy, + :weights_strategy + + def initialize name, category, backend_class, weights_strategy, partial_strategy, similarity_strategy, options = {} + super name, category, backend_class, similarity_strategy, options + + @weights_strategy = weights_strategy + @partial_strategy = partial_strategy + @key_format = options[:key_format] + @prepared = Backends::File::Text.new category.prepared_index_path + end + + # Sets up a piece of the index for the given token. + # + def initialize_inverted_index_for token + self.inverted[token] ||= [] + end + + # Generation + # + + # This method + # * Loads the base index from the "prepared..." file. + # * Generates derived indexes. + # * Dumps all the indexes into files. + # + def generate_caches_from_source + load_from_prepared_index_file + generate_caches_from_memory + end + # Generates derived indexes from the index and dumps. + # + # Note: assumes that there is something in the index + # + def generate_caches_from_memory + cache_from_memory_generation_message + generate_derived + end + def cache_from_memory_generation_message + timed_exclaim %Q{"#{identifier}": Caching from intermediate in-memory index.} + end + + # Generates the weights and similarity from the main index. + # + def generate_derived + generate_weights + generate_similarity + end + + # Load the data from the db. + # + def load_from_prepared_index_file + load_from_prepared_index_generation_message + clear + retrieve + end + def load_from_prepared_index_generation_message + timed_exclaim %Q{"#{identifier}": Loading prepared data into memory.} + end + # Retrieves the prepared index data into the index. + # + # This is in preparation for generating + # derived indexes (like weights, similarity) + # and later dumping the optimized index. + # + # TODO Move this out to the category? + # + def retrieve + format = category.key_format || :to_i # Optimization. + prepared.retrieve do |id, token| + initialize_inverted_index_for token + self.inverted[token] << id.send(format) + end + end + + # Generates a new index (writes its index) using the + # partial caching strategy of this bundle. + # + def generate_partial + generator = Generators::PartialGenerator.new self.inverted + self.inverted = generator.generate self.partial_strategy + end + # Generate a partial index from the given exact inverted index. + # + def generate_partial_from exact_inverted_index + timed_exclaim %Q{"#{identifier}": Generating partial index for index.} + self.inverted = exact_inverted_index + self.generate_partial + self + end + # Generates a new weights index (writes its index) using the + # given weight caching strategy. + # + def generate_weights + generator = Generators::WeightsGenerator.new self.inverted + self.weights = generator.generate self.weights_strategy + end + # Generates a new similarity index (writes its index) using the + # given similarity caching strategy. + # + def generate_similarity + generator = Generators::SimilarityGenerator.new self.inverted + self.similarity = generator.generate self.similarity_strategy + end + + # Saves the indexes in a dump file. + # + def dump + timed_exclaim %Q{"#{identifier}": Dumping data.} + dump_inverted + dump_similarity + dump_weights + dump_configuration + end + # Dumps the core index. + # + def dump_inverted + # timed_exclaim %Q{"#{identifier}": Dumping inverted index.} + backend.dump_inverted self.inverted + end + # Dumps the weights index. + # + def dump_weights + # timed_exclaim %Q{"#{identifier}": Dumping index weights.} + backend.dump_weights self.weights + end + # Dumps the similarity index. + # + def dump_similarity + # timed_exclaim %Q{"#{identifier}": Dumping similarity index.} + backend.dump_similarity self.similarity + end + # Dumps the similarity index. + # + def dump_configuration + # timed_exclaim %Q{"#{identifier}": Dumping configuration.} + backend.dump_configuration self.configuration + end + + # Alerts the user if an index is missing. + # + def raise_unless_cache_exists + raise_unless_index_exists + raise_unless_similarity_exists + end + # Alerts the user if one of the necessary indexes + # (core, weights) is missing. + # + def raise_unless_index_exists + if partial_strategy.saved? + warn_if_index_small + raise_unless_index_ok + end + end + # Alerts the user if the similarity + # index is missing (given that it's used). + # + def raise_unless_similarity_exists + if similarity_strategy.saved? + warn_if_similarity_small + raise_unless_similarity_ok + end + end + + # Outputs a warning for the given cache. + # + def warn_cache_small what + warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes." + end + # Raises an appropriate error message for the given cache. + # + def raise_cache_missing what + raise "Error: The #{what} cache for #{identifier} is missing." + end + + # Warns the user if the similarity index is small. + # + def warn_if_similarity_small + warn_cache_small :similarity if backend.similarity_cache_small? + end + # Alerts the user if the similarity index is not there. + # + def raise_unless_similarity_ok + raise_cache_missing :similarity unless backend.similarity_cache_ok? + end + + # Warns the user if the core or weights indexes are small. + # + def warn_if_index_small + warn_cache_small :inverted if backend.inverted_cache_small? + warn_cache_small :weights if backend.weights_cache_small? + end + # Alerts the user if the core or weights indexes are not there. + # + def raise_unless_index_ok + raise_cache_missing :inverted unless backend.inverted_cache_ok? + raise_cache_missing :weights unless backend.weights_cache_ok? + end + + end + + end + +end \ No newline at end of file diff --git a/server/lib/picky/indexing/bundle/base.rb b/server/lib/picky/indexing/bundle/base.rb deleted file mode 100644 index 5150ae23..00000000 --- a/server/lib/picky/indexing/bundle/base.rb +++ /dev/null @@ -1,242 +0,0 @@ -module Picky - - module Indexing # :nodoc:all - - # A Bundle is a number of indexes - # per [index, category] combination. - # - # At most, there are three indexes: - # * *core* index (always used) - # * *weights* index (always used) - # * *similarity* index (used with similarity) - # - # In Picky, indexing is separated from the index - # handling itself through a parallel structure. - # - # Both use methods provided by this base class, but - # have very different goals: - # - # * *Indexing*::*Bundle* is just concerned with creating index files - # and providing helper functions to e.g. check the indexes. - # - # * *Index*::*Bundle* is concerned with loading these index files into - # memory and looking up search data as fast as possible. - # - module Bundle - - # This is the indexing bundle. - # - # It does all menial tasks that have nothing to do - # with the actual index running etc. - # - class Base < Picky::Bundle - - attr_reader :backend, - :prepared - - attr_accessor :partial_strategy, - :weights_strategy - - def initialize name, category, weights_strategy, partial_strategy, similarity_strategy, options = {} - super name, category, similarity_strategy, options - - @weights_strategy = weights_strategy - @partial_strategy = partial_strategy - @key_format = options[:key_format] - @prepared = Backend::File::Text.new category.prepared_index_path - end - - # Sets up a piece of the index for the given token. - # - def initialize_inverted_index_for token - self.inverted[token] ||= [] - end - - # Generation - # - - # This method - # * Loads the base index from the "prepared..." file. - # * Generates derived indexes. - # * Dumps all the indexes into files. - # - def generate_caches_from_source - load_from_prepared_index_file - generate_caches_from_memory - end - # Generates derived indexes from the index and dumps. - # - # Note: assumes that there is something in the index - # - def generate_caches_from_memory - cache_from_memory_generation_message - generate_derived - end - def cache_from_memory_generation_message - timed_exclaim %Q{"#{identifier}": Caching from intermediate in-memory index.} - end - - # Generates the weights and similarity from the main index. - # - def generate_derived - generate_weights - generate_similarity - end - - # Load the data from the db. - # - def load_from_prepared_index_file - load_from_prepared_index_generation_message - clear - retrieve - end - def load_from_prepared_index_generation_message - timed_exclaim %Q{"#{identifier}": Loading prepared data into memory.} - end - # Retrieves the prepared index data into the index. - # - # This is in preparation for generating - # derived indexes (like weights, similarity) - # and later dumping the optimized index. - # - # TODO Move this out to the category? - # - def retrieve - format = category.key_format || :to_i # Optimization. - prepared.retrieve do |id, token| - initialize_inverted_index_for token - self.inverted[token] << id.send(format) - end - end - - # Generates a new index (writes its index) using the - # partial caching strategy of this bundle. - # - def generate_partial - generator = Generators::PartialGenerator.new self.inverted - self.inverted = generator.generate self.partial_strategy - end - # Generate a partial index from the given exact inverted index. - # - def generate_partial_from exact_inverted_index - timed_exclaim %Q{"#{identifier}": Generating partial index for index.} - self.inverted = exact_inverted_index - self.generate_partial - self - end - # Generates a new weights index (writes its index) using the - # given weight caching strategy. - # - def generate_weights - generator = Generators::WeightsGenerator.new self.inverted - self.weights = generator.generate self.weights_strategy - end - # Generates a new similarity index (writes its index) using the - # given similarity caching strategy. - # - def generate_similarity - generator = Generators::SimilarityGenerator.new self.inverted - self.similarity = generator.generate self.similarity_strategy - end - - # Saves the indexes in a dump file. - # - def dump - timed_exclaim %Q{"#{identifier}": Dumping data.} - dump_inverted - dump_similarity - dump_weights - dump_configuration - end - # Dumps the core index. - # - def dump_inverted - # timed_exclaim %Q{"#{identifier}": Dumping inverted index.} - backend.dump_inverted self.inverted - end - # Dumps the weights index. - # - def dump_weights - # timed_exclaim %Q{"#{identifier}": Dumping index weights.} - backend.dump_weights self.weights - end - # Dumps the similarity index. - # - def dump_similarity - # timed_exclaim %Q{"#{identifier}": Dumping similarity index.} - backend.dump_similarity self.similarity - end - # Dumps the similarity index. - # - def dump_configuration - # timed_exclaim %Q{"#{identifier}": Dumping configuration.} - backend.dump_configuration self.configuration - end - - # Alerts the user if an index is missing. - # - def raise_unless_cache_exists - raise_unless_index_exists - raise_unless_similarity_exists - end - # Alerts the user if one of the necessary indexes - # (core, weights) is missing. - # - def raise_unless_index_exists - if partial_strategy.saved? - warn_if_index_small - raise_unless_index_ok - end - end - # Alerts the user if the similarity - # index is missing (given that it's used). - # - def raise_unless_similarity_exists - if similarity_strategy.saved? - warn_if_similarity_small - raise_unless_similarity_ok - end - end - - # Outputs a warning for the given cache. - # - def warn_cache_small what - warn "Warning: #{what} cache for #{identifier} smaller than 16 bytes." - end - # Raises an appropriate error message for the given cache. - # - def raise_cache_missing what - raise "Error: The #{what} cache for #{identifier} is missing." - end - - # Warns the user if the similarity index is small. - # - def warn_if_similarity_small - warn_cache_small :similarity if backend.similarity_cache_small? - end - # Alerts the user if the similarity index is not there. - # - def raise_unless_similarity_ok - raise_cache_missing :similarity unless backend.similarity_cache_ok? - end - - # Warns the user if the core or weights indexes are small. - # - def warn_if_index_small - warn_cache_small :inverted if backend.inverted_cache_small? - warn_cache_small :weights if backend.weights_cache_small? - end - # Alerts the user if the core or weights indexes are not there. - # - def raise_unless_index_ok - raise_cache_missing :inverted unless backend.inverted_cache_ok? - raise_cache_missing :weights unless backend.weights_cache_ok? - end - - end - - end - - end - -end \ No newline at end of file diff --git a/server/lib/picky/indexing/bundle/memory.rb b/server/lib/picky/indexing/bundle/memory.rb deleted file mode 100644 index 50e1f1a0..00000000 --- a/server/lib/picky/indexing/bundle/memory.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Picky - - # encoding: utf-8 - # - module Indexing # :nodoc:all - - module Bundle - - # The memory version dumps its generated indexes to disk - # (mostly JSON) to load them into memory on startup. - # - # TODO Replace inheritance by passing in the backend. - # - class Memory < Base - - def initialize name, category, *args - super name, category, *args - - @backend = Backend::Files.new self - end - - end - - end - - end - -end \ No newline at end of file diff --git a/server/lib/picky/indexing/bundle/redis.rb b/server/lib/picky/indexing/bundle/redis.rb deleted file mode 100644 index 7ca8e91c..00000000 --- a/server/lib/picky/indexing/bundle/redis.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Picky - - # encoding: utf-8 - # - module Indexing # :nodoc:all - - module Bundle - - # The Redis version dumps its generated indexes to - # the Redis backend. - # - # TODO Replace inheritance by passing in the backend. - # - class Redis < Base - - def initialize name, category, *args - super name, category, *args - - @backend = Backend::Redis.new self - end - - end - - end - - end - -end \ No newline at end of file diff --git a/server/lib/picky/loader.rb b/server/lib/picky/loader.rb index eb5fb487..f3c233d1 100644 --- a/server/lib/picky/loader.rb +++ b/server/lib/picky/loader.rb @@ -110,35 +110,28 @@ def self.load_framework_internals # Index store handling. # - load_relative 'backend/base' + load_relative 'backends/base' - load_relative 'backend/redis' - load_relative 'backend/redis/basic' - load_relative 'backend/redis/list_hash' - load_relative 'backend/redis/string_hash' - load_relative 'backend/redis/float_hash' + load_relative 'backends/redis' + load_relative 'backends/redis/basic' + load_relative 'backends/redis/list_hash' + load_relative 'backends/redis/string_hash' + load_relative 'backends/redis/float_hash' - load_relative 'backend/file/basic' - load_relative 'backend/file/text' - load_relative 'backend/file/marshal' - load_relative 'backend/file/json' - - load_relative 'backend/files' + load_relative 'backends/file/basic' + load_relative 'backends/file/text' + load_relative 'backends/file/marshal' + load_relative 'backends/file/json' + load_relative 'backends/memory' # Indexing and Indexed things. # load_relative 'bundle' - load_relative 'indexing/bundle/base' - load_relative 'indexing/bundle/memory' - load_relative 'indexing/bundle/redis' - + load_relative 'indexing/bundle' load_relative 'indexing/wrappers/category/location' - load_relative 'indexed/bundle/base' - load_relative 'indexed/bundle/memory' - load_relative 'indexed/bundle/redis' - + load_relative 'indexed/bundle' load_relative 'indexed/wrappers/exact_first' # Bundle Wrapper diff --git a/server/spec/lib/backend/file/basic_spec.rb b/server/spec/lib/backend/file/basic_spec.rb index b7c6e44a..e7f42fe1 100644 --- a/server/spec/lib/backend/file/basic_spec.rb +++ b/server/spec/lib/backend/file/basic_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::File::Basic do +describe Picky::Backends::File::Basic do let(:file) { described_class.new 'some/cache/path/to/file' } @@ -18,7 +18,7 @@ describe 'to_s' do it 'returns the cache path with the default file extension' do - file.to_s.should == 'Picky::Backend::File::Basic(some/cache/path/to/file.index)' + file.to_s.should == 'Picky::Backends::File::Basic(some/cache/path/to/file.index)' end end diff --git a/server/spec/lib/backend/file/json_spec.rb b/server/spec/lib/backend/file/json_spec.rb index 2e5c2346..8f448146 100644 --- a/server/spec/lib/backend/file/json_spec.rb +++ b/server/spec/lib/backend/file/json_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::File::JSON do +describe Picky::Backends::File::JSON do let(:file) { described_class.new 'some/cache/path/to/file' } @@ -24,7 +24,7 @@ describe 'to_s' do it 'returns the cache path with the default file extension' do - file.to_s.should == 'Picky::Backend::File::JSON(some/cache/path/to/file.json)' + file.to_s.should == 'Picky::Backends::File::JSON(some/cache/path/to/file.json)' end end diff --git a/server/spec/lib/backend/file/marshal_spec.rb b/server/spec/lib/backend/file/marshal_spec.rb index 88b70b6f..dbe2249a 100644 --- a/server/spec/lib/backend/file/marshal_spec.rb +++ b/server/spec/lib/backend/file/marshal_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::File::Marshal do +describe Picky::Backends::File::Marshal do let(:file) { described_class.new 'some/cache/path/to/file' } @@ -24,7 +24,7 @@ describe 'to_s' do it 'returns the cache path with the default file extension' do - file.to_s.should == 'Picky::Backend::File::Marshal(some/cache/path/to/file.dump)' + file.to_s.should == 'Picky::Backends::File::Marshal(some/cache/path/to/file.dump)' end end diff --git a/server/spec/lib/backend/file/text_spec.rb b/server/spec/lib/backend/file/text_spec.rb index 8a2e77bd..e81c418c 100644 --- a/server/spec/lib/backend/file/text_spec.rb +++ b/server/spec/lib/backend/file/text_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::File::Text do +describe Picky::Backends::File::Text do before(:each) do @file = described_class.new "some_cache_path" diff --git a/server/spec/lib/backend/files_spec.rb b/server/spec/lib/backend/memory_spec.rb similarity index 93% rename from server/spec/lib/backend/files_spec.rb rename to server/spec/lib/backend/memory_spec.rb index a1249dc6..d13d3786 100644 --- a/server/spec/lib/backend/files_spec.rb +++ b/server/spec/lib/backend/memory_spec.rb @@ -1,12 +1,14 @@ require 'spec_helper' -describe Picky::Backend::Files do +describe Picky::Backends::Memory do before(:each) do index = Picky::Indexes::Memory.new :some_index category = Picky::Category.new :some_category, index - bundle = Picky::Indexing::Bundle::Base.new :some_bundle, category, nil, nil, nil + # This is just wrong. + # + bundle = Picky::Indexing::Bundle.new :some_bundle, category, described_class, nil, nil, nil @files = described_class.new bundle @index = @files.inverted @@ -182,7 +184,7 @@ :index_path => 'index/path', :prepared_index_path => 'prepared/index/path' - described_class.new(bundle).to_s.should == "Picky::Backend::Files(Picky::Backend::File::JSON(index/path.json), Picky::Backend::File::JSON(index/path.json), Picky::Backend::File::Marshal(index/path.dump), Picky::Backend::File::JSON(index/path.json))" + described_class.new(bundle).to_s.should == "Picky::Backends::Memory(Picky::Backends::File::JSON(index/path.json), Picky::Backends::File::JSON(index/path.json), Picky::Backends::File::Marshal(index/path.dump), Picky::Backends::File::JSON(index/path.json))" end end diff --git a/server/spec/lib/backend/redis/basic_spec.rb b/server/spec/lib/backend/redis/basic_spec.rb index 5a9529de..d1dbf2ef 100644 --- a/server/spec/lib/backend/redis/basic_spec.rb +++ b/server/spec/lib/backend/redis/basic_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::Redis::Basic do +describe Picky::Backends::Redis::Basic do let(:redis) { described_class.new "some_namespace" } @@ -66,7 +66,7 @@ describe 'to_s' do it 'returns the cache path with the default file extension' do - redis.to_s.should == 'Picky::Backend::Redis::Basic(some_namespace:*)' + redis.to_s.should == 'Picky::Backends::Redis::Basic(some_namespace:*)' end end diff --git a/server/spec/lib/backend/redis/float_hash_spec.rb b/server/spec/lib/backend/redis/float_hash_spec.rb index bcfa225f..92c4469b 100644 --- a/server/spec/lib/backend/redis/float_hash_spec.rb +++ b/server/spec/lib/backend/redis/float_hash_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::Redis::FloatHash do +describe Picky::Backends::Redis::FloatHash do let(:index) { described_class.new :some_namespace } @@ -34,7 +34,7 @@ describe 'to_s' do it 'returns the cache path with the default file extension' do - index.to_s.should == 'Picky::Backend::Redis::FloatHash(some_namespace:*)' + index.to_s.should == 'Picky::Backends::Redis::FloatHash(some_namespace:*)' end end diff --git a/server/spec/lib/backend/redis/list_hash_spec.rb b/server/spec/lib/backend/redis/list_hash_spec.rb index af931ebe..2e64bda8 100644 --- a/server/spec/lib/backend/redis/list_hash_spec.rb +++ b/server/spec/lib/backend/redis/list_hash_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::Redis::ListHash do +describe Picky::Backends::Redis::ListHash do let(:index) { described_class.new :some_namespace } @@ -25,7 +25,7 @@ describe 'to_s' do it 'returns the cache path with the default file extension' do - index.to_s.should == 'Picky::Backend::Redis::ListHash(some_namespace:*)' + index.to_s.should == 'Picky::Backends::Redis::ListHash(some_namespace:*)' end end diff --git a/server/spec/lib/backend/redis/string_hash_spec.rb b/server/spec/lib/backend/redis/string_hash_spec.rb index d5a205a7..299a150b 100644 --- a/server/spec/lib/backend/redis/string_hash_spec.rb +++ b/server/spec/lib/backend/redis/string_hash_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Picky::Backend::Redis::StringHash do +describe Picky::Backends::Redis::StringHash do let(:index) { described_class.new :some_namespace } @@ -34,7 +34,7 @@ describe 'to_s' do it 'returns the cache path with the default file extension' do - index.to_s.should == 'Picky::Backend::Redis::StringHash(some_namespace:*)' + index.to_s.should == 'Picky::Backends::Redis::StringHash(some_namespace:*)' end end diff --git a/server/spec/lib/backend/redis_spec.rb b/server/spec/lib/backend/redis_spec.rb index 771d6ef7..ba8b1c0c 100644 --- a/server/spec/lib/backend/redis_spec.rb +++ b/server/spec/lib/backend/redis_spec.rb @@ -1,5 +1,5 @@ require 'spec_helper' -describe Picky::Backend::Redis do +describe Picky::Backends::Redis do end \ No newline at end of file diff --git a/server/spec/lib/category_indexed_spec.rb b/server/spec/lib/category_indexed_spec.rb index 602b6ad4..5fdf4bc6 100644 --- a/server/spec/lib/category_indexed_spec.rb +++ b/server/spec/lib/category_indexed_spec.rb @@ -31,12 +31,12 @@ @partial_strategy.stub! :use_exact_for_partial? => true end it 'returns the partial index' do - @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle::Memory) + @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle) end end context 'with a partial strategy that uses the partial index (default)' do it 'returns the partial index' do - @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle::Memory) + @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle) end end end @@ -52,12 +52,12 @@ @partial_strategy.stub! :use_exact_for_partial? => true end it 'returns the partial index' do - @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle::Redis) + @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle) end end context 'with a partial strategy that uses the partial index (default)' do it 'returns the partial index' do - @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle::Redis) + @category.indexed_partial.should be_kind_of(Picky::Indexed::Bundle) end end end diff --git a/server/spec/lib/indexed/bundle/redis_spec.rb b/server/spec/lib/indexed/bundle/redis_spec.rb deleted file mode 100644 index 2e641657..00000000 --- a/server/spec/lib/indexed/bundle/redis_spec.rb +++ /dev/null @@ -1,17 +0,0 @@ -require 'spec_helper' - -describe Picky::Indexed::Bundle::Redis do - - before(:each) do - @backend = stub :backend - - Picky::Backend::Redis.stub! :new => @backend - - @index = Picky::Indexes::Memory.new :some_index - @category = Picky::Category.new :some_category, @index - - @similarity = stub :similarity - @bundle = described_class.new :some_name, @category, @similarity - end - -end \ No newline at end of file diff --git a/server/spec/lib/indexed/bundle/memory_spec.rb b/server/spec/lib/indexed/memory_spec.rb similarity index 93% rename from server/spec/lib/indexed/bundle/memory_spec.rb rename to server/spec/lib/indexed/memory_spec.rb index 13987ddd..de3bc02c 100644 --- a/server/spec/lib/indexed/bundle/memory_spec.rb +++ b/server/spec/lib/indexed/memory_spec.rb @@ -1,18 +1,18 @@ require 'spec_helper' -describe Picky::Indexed::Bundle::Memory do +describe Picky::Indexed::Bundle do before(:each) do @index = Picky::Indexes::Memory.new :some_index @category = Picky::Category.new :some_category, @index @similarity = stub :similarity - @bundle = described_class.new :some_name, @category, @similarity + @bundle = described_class.new :some_name, @category, Picky::Backends::Memory, @similarity end describe 'to_s' do it 'does something' do - @bundle.to_s.should == "Picky::Indexed::Bundle::Memory(test:some_index:some_category:some_name)" + @bundle.to_s.should == "Picky::Indexed::Bundle(test:some_index:some_category:some_name)" end end @@ -156,7 +156,7 @@ @index = Picky::Indexes::Memory.new :some_index @category = Picky::Category.new :some_category, @index - @bundle = described_class.new :some_name, @category, :similarity + @bundle = described_class.new :some_name, @category, Picky::Backends::Memory, :similarity end it 'should initialize the index correctly' do @bundle.inverted.should == {} diff --git a/server/spec/lib/indexing/bundle/base_spec.rb b/server/spec/lib/indexing/bundle/base_spec.rb deleted file mode 100644 index 80fc1dd6..00000000 --- a/server/spec/lib/indexing/bundle/base_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -require 'spec_helper' - -describe Picky::Indexing::Bundle::Base do - - before(:each) do - @category = stub :category, - :prepared_index_path => 'prepared/index/path', - :identifier => 'some_identifier' - Picky::Backend::Files.stub! :new - @similarity = Picky::Similarity::DoubleMetaphone.new 3 - @bundle = described_class.new :some_name, @category, :some_weights, :some_partial, @similarity - end - - describe 'identifier' do - it 'is correct' do - @bundle.identifier.should == 'some_identifier:some_name' - end - end - - describe 'similar' do - before(:each) do - @bundle.similarity = @similarity.generate_from( :dragon => [1,2,3], :dargon => [4,5,6] ) - end - it 'returns the right similars (not itself)' do - @bundle.similar(:dragon).should == [:dargon] - end - it 'returns the right similars' do - @bundle.similar(:trkn).should == [:dragon, :dargon] - end - it 'performs' do - performance_of { @bundle.similar(:dragon) }.should < 0.000075 - end - it 'performs' do - performance_of { @bundle.similar(:trkn) }.should < 0.00006 - end - end - -end \ No newline at end of file diff --git a/server/spec/lib/indexing/bundle/redis_spec.rb b/server/spec/lib/indexing/bundle/redis_spec.rb deleted file mode 100644 index 5692a3da..00000000 --- a/server/spec/lib/indexing/bundle/redis_spec.rb +++ /dev/null @@ -1,283 +0,0 @@ -require 'spec_helper' - -describe Picky::Indexing::Bundle::Redis do - - before(:each) do - @index = Picky::Indexes::Memory.new :some_index - @category = Picky::Category.new :some_category, @index - - @partial = stub :partial - @weights = stub :weights - @similarity = stub :similarity - end - let(:index) { described_class.new :some_name, @category, @weights, @partial, @similarity } - - describe 'raise_cache_missing' do - it 'does something' do - expect { - index.raise_cache_missing :similarity - }.to raise_error("Error: The similarity cache for test:some_index:some_category:some_name is missing.") - end - end - - describe 'warn_cache_small' do - it 'warns the user' do - index.should_receive(:warn).once.with "Warning: similarity cache for test:some_index:some_category:some_name smaller than 16 bytes." - - index.warn_cache_small :similarity - end - end - - describe 'identifier' do - it 'should return a specific identifier' do - index.identifier.should == 'test:some_index:some_category:some_name' - end - end - - describe 'initialize_index_for' do - context 'token not yet assigned' do - before(:each) do - index.stub! :inverted => {} - end - it 'should assign it an empty array' do - index.initialize_inverted_index_for :some_token - - index.inverted[:some_token].should == [] - end - end - context 'token already assigned' do - before(:each) do - index.stub! :index => { :some_token => :already_assigned } - end - it 'should not assign it anymore' do - index.initialize_inverted_index_for :some_token - - index.index[:some_token].should == :already_assigned - end - end - end - - describe 'retrieve' do - before(:each) do - prepared = stub :prepared - prepared.should_receive(:retrieve).once.and_yield ' 1234', :some_token - index.stub! :prepared => prepared - - @ary = stub :ary - inverted = stub :inverted, :[] => @ary - index.stub! :inverted => inverted - end - context 'id key format' do - before(:each) do - @category.stub! :key_format => :to_i - end - it 'should call the other methods correctly' do - @ary.should_receive(:<<).once.with 1234 - - index.retrieve - end - end - context 'other key format' do - before(:each) do - @category.stub! :key_format => :strip - end - it 'should call the other methods correctly' do - @ary.should_receive(:<<).once.with '1234' - - index.retrieve - end - end - context 'no key format - default' do - before(:each) do - @category.stub! :key_format => nil - end - it 'should call the other methods correctly' do - @ary.should_receive(:<<).once.with 1234 - - index.retrieve - end - end - end - - describe 'load_from_prepared_index_file' do - it 'should call two methods in order' do - index.should_receive(:load_from_prepared_index_generation_message).once.ordered - index.should_receive(:clear).once.ordered - index.should_receive(:retrieve).once.ordered - - index.load_from_prepared_index_file - end - end - - describe 'generate_derived' do - it 'should call two methods in order' do - index.should_receive(:generate_weights).once.ordered - index.should_receive(:generate_similarity).once.ordered - - index.generate_derived - end - end - - describe 'generate_caches_from_memory' do - it 'should call two methods in order' do - index.should_receive(:cache_from_memory_generation_message).once.ordered - index.should_receive(:generate_derived).once.ordered - - index.generate_caches_from_memory - end - end - - describe 'generate_caches_from_source' do - it 'should call two methods in order' do - index.should_receive(:load_from_prepared_index_file).once.ordered - index.should_receive(:generate_caches_from_memory).once.ordered - - index.generate_caches_from_source - end - end - - describe 'dump' do - it 'should trigger dumps' do - index.stub! :timed_exclaim - - index.should_receive(:dump_inverted).once.with - index.should_receive(:dump_weights).once.with - index.should_receive(:dump_similarity).once.with - index.should_receive(:dump_configuration).once.with - - index.dump - end - end - - describe 'raise_unless_cache_exists' do - it "calls methods in order" do - index.should_receive(:raise_unless_index_exists).once.ordered - index.should_receive(:raise_unless_similarity_exists).once.ordered - - index.raise_unless_cache_exists - end - end - describe 'raise_unless_index_exists' do - context 'partial strategy saved' do - before(:each) do - strategy = stub :strategy, :saved? => true - index.stub! :partial_strategy => strategy - end - it "calls the methods in order" do - index.should_receive(:warn_if_index_small).once.ordered - index.should_receive(:raise_unless_index_ok).once.ordered - - index.raise_unless_index_exists - end - end - context 'partial strategy not saved' do - before(:each) do - strategy = stub :strategy, :saved? => false - index.stub! :partial_strategy => strategy - end - it "calls nothing" do - index.should_receive(:warn_if_index_small).never - index.should_receive(:raise_unless_index_ok).never - - index.raise_unless_index_exists - end - end - end - describe 'raise_unless_similarity_exists' do - context 'similarity strategy saved' do - before(:each) do - strategy = stub :strategy, :saved? => true - index.stub! :similarity_strategy => strategy - end - it "calls the methods in order" do - index.should_receive(:warn_if_similarity_small).once.ordered - index.should_receive(:raise_unless_similarity_ok).once.ordered - - index.raise_unless_similarity_exists - end - end - context 'similarity strategy not saved' do - before(:each) do - strategy = stub :strategy, :saved? => false - index.stub! :similarity_strategy => strategy - end - it "calls nothing" do - index.should_receive(:warn_if_similarity_small).never - index.should_receive(:raise_unless_similarity_ok).never - - index.raise_unless_similarity_exists - end - end - end - describe 'warn_if_similarity_small' do - let(:backend) { index.backend } - context "files similarity cache small" do - before(:each) do - backend.stub! :similarity_cache_small? => true - end - it "warns" do - index.should_receive(:warn_cache_small).once.with :similarity - - index.warn_if_similarity_small - end - end - context "files similarity cache not small" do - before(:each) do - backend.stub! :similarity_cache_small? => false - end - it "does not warn" do - index.should_receive(:warn_cache_small).never - - index.warn_if_similarity_small - end - end - end - describe 'raise_unless_similarity_ok' do - let(:backend) { index.backend } - context "files similarity cache ok" do - before(:each) do - backend.stub! :similarity_cache_ok? => true - end - it "warns" do - index.should_receive(:raise_cache_missing).never - - index.raise_unless_similarity_ok - end - end - context "files similarity cache not ok" do - before(:each) do - backend.stub! :similarity_cache_ok? => false - end - it "does not warn" do - index.should_receive(:raise_cache_missing).once.with :similarity - - index.raise_unless_similarity_ok - end - end - end - - describe 'initialization' do - it 'should initialize the index correctly' do - index.inverted.should == {} - end - it 'should initialize the weights index correctly' do - index.weights.should == {} - end - it 'should initialize the similarity index correctly' do - index.similarity.should == {} - end - it 'should initialize the configuration correctly' do - index.configuration.should == {} - end - it 'should initialize the partial strategy correctly' do - index.partial_strategy.should == @partial - end - it 'should initialize the weights strategy correctly' do - index.weights_strategy.should == @weights - end - it 'should initialize the similarity strategy correctly' do - index.similarity_strategy.should == @similarity - end - end - -end \ No newline at end of file diff --git a/server/spec/lib/indexing/bundle/memory_partial_generation_speed_spec.rb b/server/spec/lib/indexing/bundle_partial_generation_speed_spec.rb similarity index 91% rename from server/spec/lib/indexing/bundle/memory_partial_generation_speed_spec.rb rename to server/spec/lib/indexing/bundle_partial_generation_speed_spec.rb index ec889202..f5f2ec99 100644 --- a/server/spec/lib/indexing/bundle/memory_partial_generation_speed_spec.rb +++ b/server/spec/lib/indexing/bundle_partial_generation_speed_spec.rb @@ -1,13 +1,13 @@ require 'spec_helper' -describe Picky::Indexing::Bundle::Memory do +describe Picky::Indexing::Bundle do before(:each) do @index = Picky::Indexes::Memory.new :some_index @category = Picky::Category.new :some_category, @index @partial_strategy = Picky::Generators::Partial::Substring.new :from => 1 - @exact = described_class.new :some_name, @category, nil, @partial_strategy, nil + @exact = described_class.new :some_name, @category, Picky::Backends::Memory, nil, @partial_strategy, nil end def generate_random_keys amount diff --git a/server/spec/lib/indexing/bundle/memory_spec.rb b/server/spec/lib/indexing/bundle_spec.rb similarity index 88% rename from server/spec/lib/indexing/bundle/memory_spec.rb rename to server/spec/lib/indexing/bundle_spec.rb index a784ddcf..59eb588c 100644 --- a/server/spec/lib/indexing/bundle/memory_spec.rb +++ b/server/spec/lib/indexing/bundle_spec.rb @@ -1,16 +1,38 @@ require 'spec_helper' -describe Picky::Indexing::Bundle::Memory do +describe Picky::Indexing::Bundle do before(:each) do @index = Picky::Indexes::Memory.new :some_index @category = Picky::Category.new :some_category, @index - - @partial = stub :partial - @weights = stub :weights - @similarity = stub :similarity + @similarity = Picky::Similarity::DoubleMetaphone.new 3 + @bundle = described_class.new :some_name, @category, Picky::Backends::Memory, :some_weights, :some_partial, @similarity + end + let(:index) { described_class.new :some_name, @category, Picky::Backends::Memory, @weights, @partial, @similarity } + + describe 'identifier' do + it 'is correct' do + @bundle.identifier.should == 'test:some_index:some_category:some_name' + end + end + + describe 'similar' do + before(:each) do + @bundle.similarity = @similarity.generate_from( :dragon => [1,2,3], :dargon => [4,5,6] ) + end + it 'returns the right similars (not itself)' do + @bundle.similar(:dragon).should == [:dargon] + end + it 'returns the right similars' do + @bundle.similar(:trkn).should == [:dragon, :dargon] + end + it 'performs' do + performance_of { @bundle.similar(:dragon) }.should < 0.000075 + end + it 'performs' do + performance_of { @bundle.similar(:trkn) }.should < 0.00006 + end end - let(:index) { described_class.new :some_name, @category, @weights, @partial, @similarity } describe 'raise_cache_missing' do it 'does something' do @@ -283,5 +305,5 @@ index.similarity_strategy.should == @similarity end end - + end \ No newline at end of file