+ moved source/tokenizer factory methods from Category into fitting c…

…lasses

server/lib/picky/category.rb

@@ -43,9 +43,13 @@ def initialize name, index, options = {}
     def configure_from options
       # Indexing.
       #
-      @source    = Generators::Source.from options[:source], true
       @from      = options[:from]
-      @tokenizer = extract_tokenizer options[:indexing]
+
+      # TODO Can this be replaced?
+      #
+      @source    = Source.from options[:source], true, @index.name
+
+      @tokenizer = Tokenizer.from options[:indexing], @index.name, name
 
       @key_format = options.delete :key_format
       @backend    = options.delete :backend

server/lib/picky/category_indexing.rb

@@ -64,7 +64,7 @@ def retrieve
     # If we have no explicit source, we'll check the index for one.
     #
     def source
-      (@source = Generators::Source.from(@source, true, @index && @index.name)) || @index.source
+      (@source = Source.from(@source, true, @index && @index.name)) || @index.source
     end
 
     # Return the key format.

server/lib/picky/generators/partial.rb

@@ -3,7 +3,7 @@ module Picky
   module Generators
 
     module Partial
-      extend Helpers
+      extend Helpers::Identification
 
       def self.from thing, index_name = nil, category_name = nil
         return Default unless thing

server/lib/picky/generators/similarity.rb

@@ -3,18 +3,14 @@ module Picky
   module Generators
 
     module Similarity
-      extend Helpers
+      extend Helpers::Identification
 
       def self.from thing, index_name = nil, category_name = nil
         return Default unless thing
 
         if thing.respond_to?(:encode) && thing.respond_to?(:prioritize)
           thing
         else
-          specifics = ""
-          specifics << index_name if index_name
-          specifics << ":#{category_name}" if category_name
-          specifics = "for #{specifics} " unless specifics.empty?
           raise <<-ERROR
 similarity options #{identifier_for(index_name, category_name)}should be either
 * for example a Similarity::Phonetic.new(n), Similarity::Metaphone.new(n), Similarity::DoubleMetaphone.new(n) etc.

server/lib/picky/generators/weights.rb

@@ -3,7 +3,7 @@ module Picky
   module Generators
 
     module Weights
-      extend Helpers
+      extend Helpers::Identification
 
       # Factory method to return a fitting
       # weight handling thing for the given thing.

server/lib/picky/generators/helpers.rb → server/lib/picky/helpers/identification.rb

@@ -1,8 +1,8 @@
 module Picky
 
-  module Generators
+  module Helpers
 
-    module Helpers
+    module Identification
 
       def identifier_for index_name = nil, category_name = nil
         specifics = ""

server/lib/picky/index_indexing.rb

@@ -3,8 +3,6 @@ module Picky
   #
   #
   class Index
-
-    include API::Tokenizer
     include Helpers::Indexing
 
     # Delegators for indexing.
@@ -18,7 +16,7 @@ class Index
     # Parameters are the exact same as for indexing.
     #
     def indexing options = {}
-      @tokenizer = extract_tokenizer options
+      @tokenizer = Tokenizer.from options
     end
 
     # Calling prepare on an index will call prepare
@@ -84,7 +82,7 @@ def tokenizer
     #
     def source some_source = nil, &block
       some_source ||= block
-      some_source ? (@source = Generators::Source.from(some_source, false, name)) : unblock_source
+      some_source ? (@source = Source.from(some_source, false, name)) : unblock_source
     end
     # Get the actual source if it is wrapped in a time
     # capsule, i.e. a block/lambda.

server/lib/picky/loader.rb

@@ -68,7 +68,8 @@ def load_extensions
       end
       def load_helpers
         load_relative 'helpers/measuring',
-                      'helpers/indexing'
+                      'helpers/indexing',
+                      'helpers/identification'
       end
       def load_index_generation_strategies
         load_relative 'indexers/base',
@@ -198,8 +199,7 @@ def load_framework_internals
       # All things API related.
       #
       def load_api
-        load_relative 'api/tokenizer',
-                      'api/tokenizer/character_substituter',
+        load_relative 'api/tokenizer/character_substituter',
                       'api/search/boost'
       end
 
@@ -211,11 +211,9 @@ def load_logging
       end
 
       def load_generators
-        load_relative 'generators/helpers'
         load_relative 'generators/weights'
         load_relative 'generators/partial'
         load_relative 'generators/similarity'
-        load_relative 'generators/source'
         load_relative 'generators/aliases'
       end
 
@@ -262,9 +260,12 @@ def load_interfaces
 
       # Loads the user interface parts.
       #
+      # TODO Move tokenizer etc.?
+      #
       def load_user_interface
         load_api
         load_logging
+        load_relative 'source'
         load_relative 'tokenizer'
         load_relative 'rack/harakiri'
         load_relative 'character_substituters/west_european'

server/lib/picky/source.rb

@@ -0,0 +1,23 @@
+module Picky
+
+  module Source
+    extend Helpers::Identification
+
+    # Either a thing responding to #each or a block is fine.
+    #
+    def self.from thing, nil_ok, index_name = nil
+      if thing.respond_to?(:each) || thing.respond_to?(:call)
+        thing
+      else
+        return if nil_ok
+
+        raise ArgumentError.new(<<-ERROR)
+The source #{identifier_for(index_name)}should respond to either the method #each or
+it can be a lambda/block, returning such a source.
+ERROR
+      end
+    end
+
+  end
+
+end

server/lib/picky/tokenizer.rb

@@ -6,23 +6,41 @@ module Picky
   #
   class Tokenizer
 
-    extend API::Tokenizer
-
+    extend Picky::Helpers::Identification
     include API::Tokenizer::CharacterSubstituter
 
     def self.default_indexing_with options = {}
-      @indexing = extract_tokenizer options
+      @indexing = from options
     end
     def self.indexing
       @indexing ||= new
     end
 
     def self.default_searching_with options = {}
-      @searching = extract_tokenizer options
+      @searching = from options
     end
     def self.searching
       @searching ||= new
     end
+
+    def self.from thing, index_name = nil, category_name = nil
+      return unless thing
+
+      if thing.respond_to? :tokenize
+        thing
+      else
+        if thing.respond_to? :[]
+          Picky::Tokenizer.new thing
+        else
+          raise <<-ERROR
+indexing options #{identifier_for(index_name, category_name)}should be either
+* a Hash
+or
+* an object that responds to #tokenize(text) => [[token1, token2, ...], [original1, original2, ...]]
+ERROR
+        end
+      end
+    end
 
     def to_s
       reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]

server/spec/lib/category_spec.rb

@@ -59,7 +59,7 @@ def tokenize text
 indexing options for some_index:some_category should be either
 * a Hash
 or
-* an object that responds to #tokenize(text) => [[token1, ...], [original1, ...]]
+* an object that responds to #tokenize(text) => [[token1, token2, ...], [original1, original2, ...]]
 ERROR
       end
     end