Permalink
Browse files

+ moved source/tokenizer factory methods from Category into fitting c…

…lasses
  • Loading branch information...
1 parent 93ba68a commit cfa9eb372ca3b8131ee6278dc2e590bd1e81b3ce @floere committed Sep 12, 2012
@@ -1,37 +0,0 @@
-module Picky
- module API
-
- module Tokenizer
-
- def extract_tokenizer thing
- return unless thing
- if thing.respond_to? :tokenize
- thing
- else
- if thing.respond_to? :[]
- Picky::Tokenizer.new thing
- else
- if respond_to? :name
- location = ' for '
- if @index
- location += "#{@index.name}:#{name}"
- else
- location += "#{name}"
- end
- else
- location = ''
- end
- raise <<-ERROR
-indexing options#{location} should be either
-* a Hash
-or
-* an object that responds to #tokenize(text) => [[token1, ...], [original1, ...]]
-ERROR
- end
- end
- end
-
- end
-
- end
-end
@@ -43,9 +43,13 @@ def initialize name, index, options = {}
def configure_from options
# Indexing.
#
- @source = Generators::Source.from options[:source], true
@from = options[:from]
- @tokenizer = extract_tokenizer options[:indexing]
+
+ # TODO Can this be replaced?
+ #
+ @source = Source.from options[:source], true, @index.name
+
+ @tokenizer = Tokenizer.from options[:indexing], @index.name, name
@key_format = options.delete :key_format
@backend = options.delete :backend
@@ -64,7 +64,7 @@ def retrieve
# If we have no explicit source, we'll check the index for one.
#
def source
- (@source = Generators::Source.from(@source, true, @index && @index.name)) || @index.source
+ (@source = Source.from(@source, true, @index && @index.name)) || @index.source
end
# Return the key format.
@@ -3,7 +3,7 @@ module Picky
module Generators
module Partial
- extend Helpers
+ extend Helpers::Identification
def self.from thing, index_name = nil, category_name = nil
return Default unless thing
@@ -3,18 +3,14 @@ module Picky
module Generators
module Similarity
- extend Helpers
+ extend Helpers::Identification
def self.from thing, index_name = nil, category_name = nil
return Default unless thing
if thing.respond_to?(:encode) && thing.respond_to?(:prioritize)
thing
else
- specifics = ""
- specifics << index_name if index_name
- specifics << ":#{category_name}" if category_name
- specifics = "for #{specifics} " unless specifics.empty?
raise <<-ERROR
similarity options #{identifier_for(index_name, category_name)}should be either
* for example a Similarity::Phonetic.new(n), Similarity::Metaphone.new(n), Similarity::DoubleMetaphone.new(n) etc.
@@ -1,27 +0,0 @@
-module Picky
-
- module Generators
-
- module Source
- extend Helpers
-
- # Either a thing responding to #each or a block is fine.
- #
- def self.from thing, nil_ok, index_name = nil
- if thing.respond_to?(:each) || thing.respond_to?(:call)
- thing
- else
- return if nil_ok
-
- raise ArgumentError.new(<<-ERROR)
-The source #{identifier_for(index_name)}should respond to either the method #each or
-it can be a lambda/block, returning such a source.
-ERROR
- end
- end
-
- end
-
- end
-
-end
@@ -3,7 +3,7 @@ module Picky
module Generators
module Weights
- extend Helpers
+ extend Helpers::Identification
# Factory method to return a fitting
# weight handling thing for the given thing.
@@ -1,8 +1,8 @@
module Picky
- module Generators
+ module Helpers
- module Helpers
+ module Identification
def identifier_for index_name = nil, category_name = nil
specifics = ""
@@ -3,8 +3,6 @@ module Picky
#
#
class Index
-
- include API::Tokenizer
include Helpers::Indexing
# Delegators for indexing.
@@ -18,7 +16,7 @@ class Index
# Parameters are the exact same as for indexing.
#
def indexing options = {}
- @tokenizer = extract_tokenizer options
+ @tokenizer = Tokenizer.from options
end
# Calling prepare on an index will call prepare
@@ -84,7 +82,7 @@ def tokenizer
#
def source some_source = nil, &block
some_source ||= block
- some_source ? (@source = Generators::Source.from(some_source, false, name)) : unblock_source
+ some_source ? (@source = Source.from(some_source, false, name)) : unblock_source
end
# Get the actual source if it is wrapped in a time
# capsule, i.e. a block/lambda.
View
@@ -68,7 +68,8 @@ def load_extensions
end
def load_helpers
load_relative 'helpers/measuring',
- 'helpers/indexing'
+ 'helpers/indexing',
+ 'helpers/identification'
end
def load_index_generation_strategies
load_relative 'indexers/base',
@@ -198,8 +199,7 @@ def load_framework_internals
# All things API related.
#
def load_api
- load_relative 'api/tokenizer',
- 'api/tokenizer/character_substituter',
+ load_relative 'api/tokenizer/character_substituter',
'api/search/boost'
end
@@ -211,11 +211,9 @@ def load_logging
end
def load_generators
- load_relative 'generators/helpers'
load_relative 'generators/weights'
load_relative 'generators/partial'
load_relative 'generators/similarity'
- load_relative 'generators/source'
load_relative 'generators/aliases'
end
@@ -262,9 +260,12 @@ def load_interfaces
# Loads the user interface parts.
#
+ # TODO Move tokenizer etc.?
+ #
def load_user_interface
load_api
load_logging
+ load_relative 'source'
load_relative 'tokenizer'
load_relative 'rack/harakiri'
load_relative 'character_substituters/west_european'
View
@@ -0,0 +1,23 @@
+module Picky
+
+ module Source
+ extend Helpers::Identification
+
+ # Either a thing responding to #each or a block is fine.
+ #
+ def self.from thing, nil_ok, index_name = nil
+ if thing.respond_to?(:each) || thing.respond_to?(:call)
+ thing
+ else
+ return if nil_ok
+
+ raise ArgumentError.new(<<-ERROR)
+The source #{identifier_for(index_name)}should respond to either the method #each or
+it can be a lambda/block, returning such a source.
+ERROR
+ end
+ end
+
+ end
+
+end
@@ -6,23 +6,41 @@ module Picky
#
class Tokenizer
- extend API::Tokenizer
-
+ extend Picky::Helpers::Identification
include API::Tokenizer::CharacterSubstituter
def self.default_indexing_with options = {}
- @indexing = extract_tokenizer options
+ @indexing = from options
end
def self.indexing
@indexing ||= new
end
def self.default_searching_with options = {}
- @searching = extract_tokenizer options
+ @searching = from options
end
def self.searching
@searching ||= new
end
+
+ def self.from thing, index_name = nil, category_name = nil
+ return unless thing
+
+ if thing.respond_to? :tokenize
+ thing
+ else
+ if thing.respond_to? :[]
+ Picky::Tokenizer.new thing
+ else
+ raise <<-ERROR
+indexing options #{identifier_for(index_name, category_name)}should be either
+* a Hash
+or
+* an object that responds to #tokenize(text) => [[token1, token2, ...], [original1, original2, ...]]
+ERROR
+ end
+ end
+ end
def to_s
reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
@@ -1,42 +0,0 @@
-require 'spec_helper'
-
-describe Picky::API::Tokenizer do
- let(:object) do
- Class.new do
- include Picky::API::Tokenizer
- end.new
- end
- context 'extract_tokenizer' do
- context 'options hash' do
- it 'creates a tokenizer' do
- object.extract_tokenizer(splits_text_on: /\t/).
- tokenize("hello\tworld").should == [['hello', 'world'], ['hello', 'world']]
- end
- end
- context 'tokenizer' do
- let(:tokenizer) do
- Class.new do
- def tokenize text
- ['unmoved', 'by', 'your', 'texts']
- end
- end.new
- end
- it 'creates a tokenizer' do
- object.extract_tokenizer(tokenizer).
- tokenize("hello\tworld").should == ['unmoved', 'by', 'your', 'texts']
- end
- end
- context 'invalid tokenizer' do
- it 'raises with a nice error message' do
- expect {
- object.extract_tokenizer Object.new
- }.to raise_error(<<-ERROR)
-indexing options should be either
-* a Hash
-or
-* an object that responds to #tokenize(text) => [[token1, ...], [original1, ...]]
-ERROR
- end
- end
- end
-end
@@ -59,7 +59,7 @@ def tokenize text
indexing options for some_index:some_category should be either
* a Hash
or
-* an object that responds to #tokenize(text) => [[token1, ...], [original1, ...]]
+* an object that responds to #tokenize(text) => [[token1, token2, ...], [original1, original2, ...]]
ERROR
end
end
Oops, something went wrong.

0 comments on commit cfa9eb3

Please sign in to comment.