diff --git a/server/Gemfile b/server/Gemfile index 262e9a80..3c375a05 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -9,7 +9,7 @@ group :development do gem 'rack' gem 'text' gem 'procrastinate' - + gem 'stackprof' gem 'perfer' end @@ -17,7 +17,7 @@ end group :test do gem 'rspec', '~> 3.0.0.beta' gem 'rake-compiler' - + # Needed for testing itself. # gem 'simplecov', :require => false # Will install simplecov-html as a dependency @@ -34,7 +34,7 @@ group :test do gem 'hiredis' gem 'redis' gem 'sqlite3' - + gem 'fast-stemmer', :require => 'stemmer' gem 'ruby-stemmer', :require => 'lingua/stemmer' diff --git a/server/Gemfile.lock b/server/Gemfile.lock index c347e61b..b45cae76 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -1,8 +1,8 @@ PATH remote: . specs: - picky (4.20.0) - activesupport (~> 3) + picky (4.21.0) + activesupport multi_json (~> 1.3) rack_fast_escape (~> 2009.0) text (~> 1.2) @@ -10,87 +10,92 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (3.2.16) - activesupport (= 3.2.16) - builder (~> 3.0.0) - activerecord (3.2.16) - activemodel (= 3.2.16) - activesupport (= 3.2.16) - arel (~> 3.0.2) - tzinfo (~> 0.3.29) - activesupport (3.2.16) - i18n (~> 0.6, >= 0.6.4) - multi_json (~> 1.0) - arel (3.0.3) + activemodel (4.1.0) + activesupport (= 4.1.0) + builder (~> 3.1) + activerecord (4.1.0) + activemodel (= 4.1.0) + activesupport (= 4.1.0) + arel (~> 5.0.0) + activesupport (4.1.0) + i18n (~> 0.6, >= 0.6.9) + json (~> 1.7, >= 1.7.7) + minitest (~> 5.1) + thread_safe (~> 0.1) + tzinfo (~> 1.1) + arel (5.0.1.20140414130214) backports (2.6.7) - builder (3.0.4) - coverband (0.0.15) + builder (3.2.2) + coverband (0.0.24) json redis simplecov diff-lcs (1.2.5) - docile (1.1.2) + docile (1.1.3) fast-stemmer (1.0.2) ffi (1.0.11) - highline (1.6.20) - hiredis (0.4.5) + highline (1.6.21) + hiredis (0.5.2) hitimes (1.1.1) i18n (0.6.9) json (1.8.1) - mime-types (2.1) - multi_json (1.8.4) + mime-types (2.2) + minitest (5.3.3) + multi_json (1.9.3) path (1.3.3) perfer (0.2.1) backports (~> 2.6.3) ffi (~> 1.0.11) hitimes (~> 1.1.1) path (~> 1.3.1) - picky-client (4.19.7) - activesupport (~> 3) + picky-client (4.21.0) + activesupport yajl-ruby (~> 1.2) procrastinate (0.6.0) state_machine (~> 1.1) rack (1.5.2) rack-mount (0.8.3) rack (>= 1.0.0) - rack-protection (1.5.2) + rack-protection (1.5.3) rack rack_fast_escape (2009.06.24) url_escape - rake (10.1.1) + rake (10.3.1) rake-compiler (0.9.2) rake redis (3.0.7) rest-client (1.6.7) mime-types (>= 1.16) - rspec (3.0.0.beta1) - rspec-core (= 3.0.0.beta1) - rspec-expectations (= 3.0.0.beta1) - rspec-mocks (= 3.0.0.beta1) - rspec-core (3.0.0.beta1) - rspec-support (= 3.0.0.beta1) - rspec-expectations (3.0.0.beta1) - diff-lcs (>= 1.1.3, < 2.0) - rspec-support (= 3.0.0.beta1) - rspec-mocks (3.0.0.beta1) - rspec-support (= 3.0.0.beta1) - rspec-support (3.0.0.beta1) + rspec (3.0.0.beta2) + rspec-core (= 3.0.0.beta2) + rspec-expectations (= 3.0.0.beta2) + rspec-mocks (= 3.0.0.beta2) + rspec-core (3.0.0.beta2) + rspec-support (= 3.0.0.beta2) + rspec-expectations (3.0.0.beta2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (= 3.0.0.beta2) + rspec-mocks (3.0.0.beta2) + rspec-support (= 3.0.0.beta2) + rspec-support (3.0.0.beta2) ruby-stemmer (0.9.3) simplecov (0.8.2) docile (~> 1.1.0) multi_json simplecov-html (~> 0.8.0) simplecov-html (0.8.0) - sinatra (1.4.4) + sinatra (1.4.5) rack (~> 1.4) rack-protection (~> 1.4) tilt (~> 1.3, >= 1.3.4) - sqlite3 (1.3.8) - stackprof (0.2.5) + sqlite3 (1.3.9) + stackprof (0.2.6) state_machine (1.2.0) text (1.2.3) + thread_safe (0.3.3) tilt (1.4.1) - tzinfo (0.3.38) + tzinfo (1.1.0) + thread_safe (~> 0.1) url_escape (2009.06.24) uuidtools (2.1.4) www-delicious (0.4.0) diff --git a/server/lib/picky/category_indexing.rb b/server/lib/picky/category_indexing.rb index 1a1f7be6..b5b8fe3a 100644 --- a/server/lib/picky/category_indexing.rb +++ b/server/lib/picky/category_indexing.rb @@ -73,7 +73,17 @@ def key_format def key_format? key_format end - + + # Return the key format. + # + # If no key_format is defined on the category + # and the source has no key format, ask + # the index for one. + # + def id + @id ||= @index.id + end + def static? @index.static? # || @static end @@ -83,7 +93,7 @@ def static? def from @from || name end - + # Return an appropriate source. # # If we have no explicit source, we'll check the index for one. diff --git a/server/lib/picky/category_realtime.rb b/server/lib/picky/category_realtime.rb index ca243234..447a158b 100644 --- a/server/lib/picky/category_realtime.rb +++ b/server/lib/picky/category_realtime.rb @@ -1,7 +1,7 @@ module Picky class Category - + class Picky::IdNotGivenException < StandardError; end # Removes an indexed object with the @@ -20,9 +20,9 @@ def remove id # def add object, where = :unshift if from.respond_to? :call - add_text object.id, from.call(object), where + add_text object.send(id), from.call(object), where else - add_text object.id, object.send(from), where + add_text object.send(id), object.send(from), where end end @@ -30,20 +30,20 @@ def add object, where = :unshift # adds it again. # def replace object, where = :unshift - remove object.id + remove object.send id add object, where end - + # Replaces just part of the indexed data. # # Note: Takes a hash as opposed to the add/replace method. # def replace_from hash #, id = (hash[:id] || hash['id'] || raise(IdNotGivenException.new)).send(key_format) return unless text = hash[from] || hash[from.to_s] - + raise IdNotGivenException.new unless id = hash[:id] || hash['id'] id = id.send key_format if key_format? - + remove id add_text id, text end @@ -71,13 +71,13 @@ def add_text id, text_or_tokens, where = :unshift else tokens = text_or_tokens end - + format = key_format? tokens.each { |text| add_tokenized_token id, text, where, format } rescue NoMethodError => e show_informative_add_text_error_message_for e end - + def show_informative_add_text_error_message_for e if e.name == :each raise %Q{#{e.message}. You probably set tokenize: false on category "#{name}". It will need an Enumerator of previously tokenized tokens.} @@ -90,7 +90,7 @@ def show_informative_add_text_error_message_for e # def add_tokenized_token id, text, where = :unshift, format = true, static = false return unless text - + id = id.send key_format if format # text = text.to_sym if @symbols # SYMBOLS. id.freeze diff --git a/server/lib/picky/index.rb b/server/lib/picky/index.rb index 988e7374..b4a522d7 100644 --- a/server/lib/picky/index.rb +++ b/server/lib/picky/index.rb @@ -126,7 +126,7 @@ def initialize name instance_eval(&Proc.new) if block_given? end - + # TODO Doc. # def static @@ -149,7 +149,7 @@ def backend backend = nil @backend ||= Backends::Memory.new end end - + # The directory used by this index. # # Note: Used @directory ||=, but needs to be dynamic. @@ -157,7 +157,7 @@ def backend backend = nil def directory ::File.join(Picky.root, 'index', PICKY_ENVIRONMENT, name.to_s) end - + # Restrict categories to the given ones. # # Functionally equivalent as if indexes didn't @@ -380,7 +380,7 @@ def to_s ].compact "#{self.class}(#{s.join(', ')})" end - + # Displays the structure as a tree. # def to_tree_s indent = 0 diff --git a/server/lib/picky/index_indexing.rb b/server/lib/picky/index_indexing.rb index 0573feb8..0e6e459b 100644 --- a/server/lib/picky/index_indexing.rb +++ b/server/lib/picky/index_indexing.rb @@ -89,6 +89,17 @@ def unblock_source @source.respond_to?(:call) ? @source.call : @source end + # API method. + # + # Defines the name of the ID method to use on the indexed object. + # + # === Parameters + # * name: Method name of the ID. + # + def id name = nil + @id_name = name || @id_name || :id + end + # Define a key_format on the index. # # Parameter is a method name to use on the key (e.g. :to_i, :to_s, :strip, :split). diff --git a/server/lib/picky/indexers/parallel.rb b/server/lib/picky/indexers/parallel.rb index 7bd28914..1f8ea993 100644 --- a/server/lib/picky/indexers/parallel.rb +++ b/server/lib/picky/indexers/parallel.rb @@ -58,15 +58,17 @@ def process source_for_prepare, categories, scheduler = Scheduler.new def index_flush objects, file, category, cache, tokenizer comma = ?, newline = ?\n - + # Optimized, therefore duplicate code. # + id = category.id + from = category.from objects.each do |object| - tokens = object.send category.from + tokens = object.send from tokens, _ = tokenizer.tokenize tokens if tokenizer # Note: Originals not needed. TODO Optimize? tokens.each do |token_text| next unless token_text - cache << object.id << comma << token_text << newline + cache << object.send(id) << comma << token_text << newline end end diff --git a/server/spec/functional/id_spec.rb b/server/spec/functional/id_spec.rb new file mode 100644 index 00000000..c3695dfa --- /dev/null +++ b/server/spec/functional/id_spec.rb @@ -0,0 +1,66 @@ +# encoding: utf-8 +# +require 'spec_helper' + +describe "id option" do + + it 'can be given a different id (in-ruby based)' do + data = Picky::Index.new :id do + id :number + category :text + end + + require 'ostruct' + + thing = OpenStruct.new number: 1, text: "ohai" + other = OpenStruct.new number: 2, text: "ohai kthxbye" + + data.add thing + data.add other + + try = Picky::Search.new data + + try.search("text:kthxbye").ids.should == [2] + end + + it 'can be given a different id (source based)' do + require 'ostruct' + + things = [] + things << OpenStruct.new(number: 1, text: "ohai") + things << OpenStruct.new(number: 2, text: "ohai kthxbye") + + data = Picky::Index.new :id do + source { things } + + key_format :to_i + id :number # TODO :format => :to_i + category :text + end + + data.index + + try = Picky::Search.new data + + try.search("text:kthxbye").ids.should == [2] + end + + it 'default is id' do + index = Picky::Index.new :id do + category :text + end + + require 'ostruct' + + thing = OpenStruct.new id: 1, text: "ohai" + other = OpenStruct.new id: 2, text: "ohai kthxbye" + + index.add thing + index.add other + + try = Picky::Search.new index + + try.search("text:kthxbye").ids.should == [2] + end + +end \ No newline at end of file