@@ -11,7 +11,7 @@ class SearchController < ApplicationController def search @query = params[:q] @users = User.search(@query) - # @assets = Asset.search(@query) + @assets = Asset.search(@query) render :partial => "results" end app/controllers/search_controller.rb @@ -1,4 +1,13 @@ ## +# SphinxSearch Installation +# wget http://sphinxsearch.com/downloads/sphinx-0.9.8.1.tar.gz +# tar vxfz sphinx-0.9.8.1.tar.gz +# cd sphinx-0.9.8.1 +# ./configure +# make +# sudo make install + +## # Indexing class Asset < ActiveRecord::Base define_index do app/models/sphinx_search.rb @@ -92,3 +92,4 @@ Since I first released this library, there's been quite a few people who have su - Jim Remsik - Kennon Ballou - Henrik Nyh +- Emil Tin vendor/plugins/thinking-sphinx/README @@ -92,3 +92,4 @@ Since I first released this library, there's been quite a few people who have su * Jim Remsik * Kennon Ballou * Henrik Nyh +* Emil Tin vendor/plugins/thinking-sphinx/README.textile @@ -40,6 +40,15 @@ module ThinkingSphinx class ConnectionError < StandardError end + # A StaleIdsException is thrown by Collection.instances_from_matches if there + # are records in Sphinx but not in the database, so the search can be retried. + class StaleIdsException < StandardError + attr_accessor :ids + def initialize(ids) + self.ids = ids + end + end + # The collection of indexed models. Keep in mind that Rails lazily loads # its classes, so this may not actually be populated with _all_ the models # that have Sphinx indexes. vendor/plugins/thinking-sphinx/lib/thinking_sphinx.rb @@ -83,6 +83,10 @@ module ThinkingSphinx end alias_method :sphinx_index, :define_index + def sphinx_index_options + sphinx_indexes.last.options + end + # Generate a unique CRC value for the model's name, to use to # determine which Sphinx documents belong to which AR records. # @@ -155,6 +159,8 @@ module ThinkingSphinx ) if ThinkingSphinx.deltas_enabled? && self.class.sphinx_indexes.any? { |index| index.delta? } && self.delta? + rescue ::ThinkingSphinx::ConnectionError + # nothing end def sphinx_document_id vendor/plugins/thinking-sphinx/lib/thinking_sphinx/active_record.rb @@ -1,7 +1,10 @@ module ThinkingSphinx class Collection < ::Array - attr_reader :total_entries, :total_pages, :current_page + attr_reader :total_entries, :total_pages, :current_page, :per_page attr_accessor :results + + # Compatibility with older versions of will_paginate + alias_method :page_count, :total_pages def initialize(page, per_page, entries, total_entries) @current_page, @per_page, @total_entries = page, per_page, total_entries @@ -34,25 +37,36 @@ module ThinkingSphinx instance_from_match match, options } unless klass = options[:class] + index_options = klass.sphinx_index_options + ids = matches.collect { |match| match[:attributes]["sphinx_internal_id"] } instances = ids.length > 0 ? klass.find( :all, :conditions => {klass.primary_key.to_sym => ids}, - :include => options[:include], - :select => options[:select] + :include => (options[:include] || index_options[:include]), + :select => (options[:select] || index_options[:select]) ) : [] + + # Raise an exception if we find records in Sphinx but not in the DB, so the search method + # can retry without them. See ThinkingSphinx::Search.retry_search_on_stale_index. + if options[:raise_on_stale] && instances.length < ids.length + stale_ids = ids - instances.map {|i| i.id } + raise StaleIdsException, stale_ids + end + ids.collect { |obj_id| instances.detect { |obj| obj.id == obj_id } } end def self.instance_from_match(match, options) - # puts "ARGS: #{match[:attributes]["sphinx_internal_id"].inspect}, {:include => #{options[:include].inspect}, :select => #{options[:select].inspect}}" class_from_crc(match[:attributes]["class_crc"]).find( match[:attributes]["sphinx_internal_id"], :include => options[:include], :select => options[:select] ) + rescue ::ActiveRecord::RecordNotFound + nil end def self.class_from_crc(crc) vendor/plugins/thinking-sphinx/lib/thinking_sphinx/collection.rb @@ -394,7 +394,7 @@ GROUP BY #{ ( when :postgres "COALESCE(crc32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})" when :mysql - "IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})" + "CAST(IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s}) AS UNSIGNED)" end else @model.to_crc32.to_s vendor/plugins/thinking-sphinx/lib/thinking_sphinx/index.rb @@ -176,6 +176,9 @@ module ThinkingSphinx # # set_property :delta => true # set_property :field_weights => {"name" => 100} + # set_property :order => "name ASC" + # set_property :include => :picture + # set_property :select => 'name' # # Also, the following two properties are particularly relevant for # geo-location searching - latitude_attr and longitude_attr. If your vendor/plugins/thinking-sphinx/lib/thinking_sphinx/index/builder.rb @@ -12,6 +12,10 @@ module ThinkingSphinx # return the ids for the matching objects. See #search for syntax # examples. # + # Note that this only searches the Sphinx index, with no ActiveRecord + # queries. Thus, if your index is not in sync with the database, this + # method may return ids that no longer exist there. + # def search_for_ids(*args) results, client = search_results(*args.clone) @@ -113,7 +117,52 @@ module ThinkingSphinx # attribute values to exclude. This is done with the :without option: # # User.search :without => {:role_id => 1} - # + # + # == Excluding by Primary Key + # + # There is a shortcut to exclude records by their ActiveRecord primary key: + # + # User.search :without_ids => 1 + # + # Pass an array or a single value. + # + # The primary key must be an integer as a negative filter is used. Note + # that for multi-model search, an id may occur in more than one model. + # + # == Infix (Star) Searching + # + # By default, Sphinx uses English stemming, e.g. matching "shoes" if you + # search for "shoe". It won't find "Melbourne" if you search for + # "elbourn", though. + # + # Enable infix searching by something like this in config/sphinx.yml: + # + # development: + # enable_star: 1 + # min_infix_length: 2 + # + # Note that this will make indexing take longer. + # + # With those settings (and after reindexing), wildcard asterisks can be used + # in queries: + # + # Location.search "*elbourn*" + # + # To automatically add asterisks around every token (but not operators), + # pass the :star option: + # + # Location.search "elbourn -ustrali", :star => true, :match_mode => :boolean + # + # This would become "*elbourn* -*ustrali*". The :star option only adds the + # asterisks. You need to make the config/sphinx.yml changes yourself. + # + # By default, the tokens are assumed to match the regular expression /\w+/u. + # If you've modified the charset_table, pass another regular expression, e.g. + # + # User.search("oo@bar.c", :star => /[\w@.]+/u) + # + # to search for "*oo@bar.c*" and not "*oo*@*bar*.*c*". + # # == Sorting # # Sphinx can only sort by attributes, so generally you will need to avoid @@ -198,18 +247,75 @@ module ThinkingSphinx # The distance value is returned as a float, representing the distance in # metres. # + # == Handling a Stale Index + # + # Especially if you don't use delta indexing, you risk having records in the + # Sphinx index that are no longer in the database. By default, those will simply + # come back as nils: + # + # >> pat_user.delete + # >> User.search("pat") + # Sphinx Result: [1,2] + # => [nil, <#User id: 2>] + # + # (If you search across multiple models, you'll get ActiveRecord::RecordNotFound.) + # + # You can simply Array#compact these results or handle the nils in some other way, but + # Sphinx will still report two results, and the missing records may upset your layout. + # + # If you pass :retry_stale => true to a single-model search, missing records will + # cause Thinking Sphinx to retry the query but excluding those records. Since search + # is paginated, the new search could potentially include missing records as well, so by + # default Thinking Sphinx will retry three times. Pass :retry_stale => 5 to retry five + # times, and so on. If there are still missing ids on the last retry, they are + # shown as nils. + # def search(*args) - results, client = search_results(*args.clone) + query = args.clone # an array + options = query.extract_options! - ::ActiveRecord::Base.logger.error( - "Sphinx Error: #{results[:error]}" - ) if results[:error] + retry_search_on_stale_index(query, options) do + results, client = search_results(*(query + [options])) - options = args.extract_options! - klass = options[:class] - page = options[:page] ? options[:page].to_i : 1 + ::ActiveRecord::Base.logger.error( + "Sphinx Error: #{results[:error]}" + ) if results[:error] - ThinkingSphinx::Collection.create_from_results(results, page, client.limit, options) + klass = options[:class] + page = options[:page] ? options[:page].to_i : 1 + + ThinkingSphinx::Collection.create_from_results(results, page, client.limit, options) + end + end + + def retry_search_on_stale_index(query, options, &block) + stale_ids = [] + stale_retries_left = case options[:retry_stale] + when true: 3 # default to three retries + when nil, false: 0 # no retries + else options[:retry_stale].to_i + end + begin + # Passing this in an option so Collection.create_from_results can see it. + # It should only raise on stale records if there are any retries left. + options[:raise_on_stale] = stale_retries_left > 0 + block.call + # If ThinkingSphinx::Collection.create_from_results found records in Sphinx but not + # in the DB and the :raise_on_stale option is set, this exception is raised. We retry + # a limited number of times, excluding the stale ids from the search. + rescue StaleIdsException => e + stale_retries_left -= 1 + + stale_ids |= e.ids # For logging + options[:without_ids] = Array(options[:without_ids]) | e.ids # Actual exclusion + + tries = stale_retries_left + ::ActiveRecord::Base.logger.debug("Sphinx Stale Ids (%s %s left): %s" % [ + tries, (tries==1 ? 'try' : 'tries'), stale_ids.join(', ') + ]) + + retry + end end def count(*args) @@ -254,14 +360,17 @@ module ThinkingSphinx # def search_results(*args) options = args.extract_options! + query = args.join(' ') client = client_from_options options - query, filters = search_conditions( + query = star_query(query, options[:star]) if options[:star] + + extra_query, filters = search_conditions( options[:class], options[:conditions] || {} ) client.filters += filters - client.match_mode = :extended unless query.empty? - query = (args + [query]).join(' ') + client.match_mode = :extended unless extra_query.empty? + query = [query, extra_query].join(' ') query.strip! # Because "" and " " are not equivalent set_sort_options! client, options @@ -288,14 +397,27 @@ module ThinkingSphinx config = ThinkingSphinx::Configuration.instance client = Riddle::Client.new config.address, config.port klass = options[:class] - index_options = klass ? klass.sphinx_indexes.last.options : {} + index_options = klass ? klass.sphinx_index_options : {} + + # The Riddle default is per-query max_matches=1000. If we set the + # per-server max to a smaller value in sphinx.yml, we need to override + # the Riddle default or else we get search errors like + # "per-query max_matches=1000 out of bounds (per-server max_matches=200)" + if per_server_max_matches = config.searchd_options[:max_matches] + options[:max_matches] ||= per_server_max_matches + end # Turn :index_weights => { "foo" => 2, User => 1 } - # into :index_weights => { "foo" => 2, "user_core" => 1 } + # into :index_weights => { "foo" => 2, "user_core" => 1, "user_delta" => 1 } if iw = options[:index_weights] options[:index_weights] = iw.inject({}) do |hash, (index,weight)| - key = index.is_a?(Class) ? "#{ThinkingSphinx::Index.name(index)}_core" : index - hash[key] = weight + if index.is_a?(Class) + name = ThinkingSphinx::Index.name(index) + hash["#{name}_core"] = weight + hash["#{name}_delta"] = weight + else + hash[index] = weight + end hash end end @@ -335,9 +457,30 @@ module ThinkingSphinx Riddle::Client::Filter.new attr.to_s, filter_value(val), true } if options[:without] + # exclusive attribute filter on primary key + client.filters += Array(options[:without_ids]).collect { |id| + Riddle::Client::Filter.new 'sphinx_internal_id', filter_value(id), true + } if options[:without_ids] + client end + def star_query(query, custom_token = nil) + token = custom_token.is_a?(Regexp) ? custom_token : /\w+/u + + query.gsub(/("#{token}(.*?#{token})?"|(?![!-])#{token})/u) do + pre, proper, post = $`, $&, $' + is_operator = pre.match(%r{(\W|^)[@~/]\Z}) # E.g. "@foo", "/2", "~3", but not as part of a token + is_quote = proper.starts_with?('"') && proper.ends_with?('"') # E.g. "foo bar", with quotes + has_star = pre.ends_with?("*") || post.starts_with?("*") + if is_operator || is_quote || has_star + proper + else + "*#{proper}*" + end + end + end + def filter_value(value) case value when Range @@ -424,8 +567,10 @@ module ThinkingSphinx fields = klass ? klass.sphinx_indexes.collect { |index| index.fields.collect { |field| field.unique_name } }.flatten : [] - - case order = options[:order] + index_options = klass ? klass.sphinx_index_options : {} + + order = options[:order] || index_options[:order] + case order when Symbol client.sort_mode = :attr_asc if client.sort_mode == :relevance || client.sort_mode.nil? if fields.include?(order) vendor/plugins/thinking-sphinx/lib/thinking_sphinx/search.rb @@ -1 +1,6 @@ -require 'thinking_sphinx' \ No newline at end of file +require 'thinking_sphinx' +require 'action_controller/dispatcher' + +ActionController::Dispatcher.to_prepare :thinking_sphinx do + ThinkingSphinx::Configuration.instance.load_models +end vendor/plugins/thinking-sphinx/rails/init.rb @@ -11,6 +11,55 @@ describe ThinkingSphinx::Search do @sphinx.stop end + describe "search method" do + before :each do + @client = Riddle::Client.stub_instance( + :filters => [], + :filters= => true, + :id_range= => true, + :sort_mode => :asc, + :limit => 5, + :offset= => 0, + :sort_mode= => true, + :query => { + :matches => [], + :total => 50 + } + ) + + ThinkingSphinx::Search.stub_methods( + :client_from_options => @client, + :search_conditions => ["", []] + ) + end + + describe ":star option" do + + it "should not apply by default" do + ThinkingSphinx::Search.search "foo bar" + @client.should have_received(:query).with("foo bar") + end + + it "should apply when passed, and handle full extended syntax" do + input = %{a b* c (d | e) 123 5&6 (f_f g) !h "i j" "k l"~10 "m n"/3 @o p -(q|r)} + expected = %{*a* b* *c* (*d* | *e*) *123* *5*&*6* (*f_f* *g*) !*h* "i j" "k l"~10 "m n"/3 @o *p* -(*q*|*r*)} + ThinkingSphinx::Search.search input, :star => true + @client.should have_received(:query).with(expected) + end + + it "should default to /\w+/ as token" do + ThinkingSphinx::Search.search "foo@bar.com", :star => true + @client.should have_received(:query).with("*foo*@*bar*.*com*") + end + + it "should honour custom token" do + ThinkingSphinx::Search.search "foo@bar.com -foo-bar", :star => /[\w@.-]+/u + @client.should have_received(:query).with("*foo@bar.com* -*foo-bar*") + end + + end + end + describe "search_for_id method" do before :each do @client = Riddle::Client.stub_instance( @@ -201,3 +250,39 @@ describe ThinkingSphinx::Search do Beta.search("two").should be_empty end end + + +# We'll be removing stuff, so best to set up things afresh. +describe ThinkingSphinx::Search, "handling stale index" do + before :all do + @sphinx.setup_mysql + @sphinx.setup_sphinx + @sphinx.start + end + + after :all do + @sphinx.stop + end + + it "should work" do + Alpha.search().should_not be_empty + + two = Alpha.find(:first, :conditions => {:name => "two"}) + three = Alpha.find(:first, :conditions => {:name => "three"}) + + defaults = { :per_page => 1, :order => 'sphinx_internal_id ASC' } + + Alpha.delete(1) # remove without callbacks + # If the first record has gone missing, you should get a nil back. + Alpha.search(defaults.merge(:retry_stale => false)).should == [nil] + # If you retry once, you find the second item which still exists. + Alpha.search(defaults.merge(:retry_stale => 1)).should == [two] + + Alpha.delete(2) # remove without callbacks + # If the second item has gone as well, you get a nil there too. + Alpha.search(defaults.merge(:retry_stale => 1)).should == [nil] + # But :retry_stale => true (three retries) overcomes that as well. + Alpha.search(defaults.merge(:retry_stale => true)).should == [three] + end + +end vendor/plugins/thinking-sphinx/spec/unit/thinking_sphinx/search_spec.rb 90c8e62133a208e2a8d923281c486866e0b8f040 Tien Dung dungtn@gmail.com http://github.com/tiendung/alonetone/commit/692bc4e4ea13aa0b038afbe21d64cc3532c0d16f 692bc4e4ea13aa0b038afbe21d64cc3532c0d16f 2008-11-10T17:55:55-08:00 2008-11-10T17:55:55-08:00 update thinking-sphinx; add sphinxsearch installation 1875962e66bc4706c304998208693a7b2c457db1 Tien Dung dungtn@gmail.com