ginger_scenarios.rb lib/thinking_sphinx/adapters/abstract_adapter.rb lib/thinking_sphinx/adapters/mysql_adapter.rb lib/thinking_sphinx/adapters/postgresql_adapter.rb spec/unit/thinking_sphinx/collection_spec.rb vendor/after_commit/.gitignore vendor/after_commit/LICENSE vendor/after_commit/README vendor/after_commit/Rakefile vendor/after_commit/init.rb vendor/after_commit/lib/after_commit.rb vendor/after_commit/lib/after_commit/active_record.rb vendor/after_commit/lib/after_commit/connection_adapters.rb vendor/after_commit/test/after_commit_test.rb vendor/riddle/lib/riddle.rb vendor/riddle/lib/riddle/client.rb vendor/riddle/lib/riddle/client/filter.rb vendor/riddle/lib/riddle/client/message.rb vendor/riddle/lib/riddle/client/response.rb @@ -16,16 +16,33 @@ To get the spec suite running, you will need to install the not-a-mock gem if yo cd not-a-mock rake gem gem install pkg/not_a_mock-1.1.0.gem + +Then install the ginger gem. The steps are the same, except that you might need to sudo the gem install: -Then set up your database + git clone git://github.com/freelancing-god/ginger.git + cd ginger + rake gem + sudo gem install pkg/ginger-1.1.0.gem + +Then set up your database: cp spec/fixtures/database.yml.default spec/fixtures/database.yml mysqladmin -u root create thinking_sphinx +Make sure you don't have another Sphinx daemon (searchd) running. If you do, quit it with "rake ts:stop" +in the app root. + You should now have a passing test suite from which to build your patch on. rake spec +If you get the message "Failed to start searchd daemon", run the spec with sudo: + + sudo rake spec + +If you quit the spec suite before it's completed, you may be left with data in the test +database, causing the next run to have failures. Let that run complete and then try again. + h2. Contributors Since I first released this library, there's been quite a few people who have submitted patches, to my immense gratitude. Others have suggested syntax changes and general improvements. So my thanks to the following people: @@ -63,3 +80,16 @@ Since I first released this library, there's been quite a few people who have su * David Eisinger * Shay Arnett * Minh Tran +* Jeremy Durham +* Piotr Sarnacki +* Matt Johnson +* Nicolas Blanco +* Max Lapshin +* Josh Natanson +* Philip Hallstrom +* Christian Rishøj +* Mike Flester +* Jim Remsik +* Kennon Ballou +* Henrik Nyh +* Emil Tin README.textile @@ -18,7 +18,7 @@ require 'thinking_sphinx' desc 'Generate documentation' Rake::RDocTask.new(:rdoc) do |rdoc| rdoc.rdoc_dir = 'rdoc' - rdoc.title = 'Riddle - Ruby Sphinx Client' + rdoc.title = 'Thinking Sphinx - ActiveRecord Sphinx Plugin' rdoc.options << '--line-numbers' << '--inline-source' rdoc.rdoc_files.include('README') rdoc.rdoc_files.include('lib/**/*.rb') @@ -56,7 +56,8 @@ spec = Gem::Specification.new do |s| "LICENCE", "README", "tasks/**/*.rb", - "tasks/**/*.rake" + "tasks/**/*.rake", + "vendor/**/*" ] end Rakefile @@ -1,3 +1,10 @@ require 'thinking_sphinx' +require 'action_controller/dispatcher' -ThinkingSphinx::Configuration.new.load_models \ No newline at end of file +if Rails::VERSION::STRING.to_f < 2.1 + ThinkingSphinx::Configuration.instance.load_models +end + +ActionController::Dispatcher.to_prepare :thinking_sphinx do + ThinkingSphinx::Configuration.instance.load_models +end \ No newline at end of file init.rb @@ -1,5 +1,10 @@ +Dir[File.join(File.dirname(__FILE__), '../vendor/*/lib')].each do |path| + $LOAD_PATH.unshift path +end + require 'active_record' require 'riddle' +require 'after_commit' require 'thinking_sphinx/active_record' require 'thinking_sphinx/association' @@ -11,6 +16,10 @@ require 'thinking_sphinx/index' require 'thinking_sphinx/rails_additions' require 'thinking_sphinx/search' +require 'thinking_sphinx/adapters/abstract_adapter' +require 'thinking_sphinx/adapters/mysql_adapter' +require 'thinking_sphinx/adapters/postgresql_adapter' + ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord) Merb::Plugins.add_rakefiles( @@ -21,7 +30,7 @@ module ThinkingSphinx module Version #:nodoc: Major = 0 Minor = 9 - Tiny = 9 + Tiny = 12 String = [Major, Minor, Tiny].join('.') end @@ -31,6 +40,15 @@ module ThinkingSphinx class ConnectionError < StandardError end + # A StaleIdsException is thrown by Collection.instances_from_matches if there + # are records in Sphinx but not in the database, so the search can be retried. + class StaleIdsException < StandardError + attr_accessor :ids + def initialize(ids) + self.ids = ids + end + end + # The collection of indexed models. Keep in mind that Rails lazily loads # its classes, so this may not actually be populated with _all_ the models # that have Sphinx indexes. @@ -100,4 +118,15 @@ module ThinkingSphinx "SELECT @@global.sql_mode, @@session.sql_mode;" ).all? { |key,value| value.nil? || value[/ONLY_FULL_GROUP_BY/].nil? } end + + def self.sphinx_running? + pid_file = ThinkingSphinx::Configuration.instance.pid_file + + if File.exists?(pid_file) + pid = `cat #{pid_file}`[/\d+/] + `ps -p #{pid} | wc -l`.to_i > 1 + else + false + end + end end lib/thinking_sphinx.rb @@ -5,12 +5,12 @@ require 'thinking_sphinx/active_record/has_many_association' module ThinkingSphinx # Core additions to ActiveRecord models - define_index for creating indexes # for models. If you want to interrogate the index objects created for the - # model, you can use the class-level accessor :indexes. + # model, you can use the class-level accessor :sphinx_indexes. # module ActiveRecord def self.included(base) base.class_eval do - class_inheritable_array :indexes + class_inheritable_array :sphinx_indexes class << self # Allows creation of indexes for Sphinx. If you don't do this, there # isn't much point trying to search (or using this plugin at all, @@ -64,10 +64,10 @@ module ThinkingSphinx def define_index(&block) return unless ThinkingSphinx.define_indexes? - self.indexes ||= [] + self.sphinx_indexes ||= [] index = Index.new(self, &block) - self.indexes << index + self.sphinx_indexes << index unless ThinkingSphinx.indexed_models.include?(self.name) ThinkingSphinx.indexed_models << self.name end @@ -83,6 +83,10 @@ module ThinkingSphinx end alias_method :sphinx_index, :define_index + def sphinx_index_options + sphinx_indexes.last.options + end + # Generate a unique CRC value for the model's name, to use to # determine which Sphinx documents belong to which AR records. # @@ -103,6 +107,18 @@ module ThinkingSphinx def to_crc32s (subclasses << self).collect { |klass| klass.to_crc32 } end + + def source_of_sphinx_index + possible_models = self.sphinx_indexes.collect { |index| index.model } + return self if possible_models.include?(self) + + parent = self.superclass + while !possible_models.include?(parent) && parent != ::ActiveRecord::Base + parent = parent.superclass + end + + return parent + end end end @@ -118,28 +134,38 @@ module ThinkingSphinx end def in_core_index? - @in_core_index ||= self.class.search_for_id(self.id, "#{self.class.name.downcase}_core") + self.class.search_for_id( + self.sphinx_document_id, + "#{self.class.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_core" + ) end def toggle_deleted - return unless ThinkingSphinx.updates_enabled? + return unless ThinkingSphinx.updates_enabled? && ThinkingSphinx.sphinx_running? - config = ThinkingSphinx::Configuration.new + config = ThinkingSphinx::Configuration.instance client = Riddle::Client.new config.address, config.port client.update( - "#{self.class.indexes.first.name}_core", + "#{self.class.sphinx_indexes.first.name}_core", ['sphinx_deleted'], - {self.id => 1} + {self.sphinx_document_id => 1} ) if self.in_core_index? client.update( - "#{self.class.indexes.first.name}_delta", + "#{self.class.sphinx_indexes.first.name}_delta", ['sphinx_deleted'], - {self.id => 1} + {self.sphinx_document_id => 1} ) if ThinkingSphinx.deltas_enabled? && - self.class.indexes.any? { |index| index.delta? } && + self.class.sphinx_indexes.any? { |index| index.delta? } && self.delta? + rescue ::ThinkingSphinx::ConnectionError + # nothing + end + + def sphinx_document_id + (self.id * ThinkingSphinx.indexed_models.size) + + ThinkingSphinx.indexed_models.index(self.class.source_of_sphinx_index.name) end end end lib/thinking_sphinx/active_record.rb @@ -11,57 +11,55 @@ module ThinkingSphinx # def self.included(base) base.class_eval do - # The define_callbacks method was added post Rails 2.0.2 - if it - # doesn't exist, we define the callback manually - # - if respond_to?(:define_callbacks) - define_callbacks :after_commit - else - class << self - # Handle after_commit callbacks - call all the registered callbacks. - # - def after_commit(*callbacks, &block) - callbacks << block if block_given? - write_inheritable_array(:after_commit, callbacks) + class << self + # Temporarily disable delta indexing inside a block, then perform a single + # rebuild of index at the end. + # + # Useful when performing updates to batches of models to prevent + # the delta index being rebuilt after each individual update. + # + # In the following example, the delta index will only be rebuilt once, + # not 10 times. + # + # SomeModel.suspended_delta do + # 10.times do + # SomeModel.create( ... ) + # end + # end + # + def suspended_delta(reindex_after = true, &block) + original_setting = ThinkingSphinx.deltas_enabled? + ThinkingSphinx.deltas_enabled = false + begin + yield + ensure + ThinkingSphinx.deltas_enabled = original_setting + self.index_delta if reindex_after end end + + # Build the delta index for the related model. This won't be called + # if running in the test environment. + # + def index_delta(instance = nil) + return true unless ThinkingSphinx.updates_enabled? && + ThinkingSphinx.deltas_enabled? + + config = ThinkingSphinx::Configuration.instance + client = Riddle::Client.new config.address, config.port + + client.update( + "#{self.sphinx_indexes.first.name}_core", + ['sphinx_deleted'], + {instance.sphinx_document_id => 1} + ) if instance && instance.in_core_index? + + system "#{config.bin_path}indexer --config #{config.config_file} --rotate #{self.sphinx_indexes.first.name}_delta" + + true + end end - def after_commit - # Deliberately blank. - end - - # Normal boolean save wrapped in a handler for the after_commit - # callback. - # - def save_with_after_commit_callback(*args) - value = save_without_after_commit_callback(*args) - callback(:after_commit) if value - return value - end - - alias_method_chain :save, :after_commit_callback - - # Forceful save wrapped in a handler for the after_commit callback. - # - def save_with_after_commit_callback!(*args) - value = save_without_after_commit_callback!(*args) - callback(:after_commit) if value - return value - end - - alias_method_chain :save!, :after_commit_callback - - # Normal destroy wrapped in a handler for the after_commit callback. - # - def destroy_with_after_commit_callback - value = destroy_without_after_commit_callback - callback(:after_commit) if value - return value - end - - alias_method_chain :destroy, :after_commit_callback - private # Set the delta value for the model to be true. @@ -73,22 +71,7 @@ module ThinkingSphinx # if running in the test environment. # def index_delta - return true unless ThinkingSphinx.updates_enabled? && - ThinkingSphinx.deltas_enabled? - - config = ThinkingSphinx::Configuration.new - client = Riddle::Client.new config.address, config.port - - client.update( - "#{self.class.indexes.first.name}_core", - ['sphinx_deleted'], - {self.id => 1} - ) if self.in_core_index? - - configuration = ThinkingSphinx::Configuration.new - system "indexer --config #{configuration.config_file} --rotate #{self.class.indexes.first.name}_delta" - - true + self.class.index_delta(self) end end end lib/thinking_sphinx/active_record/delta.rb @@ -6,7 +6,7 @@ module ThinkingSphinx stack = [@reflection.options[:through]].compact attribute = nil - (@reflection.klass.indexes || []).each do |index| + (@reflection.klass.sphinx_indexes || []).each do |index| attribute = index.attributes.detect { |attrib| attrib.columns.length == 1 && attrib.columns.first.__name == foreign_key.to_sym && lib/thinking_sphinx/active_record/has_many_association.rb @@ -1,6 +1,10 @@ module ThinkingSphinx class Collection < ::Array - attr_reader :total_entries, :total_pages, :current_page + attr_reader :total_entries, :total_pages, :current_page, :per_page + attr_accessor :results + + # Compatibility with older versions of will_paginate + alias_method :page_count, :total_pages def initialize(page, per_page, entries, total_entries) @current_page, @per_page, @total_entries = page, per_page, total_entries @@ -8,6 +12,74 @@ module ThinkingSphinx @total_pages = (entries / @per_page.to_f).ceil end + def self.ids_from_results(results, page, limit, options) + collection = self.new(page, limit, + results[:total] || 0, results[:total_found] || 0 + ) + collection.results = results + collection.replace results[:matches].collect { |match| + match[:attributes]["sphinx_internal_id"] + } + return collection + end + + def self.create_from_results(results, page, limit, options) + collection = self.new(page, limit, + results[:total] || 0, results[:total_found] || 0 + ) + collection.results = results + collection.replace instances_from_matches(results[:matches], options) + return collection + end + + def self.instances_from_matches(matches, options = {}) + return matches.collect { |match| + instance_from_match match, options + } unless klass = options[:class] + + index_options = klass.sphinx_index_options + + ids = matches.collect { |match| match[:attributes]["sphinx_internal_id"] } + instances = ids.length > 0 ? klass.find( + :all, + :conditions => {klass.primary_key.to_sym => ids}, + :include => (options[:include] || index_options[:include]), + :select => (options[:select] || index_options[:select]) + ) : [] + + # Raise an exception if we find records in Sphinx but not in the DB, so the search method + # can retry without them. See ThinkingSphinx::Search.retry_search_on_stale_index. + if options[:raise_on_stale] && instances.length < ids.length + stale_ids = ids - instances.map {|i| i.id } + raise StaleIdsException, stale_ids + end + + ids.collect { |obj_id| + instances.detect { |obj| obj.id == obj_id } + } + end + + def self.instance_from_match(match, options) + class_from_crc(match[:attributes]["class_crc"]).find( + match[:attributes]["sphinx_internal_id"], + :include => options[:include], + :select => options[:select] + ) + rescue ::ActiveRecord::RecordNotFound + nil + end + + def self.class_from_crc(crc) + @@models_by_crc ||= ThinkingSphinx.indexed_models.inject({}) do |hash, model| + hash[model.constantize.to_crc32] = model + model.constantize.subclasses.each { |subclass| + hash[subclass.to_crc32] = subclass.name + } + hash + end + @@models_by_crc[crc].constantize + end + def previous_page current_page > 1 ? (current_page - 1) : nil end @@ -19,5 +91,29 @@ module ThinkingSphinx def offset (current_page - 1) * @per_page end + + def method_missing(method, *args, &block) + super unless method.to_s[/^each_with_.*/] + + each_with_attribute method.to_s.gsub(/^each_with_/, ''), &block + end + + def each_with_group_and_count(&block) + results[:matches].each_with_index do |match, index| + yield self[index], match[:attributes]["@group"], match[:attributes]["@count"] + end + end + + def each_with_attribute(attribute, &block) + results[:matches].each_with_index do |match, index| + yield self[index], (match[:attributes][attribute] || match[:attributes]["@#{attribute}"]) + end + end + + def each_with_weighting(&block) + results[:matches].each_with_index do |match, index| + yield self[index], match[:weight] + end + end end end \ No newline at end of file lib/thinking_sphinx/collection.rb @@ -1,4 +1,5 @@ require 'erb' +require 'singleton' module ThinkingSphinx # This class both keeps track of the configuration settings for Sphinx and @@ -41,11 +42,28 @@ module ThinkingSphinx # don't set allow_star to true. # class Configuration + include Singleton + + SourceOptions = %w( mysql_connect_flags sql_range_step sql_query_pre + sql_query_post sql_ranged_throttle sql_query_post_index ) + + IndexOptions = %w( charset_table charset_type docinfo enable_star + exceptions html_index_attrs html_remove_elements html_strip ignore_chars + min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars + ngram_len phrase_boundary phrase_boundary_step preopen stopwords + wordforms ) + + IndexerOptions = %w( max_iops max_iosize mem_limit ) + + SearchdOptions = %w( read_timeout max_children max_matches seamless_rotate + preopen_indexes unlink_old ) + attr_accessor :config_file, :searchd_log_file, :query_log_file, - :pid_file, :searchd_file_path, :address, :port, :enable_star, - :allow_star, :min_prefix_len, :min_infix_len, :mem_limit, :max_matches, - :morphology, :charset_type, :charset_table, :ignore_chars, :html_strip, - :html_remove_elements, :database_yml_file, :app_root + :pid_file, :searchd_file_path, :address, :port, :allow_star, + :database_yml_file, :app_root, :bin_path + + attr_accessor :source_options, :index_options, :indexer_options, + :searchd_options attr_reader :environment @@ -53,6 +71,10 @@ module ThinkingSphinx # and parse it according to the current environment. # def initialize(app_root = Dir.pwd) + self.reset + end + + def reset self.app_root = RAILS_ROOT if defined?(RAILS_ROOT) self.app_root = Merb.root if defined?(Merb) self.app_root ||= app_root @@ -66,24 +88,24 @@ module ThinkingSphinx self.address = "127.0.0.1" self.port = 3312 self.allow_star = false - self.enable_star = false - self.min_prefix_len = nil - self.min_infix_len = nil - self.mem_limit = "64M" - self.max_matches = 1000 - self.morphology = "stem_en" - self.charset_type = "utf-8" - self.charset_table = nil - self.ignore_chars = nil - self.html_strip = false - self.html_remove_elements = "" + self.bin_path = "" + + self.source_options = {} + self.index_options = { + :charset_type => "utf-8", + :morphology => "stem_en" + } + self.indexer_options = {} + self.searchd_options = {} parse_config + + self end def self.environment @@environment ||= ( - defined?(Merb) ? ENV['MERB_ENV'] : ENV['RAILS_ENV'] + defined?(Merb) ? Merb.environment : ENV['RAILS_ENV'] ) || "development" end @@ -107,7 +129,7 @@ module ThinkingSphinx file.write <<-CONFIG indexer { - mem_limit = #{self.mem_limit} +#{hash_to_config(self.indexer_options)} } searchd @@ -116,10 +138,8 @@ searchd port = #{self.port} log = #{self.searchd_log_file} query_log = #{self.query_log_file} - read_timeout = 5 - max_children = 30 pid_file = #{self.pid_file} - max_matches = #{self.max_matches} +#{hash_to_config(self.searchd_options)} } CONFIG @@ -130,10 +150,11 @@ searchd prefixed_fields = [] infixed_fields = [] - model.indexes.select { |index| index.model == model }.each_with_index do |index, i| - file.write index.to_config(model, i, database_conf, charset_type, model_index) + model.sphinx_indexes.select { |index| index.model == model }.each_with_index do |index, i| + file.write index.to_config(model, i, database_conf, model_index) + + index.adapter_object.setup - create_array_accum if index.adapter == :postgres sources << "#{ThinkingSphinx::Index.name(model)}_#{i}_core" delta_sources << "#{ThinkingSphinx::Index.name(model)}_#{i}_delta" if index.delta? end @@ -177,6 +198,31 @@ searchd end end + def hash_to_config(hash) + hash.collect { |key, value| + translated_value = case value + when TrueClass + "1" + when FalseClass + "0" + when NilClass, "" + next + else + value + end + " #{key} = #{translated_value}" + }.join("\n") + end + + def self.options_merge(base, extra) + base = base.clone + extra.each do |key, value| + next if value.nil? || value == "" + base[key] = value + end + base + end + private # Parse the config/sphinx.yml file - if it exists - then use the attribute @@ -190,7 +236,14 @@ searchd conf.each do |key,value| self.send("#{key}=", value) if self.methods.include?("#{key}=") + + self.source_options[key.to_sym] = value if SourceOptions.include?(key.to_s) + self.index_options[key.to_sym] = value if IndexOptions.include?(key.to_s) + self.indexer_options[key.to_sym] = value if IndexerOptions.include?(key.to_s) + self.searchd_options[key.to_sym] = value if SearchdOptions.include?(key.to_s) end unless conf.nil? + + self.bin_path += '/' unless self.bin_path.blank? end def core_index_for_model(model, sources) @@ -200,44 +253,31 @@ index #{ThinkingSphinx::Index.name(model)}_core { #{sources} path = #{self.searchd_file_path}/#{ThinkingSphinx::Index.name(model)}_core -charset_type = #{self.charset_type} INDEX - morphology = model.indexes.inject(self.morphology) { |morph, index| - index.options[:morphology] || morph - } - output += " morphology = #{morphology}\n" unless morphology.blank? - output += " charset_table = #{self.charset_table}\n" unless self.charset_table.nil? - output += " ignore_chars = #{self.ignore_chars}\n" unless self.ignore_chars.nil? + unless combined_index_options(model).empty? + output += hash_to_config(combined_index_options(model)) + end if self.allow_star # Ye Olde way of turning on enable_star output += " enable_star = 1\n" - output += " min_prefix_len = #{self.min_prefix_len}\n" - else - # New, better way of turning on enable_star - thanks to James Healy - output += " enable_star = 1\n" if self.enable_star - output += " min_prefix_len = #{self.min_prefix_len}\n" unless self.min_prefix_len.nil? - output += " min_infix_len = #{self.min_infix_len}\n" unless self.min_infix_len.nil? + output += " min_prefix_len = #{combined_index_options[:min_prefix_len]}\n" end - - output += " html_strip = 1\n" if self.html_strip - output += " html_remove_elements = #{self.html_remove_elements}\n" unless self.html_remove_elements.blank? - - unless model.indexes.collect(&:prefix_fields).flatten.empty? - output += " prefix_fields = #{model.indexes.collect(&:prefix_fields).flatten.map(&:unique_name).join(', ')}\n" + unless model.sphinx_indexes.collect(&:prefix_fields).flatten.empty? + output += " prefix_fields = #{model.sphinx_indexes.collect(&:prefix_fields).flatten.map(&:unique_name).join(', ')}\n" else - output += " prefix_fields = _\n" unless model.indexes.collect(&:infix_fields).flatten.empty? + output += " prefix_fields = _\n" unless model.sphinx_indexes.collect(&:infix_fields).flatten.empty? end - unless model.indexes.collect(&:infix_fields).flatten.empty? - output += " infix_fields = #{model.indexes.collect(&:infix_fields).flatten.map(&:unique_name).join(', ')}\n" + unless model.sphinx_indexes.collect(&:infix_fields).flatten.empty? + output += " infix_fields = #{model.sphinx_indexes.collect(&:infix_fields).flatten.map(&:unique_name).join(', ')}\n" else - output += " infix_fields = -\n" unless model.indexes.collect(&:prefix_fields).flatten.empty? + output += " infix_fields = -\n" unless model.sphinx_indexes.collect(&:prefix_fields).flatten.empty? end - output + "}\n" + output + "\n}\n" end def delta_index_for_model(model, sources) @@ -252,7 +292,7 @@ index #{ThinkingSphinx::Index.name(model)}_delta : #{ThinkingSphinx::Index.name( def distributed_index_for_model(model) sources = ["local = #{ThinkingSphinx::Index.name(model)}_core"] - if model.indexes.any? { |index| index.delta? } + if model.sphinx_indexes.any? { |index| index.delta? } sources << "local = #{ThinkingSphinx::Index.name(model)}_delta" end @@ -261,29 +301,14 @@ index #{ThinkingSphinx::Index.name(model)} { type = distributed #{ sources.join("\n ") } - charset_type = #{self.charset_type} } INDEX end - def create_array_accum - ::ActiveRecord::Base.connection.execute "begin" - ::ActiveRecord::Base.connection.execute "savepoint ts" - begin - ::ActiveRecord::Base.connection.execute <<-SQL - CREATE AGGREGATE array_accum (anyelement) - ( - sfunc = array_append, - stype = anyarray, - initcond = '{}' - ); - SQL - rescue - raise unless $!.to_s =~ /already exists with same argument types/ - ::ActiveRecord::Base.connection.execute "rollback to savepoint ts" + def combined_index_options(model) + model.sphinx_indexes.inject(self.index_options) do |options, index| + self.class.options_merge(options, index.local_index_options) end - ::ActiveRecord::Base.connection.execute "release savepoint ts" - ::ActiveRecord::Base.connection.execute "commit" end end end lib/thinking_sphinx/configuration.rb @@ -9,7 +9,8 @@ module ThinkingSphinx # Enjoy. # class Index - attr_accessor :model, :fields, :attributes, :conditions, :delta, :options + attr_accessor :model, :fields, :attributes, :conditions, :groupings, + :delta, :options # Create a new index instance by passing in the model it is tied to, and # a block to build it with (optional but recommended). For documentation @@ -31,6 +32,7 @@ module ThinkingSphinx @fields = [] @attributes = [] @conditions = [] + @groupings = [] @options = {} @delta = false @@ -46,11 +48,11 @@ module ThinkingSphinx end def empty?(part = :core) - config = ThinkingSphinx::Configuration.new + config = ThinkingSphinx::Configuration.instance File.size?("#{config.searchd_file_path}/#{self.name}_#{part}.spa").nil? end - def to_config(model, index, database_conf, charset_type, offset) + def to_config(model, index, database_conf, offset) # Set up associations and joins add_internal_attributes link! @@ -77,15 +79,17 @@ sql_host = #{database_conf[:host] || "localhost"} sql_user = #{database_conf[:username] || database_conf[:user]} sql_pass = #{(database_conf[:password] || "").gsub('#', '\#')} sql_db = #{database_conf[:database]} +#{"sql_port = #{database_conf[:port]}" unless database_conf[:port].blank? } #{"sql_sock = #{database_conf[:socket]}" unless database_conf[:socket].blank? } -sql_query_pre = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""} +sql_query_pre = #{utf8? && adapter == :mysql ? "SET NAMES utf8" : ""} #{"sql_query_pre = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]} sql_query_pre = #{to_sql_query_pre} sql_query = #{to_sql(:offset => offset).gsub(/\n/, ' ')} sql_query_range = #{to_sql_query_range} sql_query_info = #{to_sql_query_info(offset)} #{attr_sources} +#{ThinkingSphinx::Configuration.instance.hash_to_config(self.source_options)} } SOURCE @@ -95,7 +99,7 @@ sql_query_info = #{to_sql_query_info(offset)} source #{self.class.name(model)}_#{index}_delta : #{self.class.name(model)}_#{index}_core { sql_query_pre = -sql_query_pre = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""} +sql_query_pre = #{utf8? && adapter == :mysql ? "SET NAMES utf8" : ""} #{"sql_query_pre = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]} sql_query = #{to_sql(:delta => true, :offset => offset).gsub(/\n/, ' ')} sql_query_range = #{to_sql_query_range :delta => true} @@ -155,6 +159,11 @@ sql_query_range = #{to_sql_query_range :delta => true} where_clause << " AND " << @conditions.join(" AND ") end + internal_groupings = [] + if @model.column_names.include?(@model.inheritance_column) + internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}" + end + unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}" sql = <<-SQL @@ -171,7 +180,8 @@ WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start GROUP BY #{ ( ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + @fields.collect { |field| field.to_group_sql }.compact + - @attributes.collect { |attribute| attribute.to_group_sql }.compact + @attributes.collect { |attribute| attribute.to_group_sql }.compact + + @groupings + internal_groupings ).join(", ") } SQL @@ -236,6 +246,10 @@ GROUP BY #{ ( end end + def adapter_object + @adapter_object ||= ThinkingSphinx::AbstractAdapter.detect(@model) + end + def prefix_fields @fields.select { |field| field.prefixes } end @@ -244,8 +258,37 @@ GROUP BY #{ ( @fields.select { |field| field.infixes } end + def local_index_options + @options.keys.inject({}) do |local_options, key| + if ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s) + local_options[key.to_sym] = @options[key] + end + local_options + end + end + + def index_options + all_index_options = ThinkingSphinx::Configuration.instance.index_options.clone + @options.keys.select { |key| + ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s) + }.each { |key| all_index_options[key.to_sym] = @options[key] } + all_index_options + end + + def source_options + all_source_options = ThinkingSphinx::Configuration.instance.source_options.clone + @options.keys.select { |key| + ThinkingSphinx::Configuration::SourceOptions.include?(key.to_s) + }.each { |key| all_source_options[key.to_sym] = @options[key] } + all_source_options + end + private + def utf8? + self.index_options[:charset_type] == "utf-8" + end + def quote_column(column) @model.connection.quote_column_name(column) end @@ -271,8 +314,20 @@ GROUP BY #{ ( @fields = builder.fields @attributes = builder.attributes @conditions = builder.conditions + @groupings = builder.groupings @delta = builder.properties[:delta] @options = builder.properties.except(:delta) + + # We want to make sure that if the database doesn't exist, then Thinking + # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop + # and db:migrate). It's a bit hacky, but I can't think of a better way. + rescue StandardError => err + case err.class.name + when "Mysql::Error", "ActiveRecord::StatementInvalid" + return + else + raise err + end end # Returns all associations used amongst all the fields and attributes. @@ -334,10 +389,13 @@ GROUP BY #{ ( end def crc_column - if adapter == :postgres - @model.to_crc32.to_s - elsif @model.column_names.include?(@model.inheritance_column) - "IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})" + if @model.column_names.include?(@model.inheritance_column) + case adapter + when :postgres + "COALESCE(crc32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})" + when :mysql + "CAST(IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s}) AS UNSIGNED)" + end else @model.to_crc32.to_s end @@ -345,7 +403,7 @@ GROUP BY #{ ( def add_internal_attributes @attributes << Attribute.new( - FauxColumn.new(:id), + FauxColumn.new(@model.primary_key.to_sym), :type => :integer, :as => :sphinx_internal_id ) unless @attributes.detect { |attr| attr.alias == :sphinx_internal_id } lib/thinking_sphinx/index.rb @@ -18,9 +18,14 @@ module ThinkingSphinx # rails documentation. It's not needed though, so it gets undef'd. # Hopefully the list of methods that get in the way doesn't get too # long. - undef_method :parent + HiddenMethods = [:parent, :name, :id, :type].each { |method| + define_method(method) { + caller.grep(/irb.completion/).empty? ? method_missing(method) : super + } + } - attr_accessor :fields, :attributes, :properties, :conditions + attr_accessor :fields, :attributes, :properties, :conditions, + :groupings # Set up all the collections. Consider this the equivalent of an # instance's initialize method. @@ -30,6 +35,7 @@ module ThinkingSphinx @attributes = [] @properties = {} @conditions = [] + @groupings = [] end # This is how you add fields - the strings Sphinx looks at - to your @@ -128,7 +134,7 @@ module ThinkingSphinx # when you would like to index a calculated value. Don't forget to set # the type of the attribute though: # - # indexes "age < 18", :as => :minor, :type => :boolean + # has "age < 18", :as => :minor, :type => :boolean # # If you're creating attributes for latitude and longitude, don't # forget that Sphinx expects these values to be in radians. @@ -152,6 +158,16 @@ module ThinkingSphinx @conditions += args end + # Use this method to add some manual SQL strings to the GROUP BY + # clause. You can pass in as many strings as you'd like, they'll get + # joined together with commas later on. + # + # group_by "lat", "lng" + # + def group_by(*args) + @groupings += args + end + # This is what to use to set properties on the index. Chief amongst # those is the delta property - to allow automatic updates to your # indexes as new models are added and edited - but also you can @@ -160,6 +176,9 @@ module ThinkingSphinx # # set_property :delta => true # set_property :field_weights => {"name" => 100} + # set_property :order => "name ASC" + # set_property :include => :picture + # set_property :select => 'name' # # Also, the following two properties are particularly relevant for # geo-location searching - latitude_attr and longitude_attr. If your @@ -168,7 +187,7 @@ module ThinkingSphinx # when defining the index, so you don't need to specify them for every # geo-related search. # - # set_property :latitude_attr => "lt", :longitude => "lg" + # set_property :latitude_attr => "lt", :longitude_attr => "lg" # # Please don't forget to add a boolean field named 'delta' to your # model's database table if enabling the delta index for it. @@ -189,6 +208,16 @@ module ThinkingSphinx def method_missing(method, *args) FauxColumn.new(method, *args) end + + # A method to allow adding fields from associations which have names + # that clash with method names in the Builder class (ie: properties, + # fields, attributes). + # + # Example: indexes assoc(:properties).column + # + def assoc(assoc) + FauxColumn.new(method) + end end end end lib/thinking_sphinx/index/builder.rb @@ -53,4 +53,16 @@ end ActiveRecord::Base.extend( ThinkingSphinx::ActiveRecordQuotedName -) unless ActiveRecord::Base.respond_to?("quoted_table_name") \ No newline at end of file +) unless ActiveRecord::Base.respond_to?("quoted_table_name") + +module ThinkingSphinx + module ActiveRecordStoreFullSTIClass + def store_full_sti_class + false + end + end +end + +ActiveRecord::Base.extend( + ThinkingSphinx::ActiveRecordStoreFullSTIClass +) unless ActiveRecord::Base.respond_to?(:store_full_sti_class) \ No newline at end of file lib/thinking_sphinx/rails_additions.rb @@ -12,23 +12,17 @@ module ThinkingSphinx # return the ids for the matching objects. See #search for syntax # examples. # + # Note that this only searches the Sphinx index, with no ActiveRecord + # queries. Thus, if your index is not in sync with the database, this + # method may return ids that no longer exist there. + # def search_for_ids(*args) results, client = search_results(*args.clone) options = args.extract_options! page = options[:page] ? options[:page].to_i : 1 - - begin - pager = WillPaginate::Collection.create(page, - client.limit, results[:total_found] || 0) do |collection| - collection.replace results[:matches].collect { |match| - match[:attributes]["sphinx_internal_id"] - } - collection.instance_variable_set :@total_entries, results[:total_found] - end - rescue - results[:matches].collect { |match| match[:attributes]["sphinx_internal_id"] } - end + + ThinkingSphinx::Collection.ids_from_results(results, page, client.limit, options) end # Searches through the Sphinx indexes for relevant matches. There's @@ -54,7 +48,7 @@ module ThinkingSphinx # # User.search "pat", :include => :posts # - # == Advanced Searching + # == Match Modes # # Sphinx supports 5 different matching modes. By default Thinking Sphinx # uses :all, which unsurprisingly requires all the supplied search terms @@ -72,6 +66,20 @@ module ThinkingSphinx # for more complex query syntax, refer to the sphinx documentation for further # details. # + # == Weighting + # + # Sphinx has support for weighting, where matches in one field can be considered + # more important than in another. Weights are integers, with 1 as the default. + # They can be set per-search like this: + # + # User.search "pat allan", :field_weights => { :alias => 4, :aka => 2 } + # + # If you're searching multiple models, you can set per-index weights: + # + # ThinkingSphinx::Search.search "pat", :index_weights => { User => 10 } + # + # See http://sphinxsearch.com/doc.html#weighting for further details. + # # == Searching by Fields # # If you want to step it up a level, you can limit your search terms to @@ -109,7 +117,52 @@ module ThinkingSphinx # attribute values to exclude. This is done with the :without option: # # User.search :without => {:role_id => 1} - # + # + # == Excluding by Primary Key + # + # There is a shortcut to exclude records by their ActiveRecord primary key: + # + # User.search :without_ids => 1 + # + # Pass an array or a single value. + # + # The primary key must be an integer as a negative filter is used. Note + # that for multi-model search, an id may occur in more than one model. + # + # == Infix (Star) Searching + # + # By default, Sphinx uses English stemming, e.g. matching "shoes" if you + # search for "shoe". It won't find "Melbourne" if you search for + # "elbourn", though. + # + # Enable infix searching by something like this in config/sphinx.yml: + # + # development: + # enable_star: 1 + # min_infix_length: 2 + # + # Note that this will make indexing take longer. + # + # With those settings (and after reindexing), wildcard asterisks can be used + # in queries: + # + # Location.search "*elbourn*" + # + # To automatically add asterisks around every token (but not operators), + # pass the :star option: + # + # Location.search "elbourn -ustrali", :star => true, :match_mode => :boolean + # + # This would become "*elbourn* -*ustrali*". The :star option only adds the + # asterisks. You need to make the config/sphinx.yml changes yourself. + # + # By default, the tokens are assumed to match the regular expression /\w+/u. + # If you've modified the charset_table, pass another regular expression, e.g. + # + # User.search("oo@bar.c", :star => /[\w@.]+/u) + # + # to search for "*oo@bar.c*" and not "*oo*@*bar*.*c*". + # # == Sorting # # Sphinx can only sort by attributes, so generally you will need to avoid @@ -151,8 +204,8 @@ module ThinkingSphinx # attributes. To search with that point, you can then use one of the # following syntax examples: # - # Address.search "Melbourne", :geo => [1.4, -2.217] - # Address.search "Australia", :geo => [-0.55, 3.108], + # Address.search "Melbourne", :geo => [1.4, -2.217], :order => "@geodist asc" + # Address.search "Australia", :geo => [-0.55, 3.108], :order => "@geodist asc" # :latitude_attr => "latit", :longitude_attr => "longit" # # The first example applies when your latitude and longitude attributes @@ -170,9 +223,9 @@ module ThinkingSphinx # # Now, geo-location searching really only has an affect if you have a # filter, sort or grouping clause related to it - otherwise it's just a - # normal search. To make use of the positioning difference, use the - # special attribute "@geodist" in any of your filters or sorting or grouping - # clauses. + # normal search, and _will not_ return a distance value otherwise. To + # make use of the positioning difference, use the special attribute + # "@geodist" in any of your filters or sorting or grouping clauses. # # And don't forget - both the latitude and longitude you use in your # search, and the values in your indexes, need to be stored as a float in radians, @@ -184,30 +237,85 @@ module ThinkingSphinx # # ... # end # + # Once you've got your results set, you can access the distances as + # follows: + # + # @results.each_with_geodist do |result, distance| + # # ... + # end + # + # The distance value is returned as a float, representing the distance in + # metres. + # + # == Handling a Stale Index + # + # Especially if you don't use delta indexing, you risk having records in the + # Sphinx index that are no longer in the database. By default, those will simply + # come back as nils: + # + # >> pat_user.delete + # >> User.search("pat") + # Sphinx Result: [1,2] + # => [nil, <#User id: 2>] + # + # (If you search across multiple models, you'll get ActiveRecord::RecordNotFound.) + # + # You can simply Array#compact these results or handle the nils in some other way, but + # Sphinx will still report two results, and the missing records may upset your layout. + # + # If you pass :retry_stale => true to a single-model search, missing records will + # cause Thinking Sphinx to retry the query but excluding those records. Since search + # is paginated, the new search could potentially include missing records as well, so by + # default Thinking Sphinx will retry three times. Pass :retry_stale => 5 to retry five + # times, and so on. If there are still missing ids on the last retry, they are + # shown as nils. + # def search(*args) - results, client = search_results(*args.clone) + query = args.clone # an array + options = query.extract_options! - ::ActiveRecord::Base.logger.error( - "Sphinx Error: #{results[:error]}" - ) if results[:error] + retry_search_on_stale_index(query, options) do + results, client = search_results(*(query + [options])) - options = args.extract_options! - klass = options[:class] - page = options[:page] ? options[:page].to_i : 1 + ::ActiveRecord::Base.logger.error( + "Sphinx Error: #{results[:error]}" + ) if results[:error] - # begin - pager = ThinkingSphinx::Collection.new(page, client.limit, - results[:total] || 0, results[:total_found] || 0) - pager.replace instances_from_results(results[:matches], options, klass) - pager - # pager = WillPaginate::Collection.create(page, - # client.limit, results[:total] || 0) do |collection| - # collection.replace instances_from_results(results[:matches], options, klass) - # collection.instance_variable_set :@total_entries, results[:total_found] - # end - # rescue StandardError => err - # instances_from_results(results[:matches], options, klass) - # end + klass = options[:class] + page = options[:page] ? options[:page].to_i : 1 + + ThinkingSphinx::Collection.create_from_results(results, page, client.limit, options) + end + end + + def retry_search_on_stale_index(query, options, &block) + stale_ids = [] + stale_retries_left = case options[:retry_stale] + when true: 3 # default to three retries + when nil, false: 0 # no retries + else options[:retry_stale].to_i + end + begin + # Passing this in an option so Collection.create_from_results can see it. + # It should only raise on stale records if there are any retries left. + options[:raise_on_stale] = stale_retries_left > 0 + block.call + # If ThinkingSphinx::Collection.create_from_results found records in Sphinx but not + # in the DB and the :raise_on_stale option is set, this exception is raised. We retry + # a limited number of times, excluding the stale ids from the search. + rescue StaleIdsException => e + stale_retries_left -= 1 + + stale_ids |= e.ids # For logging + options[:without_ids] = Array(options[:without_ids]) | e.ids # Actual exclusion + + tries = stale_retries_left + ::ActiveRecord::Base.logger.debug("Sphinx Stale Ids (%s %s left): %s" % [ + tries, (tries==1 ? 'try' : 'tries'), stale_ids.join(', ') + ]) + + retry + end end def count(*args) @@ -252,15 +360,19 @@ module ThinkingSphinx # def search_results(*args) options = args.extract_options! + query = args.join(' ') client = client_from_options options - query, filters = search_conditions( + query = star_query(query, options[:star]) if options[:star] + + extra_query, filters = search_conditions( options[:class], options[:conditions] || {} ) client.filters += filters - client.match_mode = :extended unless query.empty? - query = args.join(" ") + query - + client.match_mode = :extended unless extra_query.empty? + query = [query, extra_query].join(' ') + query.strip! # Because "" and " " are not equivalent + set_sort_options! client, options client.limit = options[:per_page].to_i if options[:per_page] @@ -278,54 +390,37 @@ module ThinkingSphinx return results, client end - def instances_from_results(results, options = {}, klass = nil) - if klass.nil? - results.collect { |result| instance_from_result result, options } - else - ids = results.collect { |result| result[:attributes]["sphinx_internal_id"] } - instances = ids.length > 0 ? klass.find( - :all, - :conditions => {klass.primary_key.to_sym => ids}, - :include => options[:include], - :select => options[:select] - ) : [] - ids.collect { |obj_id| instances.detect { |obj| obj.id == obj_id } } - end - end - - # Either use the provided class to instantiate a result from a model, or - # get the result's CRC value and determine the class from that. - # - def instance_from_result(result, options) - class_from_crc(result[:attributes]["class_crc"]).find( - result[:attributes]["sphinx_internal_id"], - :include => options[:include], :select => options[:select] - ) - end - - # Convert a CRC value to the corresponding class. - # - def class_from_crc(crc) - unless @models_by_crc - Configuration.new.load_models - - @models_by_crc = ThinkingSphinx.indexed_models.inject({}) do |hash, model| - hash[model.constantize.to_crc32] = model - hash - end - end - - @models_by_crc[crc].constantize - end - # Set all the appropriate settings for the client, using the provided # options hash. # def client_from_options(options = {}) - config = ThinkingSphinx::Configuration.new + config = ThinkingSphinx::Configuration.instance client = Riddle::Client.new config.address, config.port klass = options[:class] - index_options = klass ? klass.indexes.last.options : {} + index_options = klass ? klass.sphinx_index_options : {} + + # The Riddle default is per-query max_matches=1000. If we set the + # per-server max to a smaller value in sphinx.yml, we need to override + # the Riddle default or else we get search errors like + # "per-query max_matches=1000 out of bounds (per-server max_matches=200)" + if per_server_max_matches = config.searchd_options[:max_matches] + options[:max_matches] ||= per_server_max_matches + end + + # Turn :index_weights => { "foo" => 2, User => 1 } + # into :index_weights => { "foo" => 2, "user_core" => 1, "user_delta" => 1 } + if iw = options[:index_weights] + options[:index_weights] = iw.inject({}) do |hash, (index,weight)| + if index.is_a?(Class) + name = ThinkingSphinx::Index.name(index) + hash["#{name}_core"] = weight + hash["#{name}_delta"] = weight + else + hash[index] = weight + end + hash + end + end [ :max_matches, :match_mode, :sort_mode, :sort_by, :id_range, @@ -362,9 +457,30 @@ module ThinkingSphinx Riddle::Client::Filter.new attr.to_s, filter_value(val), true } if options[:without] + # exclusive attribute filter on primary key + client.filters += Array(options[:without_ids]).collect { |id| + Riddle::Client::Filter.new 'sphinx_internal_id', filter_value(id), true + } if options[:without_ids] + client end + def star_query(query, custom_token = nil) + token = custom_token.is_a?(Regexp) ? custom_token : /\w+/u + + query.gsub(/("#{token}(.*?#{token})?"|(?![!-])#{token})/u) do + pre, proper, post = $`, $&, $' + is_operator = pre.match(%r{(\W|^)[@~/]\Z}) # E.g. "@foo", "/2", "~3", but not as part of a token + is_quote = proper.starts_with?('"') && proper.ends_with?('"') # E.g. "foo bar", with quotes + has_star = pre.ends_with?("*") || post.starts_with?("*") + if is_operator || is_quote || has_star + proper + else + "*#{proper}*" + end + end + end + def filter_value(value) case value when Range @@ -380,11 +496,11 @@ module ThinkingSphinx # and filters. # def search_conditions(klass, conditions={}) - attributes = klass ? klass.indexes.collect { |index| + attributes = klass ? klass.sphinx_indexes.collect { |index| index.attributes.collect { |attrib| attrib.unique_name } }.flatten : [] - search_string = "" + search_string = [] filters = [] conditions.each do |key,val| @@ -393,11 +509,11 @@ module ThinkingSphinx key.to_s, filter_value(val) ) else - search_string << "@#{key} #{val} " + search_string << "@#{key} #{val}" end end - return search_string, filters + return search_string.join(' '), filters end # Return the appropriate latitude and longitude values, depending on @@ -405,15 +521,15 @@ module ThinkingSphinx # there's actually any values. # def anchor_conditions(klass, options) - attributes = klass ? klass.indexes.collect { |index| + attributes = klass ? klass.sphinx_indexes.collect { |index| index.attributes.collect { |attrib| attrib.unique_name } }.flatten : [] - lat_attr = klass ? klass.indexes.collect { |index| + lat_attr = klass ? klass.sphinx_indexes.collect { |index| index.options[:latitude_attr] }.compact.first : nil - lon_attr = klass ? klass.indexes.collect { |index| + lon_attr = klass ? klass.sphinx_indexes.collect { |index| index.options[:longitude_attr] }.compact.first : nil @@ -422,6 +538,7 @@ module ThinkingSphinx lat_attr ||= :latitude if attributes.include?(:latitude) lon_attr = options[:longitude_attr] if options[:longitude_attr] + lon_attr ||= :lng if attributes.include?(:lng) lon_attr ||= :lon if attributes.include?(:lon) lon_attr ||= :long if attributes.include?(:long) lon_attr ||= :longitude if attributes.include?(:longitude) @@ -435,9 +552,9 @@ module ThinkingSphinx end lat && lon ? { - :latitude_attribute => lat_attr, + :latitude_attribute => lat_attr.to_s, :latitude => lat, - :longitude_attribute => lon_attr, + :longitude_attribute => lon_attr.to_s, :longitude => lon } : nil end @@ -447,11 +564,13 @@ module ThinkingSphinx # def set_sort_options!(client, options) klass = options[:class] - fields = klass ? klass.indexes.collect { |index| + fields = klass ? klass.sphinx_indexes.collect { |index| index.fields.collect { |field| field.unique_name } }.flatten : [] - - case order = options[:order] + index_options = klass ? klass.sphinx_index_options : {} + + order = options[:order] || index_options[:order] + case order when Symbol client.sort_mode = :attr_asc if client.sort_mode == :relevance || client.sort_mode.nil? if fields.include?(order) lib/thinking_sphinx/search.rb @@ -1 +1,6 @@ -require 'thinking_sphinx' \ No newline at end of file +require 'thinking_sphinx' +require 'action_controller/dispatcher' + +ActionController::Dispatcher.to_prepare :thinking_sphinx do + ThinkingSphinx::Configuration.instance.load_models +end rails/init.rb @@ -997,4 +997,29 @@ insert into `people` (gender, first_name, middle_initial, last_name, street_addr insert into `people` (gender, first_name, middle_initial, last_name, street_address, city, state, postcode, email, birthday, team_id, team_type) values('female','Sophia','T','Hicks','94 Whitehaven Crescent','Kamma','QLD','4865','Sophia.T.Hicks@dodgit.com','1953/7/22 00:00:00', 3, 'CricketTeam'); insert into `people` (gender, first_name, middle_initial, last_name, street_address, city, state, postcode, email, birthday, team_id, team_type) values('female','Brooke','C','Lyons','4 Stillwater Avenue','Wandana','WA','6532','Brooke.C.Lyons@spambob.com','1983/4/7 00:00:00', 3, 'CricketTeam'); insert into `people` (gender, first_name, middle_initial, last_name, street_address, city, state, postcode, email, birthday, team_id, team_type) values('female','Lydia','N','Simmons','19 Jones Street','Reynolds Neck','TAS','7304','Lydia.N.Simmons@pookmail.com','1972/4/25 00:00:00', 3, 'CricketTeam'); -insert into `people` (gender, first_name, middle_initial, last_name, street_address, city, state, postcode, email, birthday, team_id, team_type) values('female','Isabella','B','Skinner','6 Hart Street','Parkville','NSW','2337','Isabella.B.Skinner@pookmail.com','1965/3/8 00:00:00', 1, 'FootballTeam'); \ No newline at end of file +insert into `people` (gender, first_name, middle_initial, last_name, street_address, city, state, postcode, email, birthday, team_id, team_type) values('female','Isabella','B','Skinner','6 Hart Street','Parkville','NSW','2337','Isabella.B.Skinner@pookmail.com','1965/3/8 00:00:00', 1, 'FootballTeam'); +insert into `alphas` (name) values ('one'); +insert into `alphas` (name) values ('two'); +insert into `alphas` (name) values ('three'); +insert into `alphas` (name) values ('four'); +insert into `alphas` (name) values ('five'); +insert into `alphas` (name) values ('six'); +insert into `alphas` (name) values ('seven'); +insert into `alphas` (name) values ('eight'); +insert into `alphas` (name) values ('nine'); +insert into `alphas` (name) values ('ten'); +insert into `betas` (name) values ('one'); +insert into `betas` (name) values ('two'); +insert into `betas` (name) values ('three'); +insert into `betas` (name) values ('four'); +insert into `betas` (name) values ('five'); +insert into `betas` (name) values ('six'); +insert into `betas` (name) values ('seven'); +insert into `betas` (name) values ('eight'); +insert into `betas` (name) values ('nine'); +insert into `betas` (name) values ('ten'); +insert into `animals` (name, type) values ('rogue', 'Cat'); +insert into `animals` (name, type) values ('nat', 'Cat'); +insert into `animals` (name, type) values ('molly', 'Cat'); +insert into `animals` (name, type) values ('jasper', 'Cat'); +insert into `animals` (name, type) values ('moggy', 'Cat'); spec/fixtures/data.sql @@ -10,13 +10,13 @@ class Person < ActiveRecord::Base indexes team.name, :as => :team_name indexes contacts.phone_number, :as => :phone_numbers - has [first_name, middle_initial, last_name], :as => :name - has team.name, :as => :team_name + has [first_name, middle_initial, last_name], :as => :name_sort + has team.name, :as => :team_name_sort has [:id, :team_id], :as => :ids has team(:id), :as => :team_id - has contacts.phone_number, :as => :phone_numbers + has contacts.phone_number, :as => :phone_number_sort has contacts(:id), :as => :contact_ids has birthday @@ -49,9 +49,35 @@ end class Friendship < ActiveRecord::Base belongs_to :person - belongs_to :friend, :class_name => "Person" + belongs_to :friend, :class_name => "Person", :foreign_key => :friend_id define_index do has person_id, friend_id end end + +class Alpha < ActiveRecord::Base + define_index do + indexes :name, :sortable => true + + set_property :field_weights => {"name" => 10} + end +end + +class Beta < ActiveRecord::Base + define_index do + indexes :name, :sortable => true + + set_property :delta => true + end +end + +class Animal < ActiveRecord::Base + define_index do + indexes name, :sortable => true + end +end + +class Cat < Animal + # +end \ No newline at end of file spec/fixtures/models.rb @@ -2,16 +2,16 @@ DROP TABLE IF EXISTS `people`; CREATE TABLE `people` ( `id` int(11) NOT NULL auto_increment, - `first_name` varchar(50) NOT NULL, - `middle_initial` varchar(10) NOT NULL, - `last_name` varchar(50) NOT NULL, - `gender` varchar(10) NOT NULL, - `street_address` varchar(200) NOT NULL, - `city` varchar(100) NOT NULL, - `state` varchar(100) NOT NULL, - `postcode` varchar(10) NOT NULL, - `email` varchar(100) NOT NULL, - `birthday` datetime NOT NULL, + `first_name` varchar(50) NULL, + `middle_initial` varchar(10) NULL, + `last_name` varchar(50) NULL, + `gender` varchar(10) NULL, + `street_address` varchar(200) NULL, + `city` varchar(100) NULL, + `state` varchar(100) NULL, + `postcode` varchar(10) NULL, + `email` varchar(100) NULL, + `birthday` datetime NULL, `team_id` int(11) NULL, `team_type` varchar(50) NULL, `type` varchar(50) NULL, @@ -57,3 +57,30 @@ CREATE TABLE `contacts` ( `person_id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `alphas`; + +CREATE TABLE `alphas` ( + `id` int(11) NOT NULL auto_increment, + `name` varchar(50) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `betas`; + +CREATE TABLE `betas` ( + `id` int(11) NOT NULL auto_increment, + `name` varchar(50) NOT NULL, + `delta` tinyint(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `animals`; + +CREATE TABLE `animals` ( + `id` int(11) NOT NULL auto_increment, + `name` varchar(50) NOT NULL, + `type` varchar(50) NOT NULL, + `delta` tinyint(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file spec/fixtures/structure.sql @@ -2,6 +2,7 @@ $:.unshift File.dirname(__FILE__) + '/../lib' require 'rubygems' require 'fileutils' +require 'ginger' require 'not_a_mock' require 'will_paginate' @@ -27,8 +28,20 @@ Spec::Runner.configure do |config| FileUtils.mkdir_p "#{Dir.pwd}/#{path}" end - sphinx.setup_sphinx - sphinx.start + @sphinx = sphinx + + ThinkingSphinx.updates_enabled = true + ThinkingSphinx.deltas_enabled = true + + ThinkingSphinx::Configuration.instance.reset + ThinkingSphinx::Configuration.instance.database_yml_file = "spec/fixtures/sphinx/database.yml" + + # Ensure after_commit plugin is loaded correctly + Object.subclasses_of(ActiveRecord::ConnectionAdapters::AbstractAdapter).each { |klass| + unless klass.ancestors.include?(AfterCommit::ConnectionAdapters) + klass.send(:include, AfterCommit::ConnectionAdapters) + end + } end config.after :each do @@ -37,8 +50,6 @@ Spec::Runner.configure do |config| end config.after :all do - sphinx.stop - FileUtils.rm_r "#{Dir.pwd}/tmp" rescue nil end end spec/spec_helper.rb @@ -43,13 +43,13 @@ class SphinxHelper File.open("spec/fixtures/data.sql") { |f| while line = f.gets - ActiveRecord::Base.connection.execute line + ActiveRecord::Base.connection.execute line unless line.blank? end } end def setup_sphinx - @configuration = ThinkingSphinx::Configuration.new + @configuration = ThinkingSphinx::Configuration.instance.reset File.open("spec/fixtures/sphinx/database.yml", "w") do |file| YAML.dump({@configuration.environment => { :adapter => 'mysql', spec/sphinx_helper.rb @@ -1,136 +1,59 @@ require 'spec/spec_helper' describe "ThinkingSphinx::ActiveRecord::Delta" do - describe "after_commit callback" do - before :each do - Person.stub_method(:write_inheritable_array => true) - end - - # This spec only passes with ActiveRecord 2.0.2 or earlier. - # it "should add callbacks" do - # Person.after_commit :toggle_delta - # - # Person.should have_received(:write_inheritable_array).with( - # :after_commit, [:toggle_delta] - # ) - # end - - it "should have an after_commit method by default" do - Person.instance_methods.should include("after_commit") - end - end - - describe "save_with_after_commit_callback method" do - before :each do - @person = Person.new - @person.stub_methods( - :save_without_after_commit_callback => true, - :callback => true - ) - end - - it "should call the normal save method" do - @person.save - - @person.should have_received(:save_without_after_commit_callback) - end + it "should call the toggle_delta method after a save" do + @beta = Beta.new(:name => 'beta') + @beta.stub_method(:toggle_delta => true) - it "should call the callbacks if the save was successful" do - @person.save - - @person.should have_received(:callback).with(:after_commit) - end + @beta.save - it "shouldn't call the callbacks if the save failed" do - @person.stub_method(:save_without_after_commit_callback => false) - - @person.save - - @person.should_not have_received(:callback) - end - - it "should return the normal save's result" do - @person.save.should be_true - - @person.stub_method(:save_without_after_commit_callback => false) - - @person.save.should be_false - end + @beta.should have_received(:toggle_delta) end - describe "save_with_after_commit_callback! method" do - before :each do - @person = Person.new - @person.stub_methods( - :save_without_after_commit_callback! => true, - :callback => true - ) - end + it "should call the toggle_delta method after a save!" do + @beta = Beta.new(:name => 'beta') + @beta.stub_method(:toggle_delta => true) - it "should call the normal save! method" do - @person.save! - - @person.should have_received(:save_without_after_commit_callback!) - end + @beta.save! - it "should call the callbacks if the save! was successful" do - @person.save! - - @person.should have_received(:callback).with(:after_commit) - end - - it "shouldn't call the callbacks if the save! failed" do - @person.stub_method(:save_without_after_commit_callback! => false) - - @person.save! - - @person.should_not have_received(:callback) - end - - it "should return the normal save's result" do - @person.save!.should be_true - - @person.stub_method(:save_without_after_commit_callback! => false) - - @person.save!.should be_false - end + @beta.should have_received(:toggle_delta) end - - describe "destroy_with_after_commit_callback method" do + + describe "suspended_delta method" do before :each do - @person = Person.new - @person.stub_methods( - :destroy_without_after_commit_callback => true, - :callback => true + ThinkingSphinx.stub_method(:deltas_enabled? => true) + end + + it "should execute the argument block with deltas disabled" do + ThinkingSphinx.should_receive(:deltas_enabled=).once.with(false) + ThinkingSphinx.should_receive(:deltas_enabled=).once.with(true) + lambda { Person.suspended_delta { raise 'i was called' } }.should( + raise_error(Exception) ) end - - it "should call the normal destroy method" do - @person.destroy - - @person.should have_received(:destroy_without_after_commit_callback) + + it "should restore deltas_enabled to its original setting" do + ThinkingSphinx.stub_method(:deltas_enabled? => false) + ThinkingSphinx.should_receive(:deltas_enabled=).twice.with(false) + Person.suspended_delta { 'no-op' } end - - it "should call the callbacks if the destroy was successful" do - @person.destroy - - @person.should have_received(:callback).with(:after_commit) + + it "should restore deltas_enabled to its original setting even if there was an exception" do + ThinkingSphinx.stub_method(:deltas_enabled? => false) + ThinkingSphinx.should_receive(:deltas_enabled=).twice.with(false) + lambda { Person.suspended_delta { raise 'bad error' } }.should( + raise_error(Exception) + ) end - - it "shouldn't call the callbacks if the destroy failed" do - @person.stub_method(:destroy_without_after_commit_callback => false) - - @person.destroy - - @person.should_not have_received(:callback) + + it "should reindex by default after the code block is run" do + Person.should_receive(:index_delta) + Person.suspended_delta { 'no-op' } end - it "should return the normal save's result" do - @person.destroy.should be_true - - @person.stub_method(:destroy_without_after_commit_callback => false) - - @person.destroy.should be_false + it "should not reindex after the code block if false is passed in" do + Person.should_not_receive(:index_delta) + Person.suspended_delta(false) { 'no-op' } end end @@ -149,8 +72,9 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do ThinkingSphinx::Configuration.stub_method(:environment => "spec") ThinkingSphinx.stub_method(:deltas_enabled? => true) - @person = Person.new - @person.stub_method(:system => true, :in_core_index? => false) + @person = Person.find(:first) + Person.stub_method(:system => true) + @person.stub_method(:in_core_index? => false) @client = Riddle::Client.stub_instance(:update => true) Riddle::Client.stub_method(:new => @client) @@ -161,7 +85,7 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do @person.send(:index_delta) - @person.should_not have_received(:system) + Person.should_not have_received(:system) @client.should_not have_received(:update) end @@ -170,23 +94,24 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do @person.send(:index_delta) - @person.should_not have_received(:system) + Person.should_not have_received(:system) end it "shouldn't index if the environment is 'test'" do ThinkingSphinx.unstub_method(:deltas_enabled?) + ThinkingSphinx.deltas_enabled = nil ThinkingSphinx::Configuration.stub_method(:environment => "test") @person.send(:index_delta) - @person.should_not have_received(:system) + Person.should_not have_received(:system) end it "should call indexer for the delta index" do @person.send(:index_delta) - @person.should have_received(:system).with( - "indexer --config #{ThinkingSphinx::Configuration.new.config_file} --rotate person_delta" + Person.should have_received(:system).with( + "#{ThinkingSphinx::Configuration.instance.bin_path}indexer --config #{ThinkingSphinx::Configuration.instance.config_file} --rotate person_delta" ) end spec/unit/thinking_sphinx/active_record/delta_spec.rb @@ -6,7 +6,7 @@ describe 'ThinkingSphinx::ActiveRecord::HasManyAssociation' do Friendship.stub_method(:search => true) @person = Person.find(:first) - @index = Friendship.indexes.first + @index = Friendship.sphinx_indexes.first end it "should raise an error if the required attribute doesn't exist" do @@ -31,7 +31,7 @@ describe 'ThinkingSphinx::ActiveRecord::HasManyAssociation' do Person.stub_method(:search => true) @person = Person.find(:first) - @index = Person.indexes.first + @index = Person.sphinx_indexes.first end it "should raise an error if the required attribute doesn't exist" do spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb @@ -1,6 +1,15 @@ require 'spec/spec_helper' describe "ThinkingSphinx::ActiveRecord" do + before :all do + @sphinx.setup_sphinx + @sphinx.start + end + + after :all do + @sphinx.stop + end + describe "define_index method" do before :each do module TestModule @@ -36,7 +45,7 @@ describe "ThinkingSphinx::ActiveRecord" do it "should add a new index to the model" do TestModule::TestModel.define_index do; end - TestModule::TestModel.indexes.length.should == 1 + TestModule::TestModel.sphinx_indexes.length.should == 1 end it "should add to ThinkingSphinx.indexed_models if the model doesn't already exist in the array" do @@ -106,22 +115,24 @@ describe "ThinkingSphinx::ActiveRecord" do person = Person.find(:first) person.in_core_index?.should == :searching_for_id - Person.should have_received(:search_for_id).with(person.id, "person_core") + Person.should have_received(:search_for_id).with(person.sphinx_document_id, "person_core") end end describe "toggle_deleted method" do before :each do - @configuration = ThinkingSphinx::Configuration.stub_instance( + ThinkingSphinx.stub_method(:sphinx_running? => true) + + @configuration = ThinkingSphinx::Configuration.instance + @configuration.stub_methods( :address => "an address", :port => 123 ) @client = Riddle::Client.stub_instance(:update => true) - @person = Person.new + @person = Person.find(:first) - ThinkingSphinx::Configuration.stub_method(:new => @configuration) Riddle::Client.stub_method(:new => @client) - Person.indexes.each { |index| index.stub_method(:delta? => false) } + Person.sphinx_indexes.each { |index| index.stub_method(:delta? => false) } @person.stub_method(:in_core_index? => true) end @@ -137,7 +148,7 @@ describe "ThinkingSphinx::ActiveRecord" do @person.toggle_deleted @client.should have_received(:update).with( - "person_core", ["sphinx_deleted"], {@person.id => 1} + "person_core", ["sphinx_deleted"], {@person.sphinx_document_id => 1} ) end @@ -147,43 +158,52 @@ describe "ThinkingSphinx::ActiveRecord" do @person.toggle_deleted @client.should_not have_received(:update).with( - "person_core", ["sphinx_deleted"], {@person.id => 1} + "person_core", ["sphinx_deleted"], {@person.sphinx_document_id => 1} ) end + it "shouldn't attempt to update the deleted flag if sphinx isn't running" do + ThinkingSphinx.stub_method(:sphinx_running? => false) + + @person.toggle_deleted + + @person.should_not have_received(:in_core_index?) + @client.should_not have_received(:update) + end + it "should update the delta index's deleted flag if delta indexes are enabled and the instance's delta is true" do ThinkingSphinx.stub_method(:deltas_enabled? => true) - Person.indexes.each { |index| index.stub_method(:delta? => true) } + Person.sphinx_indexes.each { |index| index.stub_method(:delta? => true) } @person.delta = true @person.toggle_deleted @client.should have_received(:update).with( - "person_delta", ["sphinx_deleted"], {@person.id => 1} + "person_delta", ["sphinx_deleted"], {@person.sphinx_document_id => 1} ) end it "should not update the delta index's deleted flag if delta indexes are enabled and the instance's delta is false" do ThinkingSphinx.stub_method(:deltas_enabled? => true) - Person.indexes.each { |index| index.stub_method(:delta? => true) } + Person.sphinx_indexes.each { |index| index.stub_method(:delta? => true) } @person.delta = false @person.toggle_deleted @client.should_not have_received(:update).with( - "person_delta", ["sphinx_deleted"], {@person.id => 1} + "person_delta", ["sphinx_deleted"], {@person.sphinx_document_id => 1} ) end it "should not update the delta index's deleted flag if delta indexes are enabled and the instance's delta is equivalent to false" do ThinkingSphinx.stub_method(:deltas_enabled? => true) - Person.indexes.each { |index| index.stub_method(:delta? => true) } + Person.sphinx_indexes.each { |index| index.stub_method(:delta? => true) } @person.delta = 0 @person.toggle_deleted @client.should_not have_received(:update).with( - "person_delta", ["sphinx_deleted"], {@person.id => 1} + "person_delta", ["sphinx_deleted"], {@person.sphinx_document_id => 1} ) end @@ -192,19 +212,19 @@ describe "ThinkingSphinx::ActiveRecord" do @person.toggle_deleted @client.should_not have_received(:update).with( - "person_delta", ["sphinx_deleted"], {@person.id => 1} + "person_delta", ["sphinx_deleted"], {@person.sphinx_document_id => 1} ) end it "should not update the delta index if delta indexing is disabled" do ThinkingSphinx.stub_method(:deltas_enabled? => false) - Person.indexes.each { |index| index.stub_method(:delta? => true) } + Person.sphinx_indexes.each { |index| index.stub_method(:delta? => true) } @person.delta = true @person.toggle_deleted @client.should_not have_received(:update).with( - "person_delta", ["sphinx_deleted"], {@person.id => 1} + "person_delta", ["sphinx_deleted"], {@person.sphinx_document_id => 1} ) end @@ -213,7 +233,7 @@ describe "ThinkingSphinx::ActiveRecord" do :updates_enabled? => false, :deltas_enabled => true ) - Person.indexes.each { |index| index.stub_method(:delta? => true) } + Person.sphinx_indexes.each { |index| index.stub_method(:delta? => true) } @person.delta = true @person.toggle_deleted @@ -222,15 +242,75 @@ describe "ThinkingSphinx::ActiveRecord" do end end - describe "indexes in the inheritance chain (STI)" do + describe "sphinx_indexes in the inheritance chain (STI)" do it "should hand defined indexes on a class down to its child classes" do - Child.indexes.should include(*Person.indexes) + Child.sphinx_indexes.should include(*Person.sphinx_indexes) end it "should allow associations to other STI models" do - Child.indexes.last.link! - sql = Child.indexes.last.to_sql.gsub('$start', '0').gsub('$end', '100') + Child.sphinx_indexes.last.link! + sql = Child.sphinx_indexes.last.to_sql.gsub('$start', '0').gsub('$end', '100') lambda { Child.connection.execute(sql) }.should_not raise_error(ActiveRecord::StatementInvalid) end end + + it "should return the sphinx document id as expected" do + person = Person.find(:first) + model_count = ThinkingSphinx.indexed_models.length + offset = ThinkingSphinx.indexed_models.index("Person") + + (person.id * model_count + offset).should == person.sphinx_document_id + + alpha = Alpha.find(:first) + offset = ThinkingSphinx.indexed_models.index("Alpha") + + (alpha.id * model_count + offset).should == alpha.sphinx_document_id + + beta = Beta.find(:first) + offset = ThinkingSphinx.indexed_models.index("Beta") + + (beta.id * model_count + offset).should == beta.sphinx_document_id + end + + it "should remove instances from the core index if they're in it" do + Beta.search("three").should_not be_empty + + beta = Beta.find(:first, :conditions => {:name => "three"}) + beta.destroy + + Beta.search("three").should be_empty + end + + it "should remove subclass instances from the core index if they're in it" do + Cat.search("moggy").should_not be_empty + + cat = Cat.find(:first, :conditions => {:name => "moggy"}) + cat.destroy + sleep(1) + + Cat.search("moggy").should be_empty + end + + it "should remove destroyed new instances from the delta index if they're in it" do + beta = Beta.create!(:name => "eleven") + sleep(1) # wait for Sphinx to catch up + + Beta.search("eleven").should_not be_empty + + beta.destroy + + Beta.search("eleven").should be_empty + end + + it "should remove destroyed edited instances from the delta index if they're in it" do + beta = Beta.find(:first, :conditions => {:name => "four"}) + beta.update_attributes(:name => "fourteen") + sleep(1) # wait for Sphinx to catch up + + Beta.search("fourteen").should_not be_empty + + beta.destroy + + Beta.search("fourteen").should be_empty + end end spec/unit/thinking_sphinx/active_record_spec.rb @@ -17,7 +17,7 @@ describe ThinkingSphinx::Attribute do describe "to_select_sql method with MySQL" do before :each do - @index = Person.indexes.first + @index = Person.sphinx_indexes.first @index.link! end @@ -52,7 +52,7 @@ describe ThinkingSphinx::Attribute do describe "to_select_sql method with PostgreSQL" do before :each do - @index = Person.indexes.first + @index = Person.sphinx_indexes.first Person.connection.class.stub_method( :name => "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" ) spec/unit/thinking_sphinx/attribute_spec.rb @@ -6,16 +6,15 @@ describe ThinkingSphinx::Configuration do ThinkingSphinx::Configuration.send(:class_variable_set, :@@environment, nil) ENV["RAILS_ENV"] = nil - ENV["MERB_ENV"] = nil end it "should use the Merb environment value if set" do unless defined?(Merb) module Merb; end end - + ThinkingSphinx::Configuration.stub_method(:defined? => true) - ENV["MERB_ENV"] = "merb_production" + Merb.stub_method(:environment => "merb_production") ThinkingSphinx::Configuration.environment.should == "merb_production" Object.send(:remove_const, :Merb) @@ -34,21 +33,20 @@ describe ThinkingSphinx::Configuration do describe "environment instance method" do it "should return the class method" do ThinkingSphinx::Configuration.stub_method(:environment => "spec") - ThinkingSphinx::Configuration.new.environment.should == "spec" + ThinkingSphinx::Configuration.instance.environment.should == "spec" ThinkingSphinx::Configuration.should have_received(:environment) end end describe "build method" do before :each do - @config = ThinkingSphinx::Configuration.new + @config = ThinkingSphinx::Configuration.instance @config.stub_methods( :load_models => "", :core_index_for_model => "", :delta_index_for_model => "", - :distributed_index_for_model => "", - :create_array_accum => true + :distributed_index_for_model => "" ) ThinkingSphinx.stub_method :indexed_models => ["Person", "Friendship"] @@ -58,21 +56,25 @@ describe ThinkingSphinx::Configuration do } }) + @adapter = ThinkingSphinx::MysqlAdapter.stub_instance( + :setup => true + ) + @person_index_a = ThinkingSphinx::Index.stub_instance( :to_config => "", :adapter => :mysql, :delta? => false, - :name => "person", :model => Person + :name => "person", :model => Person, :adapter_object => @adapter ) @person_index_b = ThinkingSphinx::Index.stub_instance( :to_config => "", :adapter => :mysql, :delta? => false, - :name => "person", :model => Person + :name => "person", :model => Person, :adapter_object => @adapter ) @friendship_index_a = ThinkingSphinx::Index.stub_instance( - :to_config => "", :adapter => :mysql, :delta? => false, - :name => "friendship", :model => Friendship + :to_config => "", :adapter => :mysql, :delta? => false, + :name => "friendship", :model => Friendship, :adapter_object => @adapter ) - Person.stub_method(:indexes => [@person_index_a, @person_index_b]) - Friendship.stub_method(:indexes => [@friendship_index_a]) + Person.stub_method(:sphinx_indexes => [@person_index_a, @person_index_b]) + Friendship.stub_method(:sphinx_indexes => [@friendship_index_a]) FileUtils.mkdir_p "#{@config.app_root}/config" FileUtils.touch "#{@config.app_root}/config/database.yml" @@ -82,29 +84,22 @@ describe ThinkingSphinx::Configuration do ThinkingSphinx.unstub_method :indexed_models YAML.unstub_method :load - Person.unstub_method :indexes - Friendship.unstub_method :indexes + Person.unstub_method :sphinx_indexes + Friendship.unstub_method :sphinx_indexes # # FileUtils.rm_rf "#{@config.app_root}/config" end - # it "should load the models" do - # @config.build - # - # @config.should have_received(:load_models) - # end - - it "should load in the database YAML configuration" do + it "should load the models" do @config.build - YAML.should have_received(:load) + @config.should have_received(:load_models) end - it "should set the mem limit based on the configuration" do + it "should load in the database YAML configuration" do @config.build - file = open(@config.config_file) { |f| f.read } - file.should match(/mem_limit\s+= #{@config.mem_limit}/) + YAML.should have_received(:load) end it "should use the configuration port" do @@ -129,41 +124,20 @@ describe ThinkingSphinx::Configuration do file.should match(/pid_file\s+= #{@config.pid_file}/) end - it "should set max matches from configuration" do - @config.build - - file = open(@config.config_file) { |f| f.read } - file.should match(/max_matches\s+= #{@config.max_matches}/) - end - it "should request configuration for each index for each model" do @config.build @person_index_a.should have_received(:to_config).with( - Person, 0, {:option => "value"}, @config.charset_type, 0 + Person, 0, {:option => "value"}, 0 ) @person_index_b.should have_received(:to_config).with( - Person, 1, {:option => "value"}, @config.charset_type, 0 + Person, 1, {:option => "value"}, 0 ) @friendship_index_a.should have_received(:to_config).with( - Friendship, 0, {:option => "value"}, @config.charset_type, 1 + Friendship, 0, {:option => "value"}, 1 ) end - it "should call create_array_accum if any index uses postgres" do - @person_index_a.stub_method(:adapter => :postgres) - - @config.build - - @config.should have_received(:create_array_accum) - end - - it "should not call create_array_accum if no index uses postgres" do - @config.build - - @config.should_not have_received(:create_array_accum) - end - it "should call core_index_for_model for each model" do @config.build @@ -214,7 +188,6 @@ describe ThinkingSphinx::Configuration do "searchd_file_path" => "searchd/file/path", "address" => "127.0.0.1", "port" => 3333, - "allow_star" => true, "min_prefix_len" => 2, "min_infix_len" => 3, "mem_limit" => "128M", @@ -232,9 +205,12 @@ describe ThinkingSphinx::Configuration do end it "should use the accessors to set the configuration values" do - config = ThinkingSphinx::Configuration.new - @settings["development"].each do |key, value| - config.send(key).should == value + config = ThinkingSphinx::Configuration.instance + config.send(:parse_config) + + %w(config_file searchd_log_file query_log_file pid_file searchd_file_path + address port).each do |key| + config.send(key).should == @settings["development"][key] end end @@ -245,7 +221,7 @@ describe ThinkingSphinx::Configuration do describe "core_index_for_model method" do before :each do - @config = ThinkingSphinx::Configuration.new + @config = ThinkingSphinx::Configuration.instance @model = Person end @@ -263,90 +239,61 @@ describe ThinkingSphinx::Configuration do end it "should include the charset type setting" do - @config.charset_type = "specchars" + @config.index_options[:charset_type] = "specchars" @config.send(:core_index_for_model, @model, "my sources").should match( /charset_type = specchars/ ) end it "should include the morphology setting if it isn't blank" do - @config.morphology = "morph" + @config.index_options[:morphology] = "morph" @config.send(:core_index_for_model, @model, "my sources").should match( /morphology\s+= morph/ ) end it "should not include the morphology setting if it is blank" do - @config.morphology = nil + @config.index_options[:morphology] = nil @config.send(:core_index_for_model, @model, "my sources").should_not match( /morphology\s+=/ ) - @config.morphology = "" + @config.index_options[:morphology] = "" @config.send(:core_index_for_model, @model, "my sources").should_not match( /morphology\s+=/ ) end it "should include the charset_table value if it isn't nil" do - @config.charset_table = "table_chars" + @config.index_options[:charset_table] = "table_chars" @config.send(:core_index_for_model, @model, "my sources").should match( /charset_table\s+= table_chars/ ) end it "should not set the charset_table value if it is nil" do - @config.charset_table = nil + @config.index_options[:charset_table] = nil @config.send(:core_index_for_model, @model, "my sources").should_not match( /charset_table\s+=/ ) end it "should set the ignore_chars value if it isn't nil" do - @config.ignore_chars = "ignorable" + @config.index_options[:ignore_chars] = "ignorable" @config.send(:core_index_for_model, @model, "my sources").should match( /ignore_chars\s+= ignorable/ ) end it "should not set the ignore_chars value if it is nil" do - @config.ignore_chars = nil + @config.index_options[:ignore_chars] = nil @config.send(:core_index_for_model, @model, "my sources").should_not match( /ignore_chars\s+=/ ) end - it "should include the star-related settings when allow_star is true" do - @config.allow_star = true - @config.min_prefix_len = 1 - text = @config.send(:core_index_for_model, @model, "my sources") - - text.should match(/enable_star\s+= 1/) - text.should match(/min_prefix_len\s+= 1/) - # text.should match(/min_infix_len\s+= 1/) - end - - it "should use the configuration's infix and prefix length values if set" do - @config.allow_star = true - @config.min_prefix_len = 3 - @config.min_infix_len = 2 - text = @config.send(:core_index_for_model, @model, "my sources") - - text.should match(/min_prefix_len\s+= 3/) - # text.should match(/min_infix_len\s+= 2/) - end - - it "should not include the star-related settings when allow_star is false" do - @config.allow_star = false - text = @config.send(:core_index_for_model, @model, "my sources") - - text.should_not match(/enable_star\s+=/) - text.should_not match(/min_prefix_len\s+=/) - text.should_not match(/min_infix_len\s+=/) - end - it "should set prefix_fields if any fields are flagged explicitly" do - @model.indexes.first.stub_methods( + @model.sphinx_indexes.first.stub_methods( :prefix_fields => [ ThinkingSphinx::Field.stub_instance(:unique_name => "a"), ThinkingSphinx::Field.stub_instance(:unique_name => "b"), @@ -371,7 +318,7 @@ describe ThinkingSphinx::Configuration do end it "should set infix_fields if any fields are flagged explicitly" do - @model.indexes.first.stub_methods( + @model.sphinx_indexes.first.stub_methods( :prefix_fields => [ ThinkingSphinx::Field.stub_instance(:unique_name => "a"), ThinkingSphinx::Field.stub_instance(:unique_name => "b"), @@ -396,23 +343,25 @@ describe ThinkingSphinx::Configuration do end it "should include html_strip if value is set" do - @config.html_strip = 1 + @config.index_options[:html_strip] = 1 text = @config.send(:core_index_for_model, @model, "my sources") text.should match(/html_strip\s+= 1/) end it "shouldn't include html_strip if value is not set" do + @config.index_options.delete :html_strip text = @config.send(:core_index_for_model, @model, "my sources") text.should_not match(/html_strip/) end it "should include html_remove_elements if values are set" do - @config.html_remove_elements = 'script' + @config.index_options[:html_remove_elements] = 'script' text = @config.send(:core_index_for_model, @model, "my sources") text.should match(/html_remove_elements\s+= script/) end it "shouldn't include html_remove_elements if no values are set" do + @config.index_options.delete :html_remove_elements text = @config.send(:core_index_for_model, @model, "my sources") text.should_not match(/html_remove_elements/) end @@ -420,7 +369,7 @@ describe ThinkingSphinx::Configuration do describe "delta_index_for_model method" do before :each do - @config = ThinkingSphinx::Configuration.new + @config = ThinkingSphinx::Configuration.instance @model = Person end @@ -446,7 +395,7 @@ describe ThinkingSphinx::Configuration do describe "distributed_index_for_model method" do before :each do - @config = ThinkingSphinx::Configuration.new + @config = ThinkingSphinx::Configuration.instance @model = Person end @@ -473,7 +422,7 @@ describe ThinkingSphinx::Configuration do /local = person_delta/ ) - @model.indexes.first.stub_method(:delta? => true) + @model.sphinx_indexes.first.stub_method(:delta? => true) @config.send(:distributed_index_for_model, @model).should match( /local = person_delta/ ) @@ -487,8 +436,77 @@ describe ThinkingSphinx::Configuration do ) end end + + describe "initialisation" do + it "should have a default bin_path of nothing" do + ThinkingSphinx::Configuration.instance.bin_path.should == "" + end + + it "should append a / to bin_path if one is supplied" do + @settings = { + "development" => { + "bin_path" => "path/to/somewhere" + } + } + + open("#{RAILS_ROOT}/config/sphinx.yml", "w") do |f| + f.write YAML.dump(@settings) + end + + ThinkingSphinx::Configuration.instance.send(:parse_config) + ThinkingSphinx::Configuration.instance.bin_path.should match(/\/$/) + end + end + + it "should insert set searchd options into the configuration file" do + config = ThinkingSphinx::Configuration.instance + ThinkingSphinx::Configuration::SearchdOptions.each do |option| + config.searchd_options[option.to_sym] = "something" + config.build + + file = open(config.config_file) { |f| f.read } + file.should match(/#{option}\s+= something/) + + config.searchd_options[option.to_sym] = nil + end + end + + it "should insert set indexer options into the configuration file" do + config = ThinkingSphinx::Configuration.instance + ThinkingSphinx::Configuration::IndexerOptions.each do |option| + config.indexer_options[option.to_sym] = "something" + config.build + + file = open(config.config_file) { |f| f.read } + file.should match(/#{option}\s+= something/) + + config.indexer_options[option.to_sym] = nil + end + end - describe "create_array_accum method" do - it "should create the array_accum method on PostgreSQL" + it "should insert set index options into the configuration file" do + config = ThinkingSphinx::Configuration.instance + ThinkingSphinx::Configuration::IndexOptions.each do |option| + config.index_options[option.to_sym] = "something" + config.build + + file = open(config.config_file) { |f| f.read } + file.should match(/#{option}\s+= something/) + + config.index_options[option.to_sym] = nil + end end + + it "should insert set source options into the configuration file" do + config = ThinkingSphinx::Configuration.instance + ThinkingSphinx::Configuration::SourceOptions.each do |option| + config.source_options[option.to_sym] = "something" + config.build + + file = open(config.config_file) { |f| f.read } + file.should match(/#{option}\s+= something/) + + config.source_options[option.to_sym] = nil + end + end end \ No newline at end of file spec/unit/thinking_sphinx/configuration_spec.rb @@ -17,7 +17,7 @@ describe ThinkingSphinx::Field do describe "to_select_sql method with MySQL" do before :each do - @index = Person.indexes.first + @index = Person.sphinx_indexes.first @index.link! end @@ -36,7 +36,7 @@ describe ThinkingSphinx::Field do describe "to_select_sql method with PostgreSQL" do before :each do - @index = Person.indexes.first + @index = Person.sphinx_indexes.first Person.connection.class.stub_method( :name => "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" ) spec/unit/thinking_sphinx/field_spec.rb @@ -11,6 +11,7 @@ describe ThinkingSphinx::Index::Builder do @builder.fields.should == [] @builder.attributes.should == [] @builder.conditions.should == [] + @builder.groupings.should == [] @builder.properties.should == {} end end spec/unit/thinking_sphinx/index/builder_spec.rb @@ -28,7 +28,7 @@ describe ThinkingSphinx::Index do end it "should call link!" do - @index.to_config(Person, 0, @database, "utf-8", 0) + @index.to_config(Person, 0, @database, 0) @index.should have_received(:link!) end @@ -36,17 +36,17 @@ describe ThinkingSphinx::Index do it "should raise an exception if the adapter isn't mysql or postgres" do @index.stub_method(:adapter => :sqlite) - lambda { @index.to_config(Person, 0, @database, "utf-8", 0) }.should raise_error + lambda { @index.to_config(Person, 0, @database, 0) }.should raise_error end it "should set the core source name to {model}_{index}_core" do - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_core/ ) end it "should include the database config supplied" do - conf = @index.to_config(Person, 0, @database, "utf-8", 0) + conf = @index.to_config(Person, 0, @database, 0) conf.should match(/type\s+= mysql/) conf.should match(/sql_host\s+= localhost/) conf.should match(/sql_user\s+= username/) @@ -57,86 +57,99 @@ describe ThinkingSphinx::Index do it "should use 'user' if 'username' doesn't exist in database configuration" do conf = @index.to_config(Person, 0, @database.except(:username).merge(:user => "username"), - "utf-8", 0 + 0 ) conf.should match(/sql_user\s+= username/) end it "should include the database socket if set" do - conf = @index.to_config(Person, 0, @database.merge(:socket => "dbsocket"), "utf-8", 0) + conf = @index.to_config(Person, 0, @database.merge(:socket => "dbsocket"), 0) conf.should match(/sql_sock\s+= dbsocket/) end it "should not include the database socket if not set" do - conf = @index.to_config(Person, 0, @database, "utf-8", 0) + conf = @index.to_config(Person, 0, @database, 0) conf.should_not match(/sql_sock/) end + it "should include the database port if set" do + conf = @index.to_config(Person, 0, @database.merge(:port => "dbport"), 0) + conf.should match(/sql_port\s+= dbport/) + end + + it "should not include the database socket if not set" do + conf = @index.to_config(Person, 0, @database, 0) + conf.should_not match(/sql_port/) + end + it "should have a pre query 'SET NAMES utf8' if using mysql and utf8 charset" do - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.options[:charset_type] = "utf-8" + @index.to_config(Person, 0, @database, 0).should match( /sql_query_pre\s+= SET NAMES utf8/ ) @index.stub_method(:delta? => true) - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_delta.+sql_query_pre\s+= SET NAMES utf8/m ) + @index.options[:charset_type] = "non-utf-8" @index.stub_method(:delta? => false) - @index.to_config(Person, 0, @database, "non-utf-8", 0).should_not match( + @index.to_config(Person, 0, @database, 0).should_not match( /SET NAMES utf8/ ) + @index.options[:charset_type] = "utf-8" @index.stub_method(:adapter => :postgres) - @index.to_config(Person, 0, @database, "utf-8", 0).should_not match( + @index.to_config(Person, 0, @database, 0).should_not match( /SET NAMES utf8/ ) end it "should use the pre query from the index" do - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /sql_query_pre\s+= sql_query_pre/ ) end it "should not set group_concat_max_len if not specified" do - @index.to_config(Person, 0, @database, "utf-8", 0).should_not match( + @index.to_config(Person, 0, @database, 0).should_not match( /group_concat_max_len/ ) end it "should set group_concat_max_len if specified" do @index.options.merge! :group_concat_max_len => 2056 - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /sql_query_pre\s+= SET SESSION group_concat_max_len = 2056/ ) @index.stub_method(:delta? => true) - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_delta.+sql_query_pre\s+= SET SESSION group_concat_max_len = 2056/m ) end it "should use the main query from the index" do - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /sql_query\s+= SQL/ ) end it "should use the range query from the index" do - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /sql_query_range\s+= sql_query_range/ ) end it "should use the info query from the index" do - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /sql_query_info\s+= sql_query_info/ ) end it "should include the attribute sources" do - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /attr a\n\s+attr b/ ) end @@ -144,13 +157,13 @@ describe ThinkingSphinx::Index do it "should add a delta index with name {model}_{index}_delta if requested" do @index.stub_method(:delta? => true) - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_delta/ ) end it "should not add a delta index unless requested" do - @index.to_config(Person, 0, @database, "utf-8", 0).should_not match( + @index.to_config(Person, 0, @database, 0).should_not match( /source person_0_delta/ ) end @@ -158,7 +171,7 @@ describe ThinkingSphinx::Index do it "should have the delta index inherit from the core index" do @index.stub_method(:delta? => true) - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_delta : person_0_core/ ) end @@ -166,7 +179,7 @@ describe ThinkingSphinx::Index do it "should redefine the main query for the delta index" do @index.stub_method(:delta? => true) - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_delta.+sql_query\s+= SQL/m ) end @@ -174,7 +187,7 @@ describe ThinkingSphinx::Index do it "should redefine the range query for the delta index" do @index.stub_method(:delta? => true) - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_delta.+sql_query_range\s+= sql_query_range/m ) end @@ -182,12 +195,21 @@ describe ThinkingSphinx::Index do it "should redefine the pre query for the delta index" do @index.stub_method(:delta? => true) - @index.to_config(Person, 0, @database, "utf-8", 0).should match( + @index.to_config(Person, 0, @database, 0).should match( /source person_0_delta.+sql_query_pre\s+=\s*\n/m ) end end + describe "to_sql method" do + it "should include explicit groupings if requested" do + @index = ThinkingSphinx::Index.new(Person) + + @index.groupings << "custom_sql" + @index.to_sql.should match(/GROUP BY.+custom_sql/) + end + end + describe "to_sql_query_range method" do before :each do @index = ThinkingSphinx::Index.new(Person) @@ -248,8 +270,8 @@ describe ThinkingSphinx::Index do describe "empty? method" do before :each do - @index = ThinkingSphinx::Index.new(Person) - config = ThinkingSphinx::Configuration.new + @index = ThinkingSphinx::Index.new(Contact) + config = ThinkingSphinx::Configuration.instance `mkdir -p #{config.searchd_file_path}` @file_path = "#{config.searchd_file_path}/#{@index.name}_core.spa" @@ -274,12 +296,22 @@ describe ThinkingSphinx::Index do end it "should check the delta files if specified" do - @index.should be_empty(:delta) + delta_path = @file_path.gsub(/_core.spa$/, '_delta.spa') - `echo 'a' > #{@file_path.gsub(/_core.spa$/, '_delta.spa')}` + @index.should be_empty(:delta) + `echo 'a' > #{delta_path}` @index.should_not be_empty(:delta) - `rm #{@file_path}` if File.exists?(@file_path.gsub(/_core.spa$/, '_delta.spa')) + FileUtils.rm(delta_path) + end + end + + describe "initialize_from_builder method" do + it "should copy groupings across from the builder to the index" do + @index = ThinkingSphinx::Index.new(Person) do + group_by "custom_grouping" + end + @index.groupings.should include("custom_grouping") end end end \ No newline at end of file spec/unit/thinking_sphinx/index_spec.rb @@ -2,6 +2,64 @@ require 'spec/spec_helper' require 'will_paginate/collection' describe ThinkingSphinx::Search do + before :all do + @sphinx.setup_sphinx + @sphinx.start + end + + after :all do + @sphinx.stop + end + + describe "search method" do + before :each do + @client = Riddle::Client.stub_instance( + :filters => [], + :filters= => true, + :id_range= => true, + :sort_mode => :asc, + :limit => 5, + :offset= => 0, + :sort_mode= => true, + :query => { + :matches => [], + :total => 50 + } + ) + + ThinkingSphinx::Search.stub_methods( + :client_from_options => @client, + :search_conditions => ["", []] + ) + end + + describe ":star option" do + + it "should not apply by default" do + ThinkingSphinx::Search.search "foo bar" + @client.should have_received(:query).with("foo bar") + end + + it "should apply when passed, and handle full extended syntax" do + input = %{a b* c (d | e) 123 5&6 (f_f g) !h "i j" "k l"~10 "m n"/3 @o p -(q|r)} + expected = %{*a* b* *c* (*d* | *e*) *123* *5*&*6* (*f_f* *g*) !*h* "i j" "k l"~10 "m n"/3 @o *p* -(*q*|*r*)} + ThinkingSphinx::Search.search input, :star => true + @client.should have_received(:query).with(expected) + end + + it "should default to /\w+/ as token" do + ThinkingSphinx::Search.search "foo@bar.com", :star => true + @client.should have_received(:query).with("*foo*@*bar*.*com*") + end + + it "should honour custom token" do + ThinkingSphinx::Search.search "foo@bar.com -foo-bar", :star => /[\w@.-]+/u + @client.should have_received(:query).with("*foo@bar.com* -*foo-bar*") + end + + end + end + describe "search_for_id method" do before :each do @client = Riddle::Client.stub_instance( @@ -46,128 +104,62 @@ describe ThinkingSphinx::Search do describe "instance_from_result method" do before :each do - Person.stub_method(:find => true) - ThinkingSphinx::Search.stub_method(:class_from_crc => Person) - end - - after :each do - Person.unstub_method(:find) - ThinkingSphinx::Search.unstub_method(:class_from_crc) + Person.track_methods(:find) end it "should honour the :include option" do - ThinkingSphinx::Search.send( - :instance_from_result, - { - :doc => 1, :attributes => { - "sphinx_internal_id" => 2, "class_crc" => 123 - } - }, - {:include => :assoc} - ) - - Person.should have_received(:find).with(2, :include => :assoc, :select => nil) + ellie = ThinkingSphinx::Search.search("Ellie Ford", :include => :contacts).first + pending + Person.should have_received(:find).with(ellie.id, :include => :contacts, :select => nil) end it "should honour the :select option" do - ThinkingSphinx::Search.send( - :instance_from_result, - { - :doc => 1, :attributes => { - "sphinx_internal_id" => 2, "class_crc" => 123 - } - }, - {:select => :columns} - ) - - Person.should have_received(:find).with(2, :include => nil, :select => :columns) + ellie = ThinkingSphinx::Search.search("Ellie Ford", :select => "*").first + pending + Person.should have_received(:find).with(ellie.id, :include => nil, :select => "*") end end describe "instances_from_results method" do before :each do - @person_a = Person.stub_instance - @person_b = Person.stub_instance - @person_c = Person.stub_instance - - @results = [ - {:attributes => {"sphinx_internal_id" => @person_a.id}}, - {:attributes => {"sphinx_internal_id" => @person_b.id}}, - {:attributes => {"sphinx_internal_id" => @person_c.id}} - ] + Person.track_methods(:find) - Person.stub_method( - :find => [@person_c, @person_a, @person_b] - ) - ThinkingSphinx::Search.stub_method(:instance_from_result => true) + @ellie_ids = Person.search_for_ids "Ellie" end - - after :each do - Person.unstub_method(:find) - ThinkingSphinx::Search.unstub_method(:instance_from_result) - end - - it "should pass calls to instance_from_result if no class given" do - ThinkingSphinx::Search.send( - :instances_from_results, @results - ) - - ThinkingSphinx::Search.should have_received(:instance_from_result).with( - {:attributes => {"sphinx_internal_id" => @person_a.id}}, {} - ) - ThinkingSphinx::Search.should have_received(:instance_from_result).with( - {:attributes => {"sphinx_internal_id" => @person_b.id}}, {} - ) - ThinkingSphinx::Search.should have_received(:instance_from_result).with( - {:attributes => {"sphinx_internal_id" => @person_c.id}}, {} - ) - end - + it "should call a find on all ids for the class" do - ThinkingSphinx::Search.send( - :instances_from_results, @results, {}, Person - ) + Person.search "Ellie" Person.should have_received(:find).with( :all, - :conditions => {:id => [@person_a.id, @person_b.id, @person_c.id]}, + :conditions => {:id => @ellie_ids}, :include => nil, :select => nil ) end it "should honour the :include option" do - ThinkingSphinx::Search.send( - :instances_from_results, @results, {:include => :something}, Person - ) + Person.search "Ellie", :include => :contacts Person.should have_received(:find).with( :all, - :conditions => {:id => [@person_a.id, @person_b.id, @person_c.id]}, - :include => :something, + :conditions => {:id => @ellie_ids}, + :include => :contacts, :select => nil ) end it "should honour the :select option" do - ThinkingSphinx::Search.send( - :instances_from_results, @results, {:select => :fields}, Person - ) + Person.search "Ellie", :select => "*" Person.should have_received(:find).with( :all, - :conditions => {:id => [@person_a.id, @person_b.id, @person_c.id]}, + :conditions => {:id => @ellie_ids}, :include => nil, - :select => :fields + :select => "*" ) end - - it "should sort the objects the same as the result set" do - ThinkingSphinx::Search.send( - :instances_from_results, @results, {:select => :fields}, Person - ).should == [@person_a, @person_b, @person_c] - end end describe "count method" do @@ -230,4 +222,67 @@ describe ThinkingSphinx::Search do @results.should be_kind_of(Array) end end + + it "should not return results that have been deleted" do + Alpha.search("one").should_not be_empty + + alpha = Alpha.find(:first, :conditions => {:name => "one"}) + alpha.destroy + + Alpha.search("one").should be_empty + end + + it "should still return edited results using old data if there's no delta" do + Alpha.search("two").should_not be_empty + + alpha = Alpha.find(:first, :conditions => {:name => "two"}) + alpha.update_attributes(:name => "twelve") + + Alpha.search("two").should_not be_empty + end + + it "should not return edited results using old data if there's a delta" do + Beta.search("two").should_not be_empty + + beta = Beta.find(:first, :conditions => {:name => "two"}) + beta.update_attributes(:name => "twelve") + + Beta.search("two").should be_empty + end +end + + +# We'll be removing stuff, so best to set up things afresh. +describe ThinkingSphinx::Search, "handling stale index" do + before :all do + @sphinx.setup_mysql + @sphinx.setup_sphinx + @sphinx.start + end + + after :all do + @sphinx.stop + end + + it "should work" do + Alpha.search().should_not be_empty + + two = Alpha.find(:first, :conditions => {:name => "two"}) + three = Alpha.find(:first, :conditions => {:name => "three"}) + + defaults = { :per_page => 1, :order => 'sphinx_internal_id ASC' } + + Alpha.delete(1) # remove without callbacks + # If the first record has gone missing, you should get a nil back. + Alpha.search(defaults.merge(:retry_stale => false)).should == [nil] + # If you retry once, you find the second item which still exists. + Alpha.search(defaults.merge(:retry_stale => 1)).should == [two] + + Alpha.delete(2) # remove without callbacks + # If the second item has gone as well, you get a nil there too. + Alpha.search(defaults.merge(:retry_stale => 1)).should == [nil] + # But :retry_stale => true (three retries) overcomes that as well. + Alpha.search(defaults.merge(:retry_stale => true)).should == [three] + end + end spec/unit/thinking_sphinx/search_spec.rb @@ -126,4 +126,19 @@ describe ThinkingSphinx do end end end + + it "should detect if sphinx is running" do + ThinkingSphinx.sphinx_running?.should be_false + + @sphinx.setup_sphinx + @sphinx.start + sleep(1) + + ThinkingSphinx.sphinx_running?.should be_true + + @sphinx.stop + sleep(1) + + ThinkingSphinx.sphinx_running?.should be_false + end end spec/unit/thinking_sphinx_spec.rb @@ -3,19 +3,25 @@ require 'fileutils' namespace :thinking_sphinx do task :app_env do Rake::Task[:environment].invoke if defined?(RAILS_ROOT) - Rake::Task[:merb_init].invoke if defined?(Merb) + Rake::Task[:merb_env].invoke if defined?(Merb) + end + + desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings" + task :running_start => :app_env do + Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running? + Rake::Task["thinking_sphinx:start"].invoke end desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings" task :start => :app_env do - config = ThinkingSphinx::Configuration.new + config = ThinkingSphinx::Configuration.instance FileUtils.mkdir_p config.searchd_file_path raise RuntimeError, "searchd is already running." if sphinx_running? Dir["#{config.searchd_file_path}/*.spl"].each { |file| File.delete(file) } - - cmd = "searchd --config #{config.config_file}" + + cmd = "#{config.bin_path}searchd --config #{config.config_file}" puts cmd system cmd @@ -41,15 +47,21 @@ namespace :thinking_sphinx do desc "Generate the Sphinx configuration file using Thinking Sphinx's settings" task :configure => :app_env do - ThinkingSphinx::Configuration.new.build + config = ThinkingSphinx::Configuration.instance + puts "Generating Configuration to #{config.config_file}" + config.build end desc "Index data for Sphinx using Thinking Sphinx's settings" - task :index => [:app_env, :configure] do - config = ThinkingSphinx::Configuration.new - + task :index => :app_env do + config = ThinkingSphinx::Configuration.instance + unless ENV["INDEX_ONLY"] == "true" + puts "Generating Configuration to #{config.config_file}" + config.build + end + FileUtils.mkdir_p config.searchd_file_path - cmd = "indexer --config #{config.config_file} --all" + cmd = "#{config.bin_path}indexer --config #{config.config_file} --all" cmd << " --rotate" if sphinx_running? puts cmd system cmd @@ -57,6 +69,8 @@ namespace :thinking_sphinx do end namespace :ts do + desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings" + task :run => "thinking_sphinx:running_start" desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings" task :start => "thinking_sphinx:start" desc "Stop Sphinx using Thinking Sphinx's settings" @@ -68,11 +82,13 @@ namespace :ts do desc "Restart Sphinx" task :restart => "thinking_sphinx:restart" desc "Generate the Sphinx configuration file using Thinking Sphinx's settings" + task :conf => "thinking_sphinx:configure" + desc "Generate the Sphinx configuration file using Thinking Sphinx's settings" task :config => "thinking_sphinx:configure" end def sphinx_pid - config = ThinkingSphinx::Configuration.new + config = ThinkingSphinx::Configuration.instance if File.exists?(config.pid_file) `cat #{config.pid_file}`[/\d+/] @@ -83,4 +99,4 @@ end def sphinx_running? sphinx_pid && `ps -p #{sphinx_pid} | wc -l`.to_i > 1 -end \ No newline at end of file +end tasks/thinking_sphinx_tasks.rb @@ -1,21 +1,31 @@ +# -*- encoding: utf-8 -*- + Gem::Specification.new do |s| s.name = %q{thinking-sphinx} - s.version = "0.9.9" - - s.specification_version = 2 if s.respond_to? :specification_version= + s.version = "0.9.12" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Pat Allan"] - s.date = %q{2008-08-22} + s.date = %q{2008-10-26} s.description = %q{A concise and easy-to-use Ruby library that connects ActiveRecord to the Sphinx search daemon, managing configuration, indexing and searching.} s.email = %q{pat@freelancing-gods.com} - s.files = ["lib/riddle/client/filter.rb", "lib/riddle/client/message.rb", "lib/riddle/client/response.rb", "lib/riddle/client.rb", "lib/riddle.rb", "lib/test.rb", "lib/thinking_sphinx/active_record/delta.rb", "lib/thinking_sphinx/active_record/has_many_association.rb", "lib/thinking_sphinx/active_record/search.rb", "lib/thinking_sphinx/active_record.rb", "lib/thinking_sphinx/association.rb", "lib/thinking_sphinx/attribute.rb", "lib/thinking_sphinx/collection.rb", "lib/thinking_sphinx/configuration.rb", "lib/thinking_sphinx/field.rb", "lib/thinking_sphinx/index/builder.rb", "lib/thinking_sphinx/index/faux_column.rb", "lib/thinking_sphinx/index.rb", "lib/thinking_sphinx/rails_additions.rb", "lib/thinking_sphinx/search.rb", "lib/thinking_sphinx.rb", "LICENCE", "README", "tasks/thinking_sphinx_tasks.rb", "tasks/thinking_sphinx_tasks.rake", "spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] + s.files = ["lib/thinking_sphinx/active_record/delta.rb", "lib/thinking_sphinx/active_record/has_many_association.rb", "lib/thinking_sphinx/active_record/search.rb", "lib/thinking_sphinx/active_record.rb", "lib/thinking_sphinx/adapters/abstract_adapter.rb", "lib/thinking_sphinx/adapters/mysql_adapter.rb", "lib/thinking_sphinx/adapters/postgresql_adapter.rb", "lib/thinking_sphinx/association.rb", "lib/thinking_sphinx/attribute.rb", "lib/thinking_sphinx/collection.rb", "lib/thinking_sphinx/configuration.rb", "lib/thinking_sphinx/field.rb", "lib/thinking_sphinx/index/builder.rb", "lib/thinking_sphinx/index/faux_column.rb", "lib/thinking_sphinx/index.rb", "lib/thinking_sphinx/models/abstract_model.rb", "lib/thinking_sphinx/models/active_record_model.rb", "lib/thinking_sphinx/models/data_mapper_model.rb", "lib/thinking_sphinx/orm_adapters/abstract_adapter.rb", "lib/thinking_sphinx/orm_adapters/active_record_adapter.rb", "lib/thinking_sphinx/rails_additions.rb", "lib/thinking_sphinx/search.rb", "lib/thinking_sphinx.rb", "LICENCE", "README", "tasks/thinking_sphinx_tasks.rb", "tasks/thinking_sphinx_tasks.rake", "vendor/after_commit", "vendor/after_commit/init.rb", "vendor/after_commit/lib", "vendor/after_commit/lib/after_commit", "vendor/after_commit/lib/after_commit/active_record.rb", "vendor/after_commit/lib/after_commit/connection_adapters.rb", "vendor/after_commit/lib/after_commit.rb", "vendor/after_commit/LICENSE", "vendor/after_commit/Rakefile", "vendor/after_commit/README", "vendor/after_commit/test", "vendor/after_commit/test/after_commit_test.rb", "vendor/riddle", "vendor/riddle/lib", "vendor/riddle/lib/riddle", "vendor/riddle/lib/riddle/client", "vendor/riddle/lib/riddle/client/filter.rb", "vendor/riddle/lib/riddle/client/message.rb", "vendor/riddle/lib/riddle/client/response.rb", "vendor/riddle/lib/riddle/client.rb", "vendor/riddle/lib/riddle.rb", "spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/collection_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] s.has_rdoc = true s.homepage = %q{http://ts.freelancing-gods.com} s.rdoc_options = ["--title", "Thinking Sphinx -- Rails/Merb Sphinx Plugin", "--line-numbers"] s.require_paths = ["lib"] s.rubyforge_project = %q{thinking-sphinx} - s.rubygems_version = %q{1.1.1} + s.rubygems_version = %q{1.3.0} s.summary = %q{A concise and easy-to-use Ruby library that connects ActiveRecord to the Sphinx search daemon, managing configuration, indexing and searching.} - s.test_files = ["spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] + s.test_files = ["spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/collection_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] + + if s.respond_to? :specification_version then + current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION + s.specification_version = 2 + + if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then + else + end + else + end end thinking-sphinx.gemspec lib/riddle.rb lib/riddle/client.rb lib/riddle/client/filter.rb lib/riddle/client/message.rb lib/riddle/client/response.rb lib/test.rb a0cccba1053e2e2a557bb2e065e2142a27895413 a2fc54d841658d1c4d49f53890142d35b1bd1e2d Patrick Lenz patrick+quadpro@limited-overload.de http://github.com/scoop/thinking-sphinx/commit/3f75923bd1ba298218ca6fb9b4da3e6cf878a0de 3f75923bd1ba298218ca6fb9b4da3e6cf878a0de 2008-11-22T13:52:50-08:00 2008-11-22T13:52:50-08:00 Merge commit 'freelancing-god/master' a763afc5c7ba9770ac8b087e8c76db482e3b29ba Patrick Lenz patrick+quadpro@limited-overload.de