contribute.rb features/facets.feature features/step_definitions/facet_steps.rb features/support/db/migrations/create_developers.rb features/support/db/migrations/create_tags.rb features/support/models/developer.rb features/support/models/tag.rb lib/thinking_sphinx/facet.rb lib/thinking_sphinx/facet_collection.rb lib/thinking_sphinx/tasks.rb tasks/distribution.rb tasks/rails.rake tasks/testing.rb @@ -10,6 +10,8 @@ h2. Contributing Fork on GitHub and after you've committed tested patches, send a pull request. +To quickly see if your system is ready to run the thinking sphinx specs, run the contribute.rb script found in the project root directory. Use the following instructions to install any missing requirements. + To get the spec suite running, you will need to install the not-a-mock gem if you don't already have it: git clone git://github.com/freelancing-god/not-a-mock.git @@ -48,7 +50,7 @@ h2. Contributors Since I first released this library, there's been quite a few people who have submitted patches, to my immense gratitude. Others have suggested syntax changes and general improvements. So my thanks to the following people: * Joost Hietbrink -* Jonathon Conway +* Jonathan Conway * Gregory Mirzayantz * Tung Nguyen * Sean Cribbs @@ -99,3 +101,13 @@ Since I first released this library, there's been quite a few people who have su * Thibaut Barrere * Kristopher Chambers * Dmitrij Smalko +* Aleksey Yeschenko +* Lachie Cox +* Lourens Naude +* Tom Davies +* Dan Pickett +* Alex Caudill +* Jim Benton +* John Aughey +* Keith Pitty +* Jeff Talbot README.textile @@ -1,128 +1,4 @@ -begin - require 'spec' -rescue LoadError - require 'rubygems' - require 'spec' -end +require 'rubygems' -require 'rake/rdoctask' -require 'spec/rake/spectask' -require 'rake/gempackagetask' -require 'cucumber/rake/task' - -# allow require of spec/spec_helper -$LOAD_PATH.unshift File.dirname(__FILE__) + '/../' -$LOAD_PATH.unshift File.dirname(__FILE__) + '/lib' - -require 'thinking_sphinx' - -desc 'Generate documentation' -Rake::RDocTask.new(:rdoc) do |rdoc| - rdoc.rdoc_dir = 'rdoc' - rdoc.title = 'Thinking Sphinx - ActiveRecord Sphinx Plugin' - rdoc.options << '--line-numbers' << '--inline-source' - rdoc.rdoc_files.include('README') - rdoc.rdoc_files.include('lib/**/*.rb') -end - -desc "Run the specs under spec" -Spec::Rake::SpecTask.new do |t| - t.spec_files = FileList['spec/**/*_spec.rb'] - t.spec_opts << "-c" -end - -desc "Run all feature-set configurations" -task :features do - # -end - -task :features do |t| - puts "rake features:mysql" - system "rake features:mysql" - puts "rake features:postgresql" - system "rake features:postgresql" -end - -namespace :features do - Cucumber::Rake::Task.new(:mysql, "Run feature-set against MySQL") do |t| - t.cucumber_opts = "--format pretty" - t.step_pattern = [ - "features/support/env", - "features/support/db/mysql", - "features/support/db/active_record", - "features/support/post_database", - "features/step_definitions/**.rb" - ] - end - - Cucumber::Rake::Task.new(:postgresql, "Run feature-set against PostgreSQL") do |t| - t.cucumber_opts = "--format pretty" - t.step_pattern = [ - "features/support/env", - "features/support/db/postgresql", - "features/support/db/active_record", - "features/support/post_database", - "features/step_definitions/**.rb" - ] - end -end - -desc "Generate RCov reports" -Spec::Rake::SpecTask.new(:rcov) do |t| - t.libs << 'lib' - t.spec_files = FileList['spec/**/*_spec.rb'] - t.rcov = true - t.rcov_opts = ['--exclude', 'spec', '--exclude', 'gems', '--exclude', 'riddle'] -end - -spec = Gem::Specification.new do |s| - s.name = "thinking-sphinx" - s.version = ThinkingSphinx::Version::String - s.summary = "A concise and easy-to-use Ruby library that connects ActiveRecord to the Sphinx search daemon, managing configuration, indexing and searching." - s.description = "A concise and easy-to-use Ruby library that connects ActiveRecord to the Sphinx search daemon, managing configuration, indexing and searching." - s.author = "Pat Allan" - s.email = "pat@freelancing-gods.com" - s.homepage = "http://ts.freelancing-gods.com" - s.has_rdoc = true - s.rdoc_options << "--title" << "Thinking Sphinx -- Rails/Merb Sphinx Plugin" << - "--line-numbers" - s.rubyforge_project = "thinking-sphinx" - s.test_files = FileList["spec/**/*_spec.rb"] - s.files = FileList[ - "lib/**/*.rb", - "LICENCE", - "README", - "tasks/**/*.rb", - "tasks/**/*.rake", - "vendor/**/*" - ] -end - -Rake::GemPackageTask.new(spec) do |p| - p.gem_spec = spec - p.need_tar = true - p.need_zip = true -end - -desc "Build gemspec file" -task :build do - File.open('thinking-sphinx.gemspec', 'w') { |f| f.write spec.to_ruby } -end - -desc "Build cucumber.yml file" -task :cucumber_defaults do - default_requires = %w( - --require features/support/env.rb - --require features/support/db/mysql.rb - --require features/support/db/active_record.rb - --require features/support/post_database.rb - ).join(" ") - - step_definitions = FileList["features/step_definitions/**.rb"].collect { |path| - "--require #{path}" - }.join(" ") - - File.open('cucumber.yml', 'w') { |f| - f.write "default: \"#{default_requires} #{step_definitions}\"" - } -end +require 'tasks/distribution' +require 'tasks/testing' \ No newline at end of file Rakefile @@ -1 +1 @@ -default: "--require features/support/env.rb --require features/support/db/mysql.rb --require features/support/db/active_record.rb --require features/support/post_database.rb --require features/step_definitions/alpha_steps.rb --require features/step_definitions/beta_steps.rb --require features/step_definitions/cat_steps.rb --require features/step_definitions/common_steps.rb --require features/step_definitions/datetime_delta_steps.rb --require features/step_definitions/delayed_delta_indexing_steps.rb --require features/step_definitions/find_arguments_steps.rb --require features/step_definitions/gamma_steps.rb --require features/step_definitions/search_steps.rb --require features/step_definitions/sphinx_steps.rb" \ No newline at end of file +default: "--require features/support/env.rb --require features/support/db/mysql.rb --require features/support/db/active_record.rb --require features/support/post_database.rb --require features/step_definitions/alpha_steps.rb --require features/step_definitions/beta_steps.rb --require features/step_definitions/cat_steps.rb --require features/step_definitions/common_steps.rb --require features/step_definitions/datetime_delta_steps.rb --require features/step_definitions/delayed_delta_indexing_steps.rb --require features/step_definitions/facet_steps.rb --require features/step_definitions/find_arguments_steps.rb --require features/step_definitions/gamma_steps.rb --require features/step_definitions/search_steps.rb --require features/step_definitions/sphinx_steps.rb" \ No newline at end of file cucumber.yml @@ -51,5 +51,5 @@ Feature: Datetime Delta Indexing And I search for thirteen Then I should get 1 result - When I search for the specific id of 107 in the theta_core index + When I search for the document id of theta thirteen in the theta_core index Then it should exist \ No newline at end of file features/datetime_deltas.feature @@ -11,10 +11,10 @@ Feature: Searching across multiple model Scenario: Confirming existance of a document id in a given index Given Sphinx is running - When I search for the specific id of 51 in the person_core index + When I search for the document id of alpha one in the alpha_core index Then it should exist - Scenario: Unsuccessfully confirming existance of a document id in a given index + Scenario: Retrieving results from multiple models Given Sphinx is running - When I search for the specific id of 52 in the person_core index - Then it should not exist \ No newline at end of file + When I search for ten + Then I should get 5 results features/searching_across_models.feature @@ -21,6 +21,16 @@ Feature: Searching on a single model When I search for James on first_name And I search for Chamberlain on last_name Then I should get 1 result + + Scenario: Searching on association content + Given Sphinx is running + And I am searching on posts + + When I search for "Waffles" + Then I should get 1 result + + When I search for "Turtle" + Then I should get 1 result Scenario: Searching with a filter Given Sphinx is running @@ -34,6 +44,15 @@ Feature: Searching on a single model When I filter by 2 on width And I filter by 2 on length Then I should get 1 result + + Scenario: Searching to filter multiple values on an MVA + Given Sphinx is running + And I am searching on boxes + When I filter by 11 and 12 on dimensions + Then I should get 2 results + When I clear existing filters + And I filter by both 11 and 12 on dimensions + Then I should get 1 result Scenario: Searching with ordering by attribute Given Sphinx is running features/searching_by_model.feature @@ -3,7 +3,7 @@ Feature: Checking whether Sphinx is running or not Thinking Sphinx Should be able to determine whether Sphinx is running or not - Scenario: Deleting instances from the core index + Scenario: Checking Sphinx's status Given Sphinx is running Then Sphinx should be running @@ -13,4 +13,9 @@ Feature: Checking whether Sphinx is running or not When I start Sphinx And I wait for Sphinx to catch up - Then Sphinx should be running \ No newline at end of file + Then Sphinx should be running + + Given Sphinx is running + When I kill the Sphinx process + And I wait for Sphinx to catch up + Then Sphinx should not be running \ No newline at end of file features/sphinx_detection.feature @@ -8,6 +8,7 @@ Before do @conditions = {} @with = {} @without = {} + @with_all = {} @options = {} end @@ -40,11 +41,27 @@ When /^I search for (\w+) on (\w+)$/ do |query, field| @conditions[field.to_sym] = query end +When /^I clear existing filters$/ do + @with = {} + @without = {} + @with_all = {} +end + When /^I filter by (\w+) on (\w+)$/ do |filter, attribute| @results = nil @with[attribute.to_sym] = filter.to_i end +When /^I filter by (\d+) and (\d+) on (\w+)$/ do |value_one, value_two, attribute| + @results = nil + @with[attribute.to_sym] = [value_one.to_i, value_two.to_i] +end + +When /^I filter by both (\d+) and (\d+) on (\w+)$/ do |value_one, value_two, attribute| + @results = nil + @with_all[attribute.to_sym] = [value_one.to_i, value_two.to_i] +end + When /^I filter between ([\d\.]+) and ([\d\.]+) on (\w+)$/ do |first, last, attribute| @results = nil if first[/\./].nil? && last[/\./].nil? @@ -130,7 +147,8 @@ def results @options.merge( :conditions => @conditions, :with => @with, - :without => @without + :without => @without, + :with_all => @with_all ) ) end \ No newline at end of file features/step_definitions/common_steps.rb @@ -32,10 +32,10 @@ Then "it should not exist if using Rails 2.1 or newer" do end Then /^I can iterate by result and group and count$/ do - results.each_with_group_and_count do |result, group, count| + results.each_with_groupby_and_count do |result, group, count| result.should be_kind_of(@model) count.should be_kind_of(Integer) - group.should be_kind_of(Integer) unless group.nil? + group.should be_kind_of(Integer) end end features/step_definitions/search_steps.rb @@ -2,6 +2,10 @@ Given "Sphinx is running" do ThinkingSphinx::Configuration.instance.controller.should be_running end +When "I kill the Sphinx process" do + Process.kill(9, ThinkingSphinx.sphinx_pid.to_i) +end + When "I wait for Sphinx to catch up" do sleep(0.25) end @@ -20,4 +24,4 @@ end Then "Sphinx should not be running" do ThinkingSphinx.sphinx_running?.should be_false -end \ No newline at end of file +end features/step_definitions/sphinx_steps.rb @@ -7,3 +7,7 @@ end (1..10).each do |i| Box.create :width => i, :length => i, :depth => i end + +(11..20).each do |i| + Box.create :width => i, :length => i+1, :depth => i+2 +end features/support/db/migrations/create_boxes.rb @@ -11,3 +11,13 @@ Comment.create( :content => "+1", :post_id => 1 ) + +Comment.create :name => 'A', :post_id => 1, :content => 'Es un hecho establecido hace demasiado tiempo que un lector se distraerá con el contenido del texto' +Comment.create :name => 'B', :post_id => 1, :content => 'de un sitio mientras que mira su diseño. El punto de usar Lorem Ipsum es que tiene una distribución' +Comment.create :name => 'C', :post_id => 1, :content => 'más o menos normal de las letras, al contrario de usar textos como por ejemplo "Contenido aquí' +Comment.create :name => 'D', :post_id => 1, :content => 'contenido aquí". Estos textos hacen parecerlo un español que se puede leer. Muchos paquetes de' +Comment.create :name => 'E', :post_id => 1, :content => 'autoedición y editores de páginas web usan el Lorem Ipsum como su texto por defecto, y al hacer una' +Comment.create :name => 'F', :post_id => 1, :content => 'búsqueda de "Lorem Ipsum" va a dar por resultado muchos sitios web que usan este texto si se' + +# The one we'll really want to find via Sphinx search +Comment.create :name => 'G', :post_id => 1, :content => 'Turtle' features/support/db/migrations/create_comments.rb @@ -3,4 +3,4 @@ ActiveRecord::Base.connection.create_table :posts, :force => true do |t| t.column :content, :text end -Post.create :subject => "Hello World", :content => "Um Text" +Post.create :subject => "Hello World", :content => "Um Text", :id => 1 features/support/db/migrations/create_posts.rb @@ -3,5 +3,6 @@ class Box < ActiveRecord::Base indexes width, :as => :width_field has width, length, depth + has [width, length, depth], :as => :dimensions end -end \ No newline at end of file +end features/support/models/box.rb @@ -1,8 +1,11 @@ class Post < ActiveRecord::Base - has_many :comments + has_many :comments, :dependent => :destroy + has_many :tags, :dependent => :destroy define_index do indexes subject indexes content + indexes tags.text, :as => :tags + indexes comments.content, :as => :comments end end \ No newline at end of file features/support/models/post.rb @@ -20,7 +20,7 @@ ActiveRecord::Base.logger = Logger.new open("tmp/active_record.log", "a") ThinkingSphinx.deltas_enabled = false # Load Models -Dir["features/support/models/*.rb"].each do |file| +Dir["features/support/models/*.rb"].sort.each do |file| require file.gsub(/\.rb$/, '') end features/support/post_database.rb @@ -20,5 +20,9 @@ Ginger.configure do |config| ar_2_2_0[/^active_?support$/] = "2.2.0" ar_2_2_0[/^active_?record$/] = "2.2.0" - config.scenarios << ar_1_2_6 << ar_2_0_4 << ar_2_1_2 << ar_2_2_0 + ar_2_3_0 = Ginger::Scenario.new + ar_2_3_0[/^active_?support$/] = "2.3.0" + ar_2_3_0[/^active_?record$/] = "2.3.0" + + config.scenarios << ar_1_2_6 << ar_2_0_4 << ar_2_1_2 << ar_2_2_0 << ar_2_3_0 end \ No newline at end of file ginger_scenarios.rb @@ -12,6 +12,8 @@ require 'thinking_sphinx/association' require 'thinking_sphinx/attribute' require 'thinking_sphinx/collection' require 'thinking_sphinx/configuration' +require 'thinking_sphinx/facet' +require 'thinking_sphinx/facet_collection' require 'thinking_sphinx/field' require 'thinking_sphinx/index' require 'thinking_sphinx/rails_additions' @@ -24,11 +26,15 @@ require 'thinking_sphinx/adapters/postgresql_adapter' ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord) +Merb::Plugins.add_rakefiles( + File.join(File.dirname(__FILE__), "thinking_sphinx", "tasks") +) if defined?(Merb) + module ThinkingSphinx module Version #:nodoc: Major = 1 Minor = 1 - Tiny = 2 + Tiny = 5 String = [Major, Minor, Tiny].join('.') end @@ -152,15 +158,20 @@ module ThinkingSphinx end def self.sphinx_running? - !!sphinx_pid + !!sphinx_pid && pid_active?(sphinx_pid) end def self.sphinx_pid pid_file = ThinkingSphinx::Configuration.instance.pid_file `cat #{pid_file}`[/\d+/] if File.exists?(pid_file) end -end - -Merb::Plugins.add_rakefiles( - File.join(File.dirname(__FILE__), "..", "tasks", "thinking_sphinx_tasks") -) if ThinkingSphinx.merb? \ No newline at end of file + + def self.pid_active?(pid) + begin + Process.getpgid(pid.to_i) + true + rescue Exception => e + false + end + end +end \ No newline at end of file lib/thinking_sphinx.rb @@ -10,7 +10,7 @@ module ThinkingSphinx module ActiveRecord def self.included(base) base.class_eval do - class_inheritable_array :sphinx_indexes + class_inheritable_array :sphinx_indexes, :sphinx_facets class << self # Allows creation of indexes for Sphinx. If you don't do this, there # isn't much point trying to search (or using this plugin at all, @@ -114,13 +114,18 @@ module ThinkingSphinx end def to_riddle(offset) - ThinkingSphinx::AbstractAdapter.detect(self).setup + sphinx_database_adapter.setup indexes = [to_riddle_for_core(offset)] indexes << to_riddle_for_delta(offset) if sphinx_delta? indexes << to_riddle_for_distributed end + def sphinx_database_adapter + @sphinx_database_adapter ||= + ThinkingSphinx::AbstractAdapter.detect(self) + end + private def sphinx_name lib/thinking_sphinx/active_record.rb @@ -42,6 +42,13 @@ module ThinkingSphinx args << options ThinkingSphinx::Search.search_for_id(*args) end + + def facets(*args) + options = args.extract_options! + options[:class] = self + args << options + ThinkingSphinx::Search.facets(*args) + end end end end lib/thinking_sphinx/active_record/search.rb @@ -1,27 +1,34 @@ module ThinkingSphinx class AbstractAdapter - class << self - def setup - # Deliberately blank - subclasses should do something though. Well, if - # they need to. - end - - def detect(model) - case model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - ThinkingSphinx::MysqlAdapter - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - ThinkingSphinx::PostgreSQLAdapter - else - raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL" - end - end - - protected + def initialize(model) + @model = model + end + + def setup + # Deliberately blank - subclasses should do something though. Well, if + # they need to. + end - def connection - @connection ||= ::ActiveRecord::Base.connection + def self.detect(model) + case model.connection.class.name + when "ActiveRecord::ConnectionAdapters::MysqlAdapter", + "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" + ThinkingSphinx::MysqlAdapter.new model + when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" + ThinkingSphinx::PostgreSQLAdapter.new model + else + raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL, not #{model.connection.class.name}" end end + + def quote_with_table(column) + "#{@model.quoted_table_name}.#{@model.connection.quote_column_name(column)}" + end + + protected + + def connection + @connection ||= @model.connection + end end -end \ No newline at end of file +end lib/thinking_sphinx/adapters/abstract_adapter.rb @@ -1,9 +1,53 @@ module ThinkingSphinx class MysqlAdapter < AbstractAdapter - class << self - def setup - # Does MySQL actually need to do anything? - end + def setup + # Does MySQL actually need to do anything? + end + + def sphinx_identifier + "mysql" + end + + def concatenate(clause, separator = ' ') + "CONCAT_WS('#{separator}', #{clause})" + end + + def group_concatenate(clause, separator = ' ') + "GROUP_CONCAT(DISTINCT #{clause} SEPARATOR '#{separator}')" + end + + def cast_to_string(clause) + "CAST(#{clause} AS CHAR)" + end + + def cast_to_datetime(clause) + "UNIX_TIMESTAMP(#{clause})" + end + + def cast_to_unsigned(clause) + "CAST(#{clause} AS UNSIGNED)" + end + + def convert_nulls(clause, default = '') + default = "'#{default}'" if default.is_a?(String) + + "IFNULL(#{clause}, #{default})" + end + + def boolean(value) + value ? 1 : 0 + end + + def crc(clause) + "CRC32(#{clause})" + end + + def utf8_query_pre + "SET NAMES utf8" + end + + def time_difference(diff) + "DATE_SUB(NOW(), INTERVAL #{diff} SECOND)" end end end \ No newline at end of file lib/thinking_sphinx/adapters/mysql_adapter.rb @@ -1,83 +1,129 @@ module ThinkingSphinx class PostgreSQLAdapter < AbstractAdapter - class << self - def setup - create_array_accum_function - create_crc32_function - end - - private + def setup + create_array_accum_function + create_crc32_function + end + + def sphinx_identifier + "pgsql" + end + + def concatenate(clause, separator = ' ') + clause.split(', ').collect { |field| + "COALESCE(CAST(#{field} as varchar), '')" + }.join(" || '#{separator}' || ") + end + + def group_concatenate(clause, separator = ' ') + "array_to_string(array_accum(#{clause}), '#{separator}')" + end + + def cast_to_string(clause) + clause + end + + def cast_to_datetime(clause) + "cast(extract(epoch from #{clause}) as int)" + end + + def cast_to_unsigned(clause) + clause + end + + def convert_nulls(clause, default = '') + default = "'#{default}'" if default.is_a?(String) - def execute(command, output_error = false) - connection.execute "begin" - connection.execute "savepoint ts" - begin - connection.execute command - rescue StandardError => err - puts err if output_error - connection.execute "rollback to savepoint ts" - end - connection.execute "release savepoint ts" - connection.execute "commit" + "COALESCE(#{clause}, #{default})" + end + + def boolean(value) + value ? 'TRUE' : 'FALSE' + end + + def crc(clause) + "crc32(#{clause})" + end + + def utf8_query_pre + nil + end + + def time_difference(diff) + "current_timestamp - interval '#{diff} seconds'" + end + + private + + def execute(command, output_error = false) + connection.execute "begin" + connection.execute "savepoint ts" + begin + connection.execute command + rescue StandardError => err + puts err if output_error + connection.execute "rollback to savepoint ts" end - - def create_array_accum_function - if connection.raw_connection.server_version > 80200 - execute <<-SQL - CREATE AGGREGATE array_accum (anyelement) - ( - sfunc = array_append, - stype = anyarray, - initcond = '{}' - ); - SQL - else - execute <<-SQL - CREATE AGGREGATE array_accum - ( - basetype = anyelement, - sfunc = array_append, - stype = anyarray, - initcond = '{}' - ); - SQL - end + connection.execute "release savepoint ts" + connection.execute "commit" + end + + def create_array_accum_function + if connection.raw_connection.respond_to?(:server_version) && connection.raw_connection.server_version > 80200 + execute <<-SQL + CREATE AGGREGATE array_accum (anyelement) + ( + sfunc = array_append, + stype = anyarray, + initcond = '{}' + ); + SQL + else + execute <<-SQL + CREATE AGGREGATE array_accum + ( + basetype = anyelement, + sfunc = array_append, + stype = anyarray, + initcond = '{}' + ); + SQL end - - def create_crc32_function - execute "CREATE LANGUAGE 'plpgsql';" - function = <<-SQL - CREATE OR REPLACE FUNCTION crc32(word text) - RETURNS bigint AS $$ - DECLARE tmp bigint; - DECLARE i int; - DECLARE j int; - DECLARE word_array bytea; - BEGIN - i = 0; - tmp = 4294967295; - word_array = decode(replace(word, E'\\\\', E'\\\\\\\\'), 'escape'); + end + + def create_crc32_function + execute "CREATE LANGUAGE 'plpgsql';" + function = <<-SQL + CREATE OR REPLACE FUNCTION crc32(word text) + RETURNS bigint AS $$ + DECLARE tmp bigint; + DECLARE i int; + DECLARE j int; + DECLARE word_array bytea; + BEGIN + i = 0; + tmp = 4294967295; + word_array = decode(replace(word, E'\\\\', E'\\\\\\\\'), 'escape'); + LOOP + tmp = (tmp # get_byte(word_array, i))::bigint; + i = i + 1; + j = 0; LOOP - tmp = (tmp # get_byte(word_array, i))::bigint; - i = i + 1; - j = 0; - LOOP - tmp = ((tmp >> 1) # (3988292384 * (tmp & 1)))::bigint; - j = j + 1; - IF j >= 8 THEN - EXIT; - END IF; - END LOOP; - IF i >= char_length(word) THEN + tmp = ((tmp >> 1) # (3988292384 * (tmp & 1)))::bigint; + j = j + 1; + IF j >= 8 THEN EXIT; END IF; END LOOP; - return (tmp # 4294967295); - END - $$ IMMUTABLE STRICT LANGUAGE plpgsql; - SQL - execute function, true - end + IF i >= char_length(word) THEN + EXIT; + END IF; + END LOOP; + return (tmp # 4294967295); + END + $$ IMMUTABLE STRICT LANGUAGE plpgsql; + SQL + execute function, true end end end \ No newline at end of file lib/thinking_sphinx/adapters/postgresql_adapter.rb @@ -9,7 +9,7 @@ module ThinkingSphinx # associations. Which can get messy. Use Index.link!, it really helps. # class Attribute - attr_accessor :alias, :columns, :associations, :model + attr_accessor :alias, :columns, :associations, :model, :faceted # To create a new attribute, you'll need to pass in either a single Column # or an array of them, and some (optional) options. @@ -78,10 +78,10 @@ module ThinkingSphinx separator = all_ints? ? ',' : ' ' - clause = concatenate(clause, separator) if concat_ws? - clause = group_concatenate(clause, separator) if is_many? - clause = cast_to_datetime(clause) if type == :datetime - clause = convert_nulls(clause) if type == :string + clause = adapter.concatenate(clause, separator) if concat_ws? + clause = adapter.group_concatenate(clause, separator) if is_many? + clause = adapter.cast_to_datetime(clause) if type == :datetime + clause = adapter.convert_nulls(clause) if type == :string "#{clause} AS #{quote_column(unique_name)}" end @@ -135,63 +135,31 @@ module ThinkingSphinx end end - private - - def concatenate(clause, separator = ' ') - case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - "CONCAT_WS('#{separator}', #{clause})" - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - clause.split(', ').collect { |attribute| - "COALESCE(#{attribute}, '')" - }.join(" || ' ' || ") - else - clause - end - end - - def group_concatenate(clause, separator = ' ') - case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - "GROUP_CONCAT(#{clause} SEPARATOR '#{separator}')" - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - "array_to_string(array_accum(#{clause}), '#{separator}')" + # Returns the type of the column. If that's not already set, it returns + # :multi if there's the possibility of more than one value, :string if + # there's more than one association, otherwise it figures out what the + # actual column's datatype is and returns that. + def type + @type ||= case + when is_many?, is_many_ints? + :multi + when @associations.values.flatten.length > 1 + :string else - clause + translated_type_from_database end end - def cast_to_string(clause) - case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - "CAST(#{clause} AS CHAR)" - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - clause - else - clause - end + def to_facet + return nil unless @faceted + + ThinkingSphinx::Facet.new(self) end - def cast_to_datetime(clause) - case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - "UNIX_TIMESTAMP(#{clause})" - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - "cast(extract(epoch from #{clause}) as int)" - else - clause - end - end + private - def convert_nulls(clause) - case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - "IFNULL(#{clause}, '')" - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - "COALESCE(#{clause}, '')" - else - clause - end + def adapter + @adapter ||= @model.sphinx_database_adapter end def quote_column(column) @@ -205,16 +173,7 @@ module ThinkingSphinx def concat_ws? multiple_associations? || @columns.length > 1 end - - # Checks the association tree for each column - if they're all the same, - # returns false. - # - def multiple_sources? - first = associations[@columns.first] - - !@columns.all? { |col| associations[col] == first } - end - + # Checks whether any column requires multiple associations (which only # happens for polymorphic situations). # @@ -248,6 +207,10 @@ module ThinkingSphinx associations.values.flatten.any? { |assoc| assoc.is_many? } end + def is_many_ints? + concat_ws? && all_ints? + end + # Returns true if any of the columns are string values, instead of database # column references. def is_string? @@ -342,4 +305,4 @@ explicitly convert the column's value in your define_index block: end end end -end \ No newline at end of file +end lib/thinking_sphinx/attribute.rb @@ -33,40 +33,60 @@ module ThinkingSphinx end def self.instances_from_matches(matches, options = {}) - return matches.collect { |match| - instance_from_match match, options - } unless klass = options[:class] - + if klass = options[:class] + instances_from_class klass, matches, options + else + instances_from_classes matches, options + end + end + + def self.instances_from_class(klass, matches, options = {}) index_options = klass.sphinx_index_options - + ids = matches.collect { |match| match[:attributes]["sphinx_internal_id"] } instances = ids.length > 0 ? klass.find( :all, :conditions => {klass.primary_key.to_sym => ids}, :include => (options[:include] || index_options[:include]), - :select => (options[:select] || index_options[:select]) + :select => (options[:select] || index_options[:select]), + :order => (options[:sql_order] || index_options[:sql_order]) ) : [] - - # Raise an exception if we find records in Sphinx but not in the DB, so the search method - # can retry without them. See ThinkingSphinx::Search.retry_search_on_stale_index. + + # Raise an exception if we find records in Sphinx but not in the DB, so + # the search method can retry without them. See + # ThinkingSphinx::Search.retry_search_on_stale_index. if options[:raise_on_stale] && instances.length < ids.length stale_ids = ids - instances.map {|i| i.id } raise StaleIdsException, stale_ids end + # if the user has specified an SQL order, return the collection + # without rearranging it into the Sphinx order + return instances if options[:sql_order] + ids.collect { |obj_id| instances.detect { |obj| obj.id == obj_id } } end - def self.instance_from_match(match, options) - class_from_crc(match[:attributes]["class_crc"]).find( - match[:attributes]["sphinx_internal_id"], - :include => options[:include], - :select => options[:select] - ) - rescue ::ActiveRecord::RecordNotFound - nil + # Group results by class and call #find(:all) once for each group to reduce + # the number of #find's in multi-model searches. + # + def self.instances_from_classes(matches, options = {}) + groups = matches.group_by { |match| match[:attributes]["class_crc"] } + groups.each do |crc, group| + group.replace( + instances_from_class(class_from_crc(crc), group, options) + ) + end + + matches.collect do |match| + groups.detect { |crc, group| + crc == match[:attributes]["class_crc"] + }[1].detect { |obj| + obj.id == match[:attributes]["sphinx_internal_id"] + } + end end def self.class_from_crc(crc) @@ -98,9 +118,9 @@ module ThinkingSphinx each_with_attribute method.to_s.gsub(/^each_with_/, ''), &block end - def each_with_group_and_count(&block) + def each_with_groupby_and_count(&block) results[:matches].each_with_index do |match, index| - yield self[index], match[:attributes]["@group"], match[:attributes]["@count"] + yield self[index], match[:attributes]["@groupby"], match[:attributes]["@count"] end end @@ -115,5 +135,13 @@ module ThinkingSphinx yield self[index], match[:weight] end end + + def inject_with_groupby_and_count(initial = nil, &block) + index = -1 + results[:matches].inject(initial) do |memo, match| + index += 1 + yield memo, self[index], match[:attributes]["@groupby"], match[:attributes]["@count"] + end + end end -end \ No newline at end of file +end lib/thinking_sphinx/collection.rb @@ -164,31 +164,6 @@ module ThinkingSphinx end end - def hash_to_config(hash) - hash.collect { |key, value| - translated_value = case value - when TrueClass - "1" - when FalseClass - "0" - when NilClass, "" - next - else - value - end - " #{key} = #{translated_value}" - }.join("\n") - end - - def self.options_merge(base, extra) - base = base.clone - extra.each do |key, value| - next if value.nil? || value == "" - base[key] = value - end - base - end - def address @configuration.searchd.address end lib/thinking_sphinx/configuration.rb @@ -40,22 +40,11 @@ module ThinkingSphinx def clause(model, toggled) if toggled "#{model.quoted_table_name}.#{@index.quote_column(@column.to_s)}" + - " > #{time_difference}" + " > #{adapter.time_difference(@threshold)}" else nil end end - - def time_difference - case @index.adapter - when :mysql - "DATE_SUB(NOW(), INTERVAL #{@threshold} SECOND)" - when :postgres - "current_timestamp - interval '#{@threshold} seconds'" - else - raise "Unknown Database Adapter" - end - end end end end lib/thinking_sphinx/deltas/datetime_delta.rb @@ -37,12 +37,12 @@ module ThinkingSphinx def reset_query(model) "UPDATE #{model.quoted_table_name} SET " + - "#{@index.quote_column(@column.to_s)} = #{@index.db_boolean(false)}" + "#{@index.quote_column(@column.to_s)} = #{adapter.boolean(false)}" end def clause(model, toggled) "#{model.quoted_table_name}.#{@index.quote_column(@column.to_s)}" + - " = #{@index.db_boolean(toggled)}" + " = #{adapter.boolean(toggled)}" end protected @@ -53,7 +53,13 @@ module ThinkingSphinx def delta_index_name(model) "#{model.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_delta" - end + end + + private + + def adapter + @adapter = @index.model.sphinx_database_adapter + end end end end lib/thinking_sphinx/deltas/default_delta.rb @@ -8,7 +8,8 @@ module ThinkingSphinx # associations. Which can get messy. Use Index.link!, it really helps. # class Field - attr_accessor :alias, :columns, :sortable, :associations, :model, :infixes, :prefixes + attr_accessor :alias, :columns, :sortable, :associations, :model, :infixes, + :prefixes, :faceted # To create a new field, you'll need to pass in either a single Column # or an array of them, and some (optional) options. The columns are @@ -58,10 +59,11 @@ module ThinkingSphinx raise "Cannot define a field with no columns. Maybe you are trying to index a field with a reserved name (id, name). You can fix this error by using a symbol rather than a bare name (:id instead of id)." if @columns.empty? || @columns.any? { |column| !column.respond_to?(:__stack) } - @alias = options[:as] - @sortable = options[:sortable] || false - @infixes = options[:infixes] || false - @prefixes = options[:prefixes] || false + @alias = options[:as] + @sortable = options[:sortable] || false + @infixes = options[:infixes] || false + @prefixes = options[:prefixes] || false + @faceted = options[:facet] || false end # Get the part of the SELECT clause related to this field. Don't forget @@ -75,10 +77,10 @@ module ThinkingSphinx column_with_prefix(column) }.join(', ') - clause = concatenate(clause) if concat_ws? - clause = group_concatenate(clause) if is_many? + clause = adapter.concatenate(clause) if concat_ws? + clause = adapter.group_concatenate(clause) if is_many? - "#{cast_to_string clause } AS #{quote_column(unique_name)}" + "#{adapter.cast_to_string clause } AS #{quote_column(unique_name)}" end # Get the part of the GROUP BY clause related to this field - if one is @@ -110,31 +112,17 @@ module ThinkingSphinx end end - private - - def concatenate(clause) - case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - "CONCAT_WS(' ', #{clause})" - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - clause.split(', ').collect { |field| - "COALESCE(#{field}, '')" - }.join(" || ' ' || ") - else - clause - end + def to_facet + return nil unless @faceted + + ThinkingSphinx::Facet.new(self) end - def group_concatenate(clause) - case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - "GROUP_CONCAT(#{clause} SEPARATOR ' ')" - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - "array_to_string(array_accum(#{clause}), ' ')" - else - clause - end - end + private + + def adapter + @adapter ||= @model.sphinx_database_adapter + end def cast_to_string(clause) case @model.connection.class.name @@ -158,16 +146,7 @@ module ThinkingSphinx def concat_ws? @columns.length > 1 || multiple_associations? end - - # Checks the association tree for each column - if they're all the same, - # returns false. - # - def multiple_sources? - first = associations[@columns.first] - - !@columns.all? { |col| associations[col] == first } - end - + # Checks whether any column requires multiple associations (which only # happens for polymorphic situations). # lib/thinking_sphinx/field.rb @@ -46,18 +46,13 @@ module ThinkingSphinx def self.name(model) model.name.underscore.tr(':/\\', '_') end - - def to_riddle(model, index, offset) - add_internal_attributes - link! - end def to_riddle_for_core(offset, index) add_internal_attributes link! source = Riddle::Configuration::SQLSource.new( - "#{name}_core_#{index}", riddle_adapter + "#{name}_core_#{index}", adapter.sphinx_identifier ) set_source_database_settings source @@ -73,7 +68,7 @@ module ThinkingSphinx link! source = Riddle::Configuration::SQLSource.new( - "#{name}_delta_#{index}", riddle_adapter + "#{name}_delta_#{index}", adapter.sphinx_identifier ) source.parent = "#{name}_core_#{index}" @@ -111,92 +106,6 @@ module ThinkingSphinx } end - # Generates the big SQL statement to get the data back for all the fields - # and attributes, using all the relevant association joins. If you want - # the version filtered for delta values, send through :delta => true in the - # options. Won't do much though if the index isn't set up to support a - # delta sibling. - # - # Examples: - # - # index.to_sql - # index.to_sql(:delta => true) - # - def to_sql(options={}) - assocs = all_associations - - where_clause = "" - if self.delta? && !@delta_object.clause(@model, options[:delta]).blank? - where_clause << " AND #{@delta_object.clause(@model, options[:delta])}" - end - unless @conditions.empty? - where_clause << " AND " << @conditions.join(" AND ") - end - - internal_groupings = [] - if @model.column_names.include?(@model.inheritance_column) - internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}" - end - - unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}" - - sql = <<-SQL -SELECT #{ ( - ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + - @fields.collect { |field| field.to_select_sql } + - @attributes.collect { |attribute| attribute.to_select_sql } -).join(", ") } -FROM #{ @model.table_name } - #{ assocs.collect { |assoc| assoc.to_sql }.join(' ') } -WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start - AND #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} <= $end - #{ where_clause } -GROUP BY #{ ( - ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + - @fields.collect { |field| field.to_group_sql }.compact + - @attributes.collect { |attribute| attribute.to_group_sql }.compact + - @groupings + internal_groupings -).join(", ") } - SQL - - if @model.connection.class.name == "ActiveRecord::ConnectionAdapters::MysqlAdapter" || @model.connection.class.name == "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - sql += " ORDER BY NULL" - end - - sql - end - - # Simple helper method for the query info SQL - which is a statement that - # returns the single row for a corresponding id. - # - def to_sql_query_info(offset) - "SELECT * FROM #{@model.quoted_table_name} WHERE " + - " #{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})" - end - - # Simple helper method for the query range SQL - which is a statement that - # returns minimum and maximum id values. These can be filtered by delta - - # so pass in :delta => true to get the delta version of the SQL. - # - def to_sql_query_range(options={}) - min_statement = "MIN(#{quote_column(@model.primary_key)})" - max_statement = "MAX(#{quote_column(@model.primary_key)})" - - # Fix to handle Sphinx PostgreSQL bug (it doesn't like NULLs or 0's) - if adapter == :postgres - min_statement = "COALESCE(#{min_statement}, 1)" - max_statement = "COALESCE(#{max_statement}, 1)" - end - - sql = "SELECT #{min_statement}, #{max_statement} " + - "FROM #{@model.quoted_table_name} " - if self.delta? && !@delta_object.clause(@model, options[:delta]).blank? - sql << "WHERE #{@delta_object.clause(@model, options[:delta])}" - end - - sql - end - # Flag to indicate whether this index has a corresponding delta index. # def delta? @@ -204,14 +113,7 @@ GROUP BY #{ ( end def adapter - @adapter ||= case @model.connection.class.name - when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::MysqlplusAdapter" - :mysql - when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter" - :postgres - else - raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL" - end + @adapter ||= @model.sphinx_database_adapter end def prefix_fields @@ -229,30 +131,11 @@ GROUP BY #{ ( }.each { |key| all_index_options[key.to_sym] = @options[key] } all_index_options end - - def source_options - all_source_options = ThinkingSphinx::Configuration.instance.source_options.clone - @options.keys.select { |key| - ThinkingSphinx::Configuration::SourceOptions.include?(key.to_s) - }.each { |key| all_source_options[key.to_sym] = @options[key] } - all_source_options - end - + def quote_column(column) @model.connection.quote_column_name(column) end - # Returns the proper boolean value string literal for the - # current database adapter. - # - def db_boolean(val) - if adapter == :postgres - val ? 'TRUE' : 'FALSE' - else - val ? '1' : '0' - end - end - private def utf8? @@ -276,14 +159,23 @@ GROUP BY #{ ( stored_class = @model.store_full_sti_class ? @model.name : @model.name.demodulize builder.where("#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)} = '#{stored_class}'") end - - @fields = builder.fields - @attributes = builder.attributes + + set_model = Proc.new { |item| item.model = @model } + + @fields = builder.fields &set_model + @attributes = builder.attributes.each &set_model @conditions = builder.conditions @groupings = builder.groupings @delta_object = ThinkingSphinx::Deltas.parse self, builder.properties @options = builder.properties + is_faceted = Proc.new { |item| item.faceted } + add_facet = Proc.new { |item| @model.sphinx_facets << item.to_facet } + + @model.sphinx_facets ||= [] + @fields.select( &is_faceted).each &add_facet + @attributes.select(&is_faceted).each &add_facet + # We want to make sure that if the database doesn't exist, then Thinking # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop # and db:migrate). It's a bit hacky, but I can't think of a better way. @@ -345,12 +237,10 @@ GROUP BY #{ ( def crc_column if @model.column_names.include?(@model.inheritance_column) - case adapter - when :postgres - "COALESCE(crc32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})" - when :mysql - "CAST(IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s}) AS UNSIGNED)" - end + adapter.cast_to_unsigned(adapter.convert_nulls( + adapter.crc(adapter.quote_with_table(@model.inheritance_column)), + @model.to_crc32 + )) else @model.to_crc32.to_s end @@ -383,24 +273,13 @@ GROUP BY #{ ( :as => :sphinx_deleted ) unless @attributes.detect { |attr| attr.alias == :sphinx_deleted } end - - def riddle_adapter - case adapter - when :postgres - "pgsql" - when :mysql - "mysql" - else - raise "Unsupported Database Adapter: Sphinx only supports MySQL and PosgreSQL" - end - end - + def set_source_database_settings(source) config = @model.connection.instance_variable_get(:@config) - source.sql_host = config[:host] || "localhost" - source.sql_user = config[:username] || config[:user] - source.sql_pass = (config[:password] || "").gsub('#', '\#') + source.sql_host = config[:host] || "localhost" + source.sql_user = config[:username] || config[:user] || "" + source.sql_pass = (config[:password].to_s || "").gsub('#', '\#') source.sql_db = config[:database] source.sql_port = config[:port] source.sql_sock = config[:socket] @@ -423,9 +302,7 @@ GROUP BY #{ ( source.sql_query_pre << "SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" end - if utf8? && adapter == :mysql - source.sql_query_pre << "SET NAMES utf8" - end + source.sql_query_pre += [adapter.utf8_query_pre].compact if utf8? end def set_source_settings(source) @@ -449,5 +326,89 @@ GROUP BY #{ ( def sql_query_pre_for_delta [""] end + + # Generates the big SQL statement to get the data back for all the fields + # and attributes, using all the relevant association joins. If you want + # the version filtered for delta values, send through :delta => true in the + # options. Won't do much though if the index isn't set up to support a + # delta sibling. + # + # Examples: + # + # index.to_sql + # index.to_sql(:delta => true) + # + def to_sql(options={}) + assocs = all_associations + + where_clause = "" + if self.delta? && !@delta_object.clause(@model, options[:delta]).blank? + where_clause << " AND #{@delta_object.clause(@model, options[:delta])}" + end + unless @conditions.empty? + where_clause << " AND " << @conditions.join(" AND ") + end + + internal_groupings = [] + if @model.column_names.include?(@model.inheritance_column) + internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}" + end + + unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}" + + sql = <<-SQL +SELECT #{ ( + ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + + @fields.collect { |field| field.to_select_sql } + + @attributes.collect { |attribute| attribute.to_select_sql } +).join(", ") } +FROM #{ @model.table_name } + #{ assocs.collect { |assoc| assoc.to_sql }.join(' ') } +WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start + AND #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} <= $end + #{ where_clause } +GROUP BY #{ ( + ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + + @fields.collect { |field| field.to_group_sql }.compact + + @attributes.collect { |attribute| attribute.to_group_sql }.compact + + @groupings + internal_groupings +).join(", ") } + SQL + + if @model.connection.class.name == "ActiveRecord::ConnectionAdapters::MysqlAdapter" + sql += " ORDER BY NULL" + end + + sql + end + + # Simple helper method for the query info SQL - which is a statement that + # returns the single row for a corresponding id. + # + def to_sql_query_info(offset) + "SELECT * FROM #{@model.quoted_table_name} WHERE " + + " #{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})" + end + + # Simple helper method for the query range SQL - which is a statement that + # returns minimum and maximum id values. These can be filtered by delta - + # so pass in :delta => true to get the delta version of the SQL. + # + def to_sql_query_range(options={}) + min_statement = adapter.convert_nulls( + "MIN(#{quote_column(@model.primary_key)})", 1 + ) + max_statement = adapter.convert_nulls( + "MAX(#{quote_column(@model.primary_key)})", 1 + ) + + sql = "SELECT #{min_statement}, #{max_statement} " + + "FROM #{@model.quoted_table_name} " + if self.delta? && !@delta_object.clause(@model, options[:delta]).blank? + sql << "WHERE #{@delta_object.clause(@model, options[:delta])}" + end + + sql + end end end lib/thinking_sphinx/index.rb @@ -90,13 +90,13 @@ module ThinkingSphinx args.each do |columns| fields << Field.new(FauxColumn.coerce(columns), options) - if fields.last.sortable + if fields.last.sortable || fields.last.faceted attributes << Attribute.new( fields.last.columns.collect { |col| col.clone }, options.merge( :type => :string, :as => fields.last.unique_name.to_s.concat("_sort").to_sym - ) + ).except(:facet) ) end end @@ -149,6 +149,15 @@ module ThinkingSphinx end alias_method :attribute, :has + def facet(*args) + options = args.extract_options! + options[:facet] = true + + args.each do |columns| + attributes << Attribute.new(FauxColumn.coerce(columns), options) + end + end + # Use this method to add some manual SQL conditions for your index # request. You can pass in as many strings as you like, they'll get # joined together with ANDs later on. @@ -217,8 +226,8 @@ module ThinkingSphinx # # Example: indexes assoc(:properties).column # - def assoc(assoc) - FauxColumn.new(method) + def assoc(assoc, *args) + FauxColumn.new(assoc, *args) end end end lib/thinking_sphinx/index/builder.rb @@ -21,7 +21,7 @@ module ThinkingSphinx options = args.extract_options! page = options[:page] ? options[:page].to_i : 1 - + ThinkingSphinx::Collection.ids_from_results(results, page, client.limit, options) end @@ -186,6 +186,12 @@ module ThinkingSphinx # documentation[http://sphinxsearch.com/doc.html] for that level of # detail though. # + # If desired, you can sort by a column in your model instead of a sphinx + # field or attribute. This sort only applies to the current page, so is + # most useful when performing a search with a single page of results. + # + # User.search("pat", :sql_order => "name") + # # == Grouping # # For this you can use the group_by, group_clause and group_function @@ -352,6 +358,18 @@ module ThinkingSphinx end end + def facets(*args) + hash = ThinkingSphinx::FacetCollection.new args + options = args.extract_options!.clone.merge! :group_function => :attr + + options[:class].sphinx_facets.inject(hash) do |hash, facet| + options[:group_by] = facet.attribute_name + + hash.add_from_results facet, search(*(args + [options])) + hash + end + end + private # This method handles the common search functionality, and returns both @@ -377,6 +395,7 @@ module ThinkingSphinx client.limit = options[:per_page].to_i if options[:per_page] page = options[:page] ? options[:page].to_i : 1 + page = 1 if page <= 0 client.offset = (page - 1) * client.limit begin @@ -457,6 +476,13 @@ module ThinkingSphinx Riddle::Client::Filter.new attr.to_s, filter_value(klass, attr, val), true } if options[:without] + # every-match attribute filters + client.filters += options[:with_all].collect { |attr,vals| + Array(vals).collect { |val| + Riddle::Client::Filter.new attr.to_s, filter_value(val) + } + }.flatten if options[:with_all] + # exclusive attribute filter on primary key client.filters += Array(options[:without_ids]).collect { |id| Riddle::Client::Filter.new 'sphinx_internal_id', filter_value(klass, 'sphinx_internal_id', id), true lib/thinking_sphinx/search.rb @@ -3,10 +3,12 @@ $:.unshift File.dirname(__FILE__) + '/../lib' require 'rubygems' require 'fileutils' require 'ginger' + +require 'lib/thinking_sphinx' + require 'not_a_mock' require 'will_paginate' -require 'lib/thinking_sphinx' require 'spec/sphinx_helper' ActiveRecord::Base.logger = Logger.new(StringIO.new) spec/spec_helper.rb @@ -230,7 +230,8 @@ describe "ThinkingSphinx::ActiveRecord" do it "should allow associations to other STI models" do Child.sphinx_indexes.last.link! - sql = Child.sphinx_indexes.last.to_sql.gsub('$start', '0').gsub('$end', '100') + sql = Child.sphinx_indexes.last.to_riddle_for_core(0, 0).sql_query + sql.gsub!('$start', '0').gsub!('$end', '100') lambda { Child.connection.execute(sql) }.should_not raise_error(ActiveRecord::StatementInvalid) end end spec/unit/thinking_sphinx/active_record_spec.rb @@ -161,7 +161,7 @@ describe ThinkingSphinx::Attribute do end it "should return :string if there's more than one association" do - @attribute.associations = {:a => :assoc, :b => :assoc} + @attribute.associations = {:a => [:assoc], :b => [:assoc]} @attribute.send(:type).should == :string end spec/unit/thinking_sphinx/attribute_spec.rb @@ -1,12 +1,12 @@ require 'spec/spec_helper' describe ThinkingSphinx::Index do - describe "to_sql method" do + describe "generated sql_query" do it "should include explicit groupings if requested" do @index = ThinkingSphinx::Index.new(Person) @index.groupings << "custom_sql" - @index.to_sql.should match(/GROUP BY.+custom_sql/) + @index.to_riddle_for_core(0, 0).sql_query.should match(/GROUP BY.+custom_sql/) end end spec/unit/thinking_sphinx/index_spec.rb @@ -2,22 +2,22 @@ Gem::Specification.new do |s| s.name = %q{thinking-sphinx} - s.version = "1.1.2" + s.version = "1.1.5" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Pat Allan"] - s.date = %q{2009-01-08} + s.date = %q{2009-02-09} s.description = %q{A concise and easy-to-use Ruby library that connects ActiveRecord to the Sphinx search daemon, managing configuration, indexing and searching.} s.email = %q{pat@freelancing-gods.com} - s.files = ["lib/thinking_sphinx/active_record/delta.rb", "lib/thinking_sphinx/active_record/has_many_association.rb", "lib/thinking_sphinx/active_record/search.rb", "lib/thinking_sphinx/active_record.rb", "lib/thinking_sphinx/adapters/abstract_adapter.rb", "lib/thinking_sphinx/adapters/mysql_adapter.rb", "lib/thinking_sphinx/adapters/postgresql_adapter.rb", "lib/thinking_sphinx/association.rb", "lib/thinking_sphinx/attribute.rb", "lib/thinking_sphinx/collection.rb", "lib/thinking_sphinx/configuration.rb", "lib/thinking_sphinx/deltas/datetime_delta.rb", "lib/thinking_sphinx/deltas/default_delta.rb", "lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb", "lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb", "lib/thinking_sphinx/deltas/delayed_delta/job.rb", "lib/thinking_sphinx/deltas/delayed_delta.rb", "lib/thinking_sphinx/deltas.rb", "lib/thinking_sphinx/field.rb", "lib/thinking_sphinx/index/builder.rb", "lib/thinking_sphinx/index/faux_column.rb", "lib/thinking_sphinx/index.rb", "lib/thinking_sphinx/rails_additions.rb", "lib/thinking_sphinx/search.rb", "lib/thinking_sphinx.rb", "LICENCE", "README", "tasks/thinking_sphinx_tasks.rb", "tasks/thinking_sphinx_tasks.rake", "vendor/after_commit", "vendor/after_commit/init.rb", "vendor/after_commit/lib", "vendor/after_commit/lib/after_commit", "vendor/after_commit/lib/after_commit/active_record.rb", "vendor/after_commit/lib/after_commit/connection_adapters.rb", "vendor/after_commit/lib/after_commit.rb", "vendor/after_commit/LICENSE", "vendor/after_commit/Rakefile", "vendor/after_commit/README", "vendor/after_commit/test", "vendor/after_commit/test/after_commit_test.rb", "vendor/delayed_job", "vendor/delayed_job/lib", "vendor/delayed_job/lib/delayed", "vendor/delayed_job/lib/delayed/job.rb", "vendor/delayed_job/lib/delayed/message_sending.rb", "vendor/delayed_job/lib/delayed/performable_method.rb", "vendor/delayed_job/lib/delayed/worker.rb", "vendor/riddle", "vendor/riddle/lib", "vendor/riddle/lib/riddle", "vendor/riddle/lib/riddle/client", "vendor/riddle/lib/riddle/client/filter.rb", "vendor/riddle/lib/riddle/client/message.rb", "vendor/riddle/lib/riddle/client/response.rb", "vendor/riddle/lib/riddle/client.rb", "vendor/riddle/lib/riddle/configuration", "vendor/riddle/lib/riddle/configuration/distributed_index.rb", "vendor/riddle/lib/riddle/configuration/index.rb", "vendor/riddle/lib/riddle/configuration/indexer.rb", "vendor/riddle/lib/riddle/configuration/remote_index.rb", "vendor/riddle/lib/riddle/configuration/searchd.rb", "vendor/riddle/lib/riddle/configuration/section.rb", "vendor/riddle/lib/riddle/configuration/source.rb", "vendor/riddle/lib/riddle/configuration/sql_source.rb", "vendor/riddle/lib/riddle/configuration/xml_source.rb", "vendor/riddle/lib/riddle/configuration.rb", "vendor/riddle/lib/riddle/controller.rb", "vendor/riddle/lib/riddle.rb", "spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/collection_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] + s.files = ["lib/thinking_sphinx/active_record/delta.rb", "lib/thinking_sphinx/active_record/has_many_association.rb", "lib/thinking_sphinx/active_record/search.rb", "lib/thinking_sphinx/active_record.rb", "lib/thinking_sphinx/adapters/abstract_adapter.rb", "lib/thinking_sphinx/adapters/mysql_adapter.rb", "lib/thinking_sphinx/adapters/postgresql_adapter.rb", "lib/thinking_sphinx/association.rb", "lib/thinking_sphinx/attribute.rb", "lib/thinking_sphinx/collection.rb", "lib/thinking_sphinx/configuration.rb", "lib/thinking_sphinx/core/string.rb", "lib/thinking_sphinx/deltas/datetime_delta.rb", "lib/thinking_sphinx/deltas/default_delta.rb", "lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb", "lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb", "lib/thinking_sphinx/deltas/delayed_delta/job.rb", "lib/thinking_sphinx/deltas/delayed_delta.rb", "lib/thinking_sphinx/deltas.rb", "lib/thinking_sphinx/facet.rb", "lib/thinking_sphinx/facet_collection.rb", "lib/thinking_sphinx/field.rb", "lib/thinking_sphinx/index/builder.rb", "lib/thinking_sphinx/index/faux_column.rb", "lib/thinking_sphinx/index.rb", "lib/thinking_sphinx/rails_additions.rb", "lib/thinking_sphinx/search.rb", "lib/thinking_sphinx/tasks.rb", "lib/thinking_sphinx.rb", "LICENCE", "README", "tasks/distribution.rb", "tasks/testing.rb", "tasks/rails.rake", "vendor/after_commit", "vendor/after_commit/init.rb", "vendor/after_commit/lib", "vendor/after_commit/lib/after_commit", "vendor/after_commit/lib/after_commit/active_record.rb", "vendor/after_commit/lib/after_commit/connection_adapters.rb", "vendor/after_commit/lib/after_commit.rb", "vendor/after_commit/LICENSE", "vendor/after_commit/Rakefile", "vendor/after_commit/README", "vendor/after_commit/test", "vendor/after_commit/test/after_commit_test.rb", "vendor/delayed_job", "vendor/delayed_job/lib", "vendor/delayed_job/lib/delayed", "vendor/delayed_job/lib/delayed/job.rb", "vendor/delayed_job/lib/delayed/message_sending.rb", "vendor/delayed_job/lib/delayed/performable_method.rb", "vendor/delayed_job/lib/delayed/worker.rb", "vendor/riddle", "vendor/riddle/lib", "vendor/riddle/lib/riddle", "vendor/riddle/lib/riddle/client", "vendor/riddle/lib/riddle/client/filter.rb", "vendor/riddle/lib/riddle/client/message.rb", "vendor/riddle/lib/riddle/client/response.rb", "vendor/riddle/lib/riddle/client.rb", "vendor/riddle/lib/riddle/configuration", "vendor/riddle/lib/riddle/configuration/distributed_index.rb", "vendor/riddle/lib/riddle/configuration/index.rb", "vendor/riddle/lib/riddle/configuration/indexer.rb", "vendor/riddle/lib/riddle/configuration/remote_index.rb", "vendor/riddle/lib/riddle/configuration/searchd.rb", "vendor/riddle/lib/riddle/configuration/section.rb", "vendor/riddle/lib/riddle/configuration/source.rb", "vendor/riddle/lib/riddle/configuration/sql_source.rb", "vendor/riddle/lib/riddle/configuration/xml_source.rb", "vendor/riddle/lib/riddle/configuration.rb", "vendor/riddle/lib/riddle/controller.rb", "vendor/riddle/lib/riddle.rb", "spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/collection_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/core/string_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] s.has_rdoc = true s.homepage = %q{http://ts.freelancing-gods.com} s.rdoc_options = ["--title", "Thinking Sphinx -- Rails/Merb Sphinx Plugin", "--line-numbers"] s.require_paths = ["lib"] s.rubyforge_project = %q{thinking-sphinx} - s.rubygems_version = %q{1.3.0} + s.rubygems_version = %q{1.3.1} s.summary = %q{A concise and easy-to-use Ruby library that connects ActiveRecord to the Sphinx search daemon, managing configuration, indexing and searching.} - s.test_files = ["spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/collection_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] + s.test_files = ["spec/unit/thinking_sphinx/active_record/delta_spec.rb", "spec/unit/thinking_sphinx/active_record/has_many_association_spec.rb", "spec/unit/thinking_sphinx/active_record/search_spec.rb", "spec/unit/thinking_sphinx/active_record_spec.rb", "spec/unit/thinking_sphinx/association_spec.rb", "spec/unit/thinking_sphinx/attribute_spec.rb", "spec/unit/thinking_sphinx/collection_spec.rb", "spec/unit/thinking_sphinx/configuration_spec.rb", "spec/unit/thinking_sphinx/core/string_spec.rb", "spec/unit/thinking_sphinx/field_spec.rb", "spec/unit/thinking_sphinx/index/builder_spec.rb", "spec/unit/thinking_sphinx/index/faux_column_spec.rb", "spec/unit/thinking_sphinx/index_spec.rb", "spec/unit/thinking_sphinx/search_spec.rb", "spec/unit/thinking_sphinx_spec.rb"] if s.respond_to? :specification_version then current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION thinking-sphinx.gemspec @@ -39,4 +39,4 @@ ActiveRecord::Base.send(:include, AfterCommit::ActiveRecord) Object.subclasses_of(ActiveRecord::ConnectionAdapters::AbstractAdapter).each do |klass| klass.send(:include, AfterCommit::ConnectionAdapters) -end \ No newline at end of file +end vendor/after_commit/lib/after_commit.rb @@ -71,19 +71,42 @@ module AfterCommit # after_commit callback can be made from the ConnectionAdapters when # the commit for the transaction has finally succeeded. def after_commit_callback - callback(:after_commit) + call_after_commit_callback :after_commit end def after_commit_on_create_callback - callback(:after_commit_on_create) + call_after_commit_callback :after_commit_on_create end def after_commit_on_update_callback - callback(:after_commit_on_update) + call_after_commit_callback :after_commit_on_update end def after_commit_on_destroy_callback - callback(:after_commit_on_destroy) + call_after_commit_callback :after_commit_on_destroy + end + + private + + def call_after_commit_callback(call) + if can_call_after_commit call + callback call + clear_after_commit_call call + end + end + + def can_call_after_commit(call) + @calls ||= {} + @calls[call] ||= false + if @calls[call] + return false + else + @calls[call] = true + end + end + + def clear_after_commit_call(call) + @calls[call] = false end end end vendor/after_commit/lib/after_commit/active_record.rb @@ -7,7 +7,7 @@ module AfterCommit # override it so that after this happens, any records that were saved # or destroyed within this transaction now get their after_commit # callback fired. - def commit_db_transaction_with_callback + def commit_db_transaction_with_callback commit_db_transaction_without_callback trigger_after_commit_callbacks trigger_after_commit_on_create_callbacks vendor/after_commit/lib/after_commit/connection_adapters.rb @@ -18,7 +18,7 @@ module Riddle #:nodoc: Rev = 1533 # Release number to mark my own fixes, beyond feature parity with # Sphinx itself. - Release = 3 + Release = 4 String = [Major, Minor, Tiny].join('.') GemVersion = [Major, Minor, Tiny, Rev, Release].join('.') vendor/riddle/lib/riddle.rb @@ -456,7 +456,7 @@ module Riddle header = socket.recv(8) status, version, length = header.unpack('n2N') - while response.length < length + while response.length < (length || 0) part = socket.recv(length - response.length) response << part if part end vendor/riddle/lib/riddle/client.rb @@ -20,13 +20,18 @@ module Riddle if send(setting) == "" conf = " #{setting} = " else - conf = Array(send(setting)).collect { |set| + conf = setting_to_array(setting).collect { |set| " #{setting} = #{set}" } end conf.length == 0 ? nil : conf }.flatten.compact end + + def setting_to_array(setting) + value = send(setting) + value.is_a?(Array) ? value : [value] + end end end -end \ No newline at end of file +end vendor/riddle/lib/riddle/configuration/section.rb @@ -14,7 +14,7 @@ module Riddle def start return if running? - cmd = "searchd --config #{@path}" + cmd = "searchd --pidfile --config #{@path}" `#{cmd}` sleep(1) vendor/riddle/lib/riddle/controller.rb tasks/thinking_sphinx_tasks.rake tasks/thinking_sphinx_tasks.rb 67386f3b8e1de3abfeebc00646bb189bf5c5c5bb b9e4294028a541fc58f6a5829db70be30b85fa19 Lourens Naude lourens@methodmissing.com http://github.com/methodmissing/thinking-sphinx/commit/02796217621a85253b53b784114aedd7adf730a9 02796217621a85253b53b784114aedd7adf730a9 2009-03-06T01:35:24-08:00 2009-03-06T01:35:24-08:00 sync da4e30d73218111da26b7199c062fb492c205b51 Lourens Naude lourens@methodmissing.com