From 44e247bfded06a433d48ad45d9414a21aa1eb344 Mon Sep 17 00:00:00 2001 From: Pat Allan Date: Fri, 10 Aug 2012 22:07:21 +0100 Subject: [PATCH] Sphinx 2.0.5 support --- lib/thinking_sphinx/auto_version.rb | 2 +- spec/thinking_sphinx/auto_version_spec.rb | 16 ++ thinking-sphinx.gemspec | 2 +- vendor/riddle/lib/riddle/auto_version.rb | 2 +- vendor/riddle/lib/riddle/configuration.rb | 6 + .../riddle/lib/riddle/configuration/index.rb | 70 +++++--- .../riddle/lib/riddle/configuration/parser.rb | 159 ++++++++++++++++++ vendor/riddle/lib/riddle/controller.rb | 4 + vendor/riddle/lib/riddle/query/select.rb | 28 ++- vendor/riddle/lib/riddle/version.rb | 2 +- 10 files changed, 257 insertions(+), 34 deletions(-) create mode 100644 vendor/riddle/lib/riddle/configuration/parser.rb diff --git a/lib/thinking_sphinx/auto_version.rb b/lib/thinking_sphinx/auto_version.rb index a81300472..f5d2f84ce 100644 --- a/lib/thinking_sphinx/auto_version.rb +++ b/lib/thinking_sphinx/auto_version.rb @@ -9,7 +9,7 @@ def self.detect require 'riddle/1.10' when /2.0.[12]/ require 'riddle/2.0.1' - when /2.0.3/, /2.0.4/, /2.1.\d/ + when /2.0.[^12]/, /2.1.\d/ require 'riddle/2.1.0' else documentation_link = %Q{ diff --git a/spec/thinking_sphinx/auto_version_spec.rb b/spec/thinking_sphinx/auto_version_spec.rb index cccb6ec78..3b1a75ca8 100644 --- a/spec/thinking_sphinx/auto_version_spec.rb +++ b/spec/thinking_sphinx/auto_version_spec.rb @@ -62,6 +62,22 @@ ThinkingSphinx::AutoVersion.detect end + it "should require 2.1.0 if using Sphinx 2.0.3" do + ThinkingSphinx::AutoVersion.should_receive(:require). + with('riddle/2.1.0') + + @config.stub!(:version => '2.0.4-release') + ThinkingSphinx::AutoVersion.detect + end + + it "should require 2.1.0 if using Sphinx 2.0.3" do + ThinkingSphinx::AutoVersion.should_receive(:require). + with('riddle/2.1.0') + + @config.stub!(:version => '2.0.5-release') + ThinkingSphinx::AutoVersion.detect + end + it "should require 2.1.0 if using Sphinx 2.1.0 dev" do ThinkingSphinx::AutoVersion.should_receive(:require). with('riddle/2.1.0') diff --git a/thinking-sphinx.gemspec b/thinking-sphinx.gemspec index 9a8e68b2e..51bbdae7f 100644 --- a/thinking-sphinx.gemspec +++ b/thinking-sphinx.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'activerecord', ['< 3.0.0', '>= 1.15.6'] s.add_runtime_dependency 'after_commit', '>= 1.0.7' - s.add_runtime_dependency 'riddle', '>= 1.5.2' + s.add_runtime_dependency 'riddle', '>= 1.5.3' s.add_development_dependency 'appraisal', '0.4.0' s.add_development_dependency 'cucumber', '1.0.2' diff --git a/vendor/riddle/lib/riddle/auto_version.rb b/vendor/riddle/lib/riddle/auto_version.rb index 39b7ac319..f259c46ac 100644 --- a/vendor/riddle/lib/riddle/auto_version.rb +++ b/vendor/riddle/lib/riddle/auto_version.rb @@ -10,7 +10,7 @@ def self.configure require 'riddle/1.10' when /2.0.[12]/ require 'riddle/2.0.1' - when /2.0.3/, /2.0.4/, /2.1.\d/ + when /2.0.[^12]/, /2.1.\d/ require 'riddle/2.1.0' end end diff --git a/vendor/riddle/lib/riddle/configuration.rb b/vendor/riddle/lib/riddle/configuration.rb index 811cb4a05..8e9e65393 100644 --- a/vendor/riddle/lib/riddle/configuration.rb +++ b/vendor/riddle/lib/riddle/configuration.rb @@ -10,6 +10,8 @@ require 'riddle/configuration/sql_source' require 'riddle/configuration/xml_source' +require 'riddle/configuration/parser' + module Riddle class Configuration class ConfigurationError < StandardError #:nodoc: @@ -18,6 +20,10 @@ class ConfigurationError < StandardError #:nodoc: attr_reader :indices, :searchd attr_accessor :indexer + def self.parse!(input) + Riddle::Configuration::Parser.new(input).parse! + end + def initialize @indexer = Riddle::Configuration::Indexer.new @searchd = Riddle::Configuration::Searchd.new diff --git a/vendor/riddle/lib/riddle/configuration/index.rb b/vendor/riddle/lib/riddle/configuration/index.rb index a4d2ac6a9..88a37d56f 100644 --- a/vendor/riddle/lib/riddle/configuration/index.rb +++ b/vendor/riddle/lib/riddle/configuration/index.rb @@ -16,7 +16,7 @@ def self.settings :overshort_step, :stopwords_step, :hitless_words ] end - + attr_accessor :name, :parent, :type, :sources, :path, :docinfo, :mlock, :morphologies, :dict, :index_sp, :index_zones, :min_stemming_len, :stopword_files, :wordform_files, :exception_files, :min_word_len, @@ -29,7 +29,7 @@ def self.settings :inplace_hit_gap, :inplace_docinfo_gap, :inplace_reloc_factor, :inplace_write_factor, :index_exact_words, :overshort_step, :stopwords_step, :hitless_words - + def initialize(name, *sources) @name = name @sources = sources @@ -44,86 +44,102 @@ def initialize(name, *sources) @phrase_boundaries = [] @html_remove_element_tags = [] end - + def source @sources.collect { |s| s.name } end - + def morphology nil_join @morphologies, ", " end - + def morphology=(morphology) @morphologies = nil_split morphology, /,\s?/ end - + def stopwords nil_join @stopword_files, " " end - + def stopwords=(stopwords) @stopword_files = nil_split stopwords, ' ' end - + def wordforms nil_join @wordform_files, " " end - + def wordforms=(wordforms) @wordform_files = nil_split wordforms, ' ' end - + def exceptions nil_join @exception_files, " " end - + def exceptions=(exceptions) @exception_files = nil_split exceptions, ' ' end - + def ignore_chars nil_join @ignore_characters, ", " end - + def ignore_chars=(ignore_chars) @ignore_characters = nil_split ignore_chars, /,\s?/ end - + def prefix_fields nil_join @prefix_field_names, ", " end - + + def prefix_fields=(fields) + if fields.is_a?(Array) + @prefix_field_names = fields + else + @prefix_field_names = fields.split(/,\s*/) + end + end + def infix_fields nil_join @infix_field_names, ", " end - + + def infix_fields=(fields) + if fields.is_a?(Array) + @infix_field_names = fields + else + @infix_field_names = fields.split(/,\s*/) + end + end + def ngram_chars nil_join @ngram_characters, ", " end - + def ngram_chars=(ngram_chars) @ngram_characters = nil_split ngram_chars, /,\s?/ end - + def phrase_boundary nil_join @phrase_boundaries, ", " end - + def phrase_boundary=(phrase_boundary) @phrase_boundaries = nil_split phrase_boundary, /,\s?/ end - + def html_remove_elements nil_join @html_remove_element_tags, ", " end - + def html_remove_elements=(html_remove_elements) @html_remove_element_tags = nil_split html_remove_elements, /,\s?/ end - + def render raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid? - + inherited_name = "#{name}" inherited_name << " : #{parent}" if parent ( @@ -133,17 +149,17 @@ def render ["}", ""] ).join("\n") end - + def valid? (!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?) end - + private - + def nil_split(string, pattern) (string || "").split(pattern) end - + def nil_join(array, delimiter) if array.length == 0 nil diff --git a/vendor/riddle/lib/riddle/configuration/parser.rb b/vendor/riddle/lib/riddle/configuration/parser.rb new file mode 100644 index 000000000..9f7fedc6d --- /dev/null +++ b/vendor/riddle/lib/riddle/configuration/parser.rb @@ -0,0 +1,159 @@ +# encoding: UTF-8 +require 'stringio' + +class Riddle::Configuration::Parser + SOURCE_CLASSES = { + 'mysql' => Riddle::Configuration::SQLSource, + 'pgsql' => Riddle::Configuration::SQLSource, + 'mssql' => Riddle::Configuration::SQLSource, + 'xmlpipe' => Riddle::Configuration::XMLSource, + 'xmlpipe2' => Riddle::Configuration::XMLSource, + 'odbc' => Riddle::Configuration::SQLSource + } + + INDEX_CLASSES = { + 'plain' => Riddle::Configuration::Index, + 'distributed' => Riddle::Configuration::DistributedIndex, + 'rt' => Riddle::Configuration::RealtimeIndex + } + + def initialize(input) + @input = input + end + + def parse! + set_indexer + set_searchd + set_sources + set_indices + + configuration + end + + private + + def inner + @inner ||= InnerParser.new(@input).parse! + end + + def configuration + @configuration ||= Riddle::Configuration.new + end + + def sources + @sources ||= {} + end + + def each_with_prefix(prefix) + inner.keys.select { |key| key[/^#{prefix}\s+/] }.each do |key| + yield key.gsub(/^#{prefix}\s+/, ''), inner[key] + end + end + + def set_indexer + set_settings configuration.indexer, inner['indexer'] + end + + def set_searchd + set_settings configuration.searchd, inner['searchd'] + end + + def set_sources + each_with_prefix 'source' do |name, settings| + names = name.split(/\s*:\s*/) + type = settings.delete('type').first + source = SOURCE_CLASSES[type].new names.first, type + source.parent = names.last if names.length > 1 + + set_settings source, settings + + sources[source.name] = source + end + end + + def set_indices + each_with_prefix 'index' do |name, settings| + names = name.split(/\s*:\s*/) + type = (settings.delete('type') || ['plain']).first + index = INDEX_CLASSES[type].new names.first + index.parent = names.last if names.length > 1 + + (settings.delete('source') || []).each do |source_name| + index.sources << sources[source_name] + end + + set_settings index, settings + + configuration.indices << index + end + end + + def set_settings(object, hash) + hash.each do |key, values| + values.each do |value| + set_setting object, key, value + end + end + end + + def set_setting(object, key, value) + if object.send(key).is_a?(Array) + object.send(key) << value + else + object.send "#{key}=", value + end + end + + class InnerParser + SETTING_PATTERN = /^(\w+)\s*=\s*(.*)$/ + + def initialize(input) + @stream = StringIO.new(input) + @sections = {} + end + + def parse! + while label = next_line do + @sections[label] = next_settings + end + + @sections + end + + private + + def next_line + line = @stream.gets + return line if line.nil? + + line = line.strip + line.empty? ? next_line : line + end + + def next_settings + settings = Hash.new { |hash, key| hash[key] = [] } + line = '' + while line.empty? || line == '{' do + line = next_line + end + + while line != '}' do + begin + key, value = *SETTING_PATTERN.match(line).captures + settings[key] << value + while value[/\\$/] do + value = next_line + settings[key].last << "\n" << value + end + rescue => error + raise error, "Error handling line '#{line}': #{error.message}", + error.backtrace + end + + line = next_line + end + + settings + end + end +end diff --git a/vendor/riddle/lib/riddle/controller.rb b/vendor/riddle/lib/riddle/controller.rb index 7a357504a..3bb3fafbf 100644 --- a/vendor/riddle/lib/riddle/controller.rb +++ b/vendor/riddle/lib/riddle/controller.rb @@ -73,6 +73,10 @@ def pid end end + def rotate + pid && Process.kill(:HUP, pid.to_i) + end + def running? !!pid && !!Process.kill(0, pid.to_i) rescue diff --git a/vendor/riddle/lib/riddle/query/select.rb b/vendor/riddle/lib/riddle/query/select.rb index 7298a9600..6c8506946 100644 --- a/vendor/riddle/lib/riddle/query/select.rb +++ b/vendor/riddle/lib/riddle/query/select.rb @@ -4,7 +4,9 @@ def initialize @indices = [] @matching = nil @wheres = {} + @where_alls = {} @where_nots = {} + @where_not_alls = {} @group_by = nil @order_by = nil @order_within_group_by = nil @@ -33,11 +35,21 @@ def where(filters = {}) self end + def where_all(filters = {}) + @where_alls.merge!(filters) + self + end + def where_not(filters = {}) @where_nots.merge!(filters) self end + def where_not_all(filters = {}) + @where_not_alls.merge!(filters) + self + end + def group_by(attribute) @group_by = attribute self @@ -85,13 +97,13 @@ def to_sql private def wheres? - !(@wheres.empty? && @where_nots.empty? && @matching.nil?) + !(@wheres.empty? && @where_alls.empty? && @where_nots.empty? && @where_not_alls.empty? && @matching.nil?) end def combined_wheres if @matching.nil? wheres_to_s - elsif @wheres.empty? && @where_nots.empty? + elsif @wheres.empty? && @where_nots.empty? && @where_alls.empty? && @where_not_alls.empty? "MATCH('#{@matching}')" else "MATCH('#{@matching}') AND #{wheres_to_s}" @@ -103,10 +115,20 @@ def wheres_to_s @wheres.keys.collect { |key| filter_comparison_and_value key, @wheres[key] } + + @where_alls.collect { |key, values| + values.collect { |value| + filter_comparison_and_value key, value + } + } + @where_nots.keys.collect { |key| exclusive_filter_comparison_and_value key, @where_nots[key] + } + + @where_not_alls.collect { |key, values| + '(' + values.collect { |value| + exclusive_filter_comparison_and_value key, value + }.join(' OR ') + ')' } - ).join(' AND ') + ).flatten.join(' AND ') end def filter_comparison_and_value(attribute, value) diff --git a/vendor/riddle/lib/riddle/version.rb b/vendor/riddle/lib/riddle/version.rb index c9a7150d1..dc254d035 100644 --- a/vendor/riddle/lib/riddle/version.rb +++ b/vendor/riddle/lib/riddle/version.rb @@ -1,3 +1,3 @@ module Riddle - Version = '1.5.2' + Version = '1.5.3' end