Permalink
Browse files

Merge branch 'master' into texticle_console

  • Loading branch information...
2 parents 84c4947 + aec4fbe commit 91e7008fc5dfc3bf3d6514e298c162a220a66628 @benhamill benhamill committed Jul 7, 2012
View
@@ -1,3 +1,22 @@
+== 2.1.0
+
+* 1 DEPRECATION
+
+ * `search` aliases new `advanced_search` method (same functionality as before), but will
+ alias `basic_search` in 3.0! Should print warnings.
+
+* 2 new features
+
+ * Generate full text search indexes from a rake task (sort of like in 1.x). Supply a specific
+ model name.
+ * New search methods: `basic_search`, `advanced_search` and `fuzzy_search`. Basic allows special
+ characters like &, and % in search terms. Fuzzy is based on Postgres's trigram matching extension
+ pg_trgm. Advanced is the same functionality from `search` previously.
+
+* 1 dev improvement
+
+ * Test databse configuration not automatically generated from a rake task and ignored by git.
+
== 2.0.3
* 1 new feature
View
@@ -1,6 +1,7 @@
require 'rubygems'
require 'rake'
+require 'yaml'
require 'pg'
require 'active_record'
require 'benchmark'
@@ -73,6 +74,7 @@ end
task :test do
require 'texticle_spec'
require 'texticle/searchable_spec'
+ require 'texticle/full_text_indexer_spec'
end
namespace :db do
@@ -113,7 +115,9 @@ namespace :db do
desc 'Run migrations for test database'
task :migrate do
- require 'spec_helper'
+ config = YAML.load_file File.expand_path(File.dirname(__FILE__) + '/spec/config.yml')
+ ActiveRecord::Base.establish_connection config.merge(:adapter => :postgresql)
+
ActiveRecord::Migration.instance_eval do
create_table :games do |table|
table.string :system
@@ -138,7 +142,9 @@ namespace :db do
desc 'Drop tables from test database'
task :drop do
- require 'spec_helper'
+ config = YAML.load_file File.expand_path(File.dirname(__FILE__) + '/spec/config.yml')
+ ActiveRecord::Base.establish_connection config.merge(:adapter => :postgresql)
+
ActiveRecord::Migration.instance_eval do
drop_table :games
drop_table :web_comics
View
@@ -7,24 +7,34 @@ def self.version
VERSION
end
- def search(query = "", exclusive = true)
- @similarities = []
- @conditions = []
+ def self.searchable_language
+ 'english'
+ end
- unless query.is_a?(Hash)
- exclusive = false
- query = searchable_columns.inject({}) do |terms, column|
- terms.merge column => query.to_s
- end
- end
+ def search(query = "", exclusive = true)
+ warn "[DEPRECATION] `search` is deprecated. Please use `advanced_search` instead. At the next major release `search` will become an alias for `basic_search`."
+ advanced_search(query, exclusive)
+ end
- parse_query_hash(query)
+ def basic_search(query = "", exclusive = true)
+ exclusive, query = munge_exclusive_and_query(exclusive, query)
+ parsed_query_hash = parse_query_hash(query)
+ similarities, conditions = basic_similarities_and_conditions(parsed_query_hash)
+ assemble_query(similarities, conditions, exclusive)
+ end
- rank = connection.quote_column_name('rank' + rand.to_s)
+ def advanced_search(query = "", exclusive = true)
+ exclusive, query = munge_exclusive_and_query(exclusive, query)
+ parsed_query_hash = parse_query_hash(query)
+ similarities, conditions = advanced_similarities_and_conditions(parsed_query_hash)
+ assemble_query(similarities, conditions, exclusive)
+ end
- select("#{quoted_table_name + '.*,' if scoped.select_values.empty?} #{@similarities.join(" + ")} AS #{rank}").
- where(@conditions.join(exclusive ? " AND " : " OR ")).
- order("#{rank} DESC")
+ def fuzzy_search(query = '', exclusive = true)
+ exclusive, query = munge_exclusive_and_query(exclusive, query)
+ parsed_query_hash = parse_query_hash(query)
+ similarities, conditions = fuzzy_similarities_and_conditions(parsed_query_hash)
+ assemble_query(similarities, conditions, exclusive)
end
def method_missing(method, *search_terms)
@@ -37,7 +47,7 @@ def method_missing(method, *search_terms)
query = columns.inject({}) do |query, column|
query.merge column => args.shift
end
- search(query, exclusive)
+ self.send(Helper.search_type(method), query, exclusive)
end
__send__(method, *search_terms, exclusive)
else
@@ -56,20 +66,93 @@ def respond_to?(method, include_private = false)
private
+ def munge_exclusive_and_query(exclusive, query)
+ unless query.is_a?(Hash)
+ exclusive = false
+ query = searchable_columns.inject({}) do |terms, column|
+ terms.merge column => query.to_s
+ end
+ end
+
+ [exclusive, query]
+ end
+
def parse_query_hash(query, table_name = quoted_table_name)
- language = connection.quote(searchable_language)
table_name = connection.quote_table_name(table_name)
+ results = []
+
query.each do |column_or_table, search_term|
if search_term.is_a?(Hash)
- parse_query_hash(search_term, column_or_table)
+ results += parse_query_hash(search_term, column_or_table)
else
column = connection.quote_column_name(column_or_table)
search_term = connection.quote normalize(Helper.normalize(search_term))
- @similarities << "ts_rank(to_tsvector(#{language}, #{table_name}.#{column}::text), to_tsquery(#{language}, #{search_term}::text))"
- @conditions << "to_tsvector(#{language}, #{table_name}.#{column}::text) @@ to_tsquery(#{language}, #{search_term}::text)"
+
+ results << [table_name, column, search_term]
end
end
+
+ results
+ end
+
+ def basic_similarities_and_conditions(parsed_query_hash)
+ parsed_query_hash.inject([[], []]) do |(similarities, conditions), query_args|
+ similarities << basic_similarity_string(*query_args)
+ conditions << basic_condition_string(*query_args)
+
+ [similarities, conditions]
+ end
+ end
+
+ def basic_similarity_string(table_name, column, search_term)
+ "ts_rank(to_tsvector(#{quoted_language}, #{table_name}.#{column}::text), plainto_tsquery(#{quoted_language}, #{search_term}::text))"
+ end
+
+ def basic_condition_string(table_name, column, search_term)
+ "to_tsvector(#{quoted_language}, #{table_name}.#{column}::text) @@ plainto_tsquery(#{quoted_language}, #{search_term}::text)"
+ end
+
+ def advanced_similarities_and_conditions(parsed_query_hash)
+ parsed_query_hash.inject([[], []]) do |(similarities, conditions), query_args|
+ similarities << advanced_similarity_string(*query_args)
+ conditions << advanced_condition_string(*query_args)
+
+ [similarities, conditions]
+ end
+ end
+
+ def advanced_similarity_string(table_name, column, search_term)
+ "ts_rank(to_tsvector(#{quoted_language}, #{table_name}.#{column}::text), to_tsquery(#{quoted_language}, #{search_term}::text))"
+ end
+
+ def advanced_condition_string(table_name, column, search_term)
+ "to_tsvector(#{quoted_language}, #{table_name}.#{column}::text) @@ to_tsquery(#{quoted_language}, #{search_term}::text)"
+ end
+
+ def fuzzy_similarities_and_conditions(parsed_query_hash)
+ parsed_query_hash.inject([[], []]) do |(similarities, conditions), query_args|
+ similarities << fuzzy_similarity_string(*query_args)
+ conditions << fuzzy_condition_string(*query_args)
+
+ [similarities, conditions]
+ end
+ end
+
+ def fuzzy_similarity_string(table_name, column, search_term)
+ "similarity(#{table_name}.#{column}, #{search_term})"
+ end
+
+ def fuzzy_condition_string(table_name, column, search_term)
+ "(#{table_name}.#{column} % #{search_term})"
+ end
+
+ def assemble_query(similarities, conditions, exclusive)
+ rank = connection.quote_column_name('rank' + rand.to_s)
+
+ select("#{quoted_table_name + '.*,' if scoped.select_values.empty?} #{similarities.join(" + ")} AS #{rank}").
+ where(conditions.join(exclusive ? " AND " : " OR ")).
+ order("#{rank} DESC")
end
def normalize(query)
@@ -80,8 +163,12 @@ def searchable_columns
columns.select {|column| [:string, :text].include? column.type }.map(&:name)
end
+ def quoted_language
+ @quoted_language ||= connection.quote(searchable_language)
+ end
+
def searchable_language
- 'english'
+ Texticle.searchable_language
end
module Helper
@@ -90,16 +177,24 @@ def normalize(query)
query.to_s.gsub(' ', '\\\\ ')
end
+ def method_name_regex
+ /^(?<search_type>((basic|advanced|fuzzy)_)?search)_by_(?<columns>[_a-zA-Z]\w*)$/
+ end
+
+ def search_type(method)
+ method.to_s.match(method_name_regex)[:search_type]
+ end
+
def exclusive_dynamic_search_columns(method)
- if match = method.to_s.match(/^search_by_(?<columns>[_a-zA-Z]\w*)$/)
+ if match = method.to_s.match(method_name_regex)
match[:columns].split('_and_')
else
[]
end
end
def inclusive_dynamic_search_columns(method)
- if match = method.to_s.match(/^search_by_(?<columns>[_a-zA-Z]\w*)$/)
+ if match = method.to_s.match(method_name_regex)
match[:columns].split('_or_')
else
[]
@@ -133,3 +228,5 @@ def dynamic_search_method?(method, class_columns)
end
end
end
+
+require File.expand_path(File.dirname(__FILE__) + '/texticle/full_text_indexer')
@@ -0,0 +1,79 @@
+class Texticle::FullTextIndexer
+ def generate_migration(model_name)
+ stream_output do |io|
+ io.puts(<<-MIGRATION)
+class #{model_name}FullTextSearch < ActiveRecord::Migration
+ def self.up
+ execute(<<-SQL.strip)
+ #{up_migration(model_name)}
+ SQL
+ end
+
+ def self.down
+ execute(<<-SQL.strip)
+ #{down_migration(model_name)}
+ SQL
+ end
+end
+MIGRATION
+ end
+ end
+
+ def stream_output(now = Time.now.utc, &block)
+ if !@output_stream && defined?(Rails)
+ File.open(migration_file_name(now), 'w', &block)
+ else
+ @output_stream ||= $stdout
+
+ yield @output_stream
+ end
+ end
+
+ private
+
+ def migration_file_name(now = Time.now.utc)
+ File.join(Rails.root, 'db', 'migrate',"#{now.strftime('%Y%m%d%H%M%S')}_full_text_search.rb")
+ end
+
+ def up_migration(model_name)
+ migration_with_type(model_name, :up)
+ end
+
+ def down_migration(model_name)
+ migration_with_type(model_name, :down)
+ end
+
+ def migration_with_type(model_name, type)
+ sql_lines = ''
+
+ model = Kernel.const_get(model_name)
+ model.indexable_columns.each do |column|
+ sql_lines << drop_index_sql_for(model, column)
+ sql_lines << create_index_sql_for(model, column) if type == :up
+ end
+
+ sql_lines.strip.gsub("\n","\n ")
+ end
+
+ def drop_index_sql_for(model, column)
+ "DROP index IF EXISTS #{index_name_for(model, column)};\n"
+ end
+
+ def create_index_sql_for(model, column)
+ # The spacing gets sort of wonky in here.
+
+ <<-SQL
+CREATE index #{index_name_for(model, column)}
+ ON #{model.table_name}
+ USING gin(to_tsvector("#{dictionary}", "#{model.table_name}"."#{column}"::text));
+SQL
+ end
+
+ def index_name_for(model, column)
+ "#{model.table_name}_#{column}_fts_idx"
+ end
+
+ def dictionary
+ Texticle.searchable_language
+ end
+end
@@ -10,6 +10,10 @@ def Searchable(*searchable_columns)
end
private :searchable_columns
+
+ def indexable_columns
+ searchable_columns.to_enum
+ end
end
end
View
@@ -2,6 +2,12 @@
require 'texticle'
namespace :texticle do
+ desc 'Create full text search index migration, give the model for which you want to create the indexes'
+ task :create_index_migration, [:model_name] => :environment do |task, args|
+ raise 'A model name is required' unless args[:model_name]
+ Texticle::FullTextIndexer.new.generate_migration(args[:model_name])
+ end
+
desc "Install trigram text search module"
task :install_trigram => [:environment] do
share_dir = `pg_config --sharedir`.chomp
Oops, something went wrong.

0 comments on commit 91e7008

Please sign in to comment.