initial attempt at mysql fulltext search for nodes (#1913)

* initial attempt at mysql fulltext search for nodes

* tweak

* major rails 4.x update of schema.rb, added fulltext index

* Update node.rb

* Update node_test.rb

* rename

* self.

* add title to fulltext

* plugged into typeahead

* rev title fulltext index

* fixed fulltext multi index

* updated schema example

* fix

* tweaked working fulltext in console

* removing solr test apparatus

* full text comment and user bio searching

* more conditions

* fixes

* unique results by grouping by nid

* Remove Solr stuff and add secret_token env var (#259)

* Remove Solr stuff and add secret_token env var

* Upgrade and switch database engine to Mariadb.

Rakefile

@@ -15,13 +15,9 @@ namespace :test do
   #Rake::TestTask.new(:_run) do |t|
   Rake::TestTask.new(:run) do |t|
     t.libs << "test"
-    t.test_files = FileList['test/**/*_test.rb'].exclude(
-      'test/solr/**/*_test.rb'
-    )
+    t.test_files = FileList['test/**/*_test.rb']
   end
-  #task :run => ['test:_run']
 
-  # rake test:all
   desc "Run rails and jasmine tests"
   task :all => :environment do
     require 'coveralls/rake/task'
@@ -32,10 +28,6 @@ namespace :test do
     end
     puts "Running Rails tests"
     Rake::Task["test:run"].execute
-    puts "Preparing Solr-dependent tests"
-    Rake::Task["test:solr_setup"].execute
-    Rake::Task["test:solr"].execute
-    Rake::Task["test:solr_cleanup"].execute
     puts "Running jasmine tests headlessly"
     Rake::Task["spec:javascript"].execute
     Rake::Task["coveralls:push"].execute
@@ -47,39 +39,4 @@ namespace :test do
     Rake::Task["spec:javascript"].execute
   end
 
-  desc "Prepare for Solr-specific tests"
-  # Solr is assumed running from the container or otherwise available as in sunspot.yml.
-  task :solr_setup do
-    # overwrite "diabled" in test for sunspot.yml
-    require 'yaml'
-    sunspot = YAML::load_file "config/sunspot.yml"
-    sunspot['test']['disabled'] = false
-    File.open("config/sunspot.yml", "w") do |file|
-      file.write sunspot.to_yaml
-    end
-    puts "turning on solr dependence at config/sunspot.yml"
-    puts sunspot.to_yaml
-    `RAILS_ENV=test rake SOLR_DISABLE_CHECK=1 sunspot:reindex`
-  end
-
-  desc "Clean up after solr-specific tests"
-  task :solr_cleanup do
-    # restore "diabled" to true in test for sunspot.yml
-    puts "turning solr dependence back off in tests at config/sunspot.yml"
-    require 'yaml'
-    sunspot = YAML::load_file "config/sunspot.yml"
-    sunspot['test']['disabled'] = true
-    File.open("config/sunspot.yml", "w") do |file|
-      file.write sunspot.to_yaml
-    end
-  end
-
-  desc "Run Solr-specific tests"
-  Rake::TestTask.new(:solr) do |t|
-    puts "Running Solr-dependent tests"
-    t.libs << "test"
-    t.pattern = 'test/solr/*_test.rb'
-    t.verbose = true
-  end
-
 end

app/controllers/searches_controller.rb

@@ -1,19 +1,6 @@
 require 'search'
 
 class SearchesController < ApplicationController
-  include SolrToggle
-
-  def test
-    term = params[:q] || "Chicago"
-    if solrAvailable
-      @search = Node.search do
-        fulltext term
-      end
-      render text: @search.results.to_json
-    else
-      render text: 'Solr search service offline'
-    end
-  end
 
   # Dynamic Search Page using pure JavaScript JSON RESTful API
   def dynamic

app/models/comment.rb

@@ -19,6 +19,10 @@ def self.inheritance_column
     'rails_type'
   end
 
+  def self.search(query)
+    Comment.where('MATCH(comment) AGAINST(?)', query)
+  end
+
   def self.comment_weekly_tallies(span = 52, time = Time.now)
     weeks = {}
     (0..span).each do |week|

app/models/node.rb

@@ -20,23 +20,8 @@ class Node < ActiveRecord::Base
   self.table_name = 'node'
   self.primary_key = 'nid'
 
-  include SolrToggle
-  searchable if: :shouldIndexSolr do
-    text :title
-    text :body do
-      body.to_s.gsub!(/[[:cntrl:]]/,'').to_s.slice!(0..32500)
-    end
-    string :updated_at
-    string :status
-    string :type
-    string :updated_month
-    text :comments do
-      comments.map(&:comment)
-    end
-
-    string :user_name do
-      drupal_user.name
-    end
+  def self.search(query)
+    Revision.where('MATCH(node_revisions.body, node_revisions.title) AGAINST(?)', query)
   end
 
   def updated_month

app/models/user.rb

@@ -11,14 +11,6 @@ class User < ActiveRecord::Base
   attr_accessible :username, :email, :password, :password_confirmation, :openid_identifier, :key, :photo, :photo_file_name, :bio
   alias_attribute :name, :username
 
-  include SolrToggle
-  searchable if: :shouldIndexSolr do
-    text :username, :email
-    text :bio do
-      bio.to_s.gsub!(/[[:cntrl:]]/,'').to_s.slice!(0..32500)
-    end
-  end
-
   acts_as_authentic do |c|
     c.openid_required_fields = %i[nickname email]
     c.validates_format_of_email_field_options = { with: /@/ }
@@ -49,6 +41,10 @@ class User < ActiveRecord::Base
   before_save :set_token
   after_destroy :destroy_drupal_user
 
+  def self.search(query)
+    User.where('MATCH(username, bio) AGAINST(?)', query)
+  end
+
   def create_drupal_user
     self.bio ||= ''
     if drupal_user.nil?

app/services/typeahead_service.rb

@@ -14,9 +14,15 @@ def initialize; end
   # but perhaps could simply be renamed Result.
 
   def users(input, limit = 5)
-    User.limit(limit)
-        .order('id DESC')
-        .where('username LIKE ? AND status = 1', '%' + input + '%')
+    if ActiveRecord::Base.connection.adapter_name == 'Mysql2'
+      User.search(input)
+          .limit(limit)
+          .where(status: 1)
+    else 
+      User.limit(limit)
+          .order('id DESC')
+          .where('username LIKE ? AND status = 1', '%' + input + '%')
+    end
   end
 
   def tags(input, limit = 5)
@@ -28,33 +34,48 @@ def tags(input, limit = 5)
   end
 
   def comments(input, limit = 5)
-    Comment.limit(limit)
-           .order('nid DESC')
-           .where('status = 1 AND comment LIKE ?', '%' + input + '%')
+    if ActiveRecord::Base.connection.adapter_name == 'Mysql2'
+      Comment.search(input)
+             .limit(limit)
+             .order('nid DESC')
+             .where(status: 1)
+    else 
+      Comment.limit(limit)
+             .order('nid DESC')
+             .where('status = 1 AND comment LIKE ?', '%' + input + '%')
+    end
   end
 
   def notes(input, limit = 5)
-    if solrAvailable
-      search = Node.search do
-        fulltext input
-        with :status, 1
-        #with :type, "note"
-        order_by :updated_at, :desc
-        paginate page: 1, per_page: limit
-      end
-      search.results
+    if ActiveRecord::Base.connection.adapter_name == 'Mysql2'
+      Node.search(input)
+          .group(:nid)
+          .includes(:node)
+          .references(:node)
+          .limit(limit)
+          .where("node.type": "note", "node.status": 1)
+          .order(timestamp: :desc)
     else 
       Node.limit(limit)
-          .order('nid DESC')
+          .group(:nid)
           .where(type: "note", status: 1)
+          .order(updated_at: :desc)
           .where('title LIKE ?', '%' + input + '%')
     end
   end
 
   def wikis(input, limit = 5)
-    Node.limit(limit)
-        .order('nid DESC')
-        .where('type = "page" AND node.status = 1 AND title LIKE ?', '%' + input + '%')
+    if ActiveRecord::Base.connection.adapter_name == 'Mysql2'
+      Node.search(input)
+          .includes(:node)
+          .references(:node)
+          .limit(limit)
+          .where("node.type": "page", "node.status": 1)
+    else 
+      Node.limit(limit)
+          .order('nid DESC')
+          .where('type = "page" AND node.status = 1 AND title LIKE ?', '%' + input + '%')
+    end
   end
 
   def maps(input, limit = 5)
@@ -126,7 +147,7 @@ def search_notes(srchString, limit = 5)
   def search_wikis(srchString, limit = 5)
     sresult = TagList.new
     unless srchString.nil? || srchString == 0
-      wikis(srchString, limit).select('title,type,nid,path').each do |match|
+      wikis(srchString, limit).select('node.title,node.type,node.nid,node.path').each do |match|
         tval = TagResult.new
         tval.tagId = match.nid
         tval.tagVal = match.title

db/migrate/20180103212804_revision_fulltext_index.rb

@@ -0,0 +1,17 @@
+class RevisionFulltextIndex < ActiveRecord::Migration
+  def up
+    if ActiveRecord::Base.connection.adapter_name == 'Mysql2'
+      add_index :node_revisions, [:body, :title], type: :fulltext
+      add_index :comments, :comment, type: :fulltext
+      add_index :rusers, [:username, :bio], type: :fulltext
+    end
+  end
+
+  def down
+    if ActiveRecord::Base.connection.adapter_name == 'Mysql2'
+      remove_index :node_revisions, [:body, :title]
+      remove_index :comments, :comment
+      remove_index :rusers, [:username, :bio]
+    end
+  end
+end