AO3-4952 Move autocomplete to its own Redis instance (#2889)

* Start on travis config

* Generate more config

* try and run servers

* try and run servers

* try and run servers

* Run with multiple redis servers

* Clear out the new redis instance in rspec tests

* Clear out the new redis instance in cucumber tests

* Fix typo

app/models/tag.rb

@@ -510,7 +510,7 @@ def add_to_autocomplete(score = nil)
     score ||= autocomplete_score
     if self.is_a?(Character) || self.is_a?(Relationship)
       parents.each do |parent|
-        REDIS_GENERAL.zadd("autocomplete_fandom_#{parent.name.downcase}_#{type.downcase}", score, autocomplete_value) if parent.is_a?(Fandom)
+        REDIS_AUTOCOMPLETE.zadd("autocomplete_fandom_#{parent.name.downcase}_#{type.downcase}", score, autocomplete_value) if parent.is_a?(Fandom)
       end
     end
     super
@@ -520,7 +520,7 @@ def remove_from_autocomplete
     super
     if self.is_a?(Character) || self.is_a?(Relationship)
       parents.each do |parent|
-        REDIS_GENERAL.zrem("autocomplete_fandom_#{parent.name.downcase}_#{type.downcase}", autocomplete_value) if parent.is_a?(Fandom)
+        REDIS_AUTOCOMPLETE.zrem("autocomplete_fandom_#{parent.name.downcase}_#{type.downcase}", autocomplete_value) if parent.is_a?(Fandom)
       end
     end
   end
@@ -529,7 +529,7 @@ def remove_stale_from_autocomplete
     super
     if self.is_a?(Character) || self.is_a?(Relationship)
       parents.each do |parent|
-        REDIS_GENERAL.zrem("autocomplete_fandom_#{parent.name.downcase}_#{type.downcase}", autocomplete_value_before_last_save) if parent.is_a?(Fandom)
+        REDIS_AUTOCOMPLETE.zrem("autocomplete_fandom_#{parent.name.downcase}_#{type.downcase}", autocomplete_value_before_last_save) if parent.is_a?(Fandom)
       end
     end
   end
@@ -556,10 +556,10 @@ def self.autocomplete_fandom_lookup(options = {})
     fandoms.each do |single_fandom|
       if search_param.blank?
         # just return ALL the characters
-        results += REDIS_GENERAL.zrevrange("autocomplete_fandom_#{single_fandom}_#{tag_type}", 0, -1)
+        results += REDIS_AUTOCOMPLETE.zrevrange("autocomplete_fandom_#{single_fandom}_#{tag_type}", 0, -1)
       else
         search_regex = Tag.get_search_regex(search_param)
-        results += REDIS_GENERAL.zrevrange("autocomplete_fandom_#{single_fandom}_#{tag_type}", 0, -1).select {|tag| tag.match(search_regex)}
+        results += REDIS_AUTOCOMPLETE.zrevrange("autocomplete_fandom_#{single_fandom}_#{tag_type}", 0, -1).select {|tag| tag.match(search_regex)}
       end
     end
     if options[:fallback] && results.empty? && search_param.length > 0

app/models/tagset_models/tag_set.rb

@@ -1,3 +1,4 @@
+
 class TagSet < ApplicationRecord
 
   # a complete match is numerically represented with ALL
@@ -313,22 +314,22 @@ def add_to_autocomplete(score = nil)
   end
 
   def remove_from_autocomplete
-    REDIS_GENERAL.del("autocomplete_tagset_#{self.id}")
+    REDIS_AUTOCOMPLETE.del("autocomplete_tagset_#{self.id}")
   end
 
   def add_tags_to_autocomplete(tags_to_add)
     tags_to_add.each do |tag|
       value = tag.autocomplete_value
-      REDIS_GENERAL.zadd("autocomplete_tagset_all_#{self.id}", 0, value)
-      REDIS_GENERAL.zadd("autocomplete_tagset_#{tag.type.downcase}_#{self.id}", 0, value)
+      REDIS_AUTOCOMPLETE.zadd("autocomplete_tagset_all_#{self.id}", 0, value)
+      REDIS_AUTOCOMPLETE.zadd("autocomplete_tagset_#{tag.type.downcase}_#{self.id}", 0, value)
     end
   end
 
   def remove_tags_from_autocomplete(tags_to_remove)
     tags_to_remove.each do |tag|
       value = tag.autocomplete_value
-      REDIS_GENERAL.zrem("autocomplete_tagset_all_#{self.id}", value)
-      REDIS_GENERAL.zrem("autocomplete_tagset_#{tag.type.downcase}_#{self.id}", value)
+      REDIS_AUTOCOMPLETE.zrem("autocomplete_tagset_all_#{self.id}", value)
+      REDIS_AUTOCOMPLETE.zrem("autocomplete_tagset_#{tag.type.downcase}_#{self.id}", value)
     end
   end
 
@@ -346,14 +347,14 @@ def self.autocomplete_lookup(options={})
 
     if options[:in_any]
       # get the union since we want tags in ANY of these sets
-      REDIS_GENERAL.zunionstore(combo_key, keys_to_lookup, aggregate: :max)
+      REDIS_AUTOCOMPLETE.zunionstore(combo_key, keys_to_lookup, aggregate: :max)
     else
       # take the intersection of ALL of these sets
-      REDIS_GENERAL.zinterstore(combo_key, keys_to_lookup, aggregate: :max)
+      REDIS_AUTOCOMPLETE.zinterstore(combo_key, keys_to_lookup, aggregate: :max)
     end
-    results = REDIS_GENERAL.zrevrange(combo_key, 0, -1)
+    results = REDIS_AUTOCOMPLETE.zrevrange(combo_key, 0, -1)
     # expire fast
-    REDIS_GENERAL.expire combo_key, 1
+    REDIS_AUTOCOMPLETE.expire combo_key, 1
 
     unless search_param.blank?
       search_regex = Tag.get_search_regex(search_param)

app/models/tagset_models/tag_set_association.rb

@@ -7,7 +7,7 @@ class TagSetAssociation < ApplicationRecord
   validates_presence_of :tag_id, :parent_tag_id, :owned_tag_set_id
 
   attr_accessor :create_association
-  
+
   def to_s
     "#{tag.name} (#{parent_tag.name})"
   end
@@ -21,7 +21,7 @@ def self.by_name_without_articles(fieldname = "name")
             else #{fieldname}
             end")
   end
-  
+
   def self.for_tag_set(tagset)
     where(owned_tag_set_id: tagset.id)
   end
@@ -33,7 +33,7 @@ def self.parent_names(child_type, parent_type = "fandom")
     by_name_without_articles("parent_name").
     by_name_without_articles("child_name")
   end
-  
+
   def self.names_by_parent(child_relation, child_type, parent_type = "fandom")
     hash = {}
     results = ActiveRecord::Base.connection.execute(child_relation.parent_names(child_type, parent_type).to_sql)
@@ -44,28 +44,28 @@ def self.names_by_parent(child_relation, child_type, parent_type = "fandom")
   def parent_tagname
     @parent_tagname || self.parent_tag.name
   end
-  
+
   def parent_tagname=(parent_tagname)
     self.parent_tag = Tag.find_by_name(parent_tagname)
   end
-  
+
   def make_official!
     tag.add_association(parent_tag)
     self.destroy
   end
-  
+
   after_save :add_to_autocomplete
   before_destroy :remove_from_autocomplete
-  
+
   ## AUTOCOMPLETE
   # set up autocomplete and override some methods
   include AutocompleteSource
-  
+
   def autocomplete_prefixes
     prefixes = [ ]
     prefixes
   end
-  
+
   # the value and score in autocomplete are the value/score of the child tag
   def autocomplete_value
     tag.autocomplete_value
@@ -78,21 +78,21 @@ def autocomplete_score
   def self.parse_autocomplete_value(current_autocomplete_value)
     Tag.parse_autocomplete_value(current_autocomplete_value)
   end
-      
+
   def add_to_autocomplete(score = nil)
     score ||= autocomplete_score
-    REDIS_GENERAL.zadd("autocomplete_association_#{tag.type.downcase}_#{owned_tag_set.tag_set_id}_#{parent_tag.name.downcase}", score, autocomplete_value)
+    REDIS_AUTOCOMPLETE.zadd("autocomplete_association_#{tag.type.downcase}_#{owned_tag_set.tag_set_id}_#{parent_tag.name.downcase}", score, autocomplete_value)
   end
 
   def remove_from_autocomplete
-    REDIS_GENERAL.zrem("autocomplete_association_#{tag.type.downcase}_#{owned_tag_set.tag_set_id}_#{parent_tag.name.downcase}", autocomplete_value)
+    REDIS_AUTOCOMPLETE.zrem("autocomplete_association_#{tag.type.downcase}_#{owned_tag_set.tag_set_id}_#{parent_tag.name.downcase}", autocomplete_value)
   end
-    
+
   # returns tags that have been associated with a given fandom OR wrangled
   def self.autocomplete_lookup(options = {})
     options.reverse_merge!({term: "", tag_type: "character", tag_set: "", fandom: "", include_wrangled: "true"})
     search_param = options[:term]
-    tag_type = options[:tag_type]    
+    tag_type = options[:tag_type]
     fandoms = TagSetAssociation.get_search_terms(options[:fandom])
     tag_sets = TagSetAssociation.get_search_terms(options[:tag_set])
 
@@ -110,18 +110,18 @@ def self.autocomplete_lookup(options = {})
       combo_key2 = combo_key + "2"
       combo_key3 = combo_key + "3"
       keys_for_intersect = tag_sets.map {|set| "autocomplete_tagset_#{tag_type}_#{set}"}.flatten
-      REDIS_GENERAL.zunionstore(combo_key2, keys_to_lookup, aggregate: :max)
-      REDIS_GENERAL.zunionstore(combo_key3, keys_for_intersect, aggregate: :max)
-      REDIS_GENERAL.zinterstore(combo_key, [combo_key2, combo_key3], aggregate: :max)
-      REDIS_GENERAL.expire combo_key2, 1
-      REDIS_GENERAL.expire combo_key3, 1
+      REDIS_AUTOCOMPLETE.zunionstore(combo_key2, keys_to_lookup, aggregate: :max)
+      REDIS_AUTOCOMPLETE.zunionstore(combo_key3, keys_for_intersect, aggregate: :max)
+      REDIS_AUTOCOMPLETE.zinterstore(combo_key, [combo_key2, combo_key3], aggregate: :max)
+      REDIS_AUTOCOMPLETE.expire combo_key2, 1
+      REDIS_AUTOCOMPLETE.expire combo_key3, 1
     else
-      REDIS_GENERAL.zunionstore(combo_key, keys_to_lookup, aggregate: :max)
+      REDIS_AUTOCOMPLETE.zunionstore(combo_key, keys_to_lookup, aggregate: :max)
     end
-    
-    results = REDIS_GENERAL.zrevrange(combo_key, 0, -1)
-    REDIS_GENERAL.expire combo_key, 1
-    
+
+    results = REDIS_AUTOCOMPLETE.zrevrange(combo_key, 0, -1)
+    REDIS_AUTOCOMPLETE.expire combo_key, 1
+
     unless search_param.blank?
       search_regex = Tag.get_search_regex(search_param)
       results.select! {|tag| tag.match(search_regex)}

config/redis.travis.example

@@ -6,3 +6,5 @@ redis_general:
   test: localhost:6381
 redis_rollout:
   test: localhost:6382
+redis_autocomplete:
+  test: localhost:6383

features/support/hooks.rb

@@ -21,4 +21,5 @@
   REDIS_KUDOS.flushall
   REDIS_RESQUE.flushall
   REDIS_ROLLOUT.flushall
+  REDIS_AUTOCOMPLETE.flushall
 end

spec/spec_helper.rb

@@ -87,6 +87,7 @@ def clean_the_database
   REDIS_KUDOS.flushall
   REDIS_RESQUE.flushall
   REDIS_ROLLOUT.flushall
+  REDIS_AUTOCOMPLETE.flushall
   # Finally elastic search
   Work.tire.index.delete
   Work.create_elasticsearch_index