Merge pull request #1989 from chef/praj/support_external_search_in_mi…

…gration

Add auto support for external search in migration for external elasticsearch

omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb

@@ -130,7 +130,7 @@
 ####
 # RabbitMQ
 ####
-default['private_chef']['rabbitmq']['enable'] = false
+# default['private_chef']['rabbitmq']['enable'] defined in recipes/config.rb
 default['private_chef']['rabbitmq']['ha'] = false
 default['private_chef']['rabbitmq']['dir'] = '/var/opt/opscode/rabbitmq'
 default['private_chef']['rabbitmq']['data_dir'] = '/var/opt/opscode/rabbitmq/db'
@@ -226,13 +226,12 @@
 ####
 # Chef Solr 4
 ####
-default['private_chef']['opscode-solr4']['enable'] = false
-#
+# default['private_chef']['opscode-solr4']['enable'] defined in recipes/config.rb
 # Set this to point at a solr/cloudsearch installation
 # not controlled by chef-server
 #
-default['private_chef']['opscode-solr4']['external'] = true
-default['private_chef']['opscode-solr4']['external_url'] = "http://localhost:9200"
+default['private_chef']['opscode-solr4']['external'] = false
+default['private_chef']['opscode-solr4']['external_url'] = nil
 default['private_chef']['opscode-solr4']['ha'] = false
 default['private_chef']['opscode-solr4']['dir'] = '/var/opt/opscode/opscode-solr4'
 default['private_chef']['opscode-solr4']['data_dir'] = '/var/opt/opscode/opscode-solr4/data'
@@ -268,7 +267,7 @@
 ####
 # Chef Expander
 ####
-default['private_chef']['opscode-expander']['enable'] = false
+# default['private_chef']['opscode-expander']['enable'] defined in recipes/config.rb
 default['private_chef']['opscode-expander']['ha'] = false
 default['private_chef']['opscode-expander']['dir'] = '/var/opt/opscode/opscode-expander'
 default['private_chef']['opscode-expander']['log_directory'] = '/var/log/opscode/opscode-expander'
@@ -285,7 +284,7 @@
 var_base = '/var/opt/opscode'
 log_base = '/var/log/opscode'
 
-default['private_chef']['elasticsearch']['enable'] = true
+# default['private_chef']['elasticsearch']['enable'] defined in recipes/config.rb
 elasticsearch = default['private_chef']['elasticsearch']
 
 # These attributes cannot be overridden in chef-server.rb
@@ -299,6 +298,7 @@
 elasticsearch['log_directory'] = "#{log_base}/elasticsearch"
 elasticsearch['log_rotation']['file_maxbytes'] = 104857600
 elasticsearch['log_rotation']['num_to_keep'] = 10
+elasticsearch['vip'] = '127.0.0.1'
 elasticsearch['listen'] = '127.0.0.1'
 elasticsearch['port'] = 9200
 elasticsearch['enable_gc_log'] = false
@@ -444,8 +444,8 @@
 default['private_chef']['opscode-erchef']['ibrowse_max_sessions'] = 256
 default['private_chef']['opscode-erchef']['ibrowse_max_pipeline_size'] = 1
 # general search settings used to set up chef_index
-default['private_chef']['opscode-erchef']['search_provider'] = 'elasticsearch' # solr, elasticsearch
-default['private_chef']['opscode-erchef']['search_queue_mode'] = 'batch' # rabbitmq, batch, or inline
+# default['private_chef']['opscode-erchef']['search_provider'] defined in recipes/config.rb
+# default['private_chef']['opscode-erchef']['search_queue_mode'] defined in recipes/config.rb
 default['private_chef']['opscode-erchef']['search_batch_max_size'] = '5000000'
 default['private_chef']['opscode-erchef']['search_batch_max_wait'] = '10'
 # solr_service configuration for erchef. These are used to configure an opscoderl_httpc pool

omnibus/files/private-chef-cookbooks/private-chef/libraries/helper.rb

@@ -67,28 +67,18 @@ def vip_for_uri(service)
     normalize_host(node['private_chef'][service]['vip'])
   end
 
-  # Returns scheme://host:port without any path
-  def solr_root
-    url = URI.parse(solr_url)
-    host = url.scheme + '://' + url.host
-    if url.port
-      host += ':' + url.port.to_s
-    end
-    host
-  end
-
   def elastic_search_major_version
     max_requests = 5
     current_request = 1
 
-    if node['private_chef']['opscode-solr4']['external'] && node['private_chef']['opscode-erchef']['search_provider'] == 'elasticsearch'
+    if node['private_chef']['opscode-erchef']['search_provider'] == 'elasticsearch'
       begin
-        client = Chef::HTTP.new(node['private_chef']['opscode-solr4']['external_url'])
+        client = Chef::HTTP.new(solr_url)
         response = client.get('')
       rescue => e
         # Perform a blind rescue because Net:HTTP throws a variety of exceptions - some of which are platform specific.
         if current_request == max_requests
-          raise "Failed to connect to elasticsearch service. Ensure node['private_chef']['opscode-solr4']['external_url'] is correct.\n#{e}"
+          raise "Failed to connect to elasticsearch service at #{solr_url}: #{e}"
         else
           # Chef HTTP logs the details in the debug log.
           Chef::Log.error "Failed to connect to elasticsearch service #{current_request}/#{max_requests}. Retrying."
@@ -231,11 +221,23 @@ def es_5_or_2_index
   def solr_url
     if node['private_chef']['opscode-solr4']['external']
       node['private_chef']['opscode-solr4']['external_url']
-    else
+    elsif node['private_chef']['deprecated_solr_indexing']
       "http://#{vip_for_uri('opscode-solr4')}:#{node['private_chef']['opscode-solr4']['port']}/solr"
+    else
+      "http://#{vip_for_uri('elasticsearch')}:#{node['private_chef']['elasticsearch']['port']}/"
     end
   end
 
+  # Returns scheme://host:port without any path
+  def solr_root
+    url = URI.parse(solr_url)
+    host = url.scheme + '://' + url.host
+    if url.port
+      host += ':' + url.port.to_s
+    end
+    host
+  end
+
   def bookshelf_s3_url
     # Using URI#to_s to strip ":443" for https and ":80" for http
     URI("#{node['private_chef']['nginx']['x_forwarded_proto']}://#{vip_for_uri('bookshelf')}:#{node['private_chef']['bookshelf']['vip_port']}").to_s

omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_solr_validator.rb

@@ -44,9 +44,10 @@ def run!
     verify_one_search_provider
     verify_es_disabled_if_user_set_external_solr
     verify_unused_services_are_disabled_if_using_internal_es
-    verify_no_explicit_external_disable_when_es_enabled
 
     warn_unchanged_external_flag
+    warn_on_deprecated_solr
+    warn_deprecated_indexing_change
     verify_external_url
     verify_erchef_config
   end
@@ -106,16 +107,6 @@ def verify_unused_services_are_disabled_if_using_internal_es
     end
   end
 
-  def verify_no_explicit_external_disable_when_es_enabled
-    return if cs_solr_attr['external'].nil?
-
-    # TODO(ssd) 2020-05-13: Rather than this check, we could just
-    # ignore that configuration when parsing the configuration.
-    if !cs_solr_attr['external'] && elasticsearch_enabled?
-      fail_with err_SOLR008_failed_validation
-    end
-  end
-
   def external?
     if cs_solr_attr['external'].nil?
       node_solr_attr['external']
@@ -149,10 +140,19 @@ def solr_enabled?
   def warn_unchanged_external_flag
     if OmnibusHelper.has_been_bootstrapped? && backend? && previous_run
       if cs_solr_attr.key?('external') && (cs_solr_attr['external'] != previous_run['opscode-solr4']['external'])
-        Chef::Log.warn err_SOLR009_warn_validation
+        ChefServer::Warnings.warn err_SOLR009_warn_validation
       end
-    else
-      true
+    end
+  end
+
+  def warn_deprecated_indexing_change
+    return if cs_solr_attr['external']
+    return if !previous_run
+
+    old_value = previous_run['deprecated_solr_indexing']
+    new_value = PrivateChef['deprecated_solr_indexing']
+    if old_value != new_value
+      ChefServer::Warnings.warn err_SOLR014_warn_validation
     end
   end
 
@@ -177,6 +177,10 @@ def verify_erchef_config
     end
   end
 
+  def warn_on_deprecated_solr
+    ChefServer::Warnings.warn err_SOLR013_warn_validation if PrivateChef['deprecated_solr_indexing']
+  end
+
   def err_SOLR001_failed_validation(final_min, final_max)
     <<~EOM
 
@@ -281,19 +285,6 @@ def err_SOLR007_failed_validation
     EOM
   end
 
-  def err_SOLR008_failed_validation
-    <<~EOM
-
-      SOLR008: The #{CHEF_SERVER_NAME} is configured with
-
-                   opscode_solr4['external'] = false
-
-               but this is incompatible with elasticsearch being enabled.
-
-               Please remove this line from #{CHEF_SERVER_CONFIG_FILE}.
-    EOM
-  end
-
   def err_SOLR009_warn_validation
     <<~EOM
 
@@ -336,4 +327,26 @@ def err_SOLR012_failed_validation
                elasticsearch
     EOM
   end
+
+  def err_SOLR013_warn_validation
+    <<~EOM
+      SOLR013: You have configured
+
+                  deprecated_solr_indexing true
+
+               Please note that solr indexing will be removed in a
+               future Chef Server release.
+
+               Please contact Chef Support for help moving to the
+               Elasticsearch indexing pipeline.
+    EOM
+  end
+
+  def err_SOLR014_warn_validation
+    <<~EOM
+       SOLR014: The value of deprecated_solr_indexing has been changed. Search
+                results against the new search index may be incorrect. Please
+                run `chef-server-ctl reindex --all` to ensure correct results.
+    EOM
+  end
 end

omnibus/files/private-chef-cookbooks/private-chef/libraries/private_chef.rb

@@ -121,6 +121,8 @@ module PrivateChef
 
   insecure_addon_compat true
 
+  deprecated_solr_indexing false
+
   class << self
     def from_file(filename)
       # We're overriding this here so that we can get more meaningful errors from
@@ -295,6 +297,7 @@ def generate_hash
       set_target_array_if_not_nil(results['private_chef'], 'folsom_graphite', PrivateChef['folsom_graphite'])
       set_target_array_if_not_nil(results['private_chef'], 'profiles', PrivateChef['profiles'])
       set_target_array_if_not_nil(results['private_chef'], 'insecure_addon_compat', PrivateChef['insecure_addon_compat'])
+      set_target_array_if_not_nil(results['private_chef'], 'deprecated_solr_indexing', PrivateChef['deprecated_solr_indexing'])
       results
     end
 

omnibus/files/private-chef-cookbooks/private-chef/recipes/config.rb

@@ -81,6 +81,43 @@
     PrivateChef.from_file(chef_server_path)
   end
 
+  # Add support for the external search case
+  if PrivateChef['opscode_solr4']['external']
+    # If the user has enabled external solr4 we keep these defaults
+    # more or less the same as they were before the change-over to
+    # Elasticsearch.
+    node.default['private_chef']['rabbitmq']['enable'] = true
+    node.default['private_chef']['opscode-expander']['enable'] = true
+    node.default['private_chef']['opscode-solr4']['enable'] = false
+    node.default['private_chef']['elasticsearch']['enable'] = false
+
+    node.default['private_chef']['opscode-erchef']['search_provider'] = 'solr' # solr, elasticsearch
+    node.default['private_chef']['opscode-erchef']['search_queue_mode'] = 'rabbitmq' # rabbitmq, batch, or inline
+  elsif PrivateChef['deprecated_solr_indexing']
+    # If the user has explicitly enabled the depcrecated solr mode, we
+    # start all services required to run the RabbitMQ -> Expander ->
+    # Solr indexing pipeline.
+    #
+    # This is the same as the block above, but we want to be explicit
+    # about it as the situation is very confusing.
+    node.default['private_chef']['rabbitmq']['enable'] = true
+    node.default['private_chef']['opscode-solr4']['enable'] = true
+    node.default['private_chef']['opscode-expander']['enable'] = true
+    node.default['private_chef']['elasticsearch']['enable'] = false
+
+    node.default['private_chef']['opscode-erchef']['search_provider'] = 'solr' # solr, elasticsearch
+    node.default['private_chef']['opscode-erchef']['search_queue_mode'] = 'rabbitmq' # rabbitmq, batch, or inline
+  else
+    # Our new default of Elasticsearch indexing
+    node.default['private_chef']['rabbitmq']['enable'] = false
+    node.default['private_chef']['opscode-solr4']['enable'] = false
+    node.default['private_chef']['opscode-expander']['enable'] = false
+    node.default['private_chef']['elasticsearch']['enable'] = true
+
+    node.default['private_chef']['opscode-erchef']['search_provider'] = 'elasticsearch' # solr, elasticsearch
+    node.default['private_chef']['opscode-erchef']['search_queue_mode'] = 'batch' # rabbitmq, batch, or inline
+  end
+
   # Bail out if something is wrong in our configuration.
   # NOTE: Over time, we can move the validation done in private_chef.rb
   #       here as well.

omnibus/files/private-chef-cookbooks/private-chef/recipes/elasticsearch.rb

@@ -141,10 +141,16 @@
   to elasticsearch_conf_dir
 end
 
+component_runit_service 'opscode-solr4' do
+  action :disable
+end
+
 directory '/opt/opscode/sv/opscode-solr4' do
   recursive true
   action :delete
 end
 
 # Define resource for elasticsearch component_runit_service
 component_runit_service 'elasticsearch'
+
+include_recipe 'private-chef::elasticsearch_index'

omnibus/files/private-chef-cookbooks/private-chef/recipes/elasticsearch_index.rb

@@ -0,0 +1,21 @@
+# Author:: Steven Danna
+# Copyright:: 2020 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+helper = OmnibusHelper.new(node)
+elasticsearch_index 'chef' do
+  server_url lazy { helper.solr_url }
+  index_definition lazy { helper.es_index_definition }
+end

omnibus/files/private-chef-cookbooks/private-chef/recipes/opscode-solr4-external.rb

@@ -20,8 +20,5 @@
 when 'solr'
   Chef::Log.warn('External Solr Support does not include configuring the Solr schema.')
 when 'elasticsearch'
-  elasticsearch_index 'chef' do
-    server_url node['private_chef']['opscode-solr4']['external_url']
-    index_definition lazy { helper.es_index_definition }
-  end
+  include_recipe 'private-chef::elasticsearch_index'
 end

omnibus/files/private-chef-cookbooks/private-chef/recipes/opscode-solr4.rb

@@ -165,6 +165,10 @@
 
 node.default['private_chef']['opscode-solr4']['command'] = command
 
+component_runit_service 'elasticsearch' do
+  action :disable
+end
+
 directory '/opt/opscode/sv/elasticsearch' do
   recursive true
   action :delete

omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/helper_spec.rb

@@ -117,8 +117,7 @@
     let(:client) { double(Chef::HTTP) }
 
     context 'when elastic search is disabled' do
-      let(:external) { false }
-      let(:external_url) { nil }
+      let(:provider) { 'solr' }
       let(:elastic_version) { '50.0' }
 
       it 'should return a default version 0' do