From bb96e260d3008c63a9e21f9543a8f694c2075dc8 Mon Sep 17 00:00:00 2001
From: Rolf Schmidt <rolf.schmidt@zammad.com>
Date: Fri, 13 Oct 2023 16:59:44 +0200
Subject: [PATCH] Performance: Use all cores to reindex models in
 elasticsearch.

---
 Gemfile                                         |  1 +
 Gemfile.lock                                    |  1 +
 app/models/concerns/has_search_index_backend.rb | 13 ++++++++-----
 lib/tasks/zammad/search_index_es.rake           |  8 ++++----
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/Gemfile b/Gemfile
index 9dc716935d0e..a201b5c31603 100644
--- a/Gemfile
+++ b/Gemfile
@@ -18,6 +18,7 @@ gem 'activerecord-session_store'
 gem 'bootsnap', require: false
 gem 'composite_primary_keys'
 gem 'json'
+gem 'parallel'
 
 # core - application servers
 gem 'puma', group: :puma
diff --git a/Gemfile.lock b/Gemfile.lock
index da35045106e4..060cc0ab4b24 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -734,6 +734,7 @@ DEPENDENCIES
   omniauth-weibo-oauth2!
   openssl
   overcommit
+  parallel
   pg (~> 1.2.0)
   pry-rails
   pry-remote
diff --git a/app/models/concerns/has_search_index_backend.rb b/app/models/concerns/has_search_index_backend.rb
index 68f78a55a6e2..4bbf272fd125 100644
--- a/app/models/concerns/has_search_index_backend.rb
+++ b/app/models/concerns/has_search_index_backend.rb
@@ -176,15 +176,18 @@ def search_index_attributes_relevant(*attributes)
 
 =end
 
-    def search_index_reload(silent: false)
+    def search_index_reload(silent: false, worker: 0)
       tolerance       = 10
       tolerance_count = 0
       query           = reorder(created_at: :desc)
       total           = query.count
       record_count    = 0
-      batch_size      = 100
-      query.as_batches(size: batch_size) do |record|
-        if !record.ignore_search_indexing?(:destroy)
+      batch_size      = 200
+
+      query.in_batches(of: batch_size) do |records|
+        Parallel.map(records, { in_processes: worker }) do |record|
+          next if record.ignore_search_indexing?(:destroy)
+
           begin
             record.search_index_update_backend
           rescue => e
@@ -197,7 +200,7 @@ def search_index_reload(silent: false)
 
         next if silent
 
-        record_count += 1
+        record_count += records.count
         if (record_count % batch_size).zero? || record_count == total
           print "\r    #{record_count}/#{total}" # rubocop:disable Rails/Output
         end
diff --git a/lib/tasks/zammad/search_index_es.rake b/lib/tasks/zammad/search_index_es.rake
index c3d38945c62f..8fcff8745616 100644
--- a/lib/tasks/zammad/search_index_es.rake
+++ b/lib/tasks/zammad/search_index_es.rake
@@ -34,12 +34,12 @@ namespace :zammad do
     end
 
     desc 'Reload all indexable data'
-    task reload: %i[zammad:searchindex:version_supported] do
+    task :reload, [:worker] => %i[zammad:searchindex:version_supported] do |_task, args|
       puts 'Reloading data... '
       Models.indexable.each do |model_class|
         puts "  - #{model_class}... "
         time_spent = Benchmark.realtime do
-          model_class.search_index_reload
+          model_class.search_index_reload(worker: args[:worker].to_i)
         end
         # Add whitespace at the end to overwrite text from progress indicator line.
         puts "\r    done in #{time_spent.to_i} seconds.#{' ' * 20}"
@@ -54,10 +54,10 @@ namespace :zammad do
     end
 
     desc 'Full re-creation of all search indexes and re-indexing of all data'
-    task rebuild: %i[zammad:searchindex:version_supported] do
+    task :rebuild, [:worker] => %i[zammad:searchindex:version_supported] do |_task, args|
       Rake::Task['zammad:searchindex:drop'].execute
       Rake::Task['zammad:searchindex:create'].execute
-      Rake::Task['zammad:searchindex:reload'].execute
+      Rake::Task['zammad:searchindex:reload'].execute(args)
     end
 
     task version_supported: %i[zammad:searchindex:configured] do