From 4cadb00371dfbb17402147fb81b7a97f9fa9f617 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sat, 25 Mar 2023 10:40:06 -0700 Subject: [PATCH 1/3] IndexingScheduler: fetch and import in batches --- app/workers/scheduler/indexing_scheduler.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index c423966297c31..e3eb423eef638 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -6,17 +6,20 @@ class Scheduler::IndexingScheduler sidekiq_options retry: 0 + IMPORT_BATCH_SIZE = 100 + SCAN_BATCH_SIZE = 100 * IMPORT_BATCH_SIZE + def perform return unless Chewy.enabled? indexes.each do |type| with_redis do |redis| - ids = redis.smembers("chewy:queue:#{type.name}") - - type.import!(ids) + redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| + type.import!(ids) - redis.pipelined do |pipeline| - ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } + redis.pipelined do |pipeline| + ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } + end end end end From 10ff6b43bae159bf76a63bc8ac80300ca492d88d Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Wed, 29 Mar 2023 11:27:49 -0700 Subject: [PATCH 2/3] Larger import batches, batch SREM --- app/workers/scheduler/indexing_scheduler.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index e3eb423eef638..77d29f6ccd4b1 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -6,8 +6,8 @@ class Scheduler::IndexingScheduler sidekiq_options retry: 0 - IMPORT_BATCH_SIZE = 100 - SCAN_BATCH_SIZE = 100 * IMPORT_BATCH_SIZE + IMPORT_BATCH_SIZE = 1000 + SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE def perform return unless Chewy.enabled? @@ -16,10 +16,7 @@ def perform with_redis do |redis| redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| type.import!(ids) - - redis.pipelined do |pipeline| - ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } - end + redis.srem("chewy:queue:#{type.name}", ids) end end end From c948b84b590decb16fdfdce440b1bee3a26801fb Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 30 Mar 2023 10:08:31 -0700 Subject: [PATCH 3/3] Pipeline all SREM calls for each SSCAN Co-authored-by: Claire --- app/workers/scheduler/indexing_scheduler.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index 77d29f6ccd4b1..1bbe9cd5d0102 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -14,9 +14,13 @@ def perform indexes.each do |type| with_redis do |redis| - redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| - type.import!(ids) - redis.srem("chewy:queue:#{type.name}", ids) + redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE) do |ids| + redis.pipelined do + ids.each_slice(IMPORT_BATCH_SIZE) do |slice_ids| + type.import!(slice_ids) + redis.srem("chewy:queue:#{type.name}", slice_ids) + end + end end end end