From 8c38ff76b38f146baea755bc85d333da09d0c327 Mon Sep 17 00:00:00 2001
From: Shim Shtein <sshtein@redhat.com>
Date: Mon, 28 Nov 2022 11:27:00 +0200
Subject: [PATCH] Schedule a random delay before execution

---
 .../async/delayed_start.rb                    | 51 +++++++++++++++++++
 .../async/generate_all_reports_job.rb         | 21 ++++----
 lib/foreman_rh_cloud.rb                       |  4 ++
 .../async/insights_scheduled_sync.rb          |  5 +-
 .../async/inventory_scheduled_sync.rb         |  7 ++-
 5 files changed, 76 insertions(+), 12 deletions(-)
 create mode 100644 lib/foreman_inventory_upload/async/delayed_start.rb

diff --git a/lib/foreman_inventory_upload/async/delayed_start.rb b/lib/foreman_inventory_upload/async/delayed_start.rb
new file mode 100644
index 00000000..e99ddec7
--- /dev/null
+++ b/lib/foreman_inventory_upload/async/delayed_start.rb
@@ -0,0 +1,51 @@
+module ForemanInventoryUpload
+  module Async
+    module DelayedStart
+      extend ActiveSupport::Concern
+
+      START_WINDOW = 3.hours.seconds
+
+      def after_delay(delay = nil, logger: nil, &block)
+        logger ||= self.logger if respond_to? :logger
+        delay ||= ForemanRhCloud.requests_delay || Random.new.rand(START_WINDOW)
+        delay = delay.to_i
+
+        logger&.debug("planning a delay for #{delay} seconds before the rest of the execution")
+
+        sequence do
+          plan_action(ForemanInventoryUpload::Async::DelayAction, delay)
+          concurrence(&block)
+        end
+      end
+
+      def humanized_name
+        _('Wait and %s' % super)
+      end
+    end
+
+    class DelayAction < ::Actions::EntryAction
+      Wake = Algebrick.atom
+
+      def plan(delay)
+        plan_self(delay: delay)
+      end
+
+      def run(event = nil)
+        case event
+        when nil
+          action_logger.debug("Going to sleep for #{sleep_seconds} seconds")
+          plan_event(Wake, sleep_seconds)
+          suspend
+        when Wake
+          action_logger.debug('Waking up')
+        else
+          action_logger.debug("DelayAction received unknown event #{event}")
+        end
+      end
+
+      def sleep_seconds
+        input[:delay].to_i
+      end
+    end
+  end
+end
diff --git a/lib/foreman_inventory_upload/async/generate_all_reports_job.rb b/lib/foreman_inventory_upload/async/generate_all_reports_job.rb
index 5eaf295e..e443ffff 100644
--- a/lib/foreman_inventory_upload/async/generate_all_reports_job.rb
+++ b/lib/foreman_inventory_upload/async/generate_all_reports_job.rb
@@ -2,6 +2,7 @@ module ForemanInventoryUpload
   module Async
     class GenerateAllReportsJob < ::Actions::EntryAction
       include ::Actions::RecurringAction
+      include ForemanInventoryUpload::Async::DelayedStart
 
       def plan
         unless Setting[:allow_auto_inventory_upload]
@@ -12,17 +13,19 @@ def plan
           return
         end
 
-        organizations = Organization.unscoped.all
+        after_delay do
+          organizations = Organization.unscoped.all
 
-        organizations.map do |organization|
-          total_hosts = ForemanInventoryUpload::Generators::Queries.for_org(organization.id, use_batches: false).count
+          organizations.map do |organization|
+            total_hosts = ForemanInventoryUpload::Generators::Queries.for_org(organization.id, use_batches: false).count
 
-          if total_hosts <= ForemanInventoryUpload.max_org_size
-            plan_generate_report(ForemanInventoryUpload.generated_reports_folder, organization)
-          else
-            logger.info("Skipping automatic uploads for organization #{organization.name}, too many hosts (#{total_hosts}/#{ForemanInventoryUpload.max_org_size})")
-          end
-        end.compact
+            if total_hosts <= ForemanInventoryUpload.max_org_size
+              plan_generate_report(ForemanInventoryUpload.generated_reports_folder, organization)
+            else
+              logger.info("Skipping automatic uploads for organization #{organization.name}, too many hosts (#{total_hosts}/#{ForemanInventoryUpload.max_org_size})")
+            end
+          end.compact
+        end
       end
 
       def rescue_strategy_for_self
diff --git a/lib/foreman_rh_cloud.rb b/lib/foreman_rh_cloud.rb
index 88e8ce32..afe7673f 100644
--- a/lib/foreman_rh_cloud.rb
+++ b/lib/foreman_rh_cloud.rb
@@ -120,4 +120,8 @@ def self.legacy_insights_ca
   def self.cloud_url_validator
     @cloud_url_validator ||= Regexp.new(ENV['SATELLITE_RH_CLOUD_VALIDATOR'] || 'redhat.com$')
   end
+
+  def self.requests_delay
+    @requests_delay ||= ENV['SATELLITE_RH_CLOUD_REQUESTS_DELAY']
+  end
 end
diff --git a/lib/insights_cloud/async/insights_scheduled_sync.rb b/lib/insights_cloud/async/insights_scheduled_sync.rb
index f95059e7..c90f5b80 100644
--- a/lib/insights_cloud/async/insights_scheduled_sync.rb
+++ b/lib/insights_cloud/async/insights_scheduled_sync.rb
@@ -2,6 +2,7 @@ module InsightsCloud
   module Async
     class InsightsScheduledSync < ::Actions::EntryAction
       include ::Actions::RecurringAction
+      include ForemanInventoryUpload::Async::DelayedStart
 
       def plan
         unless Setting[:allow_auto_insights_sync]
@@ -12,7 +13,9 @@ def plan
           return
         end
 
-        plan_full_sync
+        after_delay do
+          plan_full_sync
+        end
       end
 
       def plan_full_sync
diff --git a/lib/inventory_sync/async/inventory_scheduled_sync.rb b/lib/inventory_sync/async/inventory_scheduled_sync.rb
index 88f6b9eb..350f591e 100644
--- a/lib/inventory_sync/async/inventory_scheduled_sync.rb
+++ b/lib/inventory_sync/async/inventory_scheduled_sync.rb
@@ -2,6 +2,7 @@ module InventorySync
   module Async
     class InventoryScheduledSync < ::Actions::EntryAction
       include ::Actions::RecurringAction
+      include ForemanInventoryUpload::Async::DelayedStart
 
       def plan
         unless Setting[:allow_auto_inventory_upload]
@@ -12,8 +13,10 @@ def plan
           return
         end
 
-        Organization.unscoped.each do |org|
-          plan_org_sync(org)
+        after_delay do
+          Organization.unscoped.each do |org|
+            plan_org_sync(org)
+          end
         end
       end