Change max delay before crawling links to be configurable from env

Snailed-It · Jul 7, 2023 · 4aeffc9 · 4aeffc9
1 parent eb77096
commit 4aeffc9
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -52,6 +52,7 @@ Click below to **learn more** in a video:
 | RATE_LIMIT_API_SIGN_UP_PERIOD_MINUTES            | The rate limiter period (i.e. how long before the count resets) for sign up attempts via the api            |
 | RATE_LIMIT_SIGN_UP                               | Max allowed sign up attempts, via the web UI, within a rate limiter period                                  |
 | RATE_LIMIT_SIGN_UP_PERIOD_MINUTES                | The rate limiter period (i.e. how long before the count resets) for sign up attempts via the web UI         |
+| MAX_LINK_CRAWL_DELAY_SECONDS                     | Maximum delay before crawling links received from other instances.                                          |
 
 
 ## Features

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
@@ -93,7 +93,7 @@ def process_status
 
   def distribute
     # Spread out crawling randomly to avoid DDoSing the link
-    LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id)
+    LinkCrawlWorker.perform_in(rand(1..LinkCrawlWorker::MAX_DELAY).seconds, @status.id)
 
     # Distribute into home and list feeds and notify mentioned accounts
     ::DistributionWorker.perform_async(@status.id, { 'silenced_account_ids' => @silenced_account_ids }) if @options[:override_timestamps] || @status.within_realtime_window?

diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb
@@ -284,7 +284,7 @@ def already_updated_more_recently?
 
   def reset_preview_card!
     @status.preview_cards.clear
-    LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id)
+    LinkCrawlWorker.perform_in(rand(1..LinkCrawlWorker::MAX_DELAY).seconds, @status.id)
   end
 
   def broadcast_updates!

diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb
@@ -3,6 +3,8 @@
 class LinkCrawlWorker
   include Sidekiq::Worker
 
+  MAX_DELAY = ENV.fetch('MAX_LINK_CRAWL_DELAY_SECONDS', 59).to_i
+
   sidekiq_options queue: 'pull', retry: 0
 
   def perform(status_id)