add worker killer

fedibird · Jan 4, 2021 · 52b5eb6 · 52b5eb6
1 parent f253c48
commit 52b5eb6
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 0 deletions.
diff --git a/Gemfile b/Gemfile
@@ -106,6 +106,9 @@ gem 'json-ld'
 gem 'json-ld-preloaded', '~> 3.1'
 gem 'rdf-normalize', '~> 0.4'
 
+gem 'puma_worker_killer', '~> 0.3'
+gem 'sidekiq-worker-killer', '~> 1.0'
+
 group :development, :test do
   gem 'fabrication', '~> 2.21'
   gem 'fuubar', '~> 2.5'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -246,6 +246,8 @@ GEM
     fuubar (2.5.0)
       rspec-core (~> 3.0)
       ruby-progressbar (~> 1.4)
+    get_process_mem (0.2.7)
+      ffi (~> 1.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     haml (5.0.4)
@@ -443,6 +445,9 @@ GEM
     public_suffix (4.0.6)
     puma (5.1.1)
       nio4r (~> 2.0)
+    puma_worker_killer (0.3.1)
+      get_process_mem (~> 0.2)
+      puma (>= 2.7)
     pundit (2.1.0)
       activesupport (>= 3.0.0)
     raabro (1.3.3)
@@ -603,6 +608,9 @@ GEM
       concurrent-ruby (~> 1.0, >= 1.0.5)
       sidekiq (>= 4.0, < 7.0)
       thor (>= 0.20, < 2.0)
+    sidekiq-worker-killer (1.0.0)
+      get_process_mem (~> 0.2.1)
+      sidekiq (>= 5)
     simple-navigation (4.1.0)
       activesupport (>= 2.3.2)
     simple_form (5.0.3)
@@ -806,6 +814,7 @@ DEPENDENCIES
   pry-byebug (~> 3.9)
   pry-rails (~> 0.3)
   puma (~> 5.1)
+  puma_worker_killer (~> 0.3)
   pundit (~> 2.1)
   rack (~> 2.2.3)
   rack-attack (~> 6.3)
@@ -832,6 +841,7 @@ DEPENDENCIES
   sidekiq-scheduler (~> 3.0)
   sidekiq-statistic (~> 1.4)
   sidekiq-unique-jobs (~> 6.0)
+  sidekiq-worker-killer (~> 1.0)
   simple-navigation (~> 4.1)
   simple_form (~> 5.0)
   simplecov (~> 0.20)

diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'sidekiq/worker_killer'
+
 namespace    = ENV.fetch('REDIS_NAMESPACE') { nil }
 redis_params = { url: ENV['REDIS_URL'] }
 
@@ -14,6 +16,10 @@
     chain.add SidekiqErrorHandler
   end
 
+  config.server_middleware do |chain|
+    chain.add Sidekiq::WorkerKiller, max_rss: 750, grace_time: 60
+  end
+
   config.death_handlers << lambda do |job, _ex|
     digest = job['lock_digest']
     SidekiqUniqueJobs::Digests.delete_by_digest(digest) if digest

diff --git a/config/puma.rb b/config/puma.rb
@@ -20,4 +20,14 @@
   end
 end
 
+before_fork do
+  require 'puma_worker_killer'
+
+  PumaWorkerKiller.config do |config|
+    config.ram = workers * 512 # mb
+  end
+
+  PumaWorkerKiller.start
+end
+
 plugin :tmp_restart