Skip to content

Commit

Permalink
Fix slow rollout of new generation workers
Browse files Browse the repository at this point in the history
  • Loading branch information
byroot committed Aug 3, 2023
1 parent bbef923 commit 00b8222
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 0 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Unreleased

- Fix the slow rollout of workers on a new generation.
- Expose `Pitchfork::Info.fork_safe?` and `Pitchfork::Info.no_longer_fork_safe!`.

# 0.6.0
Expand Down
2 changes: 2 additions & 0 deletions lib/pitchfork/http_server.rb
Original file line number Diff line number Diff line change
Expand Up @@ -635,9 +635,11 @@ def restart_outdated_workers
outdated_workers.each do |worker|
if worker.soft_kill(:TERM)
logger.info("Sent SIGTERM to worker=#{worker.nr} pid=#{worker.pid} gen=#{worker.generation}")
workers_to_restart -= 1
else
logger.info("Failed to send SIGTERM to worker=#{worker.nr} pid=#{worker.pid} gen=#{worker.generation}")
end
break if workers_to_restart <= 0
end
end
end
Expand Down
41 changes: 41 additions & 0 deletions test/integration/test_reforking.rb
Original file line number Diff line number Diff line change
Expand Up @@ -107,5 +107,46 @@ def test_reforking_on_USR2
assert_healthy("http://#{addr}:#{port}")
assert_clean_shutdown(pid)
end

def test_slow_worker_rollout
addr, port = unused_port

pid = spawn_server(app: File.join(ROOT, "test/integration/env.ru"), config: <<~CONFIG)
listen "#{addr}:#{port}"
worker_processes 5
after_worker_fork do |_server, worker|
Kernel.at_exit do
sleep 0.1
end
end
CONFIG

assert_healthy("http://#{addr}:#{port}")
assert_stderr "worker=0 gen=0 ready"
assert_stderr "worker=4 gen=0 ready"

Process.kill(:USR2, pid)

assert_stderr "worker=0 gen=1 ready"
assert_stderr "worker=1 gen=1 ready"
assert_stderr "worker=2 gen=1 ready"
assert_stderr "worker=3 gen=1 ready"
assert_stderr "worker=4 gen=1 ready"

assert_clean_shutdown(pid)

log_lines = read_stderr.lines.drop_while { |l| !l.match?(/Terminating old mold/) }
log_lines = log_lines.take_while { |l| !l.match?(/QUIT received/) }

events = log_lines.map do |line|
case line
when /Sent SIGTERM to worker/
:term
when /registered/
:registered
end
end.compact
assert_equal([:term, :registered] * 5, events)
end
end
end

0 comments on commit 00b8222

Please sign in to comment.