diff --git a/activesupport/lib/active_support/testing/parallelization.rb b/activesupport/lib/active_support/testing/parallelization.rb
index 1caac1feb3ee0..beeb470659359 100644
--- a/activesupport/lib/active_support/testing/parallelization.rb
+++ b/activesupport/lib/active_support/testing/parallelization.rb
@@ -65,22 +65,24 @@ def run_cleanup(worker)
       def start
         @pool = @queue_size.times.map do |worker|
           fork do
-            DRb.stop_service
+            begin
+              DRb.stop_service
 
-            after_fork(worker)
+              after_fork(worker)
 
-            queue = DRbObject.new_with_uri(@url)
+              queue = DRbObject.new_with_uri(@url)
 
-            while job = queue.pop
-              klass    = job[0]
-              method   = job[1]
-              reporter = job[2]
-              result   = Minitest.run_one_method(klass, method)
+              while job = queue.pop
+                klass    = job[0]
+                method   = job[1]
+                reporter = job[2]
+                result   = Minitest.run_one_method(klass, method)
 
-              queue.record(reporter, result)
+                queue.record(reporter, result)
+              end
+            ensure
+              run_cleanup(worker)
             end
-
-            run_cleanup(worker)
           end
         end
       end