diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb
index 6b4aaa5525930c..9ca2e269e30d27 100644
--- a/test/rubygems/helper.rb
+++ b/test/rubygems/helper.rb
@@ -290,7 +290,6 @@ def setup
     FileUtils.mkdir_p @tmp
 
     @tempdir = Dir.mktmpdir("test_rubygems_", @tmp)
-    @@tempdirs << [method_name, @tempdir]
 
     ENV["GEM_VENDOR"] = nil
     ENV["GEMRC"] = nil
@@ -475,12 +474,11 @@ def teardown
 
     @back_ui.close
 
+    refute_directory_exists @tempdir, "may be still in use"
     ghosts = @@tempdirs.filter_map do |test_name, tempdir|
-      if File.exist?(tempdir)
-        FileUtils.rm_rf(tempdir)
-        test_name
-      end
+      test_name if File.exist?(tempdir)
     end
+    @@tempdirs << [method_name, @tempdir]
     assert_empty ghosts
   end