[SPARK-57952][CORE][TESTS] Retry retryOnOOM a few times to reduce SorterSuite flaky OOM

core/src/test/scala/org/apache/spark/SparkFunSuite.scala

@@ -113,13 +113,28 @@ abstract class SparkFunSuite
     }
   }
 
-  /** Run `body`; if it throws OutOfMemoryError, force a GC and retry once. */
-  protected def retryOnOOM[T](body: => T): T = {
-    try body
-    catch {
-      case _: OutOfMemoryError =>
-        runGC()
-        body
+  /**
+   * Run `body`; if it throws OutOfMemoryError, force a GC and retry, up to `maxAttempts`
+   * total attempts (default 3). A single GC+retry is not always enough in memory-constrained
+   * CI: a large array retained by a previous test may not be reclaimed by the first
+   * System.gc(), so the retried allocation OOMs again. Retrying a few times (with a full
+   * runGC() that waits for a weak reference to clear between attempts) gives the JVM
+   * additional chances to reclaim memory. This does not mask genuine leaks: an unbounded
+   * allocation still fails once the attempts are exhausted.
+   */
+  protected def retryOnOOM[T](maxAttempts: Int = 3)(body: => T): T = {
+    var attempt = 1
+    while (true) {
+      try {
+        return body
+      } catch {
+        case oom: OutOfMemoryError =>
+          if (attempt >= maxAttempts) throw oom
+          runGC()
+          attempt += 1
+      }
     }
+    // Unreachable: the loop either returns a value or rethrows the OOM.
+    throw new IllegalStateException("retryOnOOM exited its retry loop unexpectedly")
   }
 }

core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala

@@ -141,8 +141,8 @@ class SorterSuite extends SparkFunSuite {
     val arrayToSortSize = 1091482190
     // Memory held by the previous test (e.g. the ~256 MB int array in "SPARK-5984
     // TimSort bug") may not be reclaimed before this >1 GB allocation, causing flaky
-    // OOM in CI. Force a GC and retry once on OOM.
-    val arrayToSort = retryOnOOM(new Array[Byte](arrayToSortSize))
+    // OOM in CI. Force a GC and retry a few times on OOM.
+    val arrayToSort = retryOnOOM()(new Array[Byte](arrayToSortSize))
     var sum: Int = -1
     for (i <- runLengths) {
       sum += i