From 222a54dc7a6fa6915d11997f8229b496ac122907 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 6 Jul 2026 14:12:57 +0900 Subject: [PATCH] [SPARK-57951][ML][TESTS] Tolerate last-ULP FP diffs in MLTest single-prediction checks testClassificationModelSingleRawPrediction and testProbClassificationModelSingleProbPrediction compared the DataFrame transform output against the scalar predictRaw/predictProbability output with exact === equality. On arm64 macOS the two paths can round differently in the last ULP (e.g. 1.543502002724983 vs 1.5435020027249835), failing suites such as MultilayerPerceptronClassifierSuite on the Build / Maven (JDK 21, MacOS-26) lane while Linux stays bit-identical. Compare the prediction vectors with a tight absolute tolerance (1e-9) instead, which still catches any real discrepancy. Generated-by: Claude Code --- .../org/apache/spark/ml/util/MLTest.scala | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala b/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala index a35b19b2816e4..23d27162654ce 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala @@ -196,7 +196,11 @@ trait MLTest extends StreamTest with TempDirectory { self: Suite => model.transform(dataset).select(model.getFeaturesCol, model.getRawPredictionCol) .collect().foreach { case Row(features: Vector, rawPrediction: Vector) => - assert(rawPrediction === model.predictRaw(features)) + // Compare with a tight tolerance rather than exact equality: the DataFrame + // `transform` path and the scalar `predictRaw` path can round differently in the + // last ULP on some platforms (e.g. arm64 macOS, where native BLAS diverges from + // JVM scalar math), which broke the MacOS-26 lane while Linux stayed bit-identical. + assertVectorsAlmostEqual(rawPrediction, model.predictRaw(features)) } } @@ -206,7 +210,22 @@ trait MLTest extends StreamTest with TempDirectory { self: Suite => model.transform(dataset).select(model.getFeaturesCol, model.getProbabilityCol) .collect().foreach { case Row(features: Vector, probPrediction: Vector) => - assert(probPrediction === model.predictProbability(features)) + assertVectorsAlmostEqual(probPrediction, model.predictProbability(features)) + } + } + + /** + * Asserts two prediction vectors are equal up to a tight absolute tolerance. Used instead of + * exact `===` so that last-ULP rounding differences between the DataFrame `transform` path and + * the scalar `predict*` path (observed on arm64 macOS) do not fail the tests, while any real + * discrepancy is still caught. + */ + private def assertVectorsAlmostEqual(actual: Vector, expected: Vector): Unit = { + assert(actual.size === expected.size, + s"vector sizes differ: ${actual.size} vs ${expected.size}") + actual.toArray.zip(expected.toArray).zipWithIndex.foreach { case ((a, e), i) => + assert(math.abs(a - e) <= 1e-9, + s"prediction vectors differ at index $i: $a vs $e (actual=$actual, expected=$expected)") } }