From c9b258b0f46daa7aa81478cf514bb3bf3b4edf1a Mon Sep 17 00:00:00 2001 From: Kittipat Virochsiri Date: Wed, 20 Feb 2019 22:58:46 -0800 Subject: [PATCH] Fix TimelineTest in OSS Summary: Newer version of Spark has `aprrox_percentile`, which is functionally equivalent to `fb_approx_percentile`. Differential Revision: D14163262 fbshipit-source-id: 70ff24bfdfbcd3ae0cfbc5f4ab1c8a45b4f297d0 --- .../src/test/scala/com/facebook/spark/rl/TimelineTest.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/preprocessing/src/test/scala/com/facebook/spark/rl/TimelineTest.scala b/preprocessing/src/test/scala/com/facebook/spark/rl/TimelineTest.scala index 070e7a33f..b2ba837c7 100644 --- a/preprocessing/src/test/scala/com/facebook/spark/rl/TimelineTest.scala +++ b/preprocessing/src/test/scala/com/facebook/spark/rl/TimelineTest.scala @@ -549,6 +549,9 @@ class TimelineTest extends PipelineTester { import sqlCtx.implicits._ val sparkContext = sqlCtx.sparkContext + val percentileFunc = + if (sparkContext.version >= "2.3.0") "approx_percentile" else "fb_approx_percentile" + // Setup configuration val config = TimelineConfiguration("2018-01-01", "2018-01-01", @@ -559,7 +562,7 @@ class TimelineTest extends PipelineTester { null, 1, Some(0.95), - "fb_approx_percentile") + percentileFunc) // destroy previous schema Timeline.validateOrDestroyTrainingTable(sqlContext,