From b2378773474ac81cfd0bc1182e2841659527fbfc Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 25 Apr 2016 16:42:20 -0700 Subject: [PATCH] [SPARK-14907][MLLIB] Use repartition in GLMRegressionModel.save --- .../spark/mllib/regression/impl/GLMRegressionModel.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/impl/GLMRegressionModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/impl/GLMRegressionModel.scala index a6e1767fe236a..7696fdf2dcbed 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/impl/GLMRegressionModel.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/impl/GLMRegressionModel.scala @@ -48,7 +48,6 @@ private[regression] object GLMRegressionModel { weights: Vector, intercept: Double): Unit = { val sqlContext = SQLContext.getOrCreate(sc) - import sqlContext.implicits._ // Create JSON metadata. val metadata = compact(render( @@ -58,9 +57,7 @@ private[regression] object GLMRegressionModel { // Create Parquet data. val data = Data(weights, intercept) - val dataRDD: DataFrame = sc.parallelize(Seq(data), 1).toDF() - // TODO: repartition with 1 partition after SPARK-5532 gets fixed - dataRDD.write.parquet(Loader.dataPath(path)) + sqlContext.createDataFrame(Seq(data)).repartition(1).write.parquet(Loader.dataPath(path)) } /**