fix null deviance calculation and add tests

apache · Feb 10, 2017 · fc64d32 · fc64d32
1 parent afb4643
commit fc64d32
Show file tree

Hide file tree

Showing 2 changed files with 314 additions and 370 deletions.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -1279,17 +1279,19 @@ class GeneralizedLinearRegressionSummary private[regression] (
           label.minus(offset))), sum(weight)).first()
         link.link(agg.getDouble(0) / agg.getDouble(1))
       } else {
+        // Create empty feature column and fit intercept only model using param setting from model
         val featureNull = "feature_" + java.util.UUID.randomUUID.toString
-        val glr = new GeneralizedLinearRegression().copy(model.extractParamMap)
-          .setFeaturesCol(featureNull)
+        val paramMap = model.extractParamMap()
+        paramMap.put(model.featuresCol, featureNull)
+        if (family.name != "tweedie") {
+          paramMap.remove(model.variancePower)
+        }
         val emptyVectorUDF = udf{ () => Vectors.zeros(0) }
-        glr.fit(
-          predictions.withColumn(featureNull, emptyVectorUDF())
+        model.parent.fit(
+          dataset.withColumn(featureNull, emptyVectorUDF()), paramMap
         ).intercept
       }
     }
-
-    println("intercept is: " + intercept)
     predictions.select(label, offset, weight).rdd.map {
       case Row(y: Double, offset: Double, weight: Double) =>
         family.deviance(y, link.unlink(intercept + offset), weight)