diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
index 31491afb0db10..4df861f8045e5 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
@@ -291,7 +291,7 @@ object GradientDescent extends Logging {
     // This represents the difference of updated weights in the iteration.
     val solutionVecDiff: Double = norm(previousBDV - currentBDV)
 
-    solutionVecDiff < convergenceTol * norm(previousBDV)
+    solutionVecDiff < convergenceTol * Math.max(norm(currentBDV), 1.0)
   }
 
 }