address comments

apache · Jun 18, 2020 · 557eb30 · 557eb30
1 parent fd74ff9
commit 557eb30
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtils.scala
@@ -74,8 +74,8 @@ object UnsafeRowUtils {
             if ((row.getLong(index) >> 32) != 0L) return false
           case _ =>
         }
-      case (field, index) if field.dataType == NullType =>
-        if (!row.isNullAt(index) || row.getLong(index) != 0L) return false
+      case (_, index) if row.isNullAt(index) =>
+        if (row.getLong(index) != 0L) return false
       case _ =>
     }
     if (bitSetWidthInBytes + 8 * row.numFields + varLenFieldsSizeInBytes > rowSizeInBytes) {

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1241,7 +1241,8 @@ object SQLConf {
     buildConf("spark.sql.streaming.stateStore.formatValidation.enabled")
       .internal()
       .doc("When true, check if the UnsafeRow from the state store is valid or not when running " +
-        "streaming queries. This can happen if the state store format has been changed.")
+        "streaming queries. This can happen if the state store format has been changed. Note, " +
+        "the feature is only effective in the build-in HDFS state store provider now.")
       .version("3.1.0")
       .booleanConf
       .createWithDefault(true)