From 6a82f8c427733a02d162228468f4c6455e531a84 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 29 Apr 2021 10:48:23 +0800 Subject: [PATCH] [SPARK-34786][SQL][FOLLOWUP] Explicitly declare DecimalType(20, 0) for Parquet UINT_64 --- .../datasources/parquet/ParquetSchemaConverter.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala index 8c4e0881e08b5..e751c97a60c08 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala @@ -141,7 +141,9 @@ class ParquetToSparkSchemaConverter( originalType match { case INT_64 | null => LongType case DECIMAL => makeDecimalType(Decimal.MAX_LONG_DIGITS) - case UINT_64 => DecimalType.LongDecimal + // The precision to hold the largest unsigned long is: + // `java.lang.Long.toUnsignedString(-1).length` = 20 + case UINT_64 => DecimalType(20, 0) case TIMESTAMP_MICROS => TimestampType case TIMESTAMP_MILLIS => TimestampType case _ => illegalType()