Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Minor][SQL] Minor fix for CatalystSchemaConverter #7224

Closed
wants to merge 1 commit into from
Closed

[Minor][SQL] Minor fix for CatalystSchemaConverter #7224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ private[spark] object SQLConf {
val PARQUET_FOLLOW_PARQUET_FORMAT_SPEC = booleanConf(
key = "spark.sql.parquet.followParquetFormatSpec",
defaultValue = Some(false),
doc = "Wether to stick to Parquet format specification when converting Parquet schema to " +
doc = "Whether to stick to Parquet format specification when converting Parquet schema to " +
"Spark SQL schema and vice versa. Sticks to the specification if set to true; falls back " +
"to compatible mode if set to false.",
isPublic = false)
Expand Down
12 changes: 6 additions & 6 deletions sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,15 +142,15 @@ private[parquet] class CatalystSchemaConverter(
DecimalType(precision, scale)
}

field.getPrimitiveTypeName match {
typeName match {
case BOOLEAN => BooleanType

case FLOAT => FloatType

case DOUBLE => DoubleType

case INT32 =>
field.getOriginalType match {
originalType match {
case INT_8 => ByteType
case INT_16 => ShortType
case INT_32 | null => IntegerType
Expand All @@ -161,7 +161,7 @@ private[parquet] class CatalystSchemaConverter(
}

case INT64 =>
field.getOriginalType match {
originalType match {
case INT_64 | null => LongType
case DECIMAL => makeDecimalType(maxPrecisionForBytes(8))
case TIMESTAMP_MILLIS => typeNotImplemented()
Expand All @@ -176,7 +176,7 @@ private[parquet] class CatalystSchemaConverter(
TimestampType

case BINARY =>
field.getOriginalType match {
originalType match {
case UTF8 | ENUM => StringType
case null if assumeBinaryIsString => StringType
case null => BinaryType
Expand All @@ -185,7 +185,7 @@ private[parquet] class CatalystSchemaConverter(
}

case FIXED_LEN_BYTE_ARRAY =>
field.getOriginalType match {
originalType match {
case DECIMAL => makeDecimalType(maxPrecisionForBytes(field.getTypeLength))
case INTERVAL => typeNotImplemented()
case _ => illegalType()
Expand Down Expand Up @@ -261,7 +261,7 @@ private[parquet] class CatalystSchemaConverter(
// Here we implement Parquet LIST backwards-compatibility rules.
// See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules
// scalastyle:on
private def isElementType(repeatedType: Type, parentName: String) = {
private def isElementType(repeatedType: Type, parentName: String): Boolean = {
{
// For legacy 2-level list types with primitive element type, e.g.:
//
Expand Down