Skip to content
Permalink
Browse files
[CARBONDATA-4230] table properties not updated with lower-case and table
comment is not working in carbon spark3.1

Why is this PR needed?
1. table properties storing with case-sensitive and when we query table
   properties with the small case then property not able to get hence table
   create command is failed. this is induced with spark 3.1 integration changes.
2. Table comment is displayed as byte code in spark 3.1 cluster.
   CommentSpecContext is changed in 3.1

What changes were proposed in this PR?
1. convert to small case and store in table properties.
2. Get string value from commentSpec and set as table comment

Does this PR introduce any user interface change?
No

Is any new testcase added?
No, already test case is present but not failed in local ut setup as create
flow is different in local ut env and real cluster setup

This closes #4163
  • Loading branch information
maheshrajus authored and Indhumathi27 committed Jun 29, 2021
1 parent 5e2adad commit 65462ff890e90aa95adf3daf13c52f1eb9f6e80e
Showing 4 changed files with 17 additions and 13 deletions.
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkSqlAstBuilder, SparkSqlParser}
import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
import org.apache.spark.sql.parser.CarbonSparkSqlParserUtil.needToConvertToLowerCase
import org.apache.spark.sql.parser.CarbonSparkSqlParserUtil.convertPropertiesToLowercase
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
@@ -128,16 +128,8 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
None
}

val tableProperties = mutable.Map[String, String]()
val properties: Map[String, String] = getPropertyKeyValues(tablePropertyList)

properties.foreach { property =>
if (needToConvertToLowerCase(property._1)) {
tableProperties.put(property._1.toLowerCase, property._2.toLowerCase)
} else {
tableProperties.put(property._1.toLowerCase, property._2)
}
}
val tableProperties = convertPropertiesToLowercase(properties)
// validate partition clause
val partitionByStructFields = Option(partitionColumns).toSeq.flatMap(visitColTypeList)
val partitionFields = CarbonToSparkAdapter.
@@ -547,4 +547,16 @@ object CarbonSparkSqlParserUtil {
isOverwriteTable = isOverwrite.isDefined,
partition = partitionSpec)
}

def convertPropertiesToLowercase(properties: Map[String, String]): mutable.Map[String, String] = {
val tableProperties = mutable.Map[String, String]()
properties.foreach { property =>
if (needToConvertToLowerCase(property._1)) {
tableProperties.put(property._1.toLowerCase, property._2.toLowerCase)
} else {
tableProperties.put(property._1.toLowerCase, property._2)
}
}
tableProperties
}
}
@@ -44,7 +44,7 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSes
val commentSpecContext = if (commentSpec.isEmpty) {
null
} else {
commentSpec.get(0).toString()
string(commentSpec.get(0).STRING())
}
val createTableTuple = (ctx.createTableHeader, createTableClauses.skewSpec(0),
createTableClauses.bucketSpec(0), createTableClauses.partitioning, ctx.colTypeList(),
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkSqlAstBuilder, SparkSqlParser}
import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
import org.apache.spark.sql.parser.CarbonSparkSqlParserUtil.convertPropertiesToLowercase
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
@@ -125,9 +126,8 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
None
}

val tableProperties = mutable.Map[String, String]()
val properties: Map[String, String] = getPropertyKeyValues(tablePropertyList)
properties.foreach{property => tableProperties.put(property._1, property._2)}
val tableProperties = convertPropertiesToLowercase(properties)

// validate partition clause
val partitionByStructFields = Option(partitionColumns).toSeq

0 comments on commit 65462ff

Please sign in to comment.