Skip to content

Commit

Permalink
Merge 85e70d1 into 923dab1
Browse files Browse the repository at this point in the history
  • Loading branch information
@XiaotaoYi
XiaotaoYi authored Jan 6, 2019
2 parents 923dab1 + 85e70d1 commit 9c00be5
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 44 deletions.
30 changes: 15 additions & 15 deletions ...ration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ object CarbonScalaUtil {
dataType: DataType,
timeStampFormat: SimpleDateFormat,
dateFormat: SimpleDateFormat): String = {
val defaultValue = value != null && value.equalsIgnoreCase(hivedefaultpartition)
val defaultValue = value != null && value.equalsIgnoreCase(hiveDefaultPartition)
try {
dataType match {
case TimestampType if timeStampFormat != null =>
Expand Down Expand Up @@ -236,7 +236,7 @@ object CarbonScalaUtil {
}
}

private val hivedefaultpartition = "__HIVE_DEFAULT_PARTITION__"
private val hiveDefaultPartition = "__HIVE_DEFAULT_PARTITION__"

/**
* Update partition values as per the right date and time format
Expand All @@ -248,20 +248,20 @@ object CarbonScalaUtil {
val cacheProvider: CacheProvider = CacheProvider.getInstance
val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] =
cacheProvider.createCache(CacheType.FORWARD_DICTIONARY)
partitionSpec.map { case (col, pvalue) =>
partitionSpec.map { case (col, pValue) =>
// replace special string with empty value.
val value = if (pvalue == null) {
hivedefaultpartition
} else if (pvalue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
val value = if (pValue == null) {
hiveDefaultPartition
} else if (pValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
""
} else {
pvalue
pValue
}
val carbonColumn = table.getColumnByName(table.getTableName, col.toLowerCase)
val dataType =
CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(carbonColumn.getDataType)
try {
if (value.equals(hivedefaultpartition)) {
if (value.equals(hiveDefaultPartition)) {
(col, value)
} else {
val convertedString =
Expand All @@ -271,7 +271,7 @@ object CarbonScalaUtil {
forwardDictionaryCache,
table)
if (convertedString == null) {
(col, hivedefaultpartition)
(col, hiveDefaultPartition)
} else {
(col, convertedString)
}
Expand Down Expand Up @@ -518,11 +518,11 @@ object CarbonScalaUtil {
if (dictIncludeColumns.exists(x => x.equalsIgnoreCase(distCol.trim))) {
val commonColumn = (dictIncludeColumns ++ localDictColumns)
.diff((dictIncludeColumns ++ localDictColumns).distinct).distinct
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
val errorMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
commonColumn.mkString(",") +
" specified in Dictionary include. Local Dictionary will not be " +
"generated for Dictionary include columns. Please check the DDL."
throw new MalformedCarbonCommandException(errormsg)
throw new MalformedCarbonCommandException(errorMsg)
}
}
}
Expand Down Expand Up @@ -633,9 +633,9 @@ object CarbonScalaUtil {
// check if the column specified exists in table schema
localDictColumns.foreach { distCol =>
if (!fields.exists(x => x.column.equalsIgnoreCase(distCol.trim))) {
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + distCol.trim +
val errorMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + distCol.trim +
" does not exist in table. Please check the DDL."
throw new MalformedCarbonCommandException(errormsg)
throw new MalformedCarbonCommandException(errorMsg)
}
}

Expand All @@ -648,12 +648,12 @@ object CarbonScalaUtil {
!x.dataType.get.equalsIgnoreCase("STRUCT") &&
!x.dataType.get.equalsIgnoreCase("MAP") &&
!x.dataType.get.equalsIgnoreCase("ARRAY"))) {
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
val errorMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
dictColm.trim +
" is not a string/complex/varchar datatype column. LOCAL_DICTIONARY_COLUMN" +
" should be no dictionary string/complex/varchar datatype column." +
"Please check the DDL."
throw new MalformedCarbonCommandException(errormsg)
throw new MalformedCarbonCommandException(errorMsg)
}
}

Expand Down
54 changes: 27 additions & 27 deletions ...ration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -514,7 +514,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
if (!nullColumnErr.isEmpty) {
val errMsg = s"long_string_columns: ${
nullColumnErr.mkString(",")
} does not exist in table. Please check create table statement."
} does not exist in table. Please check the create table statement."
throw new MalformedCarbonCommandException(errMsg)
}

Expand All @@ -529,7 +529,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
if (!dataTypeErr.isEmpty) {
val errMsg = s"long_string_columns: ${
dataTypeErr.mkString(",")
} ,its data type is not string. Please check create table statement."
} ,its data type is not string. Please check the create table statement."
throw new MalformedCarbonCommandException(errMsg)
}
}
Expand Down Expand Up @@ -668,9 +668,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
tableProperties(CarbonCommonConstants.NO_INVERTED_INDEX).split(',').map(_.trim)
noInvertedIdxColsProps.foreach { noInvertedIdxColProp =>
if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) {
val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp +
" does not exist in table. Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
val errorMsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp +
" does not exist in table. Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
}
}
}
Expand All @@ -697,9 +697,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
tableProperties(CarbonCommonConstants.INVERTED_INDEX).split(',').map(_.trim)
invertedIdxColsProps.foreach { invertedIdxColProp =>
if (!fields.exists(x => x.column.equalsIgnoreCase(invertedIdxColProp))) {
val errormsg = "INVERTED_INDEX column: " + invertedIdxColProp +
" does not exist in table. Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
val errorMsg = "INVERTED_INDEX column: " + invertedIdxColProp +
" does not exist in table. Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
}
}
}
Expand Down Expand Up @@ -759,15 +759,15 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {

sortKey.foreach { column =>
if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
val errormsg = "sort_columns: " + column +
" does not exist in table. Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
val errorMsg = "sort_columns: " + column +
" does not exist in table. Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
} else {
val dataType = fields.find(x =>
x.column.equalsIgnoreCase(column)).get.dataType.get
if (isDataTypeSupportedForSortColumn(dataType)) {
val errormsg = s"sort_columns is unsupported for $dataType datatype column: " + column
throw new MalformedCarbonCommandException(errormsg)
val errorMsg = s"sort_columns is unsupported for $dataType datatype column: " + column
throw new MalformedCarbonCommandException(errorMsg)
}
if (varcharCols.exists(x => x.equalsIgnoreCase(column))) {
throw new MalformedCarbonCommandException(
Expand Down Expand Up @@ -796,10 +796,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
dictExcludeCols
.foreach { dictExcludeCol =>
if (!fields.exists(x => x.column.equalsIgnoreCase(dictExcludeCol))) {
val errormsg = "DICTIONARY_EXCLUDE column: " + dictExcludeCol +
val errorMsg = "DICTIONARY_EXCLUDE column: " + dictExcludeCol +
" does not exist in table or unsupported for complex child column. " +
"Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
"Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
} else {
val dataType = fields.find(x =>
x.column.equalsIgnoreCase(dictExcludeCol)).get.dataType.get
Expand All @@ -821,10 +821,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(",").map(_.trim)
dictIncludeCols.foreach { distIncludeCol =>
if (!fields.exists(x => x.column.equalsIgnoreCase(distIncludeCol.trim))) {
val errormsg = "DICTIONARY_INCLUDE column: " + distIncludeCol.trim +
val errorMsg = "DICTIONARY_INCLUDE column: " + distIncludeCol.trim +
" does not exist in table or unsupported for complex child column. " +
"Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
"Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
}
if (varcharCols.exists(x => x.equalsIgnoreCase(distIncludeCol.trim))) {
throw new MalformedCarbonCommandException(
Expand All @@ -837,9 +837,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// include cols should not contain exclude cols
dictExcludeCols.foreach { dicExcludeCol =>
if (dictIncludeCols.exists(x => x.equalsIgnoreCase(dicExcludeCol))) {
val errormsg = "DICTIONARY_EXCLUDE can not contain the same column: " + dicExcludeCol +
" with DICTIONARY_INCLUDE. Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
val errorMsg = "DICTIONARY_EXCLUDE can not contain the same column: " + dicExcludeCol +
" with DICTIONARY_INCLUDE. Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
}
}

Expand All @@ -855,7 +855,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
!dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
noDictionaryDims :+= field.column
dimFields += field
} else if (isDetectAsDimentionDatatype(field.dataType.get)) {
} else if (isDetectAsDimentionDataType(field.dataType.get)) {
dimFields += field
// consider all String cols as noDicitonaryDims by default
if (DataTypes.STRING.getName.equalsIgnoreCase(field.dataType.get)) {
Expand Down Expand Up @@ -922,7 +922,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
*
* @param dimensionDatatype
*/
def isDetectAsDimentionDatatype(dimensionDatatype: String): Boolean = {
def isDetectAsDimentionDataType(dimensionDatatype: String): Boolean = {
val dimensionType = Array("string", "array", "struct", "map", "timestamp", "date", "char")
dimensionType.exists(x => dimensionDatatype.toLowerCase.contains(x))
}
Expand Down Expand Up @@ -1138,8 +1138,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {

// Validate ESCAPECHAR length
if (options.exists(_._1.equalsIgnoreCase("ESCAPECHAR"))) {
val escapechar: String = options.get("escapechar").get.head._2
if (escapechar.length > 1 && !CarbonLoaderUtil.isValidEscapeSequence(escapechar)) {
val escapeChar: String = options.get("escapechar").get.head._2
if (escapeChar.length > 1 && !CarbonLoaderUtil.isValidEscapeSequence(escapeChar)) {
throw new MalformedCarbonCommandException("ESCAPECHAR cannot be more than one character.")
}
}
Expand Down Expand Up @@ -1192,7 +1192,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {

// check for duplicate options
val duplicateOptions = options filter {
case (_, optionlist) => optionlist.size > 1
case (_, optionList) => optionList.size > 1
}
val duplicates = StringBuilder.newBuilder
if (duplicateOptions.nonEmpty) {
Expand Down
4 changes: 2 additions & 2 deletions integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -703,9 +703,9 @@ object AlterTableUtil {
// check if the column specified exists in table schema
localDictColumns.foreach { distCol =>
if (!allColumns.exists(x => x.getColumnName.equalsIgnoreCase(distCol.trim))) {
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + distCol.trim +
val errorMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + distCol.trim +
" does not exist in table. Please check the DDL."
throw new MalformedCarbonCommandException(errormsg)
throw new MalformedCarbonCommandException(errorMsg)
}
}

Expand Down

0 comments on commit 9c00be5

Please sign in to comment.