Merge d176cfd into fa9a4ee

apache · Oct 23, 2018 · 2586e3c · 2586e3c
2 parents fa9a4ee + d176cfd
commit 2586e3c
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 3 deletions.
diff --git a/...n/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala b/...n/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.command.management
 import java.util
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
@@ -75,6 +76,9 @@ case class RefreshCarbonTableCommand(
       if (FileFactory.isFileExist(schemaFilePath, FileFactory.getFileType(schemaFilePath))) {
         // read TableInfo
         val tableInfo = SchemaReader.getTableInfo(identifier)
+        // refresh the column schema in case of store before V3
+        refreshColumnSchema(tableInfo)
+
         // 2.2 register the table with the hive check if the table being registered has
         // aggregate table then do the below steps
         // 2.2.1 validate that all the aggregate tables are copied at the store location.
@@ -118,6 +122,33 @@ case class RefreshCarbonTableCommand(
     Seq.empty
   }
 
+  /**
+   * Refresh the sort_column flag in column schema in case of old store. Before V3, sort_column
+   * option is not set but by default all dimension columns should be treated
+   * as sort columns if SORT_COLUMNS property is not defined in tblproperties
+   *
+   * @param tableInfo
+   */
+  def refreshColumnSchema(tableInfo: TableInfo): Unit = {
+    val tableProps: mutable.Map[String, String] = tableInfo.getFactTable.getTableProperties.asScala
+    val sortColumns = tableProps.get(CarbonCommonConstants.SORT_COLUMNS)
+    sortColumns match {
+      case Some(sortColumn) =>
+      // don't do anything
+      case None =>
+        // iterate over all the columns and make all the dimensions as sort columns true
+        // check for the complex data types parent and child columns to
+        // avoid adding them in SORT_COLUMNS
+        tableInfo.getFactTable.getListOfColumns.asScala collect
+        ({
+          case columnSchema if columnSchema.isDimensionColumn &&
+                               !columnSchema.getDataType.isComplexType &&
+                               columnSchema.getSchemaOrdinal != -1 =>
+            columnSchema.setSortColumn(true)
+        })
+    }
+  }
+
   /**
    * the method prepare the data type for raw column
    *

diff --git a/.../src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/.../src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -415,13 +415,13 @@ private void initSortDataRows() throws Exception {
     isVarcharDimMapping = new boolean[dimensions.size()];
     int i = 0;
     for (CarbonDimension dimension : dimensions) {
+      if (dimension.isSortColumn()) {
+        sortColumnMapping[i] = true;
+      }
       if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
         i++;
         continue;
       }
-      if (dimension.isSortColumn()) {
-        sortColumnMapping[i] = true;
-      }
       noDictionaryColMapping[i] = true;
       if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {
         isVarcharDimMapping[i] = true;