From 4af316a7cebd0f05a08c4f4519302961eccbcb1f Mon Sep 17 00:00:00 2001
From: manishgupta88 <tomanishgupta18@gmail.com>
Date: Wed, 1 Aug 2018 14:24:52 +0530
Subject: [PATCH] Problem Merge Index file not getting created with blocklet
 information for old store

Analysis
In legacy store (store <= 1.1 version), blocklet information is not written in the carbon Index files. When merge Index is created using the Alter DDL command on old store then merge Index file should be created with blocklet information which is as per the new store. This is not happening because the flag to read the carbondata file footer is not passed as true from Alter DDL command flow.

Fix
Pass the flag to read carbondataFileFooter as true while creating the merge Index file using Alter DDL command
---
 .../blockletindex/SegmentIndexFileStore.java   |  2 +-
 .../sql/events/MergeIndexEventListener.scala   | 18 ++++++++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index 16910acc0f3..c4e7f7a6ac7 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -405,7 +405,7 @@ private void readIndexAndFillBlockletInfo(CarbonFile indexFile) throws IOExcepti
       // get the index header
       org.apache.carbondata.format.IndexHeader indexHeader = indexReader.readIndexHeader();
       DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
-      String filePath = indexFile.getCanonicalPath();
+      String filePath = FileFactory.getUpdatedFilePath(indexFile.getCanonicalPath());
       String parentPath =
           filePath.substring(0, filePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR));
       while (indexReader.hasNext()) {
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala
index dff34246193..5bff9aa3004 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala
@@ -118,12 +118,18 @@ class MergeIndexEventListener extends OperationEventListener with Logging {
                   .put(loadMetadataDetails.getLoadName,
                     String.valueOf(loadMetadataDetails.getLoadStartTime))
               })
-              CommonUtil.mergeIndexFiles(sparkSession.sparkContext,
-                validSegmentIds,
-                segmentFileNameMap,
-                carbonMainTable.getTablePath,
-                carbonMainTable,
-                true)
+              // in case of merge index file creation using Alter DDL command
+              // readFileFooterFromCarbonDataFile flag should be true. This flag is check for legacy
+              // store (store <= 1.1 version) and create merge Index file as per new store so that
+              // old store is also upgraded to new store
+              CommonUtil.mergeIndexFiles(
+                sparkContext = sparkSession.sparkContext,
+                segmentIds = validSegmentIds,
+                segmentFileNameToSegmentIdMap = segmentFileNameMap,
+                tablePath = carbonMainTable.getTablePath,
+                carbonTable = carbonMainTable,
+                mergeIndexProperty = true,
+                readFileFooterFromCarbonDataFile = true)
               // clear Block dataMap Cache
               clearBlockDataMapCache(carbonMainTable, validSegmentIds)
               val requestMessage = "Compaction request completed for table "