PHOENIX-4318 Fix IndexHalfStoreFileReader and related classes(Rajeshb…

…abu)
apache · Nov 17, 2017 · 8ab7cc1 · 8ab7cc1
1 parent 5bdd3b2
commit 8ab7cc1
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 171 deletions.
diff --git a/...nix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java b/...nix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
@@ -19,13 +19,15 @@
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.Reference;
@@ -47,7 +49,7 @@
  * This file is not splitable. Calls to {@link #midkey()} return null.
  */
 
-public class IndexHalfStoreFileReader extends StoreFile.Reader {
+public class IndexHalfStoreFileReader extends StoreFileReader {
     private final boolean top;
     // This is the key we split around. Its the first possible entry on a row:
     // i.e. empty column and a timestamp of LATEST_TIMESTAMP.
@@ -56,7 +58,7 @@ public class IndexHalfStoreFileReader extends StoreFile.Reader {
     private final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers;
     private final byte[][] viewConstants;
     private final int offset;
-    private final HRegionInfo regionInfo;
+    private final RegionInfo regionInfo;
     private final byte[] regionStartKeyInHFile;
 
     /**
@@ -78,9 +80,10 @@ public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheCo
             final FSDataInputStreamWrapper in, long size, final Reference r,
             final Configuration conf,
             final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers,
-            final byte[][] viewConstants, final HRegionInfo regionInfo,
+            final byte[][] viewConstants, final RegionInfo regionInfo,
-            byte[] regionStartKeyInHFile, byte[] splitKey) throws IOException {
+            byte[] regionStartKeyInHFile, byte[] splitKey, boolean primaryReplicaStoreFile) throws IOException {
-        super(fs, p, in, size, cacheConf, conf);
+        super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, new AtomicInteger(0), false,
+                conf);
         this.splitkey = splitKey == null ? r.getSplitKey() : splitKey;
         // Is it top or bottom half?
         this.top = Reference.isTopFileRegion(r.getFileRegion());
@@ -104,7 +107,7 @@ public Map<ImmutableBytesWritable, IndexMaintainer> getIndexMaintainers() {
         return indexMaintainers;
     }
 
-    public HRegionInfo getRegionInfo() {
+    public RegionInfo getRegionInfo() {
         return regionInfo;
     }
 
@@ -123,4 +126,12 @@ public byte[] getSplitRow() {
     public boolean isTop() {
         return top;
     }
+
+    @Override
+    public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, boolean pread,
+            boolean isCompaction, long readPt, long scannerOrder,
+            boolean canOptimizeForNonNullColumn) {
+        return new LocalIndexStoreFileScanner(this, cacheBlocks, pread, isCompaction, readPt,
+                scannerOrder, canOptimizeForNonNullColumn);
+    }
 }
diff --git a/...src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java b/...src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java
@@ -17,8 +17,8 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
 
-public class LocalIndexSplitter extends BaseRegionObserver {
+public class LocalIndexSplitter implements RegionObserver {
 
 }
diff --git a/...x-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexStoreFileScanner.java b/...x-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexStoreFileScanner.java
@@ -19,13 +19,14 @@
 
 import java.io.IOException;
 import java.util.Map.Entry;
+import java.util.Optional;
 
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
-import org.apache.hadoop.hbase.regionserver.StoreFile.Reader;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.util.PhoenixKeyValueUtil;
@@ -36,10 +37,13 @@ public class LocalIndexStoreFileScanner extends StoreFileScanner{
 
     private IndexHalfStoreFileReader reader;
     private boolean changeBottomKeys;
-    public LocalIndexStoreFileScanner(Reader reader, HFileScanner hfs, boolean useMVCC,
+    @SuppressWarnings("deprecation")
-            boolean hasMVCC, long readPt) {
+    public LocalIndexStoreFileScanner(IndexHalfStoreFileReader reader, boolean cacheBlocks, boolean pread,
-        super(reader, hfs, useMVCC, hasMVCC, readPt);
+            boolean isCompaction, long readPt, long scannerOrder,
-        this.reader = ((IndexHalfStoreFileReader)super.getReader());
+            boolean canOptimizeForNonNullColumn) {
+        super(reader, reader.getScanner(cacheBlocks, pread, isCompaction), true, reader
+                .getHFileReader().hasMVCCInfo(), readPt, scannerOrder, canOptimizeForNonNullColumn);
+        this.reader = reader;
         this.changeBottomKeys =
                 this.reader.getRegionInfo().getStartKey().length == 0
                         && this.reader.getSplitRow().length != this.reader.getOffset();
@@ -114,13 +118,14 @@ public boolean reseek(Cell key) throws IOException {
     public boolean seekToPreviousRow(Cell key) throws IOException {
         KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(key);
         if (reader.isTop()) {
-            byte[] fk = reader.getFirstKey();
+            Optional<Cell> firstKey = reader.getFirstKey();
             // This will be null when the file is empty in which we can not seekBefore to
             // any key
-            if (fk == null) {
+            if (firstKey.isPresent()) {
                 return false;
             }
-            if (getComparator().compare(kv.getRowArray(), kv.getKeyOffset(), kv.getKeyLength(), fk, 0, fk.length) <= 0) {
+            byte[] fk = PhoenixKeyValueUtil.maybeCopyCell(firstKey.get()).getKey();
+            if (getComparator().compare(kv, firstKey.get()) <= 0) {
                 return super.seekToPreviousRow(key);
             }
             KeyValue replacedKey = getKeyPresentInHFiles(kv.getRowArray());
@@ -132,7 +137,8 @@ public boolean seekToPreviousRow(Cell key) throws IOException {
         } else {
             // The equals sign isn't strictly necessary just here to be consistent with
             // seekTo
-            if (getComparator().compare(kv.getRowArray(), kv.getKeyOffset(), kv.getKeyLength(), reader.getSplitkey(), 0, reader.getSplitkey().length) >= 0) {
+            KeyValue splitKeyValue = KeyValueUtil.createKeyValueFromKey(reader.getSplitkey());
+            if (getComparator().compare(kv, splitKeyValue) >= 0) {
                 boolean seekToPreviousRow = super.seekToPreviousRow(kv);
                 while(super.peek()!=null && !isSatisfiedMidKeyCondition(super.peek())) {
                     seekToPreviousRow = super.seekToPreviousRow(super.peek());
@@ -221,8 +227,9 @@ private KeyValue getKeyPresentInHFiles(byte[] key) {
     public boolean seekOrReseek(Cell cell, boolean isSeek) throws IOException{
         KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(cell);
         KeyValue keyToSeek = kv;
+        KeyValue splitKeyValue = KeyValueUtil.createKeyValueFromKey(reader.getSplitkey());
         if (reader.isTop()) {
-            if(getComparator().compare(kv.getRowArray(), kv.getKeyOffset(), kv.getKeyLength(), reader.getSplitkey(), 0, reader.getSplitkey().length) < 0){
+            if(getComparator().compare(kv, splitKeyValue) < 0){
                 if(!isSeek && realSeekDone()) {
                     return true;
                 }
@@ -231,7 +238,7 @@ public boolean seekOrReseek(Cell cell, boolean isSeek) throws IOException{
             keyToSeek = getKeyPresentInHFiles(kv.getRowArray());
             return seekOrReseekToProperKey(isSeek, keyToSeek);
         } else {
-            if (getComparator().compare(kv.getRowArray(), kv.getKeyOffset(), kv.getKeyLength(), reader.getSplitkey(), 0, reader.getSplitkey().length) >= 0) {
+            if (getComparator().compare(kv, splitKeyValue) >= 0) {
                 close();
                 return false;
             }

diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContextUtil.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContextUtil.java
@@ -19,19 +19,20 @@
 package org.apache.hadoop.hbase.regionserver;
 
 
+import java.util.List;
+
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 
-import java.util.List;
-
 /**
  * @ScannerContext has all methods package visible. To properly update the context progress for our scanners we
  * need this helper
  */
 public class ScannerContextUtil {
     public static void incrementSizeProgress(ScannerContext sc, List<Cell> cells) {
         for (Cell cell : cells) {
-            sc.incrementSizeProgress(CellUtil.estimatedHeapSizeOfWithoutTags(cell));
+            sc.incrementSizeProgress(CellUtil.estimatedSerializedSizeOf(cell),
+                    CellUtil.estimatedHeapSizeOf(cell));
         }
     }
 

diff --git a/...x-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/...x-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -1325,7 +1325,6 @@ public static byte[] serialize(List<Expression> selectExpressions) {
         }
     }
 
-
     /*
      * TODO: use waitForFlushes PHOENIX-4352
      */

diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -39,7 +39,6 @@
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Delete;
@@ -50,6 +49,7 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -541,7 +541,7 @@ public static void wrapResultUsingOffset(final RegionCoprocessorEnvironment envi
             } else {
                 TableName dataTable =
                         TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(
-                            environment.getRegion().getTableDesc().getTableName().getNameAsString()));
+                            environment.getRegion().getTableDescriptor().getTableName().getNameAsString()));
                 Table table = null;
                 try {
                     table = environment.getConnection().getTable(dataTable);
@@ -749,10 +749,10 @@ public static boolean matchingSplitKeys(byte[][] splitKeys1, byte[][] splitKeys2
     }
 
     public static boolean isLocalIndexStore(Store store) {
-        return store.getFamily().getNameAsString().startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX);
+        return store.getColumnFamilyDescriptor().getNameAsString().startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX);
     }
 
-    public static PTable getPDataTable(Connection conn, HTableDescriptor tableDesc) throws SQLException {
+    public static PTable getPDataTable(Connection conn, TableDescriptor tableDesc) throws SQLException {
         String dataTableName = Bytes.toString(tableDesc.getValue(MetaDataUtil.DATA_TABLE_NAME_PROP_BYTES));
         String physicalTableName = tableDesc.getTableName().getNameAsString();
         PTable pDataTable = null;