diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java index 35f85f71a56f..b489fe4cf155 100644 --- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.coprocessor.example; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.NavigableSet; +import java.util.OptionalInt; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -194,9 +194,7 @@ public InternalScanner preFlushScannerOpen(final ObserverContext scanners, ScanType scanType, FileDetails fd, long smallestReadPoint) throws IOException { - Scan scan = new Scan(); - scan.setMaxVersions(store.getColumnFamilyDescriptor().getMaxVersions()); - return new StoreScanner(store, store.getScanInfo(), scan, scanners, scanType, + return new StoreScanner(store, store.getScanInfo(), OptionalInt.empty(), scanners, scanType, smallestReadPoint, fd.earliestPutTs); } }; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java index c378a884cdee..2308ddfe1a7f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java @@ -55,7 +55,6 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.MobCompactPartitionPolicy; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -805,15 +804,12 @@ private Path compactDelFilesInBatch(PartitionedMobCompactionRequest request, * @throws IOException if IO failure is encountered */ private StoreScanner createScanner(List filesToCompact, ScanType scanType) - throws IOException { + throws IOException { List scanners = StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, true, false, false, HConstants.LATEST_TIMESTAMP); - Scan scan = new Scan(); - scan.setMaxVersions(column.getMaxVersions()); long ttl = HStore.determineTTLFromFamily(column); ScanInfo scanInfo = new ScanInfo(conf, column, ttl, 0, CellComparator.COMPARATOR); - return new StoreScanner(scan, scanInfo, scanType, null, scanners, 0L, - HConstants.LATEST_TIMESTAMP); + return new StoreScanner(scanInfo, scanType, scanners); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index fb837e815d3d..f38ffb5c1fae 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorSegmentsIterator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorSegmentsIterator.java index 8f481e001835..5386c7db5f0f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorSegmentsIterator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactorSegmentsIterator.java @@ -19,16 +19,16 @@ package org.apache.hadoop.hbase.regionserver; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Scan; - import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.OptionalInt; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * The MemStoreCompactorSegmentsIterator extends MemStoreSegmentsIterator @@ -106,23 +106,15 @@ public void remove() { /** * Creates the scanner for compacting the pipeline. - * * @return the scanner */ private StoreScanner createScanner(Store store, List scanners) throws IOException { - - Scan scan = new Scan(); - scan.setMaxVersions(); //Get all available versions - StoreScanner internalScanner = - new StoreScanner(store, store.getScanInfo(), scan, scanners, - ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), - HConstants.OLDEST_TIMESTAMP); - - return internalScanner; + // Get all available versions + return new StoreScanner(store, store.getScanInfo(), OptionalInt.of(Integer.MAX_VALUE), scanners, + ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP); } - /* Refill kev-value set (should be invoked only when KVS is empty) * Returns true if KVS is non-empty */ private boolean refillKVS() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedStoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedStoreScanner.java index 07f98ada8acf..86b9ea9442ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedStoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedStoreScanner.java @@ -53,11 +53,9 @@ class ReversedStoreScanner extends StoreScanner implements KeyValueScanner { } /** Constructor for testing. */ - ReversedStoreScanner(final Scan scan, ScanInfo scanInfo, ScanType scanType, - final NavigableSet columns, final List scanners) - throws IOException { - super(scan, scanInfo, scanType, columns, scanners, - HConstants.LATEST_TIMESTAMP); + ReversedStoreScanner(Scan scan, ScanInfo scanInfo, NavigableSet columns, + List scanners) throws IOException { + super(scan, scanInfo, columns, scanners); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java index 298f3d41e1bc..d29d48fef9f7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.OptionalInt; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputControlUtil; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; @@ -86,11 +85,8 @@ protected InternalScanner createScanner(List snapshotScanners, smallestReadPoint); } if (scanner == null) { - Scan scan = new Scan(); - scan.setMaxVersions(store.getScanInfo().getMaxVersions()); - scanner = new StoreScanner(store, store.getScanInfo(), scan, - snapshotScanners, ScanType.COMPACT_RETAIN_DELETES, - smallestReadPoint, HConstants.OLDEST_TIMESTAMP); + scanner = new StoreScanner(store, store.getScanInfo(), OptionalInt.empty(), snapshotScanners, + ScanType.COMPACT_RETAIN_DELETES, smallestReadPoint, HConstants.OLDEST_TIMESTAMP); } assert scanner != null; if (store.getCoprocessorHost() != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index a220f5485865..9a096f61072d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -24,6 +24,8 @@ import java.util.ArrayList; import java.util.List; import java.util.NavigableSet; +import java.util.Optional; +import java.util.OptionalInt; import java.util.concurrent.CountDownLatch; import java.util.concurrent.locks.ReentrantLock; @@ -45,7 +47,6 @@ import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState; import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler; import org.apache.hadoop.hbase.regionserver.querymatcher.CompactionScanQueryMatcher; -import org.apache.hadoop.hbase.regionserver.querymatcher.LegacyScanQueryMatcher; import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; import org.apache.hadoop.hbase.regionserver.querymatcher.UserScanQueryMatcher; import org.apache.hadoop.hbase.util.CollectionUtils; @@ -66,7 +67,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner implements KeyValueScanner, InternalScanner, ChangedReadersObserver { private static final Log LOG = LogFactory.getLog(StoreScanner.class); // In unit tests, the store could be null - protected final Store store; + protected final Optional store; private ScanQueryMatcher matcher; protected KeyValueHeap heap; private boolean cacheBlocks; @@ -166,14 +167,13 @@ enum StoreScannerCompactionRace { } /** An internal constructor. */ - protected StoreScanner(Store store, Scan scan, final ScanInfo scanInfo, - final NavigableSet columns, long readPt, boolean cacheBlocks, ScanType scanType) { + private StoreScanner(Optional store, Scan scan, ScanInfo scanInfo, + int numColumns, long readPt, boolean cacheBlocks, ScanType scanType) { this.readPt = readPt; this.store = store; this.cacheBlocks = cacheBlocks; get = scan.isGetScan(); - int numCol = columns == null ? 0 : columns.size(); - explicitColumnQuery = numCol > 0; + explicitColumnQuery = numColumns > 0; this.scan = scan; this.now = EnvironmentEdgeManager.currentTime(); this.oldestUnexpiredTS = scan.isRaw() ? 0L : now - scanInfo.getTtl(); @@ -183,13 +183,12 @@ protected StoreScanner(Store store, Scan scan, final ScanInfo scanInfo, // the seek operation. However, we also look the row-column Bloom filter // for multi-row (non-"get") scans because this is not done in // StoreFile.passesBloomFilter(Scan, SortedSet). - this.useRowColBloom = numCol > 1 || (!get && numCol == 1); - + this.useRowColBloom = numColumns > 1 || (!get && numColumns == 1); this.maxRowSize = scanInfo.getTableMaxRowSize(); if (get) { this.readType = Scan.ReadType.PREAD; this.scanUsePread = true; - } else if(scanType != scanType.USER_SCAN) { + } else if(scanType != ScanType.USER_SCAN) { // For compaction scanners never use Pread as already we have stream based scanners on the // store files to be compacted this.readType = Scan.ReadType.STREAM; @@ -207,13 +206,15 @@ protected StoreScanner(Store store, Scan scan, final ScanInfo scanInfo, this.preadMaxBytes = scanInfo.getPreadMaxBytes(); this.cellsPerHeartbeatCheck = scanInfo.getCellsPerTimeoutCheck(); // Parallel seeking is on if the config allows and more there is more than one store file. - if (this.store != null && this.store.getStorefilesCount() > 1) { - RegionServerServices rsService = ((HStore) store).getHRegion().getRegionServerServices(); - if (rsService != null && scanInfo.isParallelSeekEnabled()) { - this.parallelSeekEnabled = true; - this.executor = rsService.getExecutorService(); + this.store.ifPresent(s -> { + if (s.getStorefilesCount() > 1) { + RegionServerServices rsService = ((HStore) s).getHRegion().getRegionServerServices(); + if (rsService != null && scanInfo.isParallelSeekEnabled()) { + this.parallelSeekEnabled = true; + this.executor = rsService.getExecutorService(); + } } - } + }); } private void addCurrentScanners(List scanners) { @@ -229,21 +230,23 @@ private void addCurrentScanners(List scanners) { * @param columns which columns we are scanning * @throws IOException */ - public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, final NavigableSet columns, - long readPt) - throws IOException { - this(store, scan, scanInfo, columns, readPt, scan.getCacheBlocks(), ScanType.USER_SCAN); + public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet columns, + long readPt) throws IOException { + this(Optional.of(store), scan, scanInfo, columns != null ? columns.size() : 0, readPt, + scan.getCacheBlocks(), ScanType.USER_SCAN); if (columns != null && scan.isRaw()) { throw new DoNotRetryIOException("Cannot specify any column for a raw scan"); } matcher = UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, store.getCoprocessorHost()); - this.store.addChangedReaderObserver(this); + store.addChangedReaderObserver(this); try { // Pass columns to try to filter out unnecessary StoreFiles. - List scanners = getScannersNoCompaction(); + List scanners = selectScannersFrom(store, + store.getScanners(cacheBlocks, scanUsePread, false, matcher, scan.getStartRow(), + scan.includeStartRow(), scan.getStopRow(), scan.includeStopRow(), this.readPt)); // Seek all scanners to the start of the Row (or if the exact matching row // key does not exist, then to the start of the next matching Row). @@ -263,66 +266,61 @@ public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, final NavigableSe } catch (IOException e) { // remove us from the HStore#changedReaderObservers here or we'll have no chance to // and might cause memory leak - this.store.deleteChangedReaderObserver(this); + store.deleteChangedReaderObserver(this); throw e; } } + // a dummy scan instance for compaction. + private static final Scan SCAN_FOR_COMPACTION = new Scan(); + /** - * Used for compactions.

- * + * Used for compactions. + *

* Opens a scanner across specified StoreFiles. * @param store who we scan - * @param scan the spec * @param scanners ancillary scanners - * @param smallestReadPoint the readPoint that we should use for tracking - * versions + * @param smallestReadPoint the readPoint that we should use for tracking versions */ - public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, - List scanners, ScanType scanType, - long smallestReadPoint, long earliestPutTs) throws IOException { - this(store, scanInfo, scan, scanners, scanType, smallestReadPoint, earliestPutTs, null, null); + public StoreScanner(Store store, ScanInfo scanInfo, OptionalInt maxVersions, + List scanners, ScanType scanType, long smallestReadPoint, + long earliestPutTs) throws IOException { + this(store, scanInfo, maxVersions, scanners, scanType, smallestReadPoint, earliestPutTs, null, + null); } /** - * Used for compactions that drop deletes from a limited range of rows.

- * + * Used for compactions that drop deletes from a limited range of rows. + *

* Opens a scanner across specified StoreFiles. * @param store who we scan - * @param scan the spec * @param scanners ancillary scanners * @param smallestReadPoint the readPoint that we should use for tracking versions * @param dropDeletesFromRow The inclusive left bound of the range; can be EMPTY_START_ROW. * @param dropDeletesToRow The exclusive right bound of the range; can be EMPTY_END_ROW. */ - public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, + public StoreScanner(Store store, ScanInfo scanInfo, OptionalInt maxVersions, List scanners, long smallestReadPoint, long earliestPutTs, byte[] dropDeletesFromRow, byte[] dropDeletesToRow) throws IOException { - this(store, scanInfo, scan, scanners, ScanType.COMPACT_RETAIN_DELETES, smallestReadPoint, + this(store, scanInfo, maxVersions, scanners, ScanType.COMPACT_RETAIN_DELETES, smallestReadPoint, earliestPutTs, dropDeletesFromRow, dropDeletesToRow); } - private StoreScanner(Store store, ScanInfo scanInfo, Scan scan, + private StoreScanner(Store store, ScanInfo scanInfo, OptionalInt maxVersions, List scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs, byte[] dropDeletesFromRow, byte[] dropDeletesToRow) throws IOException { - this(store, scan, scanInfo, null, - ((HStore) store).getHRegion().getReadPoint(IsolationLevel.READ_COMMITTED), false, scanType); - if (scan.hasFilter() || (scan.getStartRow() != null && scan.getStartRow().length > 0) - || (scan.getStopRow() != null && scan.getStopRow().length > 0) - || !scan.getTimeRange().isAllTime()) { - // use legacy query matcher since we do not consider the scan object in our code. Only used to - // keep compatibility for coprocessor. - matcher = LegacyScanQueryMatcher.create(scan, scanInfo, null, scanType, smallestReadPoint, - earliestPutTs, oldestUnexpiredTS, now, dropDeletesFromRow, dropDeletesToRow, - store.getCoprocessorHost()); - } else { - matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, smallestReadPoint, - earliestPutTs, oldestUnexpiredTS, now, dropDeletesFromRow, dropDeletesToRow, - store.getCoprocessorHost()); - } + this(Optional.of(store), + maxVersions.isPresent() ? new Scan().readVersions(maxVersions.getAsInt()) + : SCAN_FOR_COMPACTION, + scanInfo, 0, ((HStore) store).getHRegion().getReadPoint(IsolationLevel.READ_COMMITTED), + false, scanType); + assert scanType != ScanType.USER_SCAN; + matcher = + CompactionScanQueryMatcher.create(scanInfo, scanType, smallestReadPoint, earliestPutTs, + oldestUnexpiredTS, now, dropDeletesFromRow, dropDeletesToRow, store.getCoprocessorHost()); // Filter the list of scanners using Bloom filters, time range, TTL, etc. - scanners = selectScannersFrom(scanners); + scanners = selectScannersFrom(store, scanners); // Seek all scanners to the initial key seekScanners(scanners, matcher.getStartKey(), false, parallelSeekEnabled); @@ -331,62 +329,46 @@ private StoreScanner(Store store, ScanInfo scanInfo, Scan scan, resetKVHeap(scanners, store.getComparator()); } - @VisibleForTesting - StoreScanner(final Scan scan, ScanInfo scanInfo, - ScanType scanType, final NavigableSet columns, - final List scanners) throws IOException { - this(scan, scanInfo, scanType, columns, scanners, - HConstants.LATEST_TIMESTAMP, - // 0 is passed as readpoint because the test bypasses Store - 0); - } - - @VisibleForTesting - StoreScanner(final Scan scan, ScanInfo scanInfo, - ScanType scanType, final NavigableSet columns, - final List scanners, long earliestPutTs) - throws IOException { - this(scan, scanInfo, scanType, columns, scanners, earliestPutTs, - // 0 is passed as readpoint because the test bypasses Store - 0); - } - - public StoreScanner(final Scan scan, ScanInfo scanInfo, ScanType scanType, - final NavigableSet columns, final List scanners, long earliestPutTs, - long readPt) throws IOException { - this(null, scan, scanInfo, columns, readPt, - scanType == ScanType.USER_SCAN ? scan.getCacheBlocks() : false, scanType); - if (scanType == ScanType.USER_SCAN) { - this.matcher = UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, - null); - } else { - if (scan.hasFilter() || (scan.getStartRow() != null && scan.getStartRow().length > 0) - || (scan.getStopRow() != null && scan.getStopRow().length > 0) - || !scan.getTimeRange().isAllTime() || columns != null) { - // use legacy query matcher since we do not consider the scan object in our code. Only used - // to keep compatibility for coprocessor. - matcher = LegacyScanQueryMatcher.create(scan, scanInfo, columns, scanType, Long.MAX_VALUE, - earliestPutTs, oldestUnexpiredTS, now, null, null, store.getCoprocessorHost()); - } else { - this.matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, Long.MAX_VALUE, - earliestPutTs, oldestUnexpiredTS, now, null, null, null); - } - } - + private void seekAllScanner(ScanInfo scanInfo, List scanners) + throws IOException { // Seek all scanners to the initial key seekScanners(scanners, matcher.getStartKey(), false, parallelSeekEnabled); addCurrentScanners(scanners); resetKVHeap(scanners, scanInfo.getComparator()); } - /** - * Get a filtered list of scanners. Assumes we are not in a compaction. - * @return list of scanners to seek - */ - private List getScannersNoCompaction() throws IOException { - return selectScannersFrom( - store.getScanners(cacheBlocks, scanUsePread, false, matcher, scan.getStartRow(), - scan.includeStartRow(), scan.getStopRow(), scan.includeStopRow(), this.readPt)); + // For mob compaction only as we do not have a Store instance when doing mob compaction. + public StoreScanner(ScanInfo scanInfo, ScanType scanType, + List scanners) throws IOException { + this(Optional.empty(), SCAN_FOR_COMPACTION, scanInfo, 0, Long.MAX_VALUE, false, scanType); + assert scanType != ScanType.USER_SCAN; + this.matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, Long.MAX_VALUE, 0L, + oldestUnexpiredTS, now, null, null, null); + seekAllScanner(scanInfo, scanners); + } + + // Used to instantiate a scanner for user scan in test + @VisibleForTesting + StoreScanner(Scan scan, ScanInfo scanInfo, NavigableSet columns, + List scanners) throws IOException { + // 0 is passed as readpoint because the test bypasses Store + this(Optional.empty(), scan, scanInfo, columns != null ? columns.size() : 0, 0L, + scan.getCacheBlocks(), ScanType.USER_SCAN); + this.matcher = + UserScanQueryMatcher.create(scan, scanInfo, columns, oldestUnexpiredTS, now, null); + seekAllScanner(scanInfo, scanners); + } + + // Used to instantiate a scanner for compaction in test + @VisibleForTesting + StoreScanner(ScanInfo scanInfo, OptionalInt maxVersions, ScanType scanType, + List scanners) throws IOException { + // 0 is passed as readpoint because the test bypasses Store + this(Optional.empty(), maxVersions.isPresent() ? new Scan().readVersions(maxVersions.getAsInt()) + : SCAN_FOR_COMPACTION, scanInfo, 0, 0L, false, scanType); + this.matcher = CompactionScanQueryMatcher.create(scanInfo, scanType, Long.MAX_VALUE, + HConstants.OLDEST_TIMESTAMP, oldestUnexpiredTS, now, null, null, null); + seekAllScanner(scanInfo, scanners); } @VisibleForTesting @@ -439,18 +421,17 @@ protected void resetKVHeap(List scanners, } /** - * Filters the given list of scanners using Bloom filter, time range, and - * TTL. + * Filters the given list of scanners using Bloom filter, time range, and TTL. *

* Will be overridden by testcase so declared as protected. */ @VisibleForTesting - protected List selectScannersFrom( - final List allScanners) { + protected List selectScannersFrom(Store store, + List allScanners) { boolean memOnly; boolean filesOnly; if (scan instanceof InternalScan) { - InternalScan iscan = (InternalScan)scan; + InternalScan iscan = (InternalScan) scan; memOnly = iscan.isCheckOnlyMemStore(); filesOnly = iscan.isCheckOnlyStoreFiles(); } else { @@ -462,7 +443,7 @@ protected List selectScannersFrom( // We can only exclude store files based on TTL if minVersions is set to 0. // Otherwise, we might have to return KVs that have technically expired. - long expiredTimestampCutoff = minVersions == 0 ? oldestUnexpiredTS: Long.MIN_VALUE; + long expiredTimestampCutoff = minVersions == 0 ? oldestUnexpiredTS : Long.MIN_VALUE; // include only those scan files which pass all filters for (KeyValueScanner kvs : allScanners) { @@ -503,10 +484,8 @@ private void close(boolean withDelayedScannersClose) { if (withDelayedScannersClose) { this.closing = true; } - // Under test, we dont have a this.store - if (this.store != null) { - this.store.deleteChangedReaderObserver(this); - } + // For mob compaction, we do not have a store. + this.store.ifPresent(s -> s.deleteChangedReaderObserver(this)); if (withDelayedScannersClose) { clearAndClose(scannersForDelayedClose); clearAndClose(memStoreScannersAfterFlush); @@ -583,7 +562,7 @@ public boolean next(List outResult, ScannerContext scannerContext) throws } // Only do a sanity-check if store and comparator are available. - CellComparator comparator = store != null ? store.getComparator() : null; + CellComparator comparator = store.map(s -> s.getComparator()).orElse(null); int count = 0; long totalBytesRead = 0; @@ -895,6 +874,8 @@ public void updateReaders(List sfs, List memStoreSca * @return if top of heap has changed (and KeyValueHeap has to try the next KV) */ protected final boolean reopenAfterFlush() throws IOException { + // here we can make sure that we have a Store instance. + Store store = this.store.get(); Cell lastTop = heap.peek(); // When we have the scan object, should we not pass it to getScanners() to get a limited set of // scanners? We did so in the constructor and we could have done it now by storing the scan @@ -906,7 +887,7 @@ protected final boolean reopenAfterFlush() throws IOException { allScanners.addAll(store.getScanners(flushedStoreFiles, cacheBlocks, get, scanUsePread, false, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt, false)); allScanners.addAll(memStoreScannersAfterFlush); - scanners = selectScannersFrom(allScanners); + scanners = selectScannersFrom(store, allScanners); // Clear the current set of flushed store files so that they don't get added again flushedStoreFiles.clear(); memStoreScannersAfterFlush.clear(); @@ -998,8 +979,8 @@ public boolean reseek(Cell kv) throws IOException { @VisibleForTesting void trySwitchToStreamRead() { - if (readType != Scan.ReadType.DEFAULT || !scanUsePread || closing || heap.peek() == null || - bytesRead < preadMaxBytes) { + if (readType != Scan.ReadType.DEFAULT || !scanUsePread || closing || + heap.peek() == null || bytesRead < preadMaxBytes) { return; } if (LOG.isDebugEnabled()) { @@ -1021,6 +1002,8 @@ void trySwitchToStreamRead() { List fileScanners = null; List newCurrentScanners; KeyValueHeap newHeap; + // We must have a store instance here + Store store = this.store.get(); try { // recreate the scanners on the current file scanners fileScanners = store.recreateScanners(scannersToClose, cacheBlocks, false, false, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index d43a75bf3720..15da298b83fa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.OptionalInt; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,7 +34,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; @@ -499,10 +499,8 @@ protected boolean performCompaction(FileDetails fd, InternalScanner scanner, Cel */ protected InternalScanner createScanner(Store store, List scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException { - Scan scan = new Scan(); - scan.setMaxVersions(store.getColumnFamilyDescriptor().getMaxVersions()); - return new StoreScanner(store, store.getScanInfo(), scan, scanners, - scanType, smallestReadPoint, earliestPutTs); + return new StoreScanner(store, store.getScanInfo(), OptionalInt.empty(), scanners, scanType, + smallestReadPoint, earliestPutTs); } /** @@ -515,11 +513,9 @@ protected InternalScanner createScanner(Store store, List scan * @return A compaction scanner. */ protected InternalScanner createScanner(Store store, List scanners, - long smallestReadPoint, long earliestPutTs, byte[] dropDeletesFromRow, - byte[] dropDeletesToRow) throws IOException { - Scan scan = new Scan(); - scan.setMaxVersions(store.getColumnFamilyDescriptor().getMaxVersions()); - return new StoreScanner(store, store.getScanInfo(), scan, scanners, smallestReadPoint, - earliestPutTs, dropDeletesFromRow, dropDeletesToRow); + long smallestReadPoint, long earliestPutTs, byte[] dropDeletesFromRow, + byte[] dropDeletesToRow) throws IOException { + return new StoreScanner(store, store.getScanInfo(), OptionalInt.empty(), scanners, + smallestReadPoint, earliestPutTs, dropDeletesFromRow, dropDeletesToRow); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/LegacyScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/LegacyScanQueryMatcher.java deleted file mode 100644 index 07fcb087a25e..000000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/LegacyScanQueryMatcher.java +++ /dev/null @@ -1,384 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver.querymatcher; - -import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions; - -import java.io.IOException; -import java.util.Arrays; -import java.util.NavigableSet; - -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.Filter.ReturnCode; -import org.apache.hadoop.hbase.io.TimeRange; -import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; -import org.apache.hadoop.hbase.regionserver.ScanInfo; -import org.apache.hadoop.hbase.regionserver.ScanType; -import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker.DeleteResult; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.Pair; - -/** - * The old query matcher implementation. Used to keep compatibility for coprocessor that could - * overwrite the StoreScanner before compaction. Should be removed once we find a better way to do - * filtering during compaction. - */ -@Deprecated -@InterfaceAudience.Private -public class LegacyScanQueryMatcher extends ScanQueryMatcher { - - private final TimeRange tr; - - private final Filter filter; - - /** Keeps track of deletes */ - private final DeleteTracker deletes; - - /** - * The following three booleans define how we deal with deletes. There are three different - * aspects: - *

    - *
  1. Whether to keep delete markers. This is used in compactions. Minor compactions always keep - * delete markers.
    2. - *
  2. Whether to keep deleted rows. This is also used in compactions, if the store is set to keep - * deleted rows. This implies keeping the delete markers as well.
    3. In this case deleted rows - * are subject to the normal max version and TTL/min version rules just like "normal" rows. - *
  3. Whether a scan can do time travel queries even before deleted marker to reach deleted - * rows.
    4. - *
- */ - /** whether to retain delete markers */ - private boolean retainDeletesInOutput; - - /** whether to return deleted rows */ - private final KeepDeletedCells keepDeletedCells; - - // By default, when hbase.hstore.time.to.purge.deletes is 0ms, a delete - // marker is always removed during a major compaction. If set to non-zero - // value then major compaction will try to keep a delete marker around for - // the given number of milliseconds. We want to keep the delete markers - // around a bit longer because old puts might appear out-of-order. For - // example, during log replication between two clusters. - // - // If the delete marker has lived longer than its column-family's TTL then - // the delete marker will be removed even if time.to.purge.deletes has not - // passed. This is because all the Puts that this delete marker can influence - // would have also expired. (Removing of delete markers on col family TTL will - // not happen if min-versions is set to non-zero) - // - // But, if time.to.purge.deletes has not expired then a delete - // marker will not be removed just because there are no Puts that it is - // currently influencing. This is because Puts, that this delete can - // influence. may appear out of order. - private final long timeToPurgeDeletes; - - /** - * This variable shows whether there is an null column in the query. There always exists a null - * column in the wildcard column query. There maybe exists a null column in the explicit column - * query based on the first column. - */ - private final boolean hasNullColumn; - - /** readPoint over which the KVs are unconditionally included */ - private final long maxReadPointToTrackVersions; - - /** - * Oldest put in any of the involved store files Used to decide whether it is ok to delete family - * delete marker of this store keeps deleted KVs. - */ - protected final long earliestPutTs; - - private final byte[] stopRow; - - private byte[] dropDeletesFromRow = null, dropDeletesToRow = null; - - private LegacyScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns, - boolean hasNullColumn, DeleteTracker deletes, ScanType scanType, long readPointToUse, - long earliestPutTs, long oldestUnexpiredTS, long now) { - super(createStartKeyFromRow(scan.getStartRow(), scanInfo), scanInfo, columns, oldestUnexpiredTS, - now); - TimeRange timeRange = scan.getColumnFamilyTimeRange().get(scanInfo.getFamily()); - if (timeRange == null) { - this.tr = scan.getTimeRange(); - } else { - this.tr = timeRange; - } - this.hasNullColumn = hasNullColumn; - this.deletes = deletes; - this.filter = scan.getFilter(); - this.maxReadPointToTrackVersions = readPointToUse; - this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes(); - this.earliestPutTs = earliestPutTs; - - /* how to deal with deletes */ - this.keepDeletedCells = scanInfo.getKeepDeletedCells(); - this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES; - this.stopRow = scan.getStopRow(); - } - - private LegacyScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns, - boolean hasNullColumn, DeleteTracker deletes, ScanType scanType, long readPointToUse, - long earliestPutTs, long oldestUnexpiredTS, long now, byte[] dropDeletesFromRow, - byte[] dropDeletesToRow) { - this(scan, scanInfo, columns, hasNullColumn, deletes, scanType, readPointToUse, earliestPutTs, - oldestUnexpiredTS, now); - this.dropDeletesFromRow = Preconditions.checkNotNull(dropDeletesFromRow); - this.dropDeletesToRow = Preconditions.checkNotNull(dropDeletesToRow); - } - - @Override - public void beforeShipped() throws IOException { - super.beforeShipped(); - deletes.beforeShipped(); - } - - @Override - public MatchCode match(Cell cell) throws IOException { - if (filter != null && filter.filterAllRemaining()) { - return MatchCode.DONE_SCAN; - } - MatchCode returnCode = preCheck(cell); - if (returnCode != null) { - return returnCode; - } - /* - * The delete logic is pretty complicated now. - * This is corroborated by the following: - * 1. The store might be instructed to keep deleted rows around. - * 2. A scan can optionally see past a delete marker now. - * 3. If deleted rows are kept, we have to find out when we can - * remove the delete markers. - * 4. Family delete markers are always first (regardless of their TS) - * 5. Delete markers should not be counted as version - * 6. Delete markers affect puts of the *same* TS - * 7. Delete marker need to be version counted together with puts - * they affect - */ - long timestamp = cell.getTimestamp(); - byte typeByte = cell.getTypeByte(); - long mvccVersion = cell.getSequenceId(); - if (CellUtil.isDelete(typeByte)) { - if (keepDeletedCells == KeepDeletedCells.FALSE - || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < oldestUnexpiredTS)) { - // first ignore delete markers if the scanner can do so, and the - // range does not include the marker - // - // during flushes and compactions also ignore delete markers newer - // than the readpoint of any open scanner, this prevents deleted - // rows that could still be seen by a scanner from being collected - boolean includeDeleteMarker = tr.withinOrAfterTimeRange(timestamp); - if (includeDeleteMarker && mvccVersion <= maxReadPointToTrackVersions) { - this.deletes.add(cell); - } - // Can't early out now, because DelFam come before any other keys - } - - if (timeToPurgeDeletes > 0 - && (EnvironmentEdgeManager.currentTime() - timestamp) <= timeToPurgeDeletes) { - return MatchCode.INCLUDE; - } else if (retainDeletesInOutput || mvccVersion > maxReadPointToTrackVersions) { - // always include or it is not time yet to check whether it is OK - // to purge deltes or not - // if this is not a user scan (compaction), we can filter this deletemarker right here - // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking - return MatchCode.INCLUDE; - } else if (keepDeletedCells == KeepDeletedCells.TRUE - || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= oldestUnexpiredTS)) { - if (timestamp < earliestPutTs) { - // keeping delete rows, but there are no puts older than - // this delete in the store files. - return columns.getNextRowOrNextColumn(cell); - } - // else: fall through and do version counting on the - // delete markers - } else { - return MatchCode.SKIP; - } - // note the following next else if... - // delete marker are not subject to other delete markers - } else if (!this.deletes.isEmpty()) { - DeleteResult deleteResult = deletes.isDeleted(cell); - switch (deleteResult) { - case FAMILY_DELETED: - case COLUMN_DELETED: - return columns.getNextRowOrNextColumn(cell); - case VERSION_DELETED: - case FAMILY_VERSION_DELETED: - return MatchCode.SKIP; - case NOT_DELETED: - break; - default: - throw new RuntimeException("UNEXPECTED"); - } - } - - int timestampComparison = tr.compare(timestamp); - if (timestampComparison >= 1) { - return MatchCode.SKIP; - } else if (timestampComparison <= -1) { - return columns.getNextRowOrNextColumn(cell); - } - - // STEP 1: Check if the column is part of the requested columns - MatchCode colChecker = columns.checkColumn(cell, typeByte); - if (colChecker == MatchCode.INCLUDE) { - ReturnCode filterResponse = ReturnCode.SKIP; - // STEP 2: Yes, the column is part of the requested columns. Check if filter is present - if (filter != null) { - // STEP 3: Filter the key value and return if it filters out - filterResponse = filter.filterKeyValue(cell); - switch (filterResponse) { - case SKIP: - return MatchCode.SKIP; - case NEXT_COL: - return columns.getNextRowOrNextColumn(cell); - case NEXT_ROW: - return MatchCode.SEEK_NEXT_ROW; - case SEEK_NEXT_USING_HINT: - return MatchCode.SEEK_NEXT_USING_HINT; - default: - //It means it is either include or include and seek next - break; - } - } - /* - * STEP 4: Reaching this step means the column is part of the requested columns and either - * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response. - * Now check the number of versions needed. This method call returns SKIP, INCLUDE, - * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL. - * - * FilterResponse ColumnChecker Desired behavior - * INCLUDE SKIP row has already been included, SKIP. - * INCLUDE INCLUDE INCLUDE - * INCLUDE INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL - * INCLUDE INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW - * INCLUDE_AND_SEEK_NEXT_COL SKIP row has already been included, SKIP. - * INCLUDE_AND_SEEK_NEXT_COL INCLUDE INCLUDE_AND_SEEK_NEXT_COL - * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL - * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW - * - * In all the above scenarios, we return the column checker return value except for - * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE) - */ - colChecker = columns.checkVersions(cell, timestamp, typeByte, - mvccVersion > maxReadPointToTrackVersions); - if (filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW) { - if (colChecker != MatchCode.SKIP) { - return MatchCode.INCLUDE_AND_SEEK_NEXT_ROW; - } - return MatchCode.SEEK_NEXT_ROW; - } - return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL && - colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL - : colChecker; - } - return colChecker; - } - - @Override - public boolean hasNullColumnInQuery() { - return hasNullColumn; - } - - /** - * Handle partial-drop-deletes. As we match keys in order, when we have a range from which we can - * drop deletes, we can set retainDeletesInOutput to false for the duration of this range only, - * and maintain consistency. - */ - private void checkPartialDropDeleteRange(Cell curCell) { - // If partial-drop-deletes are used, initially, dropDeletesFromRow and dropDeletesToRow - // are both set, and the matcher is set to retain deletes. We assume ordered keys. When - // dropDeletesFromRow is leq current kv, we start dropping deletes and reset - // dropDeletesFromRow; thus the 2nd "if" starts to apply. - if ((dropDeletesFromRow != null) - && (Arrays.equals(dropDeletesFromRow, HConstants.EMPTY_START_ROW) - || (CellComparator.COMPARATOR.compareRows(curCell, dropDeletesFromRow, 0, - dropDeletesFromRow.length) >= 0))) { - retainDeletesInOutput = false; - dropDeletesFromRow = null; - } - // If dropDeletesFromRow is null and dropDeletesToRow is set, we are inside the partial- - // drop-deletes range. When dropDeletesToRow is leq current kv, we stop dropping deletes, - // and reset dropDeletesToRow so that we don't do any more compares. - if ((dropDeletesFromRow == null) && (dropDeletesToRow != null) - && !Arrays.equals(dropDeletesToRow, HConstants.EMPTY_END_ROW) && (CellComparator.COMPARATOR - .compareRows(curCell, dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) { - retainDeletesInOutput = true; - dropDeletesToRow = null; - } - } - - @Override - protected void reset() { - checkPartialDropDeleteRange(currentRow); - } - - @Override - public boolean isUserScan() { - return false; - } - - @Override - public boolean moreRowsMayExistAfter(Cell cell) { - if (this.stopRow == null || this.stopRow.length == 0) { - return true; - } - return rowComparator.compareRows(cell, stopRow, 0, stopRow.length) < 0; - } - - @Override - public Filter getFilter() { - return filter; - } - - @Override - public Cell getNextKeyHint(Cell cell) throws IOException { - if (filter == null) { - return null; - } else { - return filter.getNextCellHint(cell); - } - } - - public static LegacyScanQueryMatcher create(Scan scan, ScanInfo scanInfo, - NavigableSet columns, ScanType scanType, long readPointToUse, long earliestPutTs, - long oldestUnexpiredTS, long now, byte[] dropDeletesFromRow, byte[] dropDeletesToRow, - RegionCoprocessorHost regionCoprocessorHost) throws IOException { - boolean hasNullColumn = - !(columns != null && columns.size() != 0 && columns.first().length != 0); - Pair trackers = getTrackers(regionCoprocessorHost, null, - scanInfo, oldestUnexpiredTS, scan); - DeleteTracker deleteTracker = trackers.getFirst(); - ColumnTracker columnTracker = trackers.getSecond(); - if (dropDeletesFromRow == null) { - return new LegacyScanQueryMatcher(scan, scanInfo, columnTracker, hasNullColumn, deleteTracker, - scanType, readPointToUse, earliestPutTs, oldestUnexpiredTS, now); - } else { - return new LegacyScanQueryMatcher(scan, scanInfo, columnTracker, hasNullColumn, deleteTracker, - scanType, readPointToUse, earliestPutTs, oldestUnexpiredTS, now, dropDeletesFromRow, - dropDeletesToRow); - } - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java index 62c8e7b808d9..8dfc8aac4e00 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.OptionalInt; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; @@ -269,19 +270,17 @@ public InternalScanner preCompactScannerOpen(ObserverContext scanners, ScanType scanType, long earliestPutTs) throws IOException { - Scan scan = new Scan(); - scan.setMaxVersions(store.getColumnFamilyDescriptor().getMaxVersions()); - return new CompactorStoreScanner(store, store.getScanInfo(), scan, scanners, scanType, - store.getSmallestReadPoint(), earliestPutTs); + return new CompactorStoreScanner(store, store.getScanInfo(), OptionalInt.empty(), scanners, + scanType, store.getSmallestReadPoint(), earliestPutTs); } } private static class CompactorStoreScanner extends StoreScanner { - public CompactorStoreScanner(Store store, ScanInfo scanInfo, Scan scan, + public CompactorStoreScanner(Store store, ScanInfo scanInfo, OptionalInt maxVersions, List scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException { - super(store, scanInfo, scan, scanners, scanType, smallestReadPoint, earliestPutTs); + super(store, scanInfo, maxVersions, scanners, scanType, smallestReadPoint, earliestPutTs); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index 48cb8126ed29..bae8d6891494 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -576,9 +576,9 @@ public MyStoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet selectScannersFrom( + protected List selectScannersFrom(Store store, List allScanners) { - List scanners = super.selectScannersFrom(allScanners); + List scanners = super.selectScannersFrom(store, allScanners); List newScanners = new ArrayList<>(scanners.size()); for (KeyValueScanner scanner : scanners) { newScanners.add(new DelegatingKeyValueScanner(scanner) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java index 26cfed748647..e9bf09b9b92c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; @@ -116,18 +117,31 @@ public KeyValueScanner preStoreScannerOpen(ObserverContext result, ScannerContext scannerContext) throws IOException { + return false; + } + + @Override + public boolean next(List results) throws IOException { + return false; + } + + @Override + public void close() throws IOException {} + }; /** * Don't allow any data in a flush by creating a custom {@link StoreScanner}. */ public static class NoDataFromFlush implements RegionObserver { + @Override public InternalScanner preFlushScannerOpen(ObserverContext c, Store store, List scanners, InternalScanner s) throws IOException { - Scan scan = new Scan(); - scan.setFilter(new NoDataFilter()); - return new StoreScanner(store, store.getScanInfo(), scan, - scanners, ScanType.COMPACT_RETAIN_DELETES, - store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP); + scanners.forEach(KeyValueScanner::close); + return NO_DATA; } } @@ -140,11 +154,8 @@ public static class NoDataFromCompaction implements RegionObserver { public InternalScanner preCompactScannerOpen(ObserverContext c, Store store, List scanners, ScanType scanType, long earliestPutTs, InternalScanner s) throws IOException { - Scan scan = new Scan(); - scan.setFilter(new NoDataFilter()); - return new StoreScanner(store, store.getScanInfo(), scan, scanners, - ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), - HConstants.OLDEST_TIMESTAMP); + scanners.forEach(KeyValueScanner::close); + return NO_DATA; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java index 2fe8085ef293..b8e1204028c3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java @@ -56,7 +56,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.MobCompactPartitionPolicy; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; @@ -883,13 +882,10 @@ private int countDelCellsInDelFiles(List paths) throws IOException { } List scanners = new ArrayList<>(StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, false, HConstants.LATEST_TIMESTAMP)); - Scan scan = new Scan(); - scan.setMaxVersions(hcd.getMaxVersions()); long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0); long ttl = HStore.determineTTLFromFamily(hcd); ScanInfo scanInfo = new ScanInfo(conf, hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR); - StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_RETAIN_DELETES, null, - scanners, 0L, HConstants.LATEST_TIMESTAMP); + StoreScanner scanner = new StoreScanner(scanInfo, ScanType.COMPACT_RETAIN_DELETES, scanners); List results = new ArrayList<>(); boolean hasMore = true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java index b090cddd94d3..5423578d5cf9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/NoOpScanPolicyObserver.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.NavigableSet; +import java.util.OptionalInt; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Scan; @@ -38,6 +38,7 @@ * of functionality still behaves as expected. */ public class NoOpScanPolicyObserver implements RegionObserver { + /** * Reimplement the default behavior */ @@ -45,11 +46,9 @@ public class NoOpScanPolicyObserver implements RegionObserver { public InternalScanner preFlushScannerOpen(final ObserverContext c, Store store, List scanners, InternalScanner s) throws IOException { ScanInfo oldSI = store.getScanInfo(); - ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(), oldSI.getTtl(), - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); - Scan scan = new Scan(); - scan.setMaxVersions(oldSI.getMaxVersions()); - return new StoreScanner(store, scanInfo, scan, scanners, + ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(), + oldSI.getTtl(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + return new StoreScanner(store, scanInfo, OptionalInt.empty(), scanners, ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP); } @@ -57,16 +56,15 @@ public InternalScanner preFlushScannerOpen(final ObserverContext c, - Store store, List scanners, ScanType scanType, long earliestPutTs, + public InternalScanner preCompactScannerOpen( + final ObserverContext c, Store store, + List scanners, ScanType scanType, long earliestPutTs, InternalScanner s) throws IOException { // this demonstrates how to override the scanners default behavior ScanInfo oldSI = store.getScanInfo(); - ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(), oldSI.getTtl(), - oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); - Scan scan = new Scan(); - scan.setMaxVersions(oldSI.getMaxVersions()); - return new StoreScanner(store, scanInfo, scan, scanners, scanType, + ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(), + oldSI.getTtl(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); + return new StoreScanner(store, scanInfo, OptionalInt.empty(), scanners, scanType, store.getSmallestReadPoint(), earliestPutTs); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index 8118e41d3970..dc3cf4d9023f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -194,25 +194,25 @@ public void testGetNextRow() throws Exception { for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { ScanInfo scanInfo = new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false); - ScanType scanType = ScanType.USER_SCAN; - InternalScanner scanner = new StoreScanner(new Scan( - Bytes.toBytes(startRowId)), scanInfo, scanType, null, - memstore.getScanners(0)); - List results = new ArrayList<>(); - for (int i = 0; scanner.next(results); i++) { - int rowId = startRowId + i; - Cell left = results.get(0); - byte[] row1 = Bytes.toBytes(rowId); - assertTrue("Row name", + try (InternalScanner scanner = + new StoreScanner(new Scan().withStartRow(Bytes.toBytes(startRowId)), scanInfo, null, + memstore.getScanners(0))) { + List results = new ArrayList<>(); + for (int i = 0; scanner.next(results); i++) { + int rowId = startRowId + i; + Cell left = results.get(0); + byte[] row1 = Bytes.toBytes(rowId); + assertTrue("Row name", CellComparator.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0); - assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); - List row = new ArrayList<>(); - for (Cell kv : results) { - row.add(kv); + assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); + List row = new ArrayList<>(); + for (Cell kv : results) { + row.add(kv); + } + isExpectedRowWithoutTimestamps(rowId, row); + // Clear out set. Otherwise row results accumulate. + results.clear(); } - isExpectedRowWithoutTimestamps(rowId, row); - // Clear out set. Otherwise row results accumulate. - results.clear(); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index 7b108467184b..b36b8fe0028e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -18,9 +18,17 @@ */ package org.apache.hadoop.hbase.regionserver; -import org.apache.hadoop.hbase.shaded.com.google.common.base.Joiner; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterables; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -55,22 +63,13 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertNotNull; - import org.junit.experimental.categories.Category; import org.junit.rules.TestName; import org.junit.rules.TestRule; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; +import org.apache.hadoop.hbase.shaded.com.google.common.base.Joiner; +import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterables; +import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; /** memstore test case */ @Category({RegionServerTests.class, MediumTests.class}) @@ -164,10 +163,8 @@ public void testScanAcrossSnapshot() throws IOException { Configuration conf = HBaseConfiguration.create(); ScanInfo scanInfo = new ScanInfo(conf, null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false); - ScanType scanType = ScanType.USER_SCAN; - StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); int count = 0; - try { + try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { while (s.next(result)) { LOG.info(result); count++; @@ -175,8 +172,6 @@ public void testScanAcrossSnapshot() throws IOException { assertEquals(rowCount, result.size()); result.clear(); } - } finally { - s.close(); } assertEquals(rowCount, count); for (KeyValueScanner scanner : memstorescanners) { @@ -185,9 +180,8 @@ public void testScanAcrossSnapshot() throws IOException { memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); // Now assert can count same number even if a snapshot mid-scan. - s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); count = 0; - try { + try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { while (s.next(result)) { LOG.info(result); // Assert the stuff is coming out in right order. @@ -201,8 +195,6 @@ public void testScanAcrossSnapshot() throws IOException { } result.clear(); } - } finally { - s.close(); } assertEquals(rowCount, count); for (KeyValueScanner scanner : memstorescanners) { @@ -211,10 +203,9 @@ public void testScanAcrossSnapshot() throws IOException { memstorescanners = this.memstore.getScanners(mvcc.getReadPoint()); // Assert that new values are seen in kvset as we scan. long ts = System.currentTimeMillis(); - s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); count = 0; int snapshotIndex = 5; - try { + try (StoreScanner s = new StoreScanner(scan, scanInfo, null, memstorescanners)) { while (s.next(result)) { LOG.info(result); // Assert the stuff is coming out in right order. @@ -225,14 +216,12 @@ public void testScanAcrossSnapshot() throws IOException { if (count == snapshotIndex) { MemStoreSnapshot snapshot = this.memstore.snapshot(); this.memstore.clearSnapshot(snapshot.getId()); - // Added more rows into kvset. But the scanner wont see these rows. + // Added more rows into kvset. But the scanner wont see these rows. addRows(this.memstore, ts); LOG.info("Snapshotted, cleared it and then added values (which wont be seen)"); } result.clear(); } - } finally { - s.close(); } assertEquals(rowCount, count); } @@ -600,27 +589,26 @@ public void testGetNextRow() throws Exception { //starting from each row, validate results should contain the starting row Configuration conf = HBaseConfiguration.create(); for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { - ScanInfo scanInfo = new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, - KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false); - ScanType scanType = ScanType.USER_SCAN; - try (InternalScanner scanner = new StoreScanner(new Scan( - Bytes.toBytes(startRowId)), scanInfo, scanType, null, - memstore.getScanners(0))) { + ScanInfo scanInfo = + new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, + HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false); + try (InternalScanner scanner = + new StoreScanner(new Scan().withStartRow(Bytes.toBytes(startRowId)), scanInfo, null, + memstore.getScanners(0))) { List results = new ArrayList<>(); for (int i = 0; scanner.next(results); i++) { int rowId = startRowId + i; Cell left = results.get(0); byte[] row1 = Bytes.toBytes(rowId); - assertTrue( - "Row name", - CellComparator.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0); + assertTrue("Row name", + CellComparator.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0); assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); List row = new ArrayList<>(); for (Cell kv : results) { row.add(kv); } isExpectedRowWithoutTimestamps(rowId, row); - // Clear out set. Otherwise row results accumulate. + // Clear out set. Otherwise row results accumulate. results.clear(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java index 21089ed50ef6..9ab1440642e2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java @@ -442,14 +442,11 @@ private int countMobCellsInMobFiles(int expectedNumDelfiles) throws IOException List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, false, HConstants.LATEST_TIMESTAMP); - Scan scan = new Scan(); - scan.setMaxVersions(hcd.getMaxVersions()); long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0); long ttl = HStore.determineTTLFromFamily(hcd); ScanInfo scanInfo = new ScanInfo(copyOfConf, hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR); - StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null, - scanners, 0L, HConstants.LATEST_TIMESTAMP); + StoreScanner scanner = new StoreScanner(scanInfo, ScanType.COMPACT_DROP_DELETES, scanners); try { size += UTIL.countRows(scanner); } finally { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java index 3782fdb44d0c..8b34a2f15c70 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java @@ -22,8 +22,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -68,6 +66,8 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; + /** * Test cases against ReversibleKeyValueScanner */ @@ -263,7 +263,6 @@ public void testReversibleStoreScanner() throws IOException { StoreFile sf2 = new HStoreFile(fs, writer2.getPath(), TEST_UTIL.getConfiguration(), cacheConf, BloomType.NONE, true); - ScanType scanType = ScanType.USER_SCAN; ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(), FAMILYNAME, 0, Integer.MAX_VALUE, Long.MAX_VALUE, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.COMPARATOR, false); @@ -271,16 +270,15 @@ public void testReversibleStoreScanner() throws IOException { // Case 1.Test a full reversed scan Scan scan = new Scan(); scan.setReversed(true); - StoreScanner storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, - scan, scanType, scanInfo, MAXMVCC); + StoreScanner storeScanner = + getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC); verifyCountAndOrder(storeScanner, QUALSIZE * ROWSIZE, ROWSIZE, false); // Case 2.Test reversed scan with a specified start row int startRowNum = ROWSIZE / 2; byte[] startRow = ROWS[startRowNum]; scan.withStartRow(startRow); - storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, - scanType, scanInfo, MAXMVCC); + storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC); verifyCountAndOrder(storeScanner, QUALSIZE * (startRowNum + 1), startRowNum + 1, false); @@ -289,16 +287,14 @@ public void testReversibleStoreScanner() throws IOException { assertTrue(QUALSIZE > 2); scan.addColumn(FAMILYNAME, QUALS[0]); scan.addColumn(FAMILYNAME, QUALS[2]); - storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, - scanType, scanInfo, MAXMVCC); + storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC); verifyCountAndOrder(storeScanner, 2 * (startRowNum + 1), startRowNum + 1, false); // Case 4.Test reversed scan with mvcc based on case 3 for (int readPoint = 0; readPoint < MAXMVCC; readPoint++) { LOG.info("Setting read point to " + readPoint); - storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, - scanType, scanInfo, readPoint); + storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, readPoint); int expectedRowCount = 0; int expectedKVCount = 0; for (int i = startRowNum; i >= 0; i--) { @@ -423,7 +419,7 @@ public void testReversibleRegionScanner() throws IOException { } private StoreScanner getReversibleStoreScanner(MemStore memstore, - StoreFile sf1, StoreFile sf2, Scan scan, ScanType scanType, + StoreFile sf1, StoreFile sf2, Scan scan, ScanInfo scanInfo, int readPoint) throws IOException { List scanners = getScanners(memstore, sf1, sf2, null, false, readPoint); @@ -434,7 +430,7 @@ private StoreScanner getReversibleStoreScanner(MemStore memstore, columns = entry.getValue(); } StoreScanner storeScanner = new ReversedStoreScanner(scan, scanInfo, - scanType, columns, scanners); + columns, scanners); return storeScanner; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index 1653728fa2cc..408281841f15 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -19,8 +19,13 @@ package org.apache.hadoop.hbase.regionserver; +import static org.apache.hadoop.hbase.CellUtil.createCell; +import static org.apache.hadoop.hbase.KeyValueTestUtil.create; import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -28,6 +33,7 @@ import java.util.Collections; import java.util.List; import java.util.NavigableSet; +import java.util.OptionalInt; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicInteger; @@ -42,7 +48,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueTestUtil; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; @@ -51,7 +56,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; -import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -66,11 +70,10 @@ public class TestStoreScanner { @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). withLookingForStuckThread(true).build(); private static final String CF_STR = "cf"; - private static final byte [] CF = Bytes.toBytes(CF_STR); + private static final byte[] CF = Bytes.toBytes(CF_STR); static Configuration CONF = HBaseConfiguration.create(); private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE, Long.MAX_VALUE, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.COMPARATOR, false); - private ScanType scanType = ScanType.USER_SCAN; /** * From here on down, we have a bunch of defines and specific CELL_GRID of Cells. The @@ -79,15 +82,15 @@ public class TestStoreScanner { * {@link StoreScanner#optimize(org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher.MatchCode, * Cell)} is not overly enthusiastic. */ - private static final byte [] ZERO = new byte [] {'0'}; - private static final byte [] ZERO_POINT_ZERO = new byte [] {'0', '.', '0'}; - private static final byte [] ONE = new byte [] {'1'}; - private static final byte [] TWO = new byte [] {'2'}; - private static final byte [] TWO_POINT_TWO = new byte [] {'2', '.', '2'}; - private static final byte [] THREE = new byte [] {'3'}; - private static final byte [] FOUR = new byte [] {'4'}; - private static final byte [] FIVE = new byte [] {'5'}; - private static final byte [] VALUE = new byte [] {'v'}; + private static final byte[] ZERO = new byte[] {'0'}; + private static final byte[] ZERO_POINT_ZERO = new byte[] {'0', '.', '0'}; + private static final byte[] ONE = new byte[] {'1'}; + private static final byte[] TWO = new byte[] {'2'}; + private static final byte[] TWO_POINT_TWO = new byte[] {'2', '.', '2'}; + private static final byte[] THREE = new byte[] {'3'}; + private static final byte[] FOUR = new byte[] {'4'}; + private static final byte[] FIVE = new byte[] {'5'}; + private static final byte[] VALUE = new byte[] {'v'}; private static final int CELL_GRID_BLOCK2_BOUNDARY = 4; private static final int CELL_GRID_BLOCK3_BOUNDARY = 11; private static final int CELL_GRID_BLOCK4_BOUNDARY = 15; @@ -100,32 +103,32 @@ public class TestStoreScanner { * We will use this to test scan does the right thing as it * we do Gets, StoreScanner#optimize, and what we do on (faked) block boundaries. */ - private static final Cell [] CELL_GRID = new Cell [] { - CellUtil.createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(ONE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(ONE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), + private static final Cell[] CELL_GRID = new Cell [] { + createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(ONE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(ONE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), // Offset 4 CELL_GRID_BLOCK2_BOUNDARY - CellUtil.createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(TWO, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(TWO, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(TWO_POINT_TWO, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(TWO_POINT_TWO, CF, ZERO_POINT_ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(TWO_POINT_TWO, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO_POINT_TWO, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO_POINT_TWO, CF, ZERO_POINT_ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO_POINT_TWO, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE), // Offset 11! CELL_GRID_BLOCK3_BOUNDARY - CellUtil.createCell(THREE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(THREE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(THREE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(THREE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(THREE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(THREE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(THREE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(THREE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), // Offset 15 CELL_GRID_BLOCK4_BOUNDARY - CellUtil.createCell(FOUR, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(FOUR, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(FOUR, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(FOUR, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(FOUR, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(FOUR, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(FOUR, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(FOUR, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE), // Offset 19 CELL_GRID_BLOCK5_BOUNDARY - CellUtil.createCell(FOUR, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(FIVE, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(FOUR, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(FIVE, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), }; private static class KeyValueHeapWithCount extends KeyValueHeap { @@ -154,11 +157,9 @@ private static class CellGridStoreScanner extends StoreScanner { AtomicInteger count; final AtomicInteger optimization = new AtomicInteger(0); - CellGridStoreScanner(final Scan scan, ScanInfo scanInfo, ScanType scanType) - throws IOException { - super(scan, scanInfo, scanType, scan.getFamilyMap().get(CF), - Arrays.asList( - new KeyValueScanner[] {new KeyValueScanFixture(CellComparator.COMPARATOR, CELL_GRID)})); + CellGridStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException { + super(scan, scanInfo, scan.getFamilyMap().get(CF), Arrays. asList( + new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.COMPARATOR, CELL_GRID) })); } protected void resetKVHeap(List scanners, @@ -204,25 +205,24 @@ public Cell getNextIndexedKey() { private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4; - private static final Cell [] CELL_WITH_VERSIONS = new Cell [] { - CellUtil.createCell(ONE, CF, ONE, 2L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(ONE, CF, TWO, 2L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + private static final Cell[] CELL_WITH_VERSIONS = new Cell [] { + createCell(ONE, CF, ONE, 2L, KeyValue.Type.Put.getCode(), VALUE), + createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(ONE, CF, TWO, 2L, KeyValue.Type.Put.getCode(), VALUE), + createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY - CellUtil.createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), - CellUtil.createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE), + createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), }; private static class CellWithVersionsStoreScanner extends StoreScanner { // Count of how often optimize is called and of how often it does an optimize. final AtomicInteger optimization = new AtomicInteger(0); - CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo, ScanType scanType) - throws IOException { - super(scan, scanInfo, scanType, scan.getFamilyMap().get(CF), Arrays - . asList(new KeyValueScanner[] { new KeyValueScanFixture( - CellComparator.COMPARATOR, CELL_WITH_VERSIONS) })); + CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException { + super(scan, scanInfo, scan.getFamilyMap().get(CF), + Arrays. asList(new KeyValueScanner[] { + new KeyValueScanFixture(CellComparator.COMPARATOR, CELL_WITH_VERSIONS) })); } protected boolean trySkipToNextColumn(Cell cell) throws IOException { @@ -246,11 +246,10 @@ private static class CellWithVersionsNoOptimizeStoreScanner extends StoreScanner // Count of how often optimize is called and of how often it does an optimize. final AtomicInteger optimization = new AtomicInteger(0); - CellWithVersionsNoOptimizeStoreScanner(final Scan scan, ScanInfo scanInfo, ScanType scanType) - throws IOException { - super(scan, scanInfo, scanType, scan.getFamilyMap().get(CF), Arrays - . asList(new KeyValueScanner[] { new KeyValueScanFixture( - CellComparator.COMPARATOR, CELL_WITH_VERSIONS) })); + CellWithVersionsNoOptimizeStoreScanner(Scan scan, ScanInfo scanInfo) throws IOException { + super(scan, scanInfo, scan.getFamilyMap().get(CF), + Arrays. asList(new KeyValueScanner[] { + new KeyValueScanFixture(CellComparator.COMPARATOR, CELL_WITH_VERSIONS) })); } protected boolean trySkipToNextColumn(Cell cell) throws IOException { @@ -272,40 +271,34 @@ public Cell getNextIndexedKey() { @Test public void testWithColumnCountGetFilter() throws Exception { Get get = new Get(ONE); - get.setMaxVersions(); + get.readAllVersions(); get.addFamily(CF); get.setFilter(new ColumnCountGetFilter(2)); - CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize = new CellWithVersionsNoOptimizeStoreScanner( - new Scan(get), this.scanInfo, this.scanType); - try { + try (CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize = + new CellWithVersionsNoOptimizeStoreScanner(new Scan(get), this.scanInfo)) { List results = new ArrayList<>(); while (scannerNoOptimize.next(results)) { continue; } - Assert.assertEquals(2, results.size()); - Assert.assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); - Assert.assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); - Assert.assertTrue("Optimize should do some optimizations", + assertEquals(2, results.size()); + assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); + assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); + assertTrue("Optimize should do some optimizations", scannerNoOptimize.optimization.get() == 0); - } finally { - scannerNoOptimize.close(); } get.setFilter(new ColumnCountGetFilter(2)); - CellWithVersionsStoreScanner scanner = new CellWithVersionsStoreScanner(new Scan(get), - this.scanInfo, this.scanType); - try { + try (CellWithVersionsStoreScanner scanner = + new CellWithVersionsStoreScanner(new Scan(get), this.scanInfo)) { List results = new ArrayList<>(); while (scanner.next(results)) { continue; } - Assert.assertEquals(2, results.size()); - Assert.assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); - Assert.assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); - Assert.assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); - } finally { - scanner.close(); + assertEquals(2, results.size()); + assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0])); + assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2])); + assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); } } @@ -329,22 +322,19 @@ public void testFullRowGetDoesNotOverreadWhenRowInsideOneBlock() throws IOExcept // row TWO_POINT_TWO. We should read one block only. Get get = new Get(TWO); Scan scan = new Scan(get); - CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo, this.scanType); - try { + try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { List results = new ArrayList<>(); while (scanner.next(results)) { continue; } // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the // TWO_POINT_TWO row does not have a a column ONE. - Assert.assertEquals(4, results.size()); + assertEquals(4, results.size()); // We should have gone the optimize route 5 times totally... an INCLUDE for the four cells // in the row plus the DONE on the end. - Assert.assertEquals(5, scanner.count.get()); + assertEquals(5, scanner.count.get()); // For a full row Get, there should be no opportunity for scanner optimization. - Assert.assertEquals(0, scanner.optimization.get()); - } finally { - scanner.close(); + assertEquals(0, scanner.optimization.get()); } } @@ -353,22 +343,19 @@ public void testFullRowSpansBlocks() throws IOException { // Do a Get against row FOUR. It spans two blocks. Get get = new Get(FOUR); Scan scan = new Scan(get); - CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo, this.scanType); - try { + try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { List results = new ArrayList<>(); while (scanner.next(results)) { continue; } // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the // TWO_POINT_TWO row does not have a a column ONE. - Assert.assertEquals(5, results.size()); + assertEquals(5, results.size()); // We should have gone the optimize route 6 times totally... an INCLUDE for the five cells // in the row plus the DONE on the end. - Assert.assertEquals(6, scanner.count.get()); + assertEquals(6, scanner.count.get()); // For a full row Get, there should be no opportunity for scanner optimization. - Assert.assertEquals(0, scanner.optimization.get()); - } finally { - scanner.close(); + assertEquals(0, scanner.optimization.get()); } } @@ -382,22 +369,19 @@ public void testOptimize() throws IOException { Scan scan = new Scan(); // A scan that just gets the first qualifier on each row of the CELL_GRID scan.addColumn(CF, ONE); - CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo, this.scanType); - try { + try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { List results = new ArrayList<>(); while (scanner.next(results)) { continue; } // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the // TWO_POINT_TWO row does not have a a column ONE. - Assert.assertEquals(4, results.size()); + assertEquals(4, results.size()); for (Cell cell: results) { assertTrue(Bytes.equals(ONE, 0, ONE.length, cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())); } - Assert.assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); - } finally { - scanner.close(); + assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0); } } @@ -416,18 +400,15 @@ public void testOptimizeAndGet() throws IOException { get.addColumn(CF, TWO); get.addColumn(CF, THREE); Scan scan = new Scan(get); - CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo, this.scanType); - try { + try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { List results = new ArrayList<>(); // For a Get there should be no more next's after the first call. - Assert.assertEquals(false, scanner.next(results)); + assertEquals(false, scanner.next(results)); // Should be one result only. - Assert.assertEquals(2, results.size()); + assertEquals(2, results.size()); // And we should have gone through optimize twice only. - Assert.assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", - 3, scanner.count.get()); - } finally { - scanner.close(); + assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 3, + scanner.count.get()); } } @@ -447,18 +428,15 @@ public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException Get get = new Get(THREE); get.addColumn(CF, TWO); Scan scan = new Scan(get); - CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo, this.scanType); - try { + try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) { List results = new ArrayList<>(); // For a Get there should be no more next's after the first call. - Assert.assertEquals(false, scanner.next(results)); + assertEquals(false, scanner.next(results)); // Should be one result only. - Assert.assertEquals(1, results.size()); + assertEquals(1, results.size()); // And we should have gone through optimize twice only. - Assert.assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", - 2, scanner.count.get()); - } finally { - scanner.close(); + assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2, + scanner.count.get()); } } @@ -467,57 +445,53 @@ public void testScanTimeRange() throws IOException { String r1 = "R1"; // returns only 1 of these 2 even though same timestamp KeyValue [] kvs = new KeyValue[] { - KeyValueTestUtil.create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), + create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"), + create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"), + create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"), + create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"), + create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), }; List scanners = Arrays.asList( new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.COMPARATOR, kvs) }); - Scan scanSpec = new Scan(Bytes.toBytes(r1)); + Scan scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); scanSpec.setTimeRange(0, 6); - scanSpec.setMaxVersions(); + scanSpec.readAllVersions(); List results = null; - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(5, results.size()); - Assert.assertEquals(kvs[kvs.length - 1], results.get(0)); + assertEquals(true, scan.next(results)); + assertEquals(5, results.size()); + assertEquals(kvs[kvs.length - 1], results.get(0)); } // Scan limited TimeRange - scanSpec = new Scan(Bytes.toBytes(r1)); + scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); scanSpec.setTimeRange(1, 3); - scanSpec.setMaxVersions(); - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { + scanSpec.readAllVersions(); + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(2, results.size()); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); } // Another range. - scanSpec = new Scan(Bytes.toBytes(r1)); + scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); scanSpec.setTimeRange(5, 10); - scanSpec.setMaxVersions(); - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { + scanSpec.readAllVersions(); + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(1, results.size()); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); } // See how TimeRange and Versions interact. // Another range. - scanSpec = new Scan(Bytes.toBytes(r1)); + scanSpec = new Scan().withStartRow(Bytes.toBytes(r1)); scanSpec.setTimeRange(0, 10); - scanSpec.setMaxVersions(3); - try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), - scanners)) { + scanSpec.readVersions(3); + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(3, results.size()); + assertEquals(true, scan.next(results)); + assertEquals(3, results.size()); } } @@ -525,22 +499,21 @@ public void testScanTimeRange() throws IOException { public void testScanSameTimestamp() throws IOException { // returns only 1 of these 2 even though same timestamp KeyValue [] kvs = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), }; List scanners = Arrays.asList( new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.COMPARATOR, kvs) }); - Scan scanSpec = new Scan(Bytes.toBytes("R1")); + Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); // this only uses maxVersions (default=1) and TimeRange (default=all) - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[0], results.get(0)); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[0], results.get(0)); } } @@ -555,30 +528,29 @@ public void testScanSameTimestamp() throws IOException { public void testWontNextToNext() throws IOException { // build the scan file: KeyValue [] kvs = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") + create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") }; List scanners = scanFixture(kvs); - Scan scanSpec = new Scan(Bytes.toBytes("R1")); + Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); // this only uses maxVersions (default=1) and TimeRange (default=all) - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List results = new ArrayList<>(); scan.next(results); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[0], results.get(0)); + assertEquals(1, results.size()); + assertEquals(kvs[0], results.get(0)); // should be ok... // now scan _next_ again. results.clear(); scan.next(results); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[2], results.get(0)); + assertEquals(1, results.size()); + assertEquals(kvs[2], results.get(0)); results.clear(); scan.next(results); - Assert.assertEquals(0, results.size()); + assertEquals(0, results.size()); } } @@ -586,16 +558,15 @@ public void testWontNextToNext() throws IOException { @Test public void testDeleteVersionSameTimestamp() throws IOException { KeyValue [] kvs = new KeyValue [] { - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), }; List scanners = scanFixture(kvs); - Scan scanSpec = new Scan(Bytes.toBytes("R1")); - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { + Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List results = new ArrayList<>(); - Assert.assertFalse(scan.next(results)); - Assert.assertEquals(0, results.size()); + assertFalse(scan.next(results)); + assertEquals(0, results.size()); } } @@ -606,207 +577,202 @@ public void testDeleteVersionSameTimestamp() throws IOException { @Test public void testDeletedRowThenGoodRow() throws IOException { KeyValue [] kvs = new KeyValue [] { - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") + create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), + create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") }; List scanners = scanFixture(kvs); - Scan scanSpec = new Scan(Bytes.toBytes("R1")); - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { + Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(0, results.size()); + assertEquals(true, scan.next(results)); + assertEquals(0, results.size()); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[2], results.get(0)); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[2], results.get(0)); - Assert.assertEquals(false, scan.next(results)); + assertEquals(false, scan.next(results)); } } public void testDeleteVersionMaskingMultiplePuts() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs1 = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") + create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; KeyValue [] kvs2 = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") + create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") }; List scanners = scanFixture(kvs1, kvs2); - try (StoreScanner scan = new StoreScanner(new Scan(Bytes.toBytes("R1")), scanInfo, scanType, - getCols("a"), scanners)) { + try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), + scanInfo, getCols("a"), scanners)) { List results = new ArrayList<>(); // the two put at ts=now will be masked by the 1 delete, and // since the scan default returns 1 version we'll return the newest // key, which is kvs[2], now-100. - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs2[1], results.get(0)); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs2[1], results.get(0)); } } public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs1 = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") + create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; KeyValue [] kvs2 = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") + create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") }; List scanners = scanFixture(kvs1, kvs2); - Scan scanSpec = new Scan(Bytes.toBytes("R1")).setMaxVersions(2); - try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, - getCols("a"), scanners)) { + Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2); + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(2, results.size()); - Assert.assertEquals(kvs2[1], results.get(0)); - Assert.assertEquals(kvs2[0], results.get(1)); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs2[1], results.get(0)); + assertEquals(kvs2[0], results.get(1)); } } @Test public void testWildCardOneVersionScan() throws IOException { KeyValue [] kvs = new KeyValue [] { - KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), + create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), }; List scanners = scanFixture(kvs); try (StoreScanner scan = - new StoreScanner(new Scan(Bytes.toBytes("R1")), scanInfo, scanType, null, scanners)) { + new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(2, results.size()); - Assert.assertEquals(kvs[0], results.get(0)); - Assert.assertEquals(kvs[1], results.get(1)); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[1], results.get(1)); } } @Test public void testWildCardScannerUnderDeletes() throws IOException { KeyValue [] kvs = new KeyValue [] { - KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc + create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc // orphaned delete column. - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), // column b - KeyValueTestUtil.create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc - KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc + create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc + create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc // column c - KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no - KeyValueTestUtil.create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), // inc + create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"), + create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no + create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), // inc // column d - KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc - KeyValueTestUtil.create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), // no - KeyValueTestUtil.create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no + create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc + create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"), + create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), // no + create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no }; List scanners = scanFixture(kvs); - try (StoreScanner scan = new StoreScanner(new Scan().setMaxVersions(2), - scanInfo, scanType, null, scanners)) { + try (StoreScanner scan = + new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(5, results.size()); - Assert.assertEquals(kvs[0], results.get(0)); - Assert.assertEquals(kvs[2], results.get(1)); - Assert.assertEquals(kvs[3], results.get(2)); - Assert.assertEquals(kvs[6], results.get(3)); - Assert.assertEquals(kvs[7], results.get(4)); + assertEquals(true, scan.next(results)); + assertEquals(5, results.size()); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[2], results.get(1)); + assertEquals(kvs[3], results.get(2)); + assertEquals(kvs[6], results.get(3)); + assertEquals(kvs[7], results.get(4)); } } @Test public void testDeleteFamily() throws IOException { - KeyValue [] kvs = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), + KeyValue[] kvs = new KeyValue[] { + create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), + create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), + create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), + create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), }; List scanners = scanFixture(kvs); try (StoreScanner scan = - new StoreScanner(new Scan().setMaxVersions(Integer.MAX_VALUE), scanInfo, scanType, null, - scanners)) { + new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(0, results.size()); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[kvs.length-1], results.get(0)); + assertEquals(true, scan.next(results)); + assertEquals(0, results.size()); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[kvs.length - 1], results.get(0)); - Assert.assertEquals(false, scan.next(results)); + assertEquals(false, scan.next(results)); } } @Test public void testDeleteColumn() throws IOException { KeyValue [] kvs = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") + create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"), + create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"), + create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") }; List scanners = scanFixture(kvs); - try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, scanType, null, - scanners)) { + try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[3], results.get(0)); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[3], results.get(0)); } } - private static final KeyValue [] kvs = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), + private static final KeyValue[] kvs = new KeyValue[] { + create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), }; @Test public void testSkipColumn() throws IOException { List scanners = scanFixture(kvs); - try (StoreScanner scan = - new StoreScanner(new Scan(), scanInfo, scanType, getCols("a", "d"), scanners)) { + try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(2, results.size()); - Assert.assertEquals(kvs[0], results.get(0)); - Assert.assertEquals(kvs[3], results.get(1)); + assertEquals(true, scan.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[3], results.get(1)); results.clear(); - Assert.assertEquals(true, scan.next(results)); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[kvs.length-1], results.get(0)); + assertEquals(true, scan.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[kvs.length - 1], results.get(0)); results.clear(); - Assert.assertEquals(false, scan.next(results)); + assertEquals(false, scan.next(results)); } } @@ -818,54 +784,48 @@ public void testSkipColumn() throws IOException { public void testWildCardTtlScan() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs = new KeyValue[] { - KeyValueTestUtil.create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care") + create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), + create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care") }; List scanners = scanFixture(kvs); Scan scan = new Scan(); - scan.setMaxVersions(1); + scan.readVersions(1); ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.COMPARATOR, false); - ScanType scanType = ScanType.USER_SCAN; - try (StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners)) { + try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scanner.next(results)); - Assert.assertEquals(2, results.size()); - Assert.assertEquals(kvs[1], results.get(0)); - Assert.assertEquals(kvs[2], results.get(1)); + assertEquals(true, scanner.next(results)); + assertEquals(2, results.size()); + assertEquals(kvs[1], results.get(0)); + assertEquals(kvs[2], results.get(1)); results.clear(); - Assert.assertEquals(true, scanner.next(results)); - Assert.assertEquals(3, results.size()); - Assert.assertEquals(kvs[4], results.get(0)); - Assert.assertEquals(kvs[5], results.get(1)); - Assert.assertEquals(kvs[6], results.get(2)); + assertEquals(true, scanner.next(results)); + assertEquals(3, results.size()); + assertEquals(kvs[4], results.get(0)); + assertEquals(kvs[5], results.get(1)); + assertEquals(kvs[6], results.get(2)); results.clear(); - Assert.assertEquals(false, scanner.next(results)); + assertEquals(false, scanner.next(results)); } } @Test public void testScannerReseekDoesntNPE() throws Exception { List scanners = scanFixture(kvs); - try (StoreScanner scan = - new StoreScanner(new Scan(), scanInfo, scanType, getCols("a", "d"), scanners)) { - - // Previously a updateReaders twice in a row would cause an NPE. In test this would also - // normally cause an NPE because scan.store is null. So as long as we get through these + try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { + // Previously a updateReaders twice in a row would cause an NPE. In test this would also + // normally cause an NPE because scan.store is null. So as long as we get through these // two calls we are good and the bug was quashed. - - scan.updateReaders(Collections.EMPTY_LIST, Collections.EMPTY_LIST); - - scan.updateReaders(Collections.EMPTY_LIST, Collections.EMPTY_LIST); - + scan.updateReaders(Collections.emptyList(), Collections.emptyList()); + scan.updateReaders(Collections.emptyList(), Collections.emptyList()); scan.peek(); } } @@ -875,15 +835,14 @@ public void testScannerReseekDoesntNPE() throws Exception { * TODO this fails, since we don't handle deletions, etc, in peek */ public void SKIP_testPeek() throws Exception { - KeyValue [] kvs = new KeyValue [] { - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), - KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), + KeyValue[] kvs = new KeyValue [] { + create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), }; List scanners = scanFixture(kvs); - Scan scanSpec = new Scan(Bytes.toBytes("R1")); - try (StoreScanner scan = - new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners)) { - Assert.assertNull(scan.peek()); + Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); + try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { + assertNull(scan.peek()); } } @@ -893,29 +852,26 @@ public void SKIP_testPeek() throws Exception { @Test public void testExpiredDeleteFamily() throws Exception { long now = System.currentTimeMillis(); - KeyValue [] kvs = new KeyValue[] { + KeyValue[] kvs = new KeyValue[] { new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000, KeyValue.Type.DeleteFamily), - KeyValueTestUtil.create("R1", "cf", "a", now-10, KeyValue.Type.Put, + create("R1", "cf", "a", now-10, KeyValue.Type.Put, "dont-care"), }; List scanners = scanFixture(kvs); Scan scan = new Scan(); - scan.setMaxVersions(1); + scan.readVersions(1); // scanner with ttl equal to 500 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.COMPARATOR, false); - ScanType scanType = ScanType.USER_SCAN; - try (StoreScanner scanner = - new StoreScanner(scan, scanInfo, scanType, null, scanners)) { - + try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List results = new ArrayList<>(); - Assert.assertEquals(true, scanner.next(results)); - Assert.assertEquals(1, results.size()); - Assert.assertEquals(kvs[1], results.get(0)); + assertEquals(true, scanner.next(results)); + assertEquals(1, results.size()); + assertEquals(kvs[1], results.get(0)); results.clear(); - Assert.assertEquals(false, scanner.next(results)); + assertEquals(false, scanner.next(results)); } } @@ -933,38 +889,36 @@ public long currentTime() { now - 100, KeyValue.Type.DeleteFamily), // live /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000, KeyValue.Type.DeleteFamily), // expired - /*2*/ KeyValueTestUtil.create("R1", "cf", "a", now - 50, + /*2*/ create("R1", "cf", "a", now - 50, KeyValue.Type.Put, "v3"), // live - /*3*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55, + /*3*/ create("R1", "cf", "a", now - 55, KeyValue.Type.Delete, "dontcare"), // live - /*4*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55, + /*4*/ create("R1", "cf", "a", now - 55, KeyValue.Type.Put, "deleted-version v2"), // deleted - /*5*/ KeyValueTestUtil.create("R1", "cf", "a", now - 60, + /*5*/ create("R1", "cf", "a", now - 60, KeyValue.Type.Put, "v1"), // live - /*6*/ KeyValueTestUtil.create("R1", "cf", "a", now - 65, + /*6*/ create("R1", "cf", "a", now - 65, KeyValue.Type.Put, "v0"), // max-version reached - /*7*/ KeyValueTestUtil.create("R1", "cf", "a", + /*7*/ create("R1", "cf", "a", now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version - /*8*/ KeyValueTestUtil.create("R1", "cf", "b", now - 600, + /*8*/ create("R1", "cf", "b", now - 600, KeyValue.Type.DeleteColumn, "dont-care"), //expired - /*9*/ KeyValueTestUtil.create("R1", "cf", "b", now - 70, + /*9*/ create("R1", "cf", "b", now - 70, KeyValue.Type.Put, "v2"), //live - /*10*/ KeyValueTestUtil.create("R1", "cf", "b", now - 750, + /*10*/ create("R1", "cf", "b", now - 750, KeyValue.Type.Put, "v1"), //expired - /*11*/ KeyValueTestUtil.create("R1", "cf", "c", now - 500, + /*11*/ create("R1", "cf", "c", now - 500, KeyValue.Type.Delete, "dontcare"), //expired - /*12*/ KeyValueTestUtil.create("R1", "cf", "c", now - 600, + /*12*/ create("R1", "cf", "c", now - 600, KeyValue.Type.Put, "v1"), //expired - /*13*/ KeyValueTestUtil.create("R1", "cf", "c", now - 1000, + /*13*/ create("R1", "cf", "c", now - 1000, KeyValue.Type.Delete, "dontcare"), //expired - /*14*/ KeyValueTestUtil.create("R1", "cf", "d", now - 60, + /*14*/ create("R1", "cf", "d", now - 60, KeyValue.Type.Put, "expired put"), //live - /*15*/ KeyValueTestUtil.create("R1", "cf", "d", now - 100, + /*15*/ create("R1", "cf", "d", now - 100, KeyValue.Type.Delete, "not-expired delete"), //live }; List scanners = scanFixture(kvs); - Scan scan = new Scan(); - scan.setMaxVersions(2); ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"), 0 /* minVersions */, 2 /* maxVersions */, 500 /* ttl */, @@ -973,20 +927,18 @@ public long currentTime() { 200, /* timeToPurgeDeletes */ CellComparator.COMPARATOR, false); try (StoreScanner scanner = - new StoreScanner(scan, scanInfo, - ScanType.COMPACT_DROP_DELETES, null, scanners, - HConstants.OLDEST_TIMESTAMP)) { + new StoreScanner(scanInfo, OptionalInt.of(2), ScanType.COMPACT_DROP_DELETES, scanners)) { List results = new ArrayList<>(); results = new ArrayList<>(); - Assert.assertEquals(true, scanner.next(results)); - Assert.assertEquals(kvs[0], results.get(0)); - Assert.assertEquals(kvs[2], results.get(1)); - Assert.assertEquals(kvs[3], results.get(2)); - Assert.assertEquals(kvs[5], results.get(3)); - Assert.assertEquals(kvs[9], results.get(4)); - Assert.assertEquals(kvs[14], results.get(5)); - Assert.assertEquals(kvs[15], results.get(6)); - Assert.assertEquals(7, results.size()); + assertEquals(true, scanner.next(results)); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[2], results.get(1)); + assertEquals(kvs[3], results.get(2)); + assertEquals(kvs[5], results.get(3)); + assertEquals(kvs[9], results.get(4)); + assertEquals(kvs[14], results.get(5)); + assertEquals(kvs[15], results.get(6)); + assertEquals(7, results.size()); } } finally { EnvironmentEdgeManagerTestHelper.reset(); @@ -995,17 +947,17 @@ public long currentTime() { @Test public void testPreadNotEnabledForCompactionStoreScanners() throws Exception { - ScanType scanType = ScanType.COMPACT_RETAIN_DELETES; long now = System.currentTimeMillis(); KeyValue[] kvs = new KeyValue[] { new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000, KeyValue.Type.DeleteFamily), - KeyValueTestUtil.create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), }; + create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), }; List scanners = scanFixture(kvs); - Scan scan = new Scan(); ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.COMPARATOR, false); - StoreScanner storeScanner = new StoreScanner(scan, scanInfo, scanType, null, scanners); - assertFalse(storeScanner.isScanUsePread()); + try (StoreScanner storeScanner = new StoreScanner(scanInfo, OptionalInt.empty(), + ScanType.COMPACT_RETAIN_DELETES, scanners)) { + assertFalse(storeScanner.isScanUsePread()); + } } } \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java index 993a1fdf1019..2704fa453d4f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.NavigableSet; +import java.util.OptionalInt; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; @@ -251,18 +252,17 @@ public InternalScanner preFlushScannerOpen( family.getMinVersions(), newVersions == null ? family.getMaxVersions() : newVersions, newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), family.getBlocksize(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), family.isNewVersionBehavior()); - Scan scan = new Scan(); - scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions); - return new StoreScanner(store, scanInfo, scan, scanners, - ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), + return new StoreScanner(store, scanInfo, + newVersions == null ? OptionalInt.empty() : OptionalInt.of(newVersions.intValue()), + scanners, ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP); } @Override public InternalScanner preCompactScannerOpen( - final ObserverContext c, - Store store, List scanners, ScanType scanType, - long earliestPutTs, InternalScanner s) throws IOException { + final ObserverContext c, Store store, + List scanners, ScanType scanType, long earliestPutTs, + InternalScanner s) throws IOException { Long newTtl = ttls.get(store.getTableName()); Integer newVersions = versions.get(store.getTableName()); ScanInfo oldSI = store.getScanInfo(); @@ -270,11 +270,11 @@ public InternalScanner preCompactScannerOpen( ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(), family.getName(), family.getMinVersions(), newVersions == null ? family.getMaxVersions() : newVersions, newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(), - family.getBlocksize(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), family.isNewVersionBehavior()); - Scan scan = new Scan(); - scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions); - return new StoreScanner(store, scanInfo, scan, scanners, scanType, - store.getSmallestReadPoint(), earliestPutTs); + family.getBlocksize(), oldSI.getTimeToPurgeDeletes(), oldSI.getComparator(), + family.isNewVersionBehavior()); + return new StoreScanner(store, scanInfo, + newVersions == null ? OptionalInt.empty() : OptionalInt.of(newVersions.intValue()), + scanners, scanType, store.getSmallestReadPoint(), earliestPutTs); } @Override