Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.io.ByteBufferPool;
import org.junit.Test;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Tag(SmallTests.TAG)
@Tag(MiscTests.TAG)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,16 @@
*/
package org.apache.hadoop.hbase.io.encoding;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
Expand All @@ -48,24 +48,17 @@
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
import org.apache.hadoop.hbase.util.Strings;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.params.provider.Arguments;

/**
* Tests encoded seekers by loading and reading values.
*/
@Category({ IOTests.class, LargeTests.class })
@RunWith(Parameterized.class)
@org.junit.jupiter.api.Tag(IOTests.TAG)
@org.junit.jupiter.api.Tag(LargeTests.TAG)
@HBaseParameterizedTestTemplate
public class TestEncodedSeekers {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestEncodedSeekers.class);

private static final String TABLE_NAME = "encodedSeekersTable";
private static final String CF_NAME = "encodedSeekersCF";
private static final byte[] CF_BYTES = Bytes.toBytes(CF_NAME);
Expand All @@ -79,25 +72,24 @@ public class TestEncodedSeekers {
private static final int NUM_HFILES = 4;
private static final int NUM_ROWS_PER_FLUSH = NUM_ROWS / NUM_HFILES;

private final HBaseTestingUtility testUtil = HBaseTestingUtility.createLocalHTU();
private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
private final DataBlockEncoding encoding;
private final boolean includeTags;
private final boolean compressTags;

/** Enable when debugging */
private static final boolean VERBOSE = false;

@Parameters
public static Collection<Object[]> parameters() {
List<Object[]> paramList = new ArrayList<>();
public static Stream<Arguments> parameters() {
List<Arguments> params = new ArrayList<>();
for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
for (boolean includeTags : new boolean[] { false, true }) {
for (boolean compressTags : new boolean[] { false, true }) {
paramList.add(new Object[] { encoding, includeTags, compressTags });
params.add(Arguments.of(encoding, includeTags, compressTags));
}
}
}
return paramList;
return params.stream();
}

public TestEncodedSeekers(DataBlockEncoding encoding, boolean includeTags, boolean compressTags) {
Expand All @@ -106,7 +98,7 @@ public TestEncodedSeekers(DataBlockEncoding encoding, boolean includeTags, boole
this.compressTags = compressTags;
}

@Test
@TestTemplate
public void testEncodedSeeker() throws IOException {
System.err.println("Testing encoded seekers for encoding : " + encoding + ", includeTags : "
+ includeTags + ", compressTags : " + compressTags);
Expand Down Expand Up @@ -146,7 +138,7 @@ public void testEncodedSeeker() throws IOException {
private void doPuts(HRegion region) throws IOException {
LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
for (int i = 0; i < NUM_ROWS; ++i) {
byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
byte[] key = Bytes.toBytes(LoadTestKVGenerator.md5PrefixedKey(i));
for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
Put put = new Put(key);
put.setDurability(Durability.ASYNC_WAL);
Expand Down Expand Up @@ -174,7 +166,7 @@ private void doPuts(HRegion region) throws IOException {

private void doGets(Region region) throws IOException {
for (int i = 0; i < NUM_ROWS; ++i) {
final byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
final byte[] rowKey = Bytes.toBytes(LoadTestKVGenerator.md5PrefixedKey(i));
for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
final String qualStr = String.valueOf(j);
if (VERBOSE) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@
*/
package org.apache.hadoop.hbase.io.encoding;

import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
Expand All @@ -34,33 +34,23 @@
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.TestMiniClusterLoadSequential;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runners.Parameterized.Parameters;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.TestTemplate;

/**
* Uses the load tester
*/
@Category({ IOTests.class, MediumTests.class })
@Tag(IOTests.TAG)
@Tag(LargeTests.TAG)
@HBaseParameterizedTestTemplate
public class TestLoadAndSwitchEncodeOnDisk extends TestMiniClusterLoadSequential {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestLoadAndSwitchEncodeOnDisk.class);

/** We do not alternate the multi-put flag in this test. */
private static final boolean USE_MULTI_PUT = true;

/** Un-parameterize the test */
@Parameters
public static Collection<Object[]> parameters() {
return Arrays.asList(new Object[][] { new Object[0] });
}

public TestLoadAndSwitchEncodeOnDisk() {
super(USE_MULTI_PUT, DataBlockEncoding.PREFIX);
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);
Expand All @@ -72,14 +62,14 @@ protected int numKeys() {
}

@Override
@Test
@TestTemplate
public void loadTest() throws Exception {
Admin admin = TEST_UTIL.getAdmin();

compression = Compression.Algorithm.GZ; // used for table setup
super.loadTest();

HColumnDescriptor hcd = getColumnDesc(admin);
ColumnFamilyDescriptor hcd = getColumnDesc(admin);
System.err.println("\nDisabling encode-on-disk. Old column descriptor: " + hcd + "\n");
Table t = TEST_UTIL.getConnection().getTable(TABLE);
assertAllOnLine(t);
Expand Down Expand Up @@ -114,11 +104,11 @@ private void assertAllOnLine(final Table t) throws IOException {
regions = rl.getAllRegionLocations();
}
for (HRegionLocation e : regions) {
byte[] startkey = e.getRegionInfo().getStartKey();
Scan s = new Scan(startkey);
byte[] startkey = e.getRegion().getStartKey();
Scan s = new Scan().withStartRow(startkey);
ResultScanner scanner = t.getScanner(s);
Result r = scanner.next();
org.junit.Assert.assertTrue(r != null && r.size() > 0);
assertTrue(r != null && r.size() > 0);
scanner.close();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,38 +18,34 @@
package org.apache.hadoop.hbase.util;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import java.util.stream.Stream;
import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.junit.ClassRule;
import org.junit.experimental.categories.Category;
import org.junit.runners.Parameterized.Parameters;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.params.provider.Arguments;

/**
* Runs a load test on a mini HBase cluster with data block encoding turned on. Compared to other
* load-test-style unit tests, this one writes a smaller amount of data, but goes through all
* available data block encoding algorithms.
*/
@Category({ MiscTests.class, LargeTests.class })
@Tag(MiscTests.TAG)
@Tag(LargeTests.TAG)
@HBaseParameterizedTestTemplate
public class TestMiniClusterLoadEncoded extends TestMiniClusterLoadParallel {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestMiniClusterLoadEncoded.class);

/** We do not alternate the multi-put flag in this test. */
private static final boolean USE_MULTI_PUT = true;

@Parameters
public static Collection<Object[]> parameters() {
List<Object[]> parameters = new ArrayList<>();
public static Stream<Arguments> parameters() {
List<Arguments> params = new ArrayList<>();
for (DataBlockEncoding dataBlockEncoding : DataBlockEncoding.values()) {
parameters.add(new Object[] { dataBlockEncoding });
params.add(Arguments.of(dataBlockEncoding));
}
return parameters;
return params.stream();
}

public TestMiniClusterLoadEncoded(DataBlockEncoding encoding) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,36 @@
*/
package org.apache.hadoop.hbase.util;

import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;

import org.apache.hadoop.hbase.HBaseClassTestRule;
import java.util.stream.Stream;
import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.params.provider.Arguments;

/**
* A write/read/verify load test on a mini HBase cluster. Tests reading and writing at the same
* time.
*/
@Category({ MiscTests.class, LargeTests.class })
@RunWith(Parameterized.class)
@Tag(MiscTests.TAG)
@Tag(LargeTests.TAG)
@HBaseParameterizedTestTemplate
public class TestMiniClusterLoadParallel extends TestMiniClusterLoadSequential {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestMiniClusterLoadParallel.class);
public static Stream<Arguments> parameters() {
return TestMiniClusterLoadSequential.parameters();
}

public TestMiniClusterLoadParallel(boolean isMultiPut, DataBlockEncoding encoding) {
super(isMultiPut, encoding);
}

@Override
@Test
@TestTemplate
public void loadTest() throws Exception {
prepareForLoadTest();

Expand Down
Loading