Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,4 @@ tags
.codegenie/
.vscode/
**/__pycache__
.opencode
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ public boolean supportsParameter(ParameterContext pc, ExtensionContext ec)
// test with wrapper type, otherwise it will always return false
return Primitives.wrap(expectedType).isAssignableFrom(value.getClass());
}
// non-primitive type can accept null value
if (value == null) {
return true;
}
return expectedType.isAssignableFrom(value.getClass());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,20 @@
*/
package org.apache.hadoop.hbase.io;

import static org.junit.Assert.*;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

@Category(SmallTests.class)
@Tag(SmallTests.TAG)
@Tag(MiscTests.TAG)
public class TestByteBufferOutputStream {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestByteBufferOutputStream.class);

@Test
public void testByteBufferReuse() throws IOException {
byte[] someBytes = Bytes.toBytes("some bytes");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hbase.io;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.io.InputStream;
Expand All @@ -33,20 +33,16 @@
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
import org.apache.hadoop.fs.ReadOption;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.io.ByteBufferPool;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.jupiter.api.Tag;

@Category(SmallTests.class)
@Tag(SmallTests.TAG)
@Tag(MiscTests.TAG)
public class TestFSDataInputStreamWrapper {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestFSDataInputStreamWrapper.class);

@Test
public void testUnbuffer() throws Exception {
InputStream pc = new ParentClass();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@
*/
package org.apache.hadoop.hbase.io;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.FileNotFoundException;
import java.io.IOException;
Expand All @@ -31,7 +32,6 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
Expand All @@ -40,21 +40,17 @@
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.ipc.RemoteException;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

/**
* Test that FileLink switches between alternate locations when the current location moves or gets
* deleted.
*/
@Category({ IOTests.class, MediumTests.class })
@Tag(IOTests.TAG)
@Tag(MediumTests.TAG)
public class TestFileLink {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestFileLink.class);

@Test
public void testEquals() {
Path p1 = new Path("/p1");
Expand Down Expand Up @@ -160,7 +156,7 @@ public Configuration getConf() {
}
}

@Test(expected = FileNotFoundException.class)
@Test
public void testLinkReadWithMissingFile() throws Exception {
HBaseTestingUtil testUtil = new HBaseTestingUtil();
FileSystem fs = new MyDistributedFileSystem();
Expand All @@ -173,7 +169,9 @@ public void testLinkReadWithMissingFile() throws Exception {
files.add(archivedPath);

FileLink link = new FileLink(files);
link.open(fs);
assertThrows(FileNotFoundException.class, () -> {
link.open(fs);
});
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,88 +17,78 @@
*/
package org.apache.hadoop.hbase.io;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.regex.Matcher;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;

/**
* Test that FileLink switches between alternate locations when the current location moves or gets
* deleted.
*/
@Category({ IOTests.class, SmallTests.class })
@Tag(IOTests.TAG)
@Tag(SmallTests.TAG)
public class TestHFileLink {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestHFileLink.class);

@Rule
public TestName name = new TestName();

@Test
public void testValidLinkNames() {
public void testValidLinkNames(TestInfo testInfo) {
String validLinkNames[] = { "foo=fefefe-0123456", "ns=foo=abababa-fefefefe" };

for (String name : validLinkNames) {
Assert.assertTrue("Failed validating:" + name, name.matches(HFileLink.LINK_NAME_REGEX));
assertTrue(name.matches(HFileLink.LINK_NAME_REGEX), "Failed validating:" + name);
}

for (String name : validLinkNames) {
Assert.assertTrue("Failed validating:" + name, HFileLink.isHFileLink(name));
assertTrue(HFileLink.isHFileLink(name), "Failed validating:" + name);
}

String testName = name.getMethodName() + "=fefefe-0123456";
Assert.assertEquals(TableName.valueOf(name.getMethodName()),
String testName = testInfo.getTestMethod().get().getName() + "=fefefe-0123456";
assertEquals(TableName.valueOf(testInfo.getTestMethod().get().getName()),
HFileLink.getReferencedTableName(testName));
Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName));
Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName));
Assert.assertEquals(testName,
HFileLink.createHFileLinkName(TableName.valueOf(name.getMethodName()), "fefefe", "0123456"));
assertEquals("fefefe", HFileLink.getReferencedRegionName(testName));
assertEquals("0123456", HFileLink.getReferencedHFileName(testName));
assertEquals(testName, HFileLink.createHFileLinkName(
TableName.valueOf(testInfo.getTestMethod().get().getName()), "fefefe", "0123456"));

testName = "ns=" + name.getMethodName() + "=fefefe-0123456";
Assert.assertEquals(TableName.valueOf("ns", name.getMethodName()),
testName = "ns=" + testInfo.getTestMethod().get().getName() + "=fefefe-0123456";
assertEquals(TableName.valueOf("ns", testInfo.getTestMethod().get().getName()),
HFileLink.getReferencedTableName(testName));
Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName));
Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName));
Assert.assertEquals(testName, HFileLink
.createHFileLinkName(TableName.valueOf("ns", name.getMethodName()), "fefefe", "0123456"));
assertEquals("fefefe", HFileLink.getReferencedRegionName(testName));
assertEquals("0123456", HFileLink.getReferencedHFileName(testName));
assertEquals(testName, HFileLink.createHFileLinkName(
TableName.valueOf("ns", testInfo.getTestMethod().get().getName()), "fefefe", "0123456"));

for (String name : validLinkNames) {
Matcher m = HFileLink.LINK_NAME_PATTERN.matcher(name);
assertTrue(m.matches());
Assert.assertEquals(HFileLink.getReferencedTableName(name),
assertEquals(HFileLink.getReferencedTableName(name),
TableName.valueOf(m.group(1), m.group(2)));
Assert.assertEquals(HFileLink.getReferencedRegionName(name), m.group(3));
Assert.assertEquals(HFileLink.getReferencedHFileName(name), m.group(4));
assertEquals(HFileLink.getReferencedRegionName(name), m.group(3));
assertEquals(HFileLink.getReferencedHFileName(name), m.group(4));
}
}

@Test
public void testBackReference() {
public void testBackReference(TestInfo testInfo) {
Path rootDir = new Path("/root");
Path archiveDir = new Path(rootDir, ".archive");
String storeFileName = "121212";
String linkDir = FileLink.BACK_REFERENCES_DIRECTORY_PREFIX + storeFileName;
String encodedRegion = "FEFE";
String cf = "cf1";

TableName refTables[] =
{ TableName.valueOf(name.getMethodName()), TableName.valueOf("ns", name.getMethodName()) };
TableName refTables[] = { TableName.valueOf(testInfo.getTestMethod().get().getName()),
TableName.valueOf("ns", testInfo.getTestMethod().get().getName()) };

for (TableName refTable : refTables) {
Path refTableDir = CommonFSUtils.getTableDir(archiveDir, refTable);
Expand All @@ -108,28 +98,26 @@ public void testBackReference() {
String refStoreFileName = refTable.getNameAsString().replace(TableName.NAMESPACE_DELIM, '=')
+ "=" + encodedRegion + "-" + storeFileName;

TableName tableNames[] = { TableName.valueOf(name.getMethodName() + "1"),
TableName.valueOf("ns", name.getMethodName() + "2"),
TableName.valueOf(name.getMethodName() + ":" + name.getMethodName()) };
TableName tableNames[] = { TableName.valueOf(testInfo.getTestMethod().get().getName() + "1"),
TableName.valueOf("ns", testInfo.getTestMethod().get().getName() + "2"),
TableName.valueOf(testInfo.getTestMethod().get().getName() + ":"
+ testInfo.getTestMethod().get().getName()) };

for (TableName tableName : tableNames) {
Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
Path regionDir = HRegion.getRegionDir(tableDir, encodedRegion);
Path cfDir = new Path(regionDir, cf);

// Verify back reference creation
assertEquals(
encodedRegion + "." + tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='),
HFileLink.createBackReferenceName(CommonFSUtils.getTableName(tableDir).getNameAsString(),
encodedRegion));

// verify parsing back reference
Pair<TableName, String> parsedRef = HFileLink.parseBackReferenceName(encodedRegion + "."
+ tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='));
assertEquals(parsedRef.getFirst(), tableName);
assertEquals(encodedRegion, parsedRef.getSecond());

// verify resolving back reference
Path storeFileDir = new Path(refLinkDir, encodedRegion + "."
+ tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='));
Path linkPath = new Path(cfDir, refStoreFileName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@

import static org.apache.hadoop.hbase.io.ByteBuffAllocator.BUFFER_SIZE_KEY;
import static org.apache.hadoop.hbase.io.hfile.CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.nio.file.Paths;
Expand All @@ -36,7 +36,6 @@
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.ExtendedCell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
Expand All @@ -62,29 +61,29 @@
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.io.netty.util.ResourceLeakDetector;

@Category({ IOTests.class, SmallTests.class })
@Tag(IOTests.TAG)
@Tag(SmallTests.TAG)
public class TestHalfStoreFileReader {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestHalfStoreFileReader.class);
private static final Logger LOG = LoggerFactory.getLogger(TestHalfStoreFileReader.class);

private static HBaseTestingUtil TEST_UTIL;

@BeforeClass
@BeforeAll
public static void setupBeforeClass() throws Exception {
TEST_UTIL = new HBaseTestingUtil();
}

@AfterClass
@AfterAll
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.cleanupTestDir();
}
Expand All @@ -109,7 +108,7 @@ public void testHalfScanAndReseek() throws Exception {
fs.mkdirs(parentPath);
String tableName = Paths.get(root_dir).getFileName().toString();
RegionInfo splitAHri = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build();
Thread.currentThread().sleep(1000);
Thread.sleep(1000);
RegionInfo splitBHri = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build();
Path splitAPath = new Path(new Path(root_dir, splitAHri.getRegionNameAsString()), "CF");
Path splitBPath = new Path(new Path(root_dir, splitBHri.getRegionNameAsString()), "CF");
Expand Down Expand Up @@ -201,12 +200,10 @@ private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, Cach
curr = scanner.getCell();
KeyValue reseekKv = getLastOnCol(curr);
int ret = scanner.reseekTo(reseekKv);
assertTrue("reseek to returned: " + ret, ret > 0);
// System.out.println(curr + ": " + ret);
assertTrue(ret > 0, "reseek to returned: " + ret);
} while (scanner.next());

int ret = scanner.reseekTo(getLastOnCol(curr));
// System.out.println("Last reseek: " + ret);
assertTrue(ret > 0);
}

Expand Down Expand Up @@ -250,8 +247,8 @@ public void testHalfScanner() throws IOException {
}
beforeMidKey = item;
}
System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
System.out.println("beforeMidKey: " + beforeMidKey);
LOG.info("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
LOG.info("beforeMidKey: " + beforeMidKey);

// Seek on the splitKey, should be in top, not in bottom
Cell foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
Expand Down
Loading