-
Notifications
You must be signed in to change notification settings - Fork 3.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
HBASE-27495 Improve HFileLinkCleaner to validate back reference links… #4887
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -39,7 +39,9 @@ | |
import org.apache.hadoop.hbase.util.HFileArchiveUtil; | ||
import org.apache.hadoop.hbase.util.MockServer; | ||
import org.apache.hadoop.hbase.zookeeper.ZKWatcher; | ||
import org.junit.After; | ||
import org.junit.AfterClass; | ||
import org.junit.Before; | ||
import org.junit.BeforeClass; | ||
import org.junit.ClassRule; | ||
import org.junit.Rule; | ||
|
@@ -57,9 +59,28 @@ public class TestHFileLinkCleaner { | |
public static final HBaseClassTestRule CLASS_RULE = | ||
HBaseClassTestRule.forClass(TestHFileLinkCleaner.class); | ||
|
||
private Configuration conf; | ||
private Path rootDir; | ||
private FileSystem fs; | ||
private TableName tableName; | ||
private TableName tableLinkName; | ||
private String hfileName; | ||
private String familyName; | ||
private RegionInfo hri; | ||
private RegionInfo hriLink; | ||
private Path archiveDir; | ||
private Path archiveStoreDir; | ||
private Path familyPath; | ||
private Path hfilePath; | ||
private Path familyLinkPath; | ||
private String hfileLinkName; | ||
private Path linkBackRefDir; | ||
private Path linkBackRef; | ||
private FileStatus[] backRefs; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit. Can be a local variable. Only used in createLink. |
||
private HFileCleaner cleaner; | ||
private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); | ||
|
||
private static DirScanPool POOL; | ||
private static final long TTL = 1000; | ||
|
||
@Rule | ||
public TestName name = new TestName(); | ||
|
@@ -74,49 +95,71 @@ public static void tearDown() { | |
POOL.shutdownNow(); | ||
} | ||
|
||
@Test | ||
public void testHFileLinkCleaning() throws Exception { | ||
Configuration conf = TEST_UTIL.getConfiguration(); | ||
@Before | ||
public void configureDirectoriesAndLinks() throws IOException { | ||
conf = TEST_UTIL.getConfiguration(); | ||
CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); | ||
conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName()); | ||
Path rootDir = CommonFSUtils.getRootDir(conf); | ||
FileSystem fs = FileSystem.get(conf); | ||
rootDir = CommonFSUtils.getRootDir(conf); | ||
fs = FileSystem.get(conf); | ||
|
||
final TableName tableName = TableName.valueOf(name.getMethodName()); | ||
final TableName tableLinkName = TableName.valueOf(name.getMethodName() + "-link"); | ||
final String hfileName = "1234567890"; | ||
final String familyName = "cf"; | ||
tableName = TableName.valueOf(name.getMethodName()); | ||
tableLinkName = TableName.valueOf(name.getMethodName() + "-link"); | ||
hfileName = "1234567890"; | ||
familyName = "cf"; | ||
|
||
RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); | ||
RegionInfo hriLink = RegionInfoBuilder.newBuilder(tableLinkName).build(); | ||
hri = RegionInfoBuilder.newBuilder(tableName).build(); | ||
hriLink = RegionInfoBuilder.newBuilder(tableLinkName).build(); | ||
|
||
Path archiveDir = HFileArchiveUtil.getArchivePath(conf); | ||
Path archiveStoreDir = | ||
archiveDir = HFileArchiveUtil.getArchivePath(conf); | ||
archiveStoreDir = | ||
HFileArchiveUtil.getStoreArchivePath(conf, tableName, hri.getEncodedName(), familyName); | ||
|
||
// Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); | ||
Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); | ||
familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); | ||
fs.mkdirs(familyPath); | ||
Path hfilePath = new Path(familyPath, hfileName); | ||
hfilePath = new Path(familyPath, hfileName); | ||
fs.createNewFile(hfilePath); | ||
|
||
createLink(true); | ||
|
||
// Initialize cleaner | ||
conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, TTL); | ||
Server server = new DummyServer(); | ||
cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir, POOL); | ||
} | ||
|
||
private void createLink(boolean createBackReference) throws IOException { | ||
// Create link to hfile | ||
Path familyLinkPath = | ||
getFamilyDirPath(rootDir, tableLinkName, hriLink.getEncodedName(), familyName); | ||
familyLinkPath = getFamilyDirPath(rootDir, tableLinkName, hriLink.getEncodedName(), familyName); | ||
fs.mkdirs(familyLinkPath); | ||
HFileLink.create(conf, fs, familyLinkPath, hri, hfileName); | ||
Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); | ||
hfileLinkName = HFileLink.create(conf, fs, familyLinkPath, hri, hfileName, createBackReference); | ||
linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); | ||
assertTrue(fs.exists(linkBackRefDir)); | ||
FileStatus[] backRefs = fs.listStatus(linkBackRefDir); | ||
backRefs = fs.listStatus(linkBackRefDir); | ||
assertEquals(1, backRefs.length); | ||
Path linkBackRef = backRefs[0].getPath(); | ||
linkBackRef = backRefs[0].getPath(); | ||
} | ||
|
||
// Initialize cleaner | ||
final long ttl = 1000; | ||
conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); | ||
Server server = new DummyServer(); | ||
HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir, POOL); | ||
@After | ||
public void cleanup() throws IOException, InterruptedException { | ||
// HFile can be removed | ||
Thread.sleep(TTL * 2); | ||
cleaner.chore(); | ||
assertFalse("HFile should be deleted", fs.exists(hfilePath)); | ||
// Remove everything | ||
for (int i = 0; i < 4; ++i) { | ||
Thread.sleep(TTL * 2); | ||
cleaner.chore(); | ||
} | ||
assertFalse("HFile should be deleted", | ||
fs.exists(CommonFSUtils.getTableDir(archiveDir, tableName))); | ||
assertFalse("Link should be deleted", | ||
fs.exists(CommonFSUtils.getTableDir(archiveDir, tableLinkName))); | ||
} | ||
|
||
@Test | ||
public void testHFileLinkCleaning() throws Exception { | ||
// Link backref cannot be removed | ||
cleaner.chore(); | ||
assertTrue(fs.exists(linkBackRef)); | ||
|
@@ -127,21 +170,28 @@ public void testHFileLinkCleaning() throws Exception { | |
CommonFSUtils.getTableDir(archiveDir, tableLinkName)); | ||
cleaner.chore(); | ||
assertFalse("Link should be deleted", fs.exists(linkBackRef)); | ||
} | ||
|
||
// HFile can be removed | ||
Thread.sleep(ttl * 2); | ||
@Test | ||
public void testHFileLinkByRemovingReference() throws Exception { | ||
// Link backref cannot be removed | ||
cleaner.chore(); | ||
assertFalse("HFile should be deleted", fs.exists(hfilePath)); | ||
assertTrue(fs.exists(linkBackRef)); | ||
assertTrue(fs.exists(hfilePath)); | ||
|
||
// Remove everything | ||
for (int i = 0; i < 4; ++i) { | ||
Thread.sleep(ttl * 2); | ||
cleaner.chore(); | ||
} | ||
assertFalse("HFile should be deleted", | ||
fs.exists(CommonFSUtils.getTableDir(archiveDir, tableName))); | ||
assertFalse("Link should be deleted", | ||
fs.exists(CommonFSUtils.getTableDir(archiveDir, tableLinkName))); | ||
// simulate after removing the reference in data directory, the Link backref can be removed | ||
fs.delete(new Path(familyLinkPath, hfileLinkName), false); | ||
cleaner.chore(); | ||
assertFalse("Link should be deleted", fs.exists(linkBackRef)); | ||
} | ||
|
||
@Test | ||
public void testHFileLinkEmptyBackReferenceDirectory() throws Exception { | ||
// simulate and remove the back reference | ||
fs.delete(linkBackRef, false); | ||
assertTrue("back reference directory still exists", fs.exists(linkBackRefDir)); | ||
cleaner.chore(); | ||
assertFalse("back reference directory should be deleted", fs.exists(linkBackRefDir)); | ||
} | ||
|
||
private static Path getFamilyDirPath(final Path rootDir, final TableName table, | ||
|
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit. This is only used in configureDirectoriesAndLinks, can be local.