Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HDFS-7575. Upgrade should generate a unique storage ID for each volum…
…e. (Contributed by Arpit Agarwal)
- Loading branch information
Showing
14 changed files
with
274 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
139 changes: 139 additions & 0 deletions
139
...p-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeStartupFixesLegacyStorageIDs.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| /** | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.hadoop.hdfs; | ||
|
|
||
| import org.apache.hadoop.conf.Configuration; | ||
| import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; | ||
| import org.apache.hadoop.hdfs.server.protocol.StorageReport; | ||
| import org.apache.hadoop.test.GenericTestUtils; | ||
| import org.junit.Test; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
|
|
||
| import org.apache.hadoop.hdfs.TestDFSUpgradeFromImage.ClusterVerifier; | ||
|
|
||
| import static org.hamcrest.core.Is.is; | ||
| import static org.junit.Assert.assertThat; | ||
| import static org.junit.Assert.assertTrue; | ||
|
|
||
|
|
||
| /** | ||
| * The test verifies that legacy storage IDs in older DataNode | ||
| * images are replaced with UUID-based storage IDs. The startup may | ||
| * or may not involve a Datanode Layout upgrade. Each test case uses | ||
| * the following resource files. | ||
| * | ||
| * 1. testCaseName.tgz - NN and DN directories corresponding | ||
| * to a specific layout version. | ||
| * 2. testCaseName.txt - Text file listing the checksum of each file | ||
| * in the cluster and overall checksum. See | ||
| * TestUpgradeFromImage for the file format. | ||
| * | ||
| * If any test case is renamed then the corresponding resource files must | ||
| * also be renamed. | ||
| */ | ||
| public class TestDatanodeStartupFixesLegacyStorageIDs { | ||
|
|
||
| /** | ||
| * Perform a upgrade using the test image corresponding to | ||
| * testCaseName. | ||
| * | ||
| * @param testCaseName | ||
| * @param expectedStorageId if null, then the upgrade generates a new | ||
| * unique storage ID. | ||
| * @throws IOException | ||
| */ | ||
| private static void runLayoutUpgradeTest(final String testCaseName, | ||
| final String expectedStorageId) | ||
| throws IOException { | ||
| TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage(); | ||
| upgrade.unpackStorage(testCaseName + ".tgz", testCaseName + ".txt"); | ||
| Configuration conf = new Configuration(TestDFSUpgradeFromImage.upgradeConf); | ||
| initStorageDirs(conf, testCaseName); | ||
| upgradeAndVerify(upgrade, conf, new ClusterVerifier() { | ||
| @Override | ||
| public void verifyClusterPostUpgrade(MiniDFSCluster cluster) throws IOException { | ||
| // Verify that a GUID-based storage ID was generated. | ||
| final String bpid = cluster.getNamesystem().getBlockPoolId(); | ||
| StorageReport[] reports = | ||
| cluster.getDataNodes().get(0).getFSDataset().getStorageReports(bpid); | ||
| assertThat(reports.length, is(1)); | ||
| final String storageID = reports[0].getStorage().getStorageID(); | ||
| assertTrue(DatanodeStorage.isValidStorageId(storageID)); | ||
|
|
||
| if (expectedStorageId != null) { | ||
| assertThat(storageID, is(expectedStorageId)); | ||
| } | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| private static void initStorageDirs(final Configuration conf, | ||
| final String testName) { | ||
| conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, | ||
| System.getProperty("test.build.data") + File.separator + | ||
| testName + File.separator + "dfs" + File.separator + "data"); | ||
| conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, | ||
| System.getProperty("test.build.data") + File.separator + | ||
| testName + File.separator + "dfs" + File.separator + "name"); | ||
|
|
||
| } | ||
|
|
||
| private static void upgradeAndVerify(final TestDFSUpgradeFromImage upgrade, | ||
| final Configuration conf, | ||
| final ClusterVerifier verifier) | ||
| throws IOException{ | ||
| upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf) | ||
| .numDataNodes(1) | ||
| .manageDataDfsDirs(false) | ||
| .manageNameDfsDirs(false), verifier); | ||
| } | ||
|
|
||
| /** | ||
| * Upgrade from 2.2 (no storage IDs per volume) correctly generates | ||
| * GUID-based storage IDs. Test case for HDFS-7575. | ||
| */ | ||
| @Test (timeout=300000) | ||
| public void testUpgradeFrom22FixesStorageIDs() throws IOException { | ||
| runLayoutUpgradeTest(GenericTestUtils.getMethodName(), null); | ||
| } | ||
|
|
||
| /** | ||
| * Startup from a 2.6-layout that has legacy storage IDs correctly | ||
| * generates new storage IDs. | ||
| * Test case for HDFS-7575. | ||
| */ | ||
| @Test (timeout=300000) | ||
| public void testUpgradeFrom22via26FixesStorageIDs() throws IOException { | ||
| runLayoutUpgradeTest(GenericTestUtils.getMethodName(), null); | ||
| } | ||
|
|
||
| /** | ||
| * Startup from a 2.6-layout that already has unique storage IDs does | ||
| * not regenerate the storage IDs. | ||
| * Test case for HDFS-7575. | ||
| */ | ||
| @Test (timeout=300000) | ||
| public void testUpgradeFrom26PreservesStorageIDs() throws IOException { | ||
| // StorageId present in the image testUpgradeFrom26PreservesStorageId.tgz | ||
| runLayoutUpgradeTest(GenericTestUtils.getMethodName(), | ||
| "DS-a0e39cfa-930f-4abd-813c-e22b59223774"); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file added
BIN
+3.18 KB
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.tgz
Binary file not shown.
25 changes: 25 additions & 0 deletions
25
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout | ||
| # upgrade test. | ||
| # Uncomment the following line to produce checksum info for a new DFS image. | ||
| #printChecksums | ||
|
|
||
| /f01 4021661486 | ||
| /f02 4021661486 | ||
| /f03 4021661486 | ||
| /f04 4021661486 | ||
| overallCRC 3193029345 |
Binary file added
BIN
+3.55 KB
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.tgz
Binary file not shown.
25 changes: 25 additions & 0 deletions
25
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout | ||
| # upgrade test. | ||
| # Uncomment the following line to produce checksum info for a new DFS image. | ||
| #printChecksums | ||
|
|
||
| /f01 4021661486 | ||
| /f02 4021661486 | ||
| /f03 4021661486 | ||
| /f04 4021661486 | ||
| overallCRC 3193029345 |
Binary file added
BIN
+3.76 KB
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom26PreservesStorageIDs.tgz
Binary file not shown.
Oops, something went wrong.