From a78430abf48bfd6db735a253c175517f81beb1ff Mon Sep 17 00:00:00 2001 From: Kitti Nanasi Date: Mon, 3 Dec 2018 09:59:56 -0800 Subject: [PATCH 1/5] HDFS-12946. Add a tool to check rack configuration against EC policies. Contributed by Kitti Nanasi. Signed-off-by: Wei-Chiu Chuang (cherry picked from commit dd5e7c6b7239a93f2391beaa11181e442a387db4) --- .../metrics/NamenodeBeanMetrics.java | 5 + .../blockmanagement/DatanodeManager.java | 7 + .../server/common/ECTopologyVerifier.java | 124 ++++++++++++++ .../namenode/ECTopologyVerifierResult.java | 45 +++++ .../namenode/ErasureCodingPolicyManager.java | 9 + .../hdfs/server/namenode/FSNamesystem.java | 19 +++ .../hdfs/server/namenode/NameNodeMXBean.java | 7 + .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 51 +++++- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 35 ++++ .../hdfs/TestErasureCodingMultipleRacks.java | 18 +- .../server/namenode/TestNameNodeMXBean.java | 19 +++ .../apache/hadoop/hdfs/tools/TestECAdmin.java | 155 ++++++++++++++++++ 12 files changed, 477 insertions(+), 17 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java index e8ebf0dd8c5a4..0ca5f737dd494 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java @@ -706,4 +706,9 @@ public String getNameDirSize() { public int getNumEncryptionZones() { return 0; } + + @Override + public String getVerifyECWithTopologyResult() { + return null; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 288bbb596db3f..771ad454fbb64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1252,6 +1252,13 @@ public int getNumDeadDataNodes() { return getDatanodeListForReport(DatanodeReportType.DEAD).size(); } + /** @return the number of datanodes. */ + public int getNumOfDataNodes() { + synchronized (this) { + return datanodeMap.size(); + } + } + /** @return list of datanodes where decommissioning is in progress. */ public List getDecommissioningNodes() { // There is no need to take namesystem reader lock as diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java new file mode 100644 index 0000000000000..3591b2deedfb1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdfs.server.common; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.server.namenode.ECTopologyVerifierResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * Class for verifying whether the cluster setup can support + * all enabled EC policies. + * + * Scenarios when the verification fails: + * 1. not enough data nodes compared to EC policy's highest data+parity number + * 2. not enough racks to satisfy BlockPlacementPolicyRackFaultTolerant + */ +@InterfaceAudience.Private +public final class ECTopologyVerifier { + + public static final Logger LOG = + LoggerFactory.getLogger(ECTopologyVerifier.class); + + private ECTopologyVerifier() {} + + /** + * Verifies whether the cluster setup can support all enabled EC policies. + * + * @param report list of data node descriptors for all data nodes + * @param policies all system and user defined erasure coding policies + * @return the status of the verification + */ + public static ECTopologyVerifierResult getECTopologyVerifierResult( + final DatanodeInfo[] report, final ErasureCodingPolicyInfo[] policies) { + final int numOfRacks = getNumberOfRacks(report); + return getECTopologyVerifierResult(policies, numOfRacks, report.length); + } + + /** + * Verifies whether the cluster setup can support all enabled EC policies. + * + * @param policies all system and user defined erasure coding policies + * @param numOfRacks number of racks + * @param numOfDataNodes number of data nodes + * @return the status of the verification + */ + public static ECTopologyVerifierResult getECTopologyVerifierResult( + final ErasureCodingPolicyInfo[] policies, final int numOfRacks, + final int numOfDataNodes) { + int minDN = 0; + int minRack = 0; + for (ErasureCodingPolicyInfo policy: policies) { + if (policy.isEnabled()) { + final int policyDN = + policy.getPolicy().getNumDataUnits() + policy.getPolicy() + .getNumParityUnits(); + minDN = Math.max(minDN, policyDN); + final int policyRack = (int) Math.ceil( + policyDN / (double) policy.getPolicy().getNumParityUnits()); + minRack = Math.max(minRack, policyRack); + } + } + if (minDN == 0 || minRack == 0) { + String resultMessage = "No erasure coding policy is enabled."; + LOG.trace(resultMessage); + return new ECTopologyVerifierResult(true, resultMessage); + } + return verifyECWithTopology(minDN, minRack, numOfRacks, numOfDataNodes); + } + + private static ECTopologyVerifierResult verifyECWithTopology( + final int minDN, final int minRack, + final int numOfRacks, final int numOfDataNodes) { + String resultMessage; + if (numOfDataNodes < minDN) { + resultMessage = "The number of DataNodes (" + numOfDataNodes + + ") is less than the minimum required number of DataNodes (" + + minDN + ") for enabled erasure coding policy."; + LOG.debug(resultMessage); + return new ECTopologyVerifierResult(false, resultMessage); + } + + if (numOfRacks < minRack) { + resultMessage = "The number of racks (" + numOfRacks + + ") is less than the minimum required number of racks (" + + minRack + ") for enabled erasure coding policy."; + LOG.debug(resultMessage); + return new ECTopologyVerifierResult(false, resultMessage); + } + return new ECTopologyVerifierResult(true, + "The cluster setup can support all enabled EC policies"); + } + + private static int getNumberOfRacks(DatanodeInfo[] report) { + final Map racks = new HashMap<>(); + for (DatanodeInfo dni : report) { + Integer count = racks.get(dni.getNetworkLocation()); + if (count == null) { + count = 0; + } + racks.put(dni.getNetworkLocation(), count + 1); + } + return racks.size(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java new file mode 100644 index 0000000000000..4757eabbaded3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Result of the verification whether the current cluster setup can + * support all enabled EC policies. + */ +@InterfaceAudience.Private +public class ECTopologyVerifierResult { + + private final String resultMessage; + private final boolean isSupported; + + public ECTopologyVerifierResult(boolean isSupported, + String resultMessage) { + this.resultMessage = resultMessage; + this.isSupported = isSupported; + } + + public String getResultMessage() { + return resultMessage; + } + + public boolean isSupported() { + return isSupported; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 66bc2c2308ee8..d3984e0a75edf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -212,6 +213,14 @@ public ErasureCodingPolicyInfo[] getPersistedPolicies() { .toArray(new ErasureCodingPolicyInfo[0]); } + public ErasureCodingPolicyInfo[] getCopyOfPolicies() { + ErasureCodingPolicyInfo[] copy; + synchronized (this) { + copy = Arrays.copyOf(allPolicies, allPolicies.length); + } + return copy; + } + /** * Get a {@link ErasureCodingPolicy} by policy ID, including system policy * and user defined policy. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 396562fa25dc4..5d1b35bfbcd65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -106,6 +106,7 @@ import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.server.common.ECTopologyVerifier; import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean; import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.apache.hadoop.util.Time.now; @@ -8191,6 +8192,24 @@ public int getNumEnteringMaintenanceDataNodes() { .size(); } + @Override // NameNodeMXBean + public String getVerifyECWithTopologyResult() { + int numOfDataNodes = getBlockManager().getDatanodeManager() + .getNumOfDataNodes(); + int numOfRacks = getBlockManager().getDatanodeManager() + .getNetworkTopology().getNumOfRacks(); + ErasureCodingPolicyInfo[] ecPolicies = + getErasureCodingPolicyManager().getCopyOfPolicies(); + ECTopologyVerifierResult result = + ECTopologyVerifier.getECTopologyVerifierResult(ecPolicies, + numOfRacks, numOfDataNodes); + + Map resultMap = new HashMap(); + resultMap.put("isSupported", Boolean.toString(result.isSupported())); + resultMap.put("resultMessage", result.getResultMessage()); + return JSON.toString(resultMap); + } + // This method logs operatoinName without super user privilege. // It should be called without holding FSN lock. void checkSuperuserPrivilege(String operationName) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java index 5c7bbbb4515cd..8bf0bcbea6c05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java @@ -303,4 +303,11 @@ public interface NameNodeMXBean { */ String getNameDirSize(); + /** + * Verifies whether the cluster setup can support all enabled EC policies. + * + * @return the result of the verification + */ + String getVerifyECWithTopologyResult(); + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 5f8626e07021e..d63d9dd5f65d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -24,9 +24,13 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; +import org.apache.hadoop.hdfs.server.common.ECTopologyVerifier; +import org.apache.hadoop.hdfs.server.namenode.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.util.ECPolicyLoader; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.tools.TableListing; @@ -588,6 +592,50 @@ public int run(Configuration conf, List args) throws IOException { } } + /** + * Command to verify the cluster setup can support all enabled EC policies. + */ + private static class VerifyClusterSetupCommand + implements AdminHelper.Command { + @Override + public String getName() { + return "-verifyClusterSetup"; + } + + @Override + public String getShortUsage() { + return "[" + getName() + "]\n"; + } + + @Override + public String getLongUsage() { + return getShortUsage() + "\n" + + "Verify the cluster setup can support all enabled erasure coding" + + " policies.\n"; + } + + @Override + public int run(Configuration conf, List args) throws IOException { + if (args.size() > 0) { + System.err.println(getName() + ": Too many arguments"); + return 1; + } + final DistributedFileSystem dfs = AdminHelper.getDFS(conf); + final ErasureCodingPolicyInfo[] policies = + dfs.getClient().getNamenode().getErasureCodingPolicies(); + final DatanodeInfo[] report = dfs.getClient().getNamenode() + .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); + + ECTopologyVerifierResult result = ECTopologyVerifier + .getECTopologyVerifierResult(report, policies); + System.out.println(result.getResultMessage()); + if (result.isSupported()) { + return 0; + } + return 2; + } + } + private static final AdminHelper.Command[] COMMANDS = { new ListECPoliciesCommand(), new AddECPoliciesCommand(), @@ -597,6 +645,7 @@ public int run(Configuration conf, List args) throws IOException { new UnsetECPolicyCommand(), new ListECCodecsCommand(), new EnableECPolicyCommand(), - new DisableECPolicyCommand() + new DisableECPolicyCommand(), + new VerifyClusterSetupCommand() }; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 6209312d705e3..b0c9acaaea1e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -2412,6 +2412,41 @@ public static HashSet closeOpenFiles( return closedFiles; } + /** + * Setup cluster with desired number of DN, racks, and specified number of + * rack that only has 1 DN. Other racks will be evenly setup with the number + * of DNs. + * + * @param conf the conf object to start the cluster. + * @param numDatanodes number of total Datanodes. + * @param numRacks number of total racks + * @param numSingleDnRacks number of racks that only has 1 DN + * @throws Exception + */ + public static MiniDFSCluster setupCluster(final Configuration conf, + final int numDatanodes, + final int numRacks, + final int numSingleDnRacks) + throws Exception { + assert numDatanodes > numRacks; + assert numRacks > numSingleDnRacks; + assert numSingleDnRacks >= 0; + final String[] racks = new String[numDatanodes]; + for (int i = 0; i < numSingleDnRacks; i++) { + racks[i] = "/rack" + i; + } + for (int i = numSingleDnRacks; i < numDatanodes; i++) { + racks[i] = + "/rack" + (numSingleDnRacks + (i % (numRacks - numSingleDnRacks))); + } + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(numDatanodes) + .racks(racks) + .build(); + cluster.waitActive(); + return cluster; + } + /** * Check the correctness of the snapshotDiff report. * Make sure all items in the passed entries are in the snapshotDiff diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java index 3e8725316be86..e47cbf0c879e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java @@ -92,23 +92,9 @@ public void setup() { */ public void setupCluster(final int numDatanodes, final int numRacks, final int numSingleDnRacks) throws Exception { - assert numDatanodes > numRacks; - assert numRacks > numSingleDnRacks; - assert numSingleDnRacks >= 0; - final String[] racks = new String[numDatanodes]; - for (int i = 0; i < numSingleDnRacks; i++) { - racks[i] = "/rack" + i; - } - for (int i = numSingleDnRacks; i < numDatanodes; i++) { - racks[i] = - "/rack" + (numSingleDnRacks + (i % (numRacks - numSingleDnRacks))); - } - cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(numDatanodes) - .racks(racks) - .build(); + cluster = DFSTestUtil + .setupCluster(conf, numDatanodes, numRacks, numSingleDnRacks); dfs = cluster.getFileSystem(); - cluster.waitActive(); dfs.setErasureCodingPolicy(new Path("/"), ecPolicy.getName()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index e5ffab08be99a..bfb3c49744bb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -1007,6 +1007,7 @@ void verifyTotalBlocksMetrics(long expectedTotalReplicatedBlocks, expectedTotalReplicatedBlocks, totalReplicaBlocks.longValue()); assertEquals("Unexpected total ec block groups!", expectedTotalECBlockGroups, totalECBlockGroups.longValue()); + verifyEcClusterSetupVerifyResult(mbs); } private String getEnabledEcPoliciesMetric() throws Exception { @@ -1016,4 +1017,22 @@ private String getEnabledEcPoliciesMetric() throws Exception { return (String) (mbs.getAttribute(mxbeanName, "EnabledEcPolicies")); } + + private void verifyEcClusterSetupVerifyResult(MBeanServer mbs) + throws Exception{ + ObjectName namenodeMXBeanName = new ObjectName( + "Hadoop:service=NameNode,name=NameNodeInfo"); + String result = (String) mbs.getAttribute(namenodeMXBeanName, + "VerifyECWithTopologyResult"); + ObjectMapper mapper = new ObjectMapper(); + Map resultMap = mapper.readValue(result, Map.class); + Boolean isSupported = Boolean.parseBoolean(resultMap.get("isSupported")); + String resultMessage = resultMap.get("resultMessage"); + + assertFalse("Test cluster does not support all enabled " + + "erasure coding policies.", isSupported); + assertTrue(resultMessage.contains("The number of racks")); + assertTrue(resultMessage.contains("is less than the minimum required " + + "number of racks")); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java new file mode 100644 index 0000000000000..44fd9c54275d5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests some ECAdmin scenarios that are hard to test from + * {@link org.apache.hadoop.cli.TestErasureCodingCLI}. + */ +public class TestECAdmin { + public static final Logger LOG = LoggerFactory.getLogger(TestECAdmin.class); + private Configuration conf = new Configuration(); + private MiniDFSCluster cluster; + private ECAdmin admin = new ECAdmin(conf); + + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private static final PrintStream OLD_OUT = System.out; + + @Rule + public Timeout globalTimeout = new Timeout(300000); + + @Before + public void setup() throws Exception { + System.setOut(new PrintStream(out)); + } + + @After + public void tearDown() throws Exception { + try { + System.out.flush(); + System.err.flush(); + out.reset(); + } finally { + System.setOut(OLD_OUT); + } + + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Test + public void testRS63MinDN() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 6, 3, 0); + String[] args = {"-verifyClusterSetup"}; + final int ret = admin.run(args); + LOG.info("Commend stdout: {}", out.toString()); + assertEquals(2, ret); + assertTrue(out.toString() + .contains("less than the minimum required number of DataNodes")); + } + + @Test + public void testRS104MinRacks() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 15, 3, 0); + cluster.getFileSystem().enableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_10_4_POLICY_ID).getName()); + String[] args = {"-verifyClusterSetup"}; + final int ret = admin.run(args); + LOG.info("Commend stdout: {}", out.toString()); + assertEquals(2, ret); + assertTrue(out.toString() + .contains("less than the minimum required number of racks")); + } + + @Test + public void testXOR21MinRacks() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0); + cluster.getFileSystem().disableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName()); + cluster.getFileSystem().enableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.XOR_2_1_POLICY_ID).getName()); + String[] args = {"-verifyClusterSetup"}; + final int ret = admin.run(args); + LOG.info("Commend stdout: {}", out.toString()); + assertEquals(2, ret); + assertTrue(out.toString() + .contains("less than the minimum required number of racks")); + } + + @Test + public void testRS32MinRacks() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0); + cluster.getFileSystem().disableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName()); + cluster.getFileSystem().enableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID).getName()); + String[] args = {"-verifyClusterSetup"}; + final int ret = admin.run(args); + LOG.info("Commend stdout: {}", out.toString()); + assertEquals(2, ret); + assertTrue(out.toString() + .contains("less than the minimum required number of racks")); + } + + @Test + public void testRS63Good() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 9, 3, 0); + String[] args = {"-verifyClusterSetup"}; + final int ret = admin.run(args); + LOG.info("Commend stdout: {}", out.toString()); + assertEquals(0, ret); + } + + @Test + public void testNoECEnabled() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 9, 3, 0); + cluster.getFileSystem().disableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName()); + String[] args = {"-verifyClusterSetup"}; + final int ret = admin.run(args); + LOG.info("Commend stdout: {}", out.toString()); + assertEquals(0, ret); + assertTrue(out.toString().contains("No erasure coding policy is enabled")); + } +} From 0dbcb81a41e601294a9b51d6aa2ad48e1f54fba4 Mon Sep 17 00:00:00 2001 From: Kitti Nanasi Date: Wed, 23 Jan 2019 14:39:56 -0800 Subject: [PATCH 2/5] HDFS-14061. Check if the cluster topology supports the EC policy before setting, enabling or adding it. Contributed by Kitti Nanasi. Signed-off-by: Wei-Chiu Chuang (cherry picked from commit 951cdd7e4cbe68284620f6805f85c51301150c58) --- .../server/common/ECTopologyVerifier.java | 59 +++++---- .../namenode/ErasureCodingPolicyManager.java | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 8 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 57 +++++++- .../apache/hadoop/hdfs/tools/TestECAdmin.java | 122 +++++++++++++++--- 5 files changed, 197 insertions(+), 55 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java index 3591b2deedfb1..8744c5480bd1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java @@ -18,13 +18,15 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.namenode.ECTopologyVerifierResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; /** * Class for verifying whether the cluster setup can support @@ -43,58 +45,57 @@ public final class ECTopologyVerifier { private ECTopologyVerifier() {} /** - * Verifies whether the cluster setup can support all enabled EC policies. + * Verifies whether the cluster setup can support the given EC policies. * * @param report list of data node descriptors for all data nodes - * @param policies all system and user defined erasure coding policies + * @param policies erasure coding policies to verify * @return the status of the verification */ public static ECTopologyVerifierResult getECTopologyVerifierResult( - final DatanodeInfo[] report, final ErasureCodingPolicyInfo[] policies) { + final DatanodeInfo[] report, final ErasureCodingPolicy... policies) { final int numOfRacks = getNumberOfRacks(report); - return getECTopologyVerifierResult(policies, numOfRacks, report.length); + return getECTopologyVerifierResult(numOfRacks, report.length, policies); } /** * Verifies whether the cluster setup can support all enabled EC policies. * - * @param policies all system and user defined erasure coding policies + * @param policies erasure coding policies to verify * @param numOfRacks number of racks * @param numOfDataNodes number of data nodes * @return the status of the verification */ public static ECTopologyVerifierResult getECTopologyVerifierResult( - final ErasureCodingPolicyInfo[] policies, final int numOfRacks, - final int numOfDataNodes) { + final int numOfRacks, final int numOfDataNodes, + final ErasureCodingPolicy... policies) { int minDN = 0; int minRack = 0; - for (ErasureCodingPolicyInfo policy: policies) { - if (policy.isEnabled()) { - final int policyDN = - policy.getPolicy().getNumDataUnits() + policy.getPolicy() - .getNumParityUnits(); - minDN = Math.max(minDN, policyDN); - final int policyRack = (int) Math.ceil( - policyDN / (double) policy.getPolicy().getNumParityUnits()); - minRack = Math.max(minRack, policyRack); - } + for (ErasureCodingPolicy policy: policies) { + final int policyDN = + policy.getNumDataUnits() + policy + .getNumParityUnits(); + minDN = Math.max(minDN, policyDN); + final int policyRack = (int) Math.ceil( + policyDN / (double) policy.getNumParityUnits()); + minRack = Math.max(minRack, policyRack); } if (minDN == 0 || minRack == 0) { - String resultMessage = "No erasure coding policy is enabled."; + String resultMessage = "No erasure coding policy is given."; LOG.trace(resultMessage); return new ECTopologyVerifierResult(true, resultMessage); } - return verifyECWithTopology(minDN, minRack, numOfRacks, numOfDataNodes); + return verifyECWithTopology(minDN, minRack, numOfRacks, numOfDataNodes, + getReadablePolicies(policies)); } private static ECTopologyVerifierResult verifyECWithTopology( final int minDN, final int minRack, - final int numOfRacks, final int numOfDataNodes) { + final int numOfRacks, final int numOfDataNodes, String readablePolicies) { String resultMessage; if (numOfDataNodes < minDN) { resultMessage = "The number of DataNodes (" + numOfDataNodes + ") is less than the minimum required number of DataNodes (" - + minDN + ") for enabled erasure coding policy."; + + minDN + ") for the erasure coding policies: " + readablePolicies; LOG.debug(resultMessage); return new ECTopologyVerifierResult(false, resultMessage); } @@ -102,12 +103,14 @@ private static ECTopologyVerifierResult verifyECWithTopology( if (numOfRacks < minRack) { resultMessage = "The number of racks (" + numOfRacks + ") is less than the minimum required number of racks (" - + minRack + ") for enabled erasure coding policy."; + + minRack + ") for the erasure coding policies: " + + readablePolicies; LOG.debug(resultMessage); return new ECTopologyVerifierResult(false, resultMessage); } return new ECTopologyVerifierResult(true, - "The cluster setup can support all enabled EC policies"); + "The cluster setup can support EC policies: " + + readablePolicies); } private static int getNumberOfRacks(DatanodeInfo[] report) { @@ -121,4 +124,12 @@ private static int getNumberOfRacks(DatanodeInfo[] report) { } return racks.size(); } + + private static String getReadablePolicies( + final ErasureCodingPolicy... policies) { + return Arrays.asList(policies) + .stream() + .map(policyInfo -> policyInfo.getName()) + .collect(Collectors.joining(", ")); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index d3984e0a75edf..d9f7e9afdc70d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -213,10 +213,10 @@ public ErasureCodingPolicyInfo[] getPersistedPolicies() { .toArray(new ErasureCodingPolicyInfo[0]); } - public ErasureCodingPolicyInfo[] getCopyOfPolicies() { - ErasureCodingPolicyInfo[] copy; + public ErasureCodingPolicy[] getCopyOfEnabledPolicies() { + ErasureCodingPolicy[] copy; synchronized (this) { - copy = Arrays.copyOf(allPolicies, allPolicies.length); + copy = Arrays.copyOf(enabledPolicies, enabledPolicies.length); } return copy; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 5d1b35bfbcd65..ff8866344a64d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -8198,11 +8198,11 @@ public String getVerifyECWithTopologyResult() { .getNumOfDataNodes(); int numOfRacks = getBlockManager().getDatanodeManager() .getNetworkTopology().getNumOfRacks(); - ErasureCodingPolicyInfo[] ecPolicies = - getErasureCodingPolicyManager().getCopyOfPolicies(); + ErasureCodingPolicy[] enabledEcPolicies = + getErasureCodingPolicyManager().getCopyOfEnabledPolicies(); ECTopologyVerifierResult result = - ECTopologyVerifier.getECTopologyVerifierResult(ecPolicies, - numOfRacks, numOfDataNodes); + ECTopologyVerifier.getECTopologyVerifierResult( + numOfRacks, numOfDataNodes, enabledEcPolicies); Map resultMap = new HashMap(); resultMap.put("isSupported", Boolean.toString(result.isSupported())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index d63d9dd5f65d6..6dccccb122c4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.hdfs.tools; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -536,6 +537,13 @@ public int run(Configuration conf, List args) throws IOException { dfs.enableErasureCodingPolicy(ecPolicyName); System.out.println("Erasure coding policy " + ecPolicyName + " is enabled"); + ECTopologyVerifierResult result = + getECTopologyVerifierResultForPolicy(dfs, ecPolicyName); + if (!result.isSupported()) { + System.err.println("Warning: The cluster setup does not support " + + "EC policy " + ecPolicyName + ". Reason: " + + result.getResultMessage()); + } } catch (IOException e) { System.err.println(AdminHelper.prettifyException(e)); return 2; @@ -621,13 +629,7 @@ public int run(Configuration conf, List args) throws IOException { return 1; } final DistributedFileSystem dfs = AdminHelper.getDFS(conf); - final ErasureCodingPolicyInfo[] policies = - dfs.getClient().getNamenode().getErasureCodingPolicies(); - final DatanodeInfo[] report = dfs.getClient().getNamenode() - .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); - - ECTopologyVerifierResult result = ECTopologyVerifier - .getECTopologyVerifierResult(report, policies); + ECTopologyVerifierResult result = getECTopologyVerifierResult(dfs); System.out.println(result.getResultMessage()); if (result.isSupported()) { return 0; @@ -636,6 +638,47 @@ public int run(Configuration conf, List args) throws IOException { } } + private static ECTopologyVerifierResult getECTopologyVerifierResult( + final DistributedFileSystem dfs) throws IOException { + final ErasureCodingPolicyInfo[] policies = + dfs.getClient().getNamenode().getErasureCodingPolicies(); + final DatanodeInfo[] report = dfs.getClient().getNamenode() + .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); + + return ECTopologyVerifier.getECTopologyVerifierResult(report, + getEnabledPolicies(policies)); + } + + private static ECTopologyVerifierResult getECTopologyVerifierResultForPolicy( + final DistributedFileSystem dfs, final String policyName) + throws IOException { + final ErasureCodingPolicy policy = + getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(), + policyName); + final DatanodeInfo[] report = dfs.getClient().getNamenode() + .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); + return ECTopologyVerifier.getECTopologyVerifierResult(report, policy); + } + + private static ErasureCodingPolicy getPolicy( + final ErasureCodingPolicyInfo[] policies, final String policyName) { + for (ErasureCodingPolicyInfo policy : policies) { + if (policyName.equals(policy.getPolicy().getName())) { + return policy.getPolicy(); + } + } + throw new HadoopIllegalArgumentException("The given erasure coding " + + "policy " + policyName + " does not exist."); + } + + private static ErasureCodingPolicy[] getEnabledPolicies( + final ErasureCodingPolicyInfo[] policies) { + return Arrays.asList(policies).stream() + .filter(policyInfo -> policyInfo.isEnabled()) + .map(ErasureCodingPolicyInfo::getPolicy) + .toArray(ErasureCodingPolicy[]::new); + } + private static final AdminHelper.Command[] COMMANDS = { new ListECPoliciesCommand(), new AddECPoliciesCommand(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java index 44fd9c54275d5..a8665d1094b66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java @@ -33,6 +33,7 @@ import java.io.PrintStream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -46,7 +47,10 @@ public class TestECAdmin { private ECAdmin admin = new ECAdmin(conf); private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private final ByteArrayOutputStream err = new ByteArrayOutputStream(); + private static final PrintStream OLD_OUT = System.out; + private static final PrintStream OLD_ERR = System.err; @Rule public Timeout globalTimeout = new Timeout(300000); @@ -54,6 +58,7 @@ public class TestECAdmin { @Before public void setup() throws Exception { System.setOut(new PrintStream(out)); + System.setErr(new PrintStream(err)); } @After @@ -62,8 +67,10 @@ public void tearDown() throws Exception { System.out.flush(); System.err.flush(); out.reset(); + err.reset(); } finally { System.setOut(OLD_OUT); + System.setErr(OLD_ERR); } if (cluster != null) { @@ -77,10 +84,13 @@ public void testRS63MinDN() throws Exception { cluster = DFSTestUtil.setupCluster(conf, 6, 3, 0); String[] args = {"-verifyClusterSetup"}; final int ret = admin.run(args); - LOG.info("Commend stdout: {}", out.toString()); - assertEquals(2, ret); - assertTrue(out.toString() + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() .contains("less than the minimum required number of DataNodes")); + assertTrue("Error output should be empty", err.toString().isEmpty()); } @Test @@ -91,10 +101,13 @@ public void testRS104MinRacks() throws Exception { .getByID(SystemErasureCodingPolicies.RS_10_4_POLICY_ID).getName()); String[] args = {"-verifyClusterSetup"}; final int ret = admin.run(args); - LOG.info("Commend stdout: {}", out.toString()); - assertEquals(2, ret); - assertTrue(out.toString() + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() .contains("less than the minimum required number of racks")); + assertTrue("Error output should be empty", err.toString().isEmpty()); } @Test @@ -108,10 +121,13 @@ public void testXOR21MinRacks() throws Exception { .getByID(SystemErasureCodingPolicies.XOR_2_1_POLICY_ID).getName()); String[] args = {"-verifyClusterSetup"}; final int ret = admin.run(args); - LOG.info("Commend stdout: {}", out.toString()); - assertEquals(2, ret); - assertTrue(out.toString() + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() .contains("less than the minimum required number of racks")); + assertTrue("Error output should be empty", err.toString().isEmpty()); } @Test @@ -125,10 +141,13 @@ public void testRS32MinRacks() throws Exception { .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID).getName()); String[] args = {"-verifyClusterSetup"}; final int ret = admin.run(args); - LOG.info("Commend stdout: {}", out.toString()); - assertEquals(2, ret); - assertTrue(out.toString() + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() .contains("less than the minimum required number of racks")); + assertTrue("Error output should be empty", err.toString().isEmpty()); } @Test @@ -136,8 +155,13 @@ public void testRS63Good() throws Exception { cluster = DFSTestUtil.setupCluster(conf, 9, 3, 0); String[] args = {"-verifyClusterSetup"}; final int ret = admin.run(args); - LOG.info("Commend stdout: {}", out.toString()); - assertEquals(0, ret); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is successful", 0, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString().contains( + "The cluster setup can support EC policies: RS-6-3-1024k")); + assertTrue("Error output should be empty", err.toString().isEmpty()); } @Test @@ -148,8 +172,72 @@ public void testNoECEnabled() throws Exception { .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName()); String[] args = {"-verifyClusterSetup"}; final int ret = admin.run(args); - LOG.info("Commend stdout: {}", out.toString()); - assertEquals(0, ret); - assertTrue(out.toString().contains("No erasure coding policy is enabled")); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is successful", 0, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", + out.toString().contains("No erasure coding policy is given")); + assertTrue("Error output should be empty", err.toString().isEmpty()); + } + + @Test + public void testUnsuccessfulEnablePolicyMessage() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0); + cluster.getFileSystem().disableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName()); + String[] args = {"-enablePolicy", "-policy", "RS-3-2-1024k"}; + + final int ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is successful", 0, ret); + assertTrue("Enabling policy should be logged", out.toString() + .contains("Erasure coding policy RS-3-2-1024k is enabled")); + assertTrue("Warning about cluster topology should be printed", + err.toString().contains("Warning: The cluster setup does not support " + + "EC policy RS-3-2-1024k. Reason:")); + assertTrue("Warning about cluster topology should be printed", + err.toString() + .contains("less than the minimum required number of racks")); + } + + @Test + public void testSuccessfulEnablePolicyMessage() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 5, 3, 0); + cluster.getFileSystem().disableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName()); + String[] args = {"-enablePolicy", "-policy", "RS-3-2-1024k"}; + + final int ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is successful", 0, ret); + assertTrue("Enabling policy should be logged", out.toString() + .contains("Erasure coding policy RS-3-2-1024k is enabled")); + assertFalse("Warning about cluster topology should not be printed", + out.toString().contains("Warning: The cluster setup does not support")); + assertTrue("Error output should be empty", err.toString().isEmpty()); + } + + @Test + public void testEnableNonExistentPolicyMessage() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 5, 3, 0); + cluster.getFileSystem().disableErasureCodingPolicy( + SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName()); + String[] args = {"-enablePolicy", "-policy", "NonExistentPolicy"}; + + final int ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is unsuccessful", 2, ret); + assertFalse("Enabling policy should not be logged when " + + "it was unsuccessful", out.toString().contains("is enabled")); + assertTrue("Error message should be printed", + err.toString().contains("RemoteException: The policy name " + + "NonExistentPolicy does not exist")); } } From 3ba40d0ccffa81dbe868659095be461ed40f7ade Mon Sep 17 00:00:00 2001 From: Kitti Nanasi Date: Mon, 4 Feb 2019 14:51:17 -0800 Subject: [PATCH 3/5] HDFS-14125. Use parameterized log format in ECTopologyVerifier. Contributed by Kitti Nanasi. Signed-off-by: Wei-Chiu Chuang (cherry picked from commit 5f15a60e329b8d0a495a977dc5255126814c5271) --- .../hdfs/server/common/ECTopologyVerifier.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java index 8744c5480bd1e..66c7c4ed2b191 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java @@ -93,24 +93,24 @@ private static ECTopologyVerifierResult verifyECWithTopology( final int numOfRacks, final int numOfDataNodes, String readablePolicies) { String resultMessage; if (numOfDataNodes < minDN) { - resultMessage = "The number of DataNodes (" + numOfDataNodes - + ") is less than the minimum required number of DataNodes (" - + minDN + ") for the erasure coding policies: " + readablePolicies; + resultMessage = String.format("The number of DataNodes (%d) is less " + + "than the minimum required number of DataNodes (%d) for the " + + "erasure coding policies: %s", numOfDataNodes, minDN, + readablePolicies); LOG.debug(resultMessage); return new ECTopologyVerifierResult(false, resultMessage); } if (numOfRacks < minRack) { - resultMessage = "The number of racks (" + numOfRacks - + ") is less than the minimum required number of racks (" - + minRack + ") for the erasure coding policies: " - + readablePolicies; + resultMessage = String.format("The number of racks (%d) is less than " + + "the minimum required number of racks (%d) for the erasure " + + "coding policies: %s", numOfRacks, minRack, readablePolicies); LOG.debug(resultMessage); return new ECTopologyVerifierResult(false, resultMessage); } return new ECTopologyVerifierResult(true, - "The cluster setup can support EC policies: " - + readablePolicies); + String.format("The cluster setup can support EC policies: %s", + readablePolicies)); } private static int getNumberOfRacks(DatanodeInfo[] report) { From 9b8eaa057261fd83ec2fdd6c8ac2592dd7bbb893 Mon Sep 17 00:00:00 2001 From: Kitti Nanasi Date: Tue, 19 Feb 2019 12:04:56 -0800 Subject: [PATCH 4/5] HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter. Contributed by Kitti Nanasi. Signed-off-by: Wei-Chiu Chuang (cherry picked from commit 14282e311be6ffcaddd2f74fa8e67c4e98a32291) --- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 40 ++++++++++--- .../src/site/markdown/HDFSCommands.md | 2 + .../src/site/markdown/HDFSErasureCoding.md | 5 ++ .../apache/hadoop/hdfs/tools/TestECAdmin.java | 58 ++++++++++++++++++- 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 6dccccb122c4f..80d1b6f40170a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.shell.CommandFormat; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -612,24 +613,32 @@ public String getName() { @Override public String getShortUsage() { - return "[" + getName() + "]\n"; + return "[" + getName() + " [-policy ...]]\n"; } @Override public String getLongUsage() { + TableListing listing = AdminHelper.getOptionDescriptionListing(); + listing.addRow("", "The name of the erasure coding policy"); return getShortUsage() + "\n" - + "Verify the cluster setup can support all enabled erasure coding" - + " policies.\n"; + + "Verify if the cluster setup can support all enabled erasure " + + "coding policies. If optional parameter -policy is specified, " + + "verify if the cluster setup can support the given policy.\n"; } @Override public int run(Configuration conf, List args) throws IOException { - if (args.size() > 0) { - System.err.println(getName() + ": Too many arguments"); - return 1; - } + boolean isPolicyOption = StringUtils.popOption("-policy", args); final DistributedFileSystem dfs = AdminHelper.getDFS(conf); - ECTopologyVerifierResult result = getECTopologyVerifierResult(dfs); + ECTopologyVerifierResult result; + if (isPolicyOption) { + CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE); + c.parse(args); + String[] parameters = args.toArray(new String[args.size()]); + result = getECTopologyResultForPolicies(dfs, parameters); + } else { + result = getECTopologyVerifierResult(dfs); + } System.out.println(result.getResultMessage()); if (result.isSupported()) { return 0; @@ -649,6 +658,21 @@ private static ECTopologyVerifierResult getECTopologyVerifierResult( getEnabledPolicies(policies)); } + private static ECTopologyVerifierResult getECTopologyResultForPolicies( + final DistributedFileSystem dfs, final String... policyNames) + throws IOException { + ErasureCodingPolicy[] policies = + new ErasureCodingPolicy[policyNames.length]; + for (int i = 0; i < policyNames.length; i++) { + policies[i] = + getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(), + policyNames[i]); + } + final DatanodeInfo[] report = dfs.getClient().getNamenode() + .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); + return ECTopologyVerifier.getECTopologyVerifierResult(report, policies); + } + private static ECTopologyVerifierResult getECTopologyVerifierResultForPolicy( final DistributedFileSystem dfs, final String policyName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 32e1a7b63d626..1f4c08866b0b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -487,6 +487,7 @@ Usage: [-listCodecs] [-enablePolicy -policy ] [-disablePolicy -policy ] + [-verifyClusterSetup -policy ...] [-help [cmd ...]] | COMMAND\_OPTION | Description | @@ -499,6 +500,7 @@ Usage: |-listCodecs| Get the list of supported erasure coding codecs and coders in system| |-enablePolicy| Enable an ErasureCoding policy in system| |-disablePolicy| Disable an ErasureCoding policy in system| +|-verifyClusterSetup| Verify if the cluster setup can support a list of erasure coding policies| Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands) for more information on this command. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 67e6b750a29ce..3a8b611a6f2c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -170,6 +170,7 @@ Deployment [-listCodecs] [-enablePolicy -policy ] [-disablePolicy -policy ] + [-verifyClusterSetup -policy ...] [-help [cmd ...]] Below are the details about each command. @@ -221,6 +222,10 @@ Below are the details about each command. Disable an erasure coding policy. +* `[-verifyClusterSetup -policy ...]` + + Verify if the cluster setup can support all enabled erasure coding policies. If optional parameter -policy is specified, verify if the cluster setup can support the given policy or policies. + Limitations ----------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java index a8665d1094b66..4da2d16c258c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java @@ -66,8 +66,7 @@ public void tearDown() throws Exception { try { System.out.flush(); System.err.flush(); - out.reset(); - err.reset(); + resetOutputs(); } finally { System.setOut(OLD_OUT); System.setErr(OLD_ERR); @@ -240,4 +239,59 @@ public void testEnableNonExistentPolicyMessage() throws Exception { err.toString().contains("RemoteException: The policy name " + "NonExistentPolicy does not exist")); } + + @Test + public void testVerifyClusterSetupWithGivenPolicies() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0); + + String[] args = new String[]{"-verifyClusterSetup", "-policy", + "RS-3-2-1024k"}; + int ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() + .contains("less than the minimum required number of racks (3) " + + "for the erasure coding policies: RS-3-2-1024k")); + assertTrue("Error output should be empty", err.toString().isEmpty()); + + resetOutputs(); + args = new String[]{"-verifyClusterSetup", "-policy", + "RS-10-4-1024k", "RS-3-2-1024k"}; + ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() + .contains( + "for the erasure coding policies: RS-10-4-1024k, RS-3-2-1024k")); + assertTrue("Error output should be empty", err.toString().isEmpty()); + + resetOutputs(); + args = new String[]{"-verifyClusterSetup", "-policy", "invalidPolicy"}; + ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", -1, ret); + assertTrue("Error message should be logged", err.toString() + .contains("The given erasure coding policy invalidPolicy " + + "does not exist.")); + + resetOutputs(); + args = new String[]{"-verifyClusterSetup", "-policy"}; + ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", -1, ret); + assertTrue("Error message should be logged", err.toString() + .contains("NotEnoughArgumentsException: Not enough arguments: " + + "expected 1 but got 0")); + } + + private void resetOutputs() { + out.reset(); + err.reset(); + } } From 544d6da3ab01789a7224e0f796a6caf5e0f51a4a Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Sun, 30 Aug 2020 10:19:26 +0530 Subject: [PATCH 5/5] HDFS-15117. EC: Add getECTopologyResultForPolicies to DistributedFileSystem. Contributed by Ayush Saxena. (#2261) --- .../org/apache/hadoop/hdfs/DFSClient.java | 12 ++ .../hadoop/hdfs/DistributedFileSystem.java | 13 ++ .../hadoop/hdfs/protocol/ClientProtocol.java | 12 ++ .../protocol}/ECTopologyVerifierResult.java | 2 +- .../ClientNamenodeProtocolTranslatorPB.java | 25 +++- .../hdfs/protocolPB/PBHelperClient.java | 16 +++ .../main/proto/ClientNamenodeProtocol.proto | 2 + .../src/main/proto/erasurecoding.proto | 8 ++ .../src/main/proto/hdfs.proto | 5 + .../hadoop/hdfs/protocol/TestReadOnly.java | 3 +- .../federation/router/ErasureCoding.java | 22 +++ .../router/RouterClientProtocol.java | 8 ++ .../federation/router/RouterRpcServer.java | 7 + .../federation/MiniRouterDFSCluster.java | 12 ++ .../router/TestRouterMultiRack.java | 129 ++++++++++++++++++ ...amenodeProtocolServerSideTranslatorPB.java | 21 +++ .../server/common/ECTopologyVerifier.java | 19 ++- .../server/namenode/FSDirErasureCodingOp.java | 27 +++- .../hdfs/server/namenode/FSNamesystem.java | 64 +++++++-- .../server/namenode/NameNodeRpcServer.java | 7 + .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 87 +++--------- .../hdfs/TestDistributedFileSystem.java | 32 +++++ 22 files changed, 438 insertions(+), 95 deletions(-) rename hadoop-hdfs-project/{hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode => hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol}/ECTopologyVerifierResult.java (96%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMultiRack.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 187143e7bcf36..c3a00c923ec67 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -116,6 +116,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; @@ -2770,6 +2771,17 @@ public void unsetErasureCodingPolicy(String src) throws IOException { } } + public ECTopologyVerifierResult getECTopologyResultForPolicies( + final String... policyNames) throws IOException { + checkOpen(); + try { + return namenode.getECTopologyResultForPolicies(policyNames); + } catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + SafeModeException.class); + } + } + public void setXAttr(String src, String name, byte[] value, EnumSet flag) throws IOException { checkOpen(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 7a218bf09fd21..b04004066ad8c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -80,6 +80,7 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; @@ -3096,6 +3097,18 @@ public Void next(final FileSystem fs, final Path p) throws IOException { }.resolve(this, absF); } + /** + * Verifies if the given policies are supported in the given cluster setup. + * If not policy is specified checks for all enabled policies. + * @param policyNames name of policies. + * @return the result if the given policies are supported in the cluster setup + * @throws IOException + */ + public ECTopologyVerifierResult getECTopologyResultForPolicies( + final String... policyNames) throws IOException { + return dfs.getECTopologyResultForPolicies(policyNames); + } + /** * Get the root directory of Trash for a path in HDFS. * 1. File in encryption zone returns /ez1/.Trash/username diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 953e48a932c5e..dae77a4ea8f11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1741,6 +1741,18 @@ AddErasureCodingPolicyResponse[] addErasureCodingPolicies( @AtMostOnce void unsetErasureCodingPolicy(String src) throws IOException; + /** + * Verifies if the given policies are supported in the given cluster setup. + * If not policy is specified checks for all enabled policies. + * @param policyNames name of policies. + * @return the result if the given policies are supported in the cluster setup + * @throws IOException + */ + @Idempotent + @ReadOnly + ECTopologyVerifierResult getECTopologyResultForPolicies(String... policyNames) + throws IOException; + /** * Get {@link QuotaUsage} rooted at the specified directory. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECTopologyVerifierResult.java similarity index 96% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECTopologyVerifierResult.java index 4757eabbaded3..159688c28e149 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECTopologyVerifierResult.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs.server.namenode; +package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 65ebc2cc89736..af4abb0c0792d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; @@ -216,6 +217,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.CodecProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECTopologyResultForPoliciesRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECTopologyResultForPoliciesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto; @@ -1611,10 +1614,9 @@ public void setErasureCodingPolicy(String src, String ecPolicyName) } @Override - public void unsetErasureCodingPolicy(String src) - throws IOException { + public void unsetErasureCodingPolicy(String src) throws IOException { final UnsetErasureCodingPolicyRequestProto.Builder builder = - ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto.newBuilder(); + UnsetErasureCodingPolicyRequestProto.newBuilder(); builder.setSrc(src); UnsetErasureCodingPolicyRequestProto req = builder.build(); try { @@ -1624,6 +1626,23 @@ public void unsetErasureCodingPolicy(String src) } } + @Override + public ECTopologyVerifierResult getECTopologyResultForPolicies( + final String... policyNames) throws IOException { + final GetECTopologyResultForPoliciesRequestProto.Builder builder = + GetECTopologyResultForPoliciesRequestProto.newBuilder(); + builder.addAllPolicies(Arrays.asList(policyNames)); + GetECTopologyResultForPoliciesRequestProto req = builder.build(); + try { + GetECTopologyResultForPoliciesResponseProto response = + rpcProxy.getECTopologyResultForPolicies(null, req); + return PBHelperClient + .convertECTopologyVerifierResultProto(response.getResponse()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public void reencryptEncryptionZone(String zone, ReencryptAction action) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 3d43c97474a7a..cf1a92fc5b461 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -84,6 +84,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; @@ -3337,6 +3338,21 @@ public static HdfsProtos.DatanodeInfosProto convertToProto( return builder.build(); } + public static ECTopologyVerifierResult convertECTopologyVerifierResultProto( + HdfsProtos.ECTopologyVerifierResultProto resp) { + return new ECTopologyVerifierResult(resp.getIsSupported(), + resp.getResultMessage()); + } + + public static HdfsProtos.ECTopologyVerifierResultProto convertECTopologyVerifierResult( + ECTopologyVerifierResult resp) { + final HdfsProtos.ECTopologyVerifierResultProto.Builder builder = + HdfsProtos.ECTopologyVerifierResultProto.newBuilder() + .setIsSupported(resp.isSupported()) + .setResultMessage(resp.getResultMessage()); + return builder.build(); + } + public static EnumSet convertAddBlockFlags( List addBlockFlags) { EnumSet flags = diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index ce78d1f64fe42..f353c033a5039 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -1016,6 +1016,8 @@ service ClientNamenodeProtocol { returns(SetErasureCodingPolicyResponseProto); rpc unsetErasureCodingPolicy(UnsetErasureCodingPolicyRequestProto) returns(UnsetErasureCodingPolicyResponseProto); + rpc getECTopologyResultForPolicies(GetECTopologyResultForPoliciesRequestProto) + returns(GetECTopologyResultForPoliciesResponseProto); rpc getCurrentEditLogTxid(GetCurrentEditLogTxidRequestProto) returns(GetCurrentEditLogTxidResponseProto); rpc getEditsFromTxid(GetEditsFromTxidRequestProto) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto index de3bf4a9d8dbd..d92dd4cb84c97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto @@ -89,6 +89,14 @@ message UnsetErasureCodingPolicyRequestProto { message UnsetErasureCodingPolicyResponseProto { } +message GetECTopologyResultForPoliciesRequestProto { + repeated string policies = 1; +} + +message GetECTopologyResultForPoliciesResponseProto { + required ECTopologyVerifierResultProto response = 1; +} + /** * Block erasure coding reconstruction info */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 1f0e179f20c70..38459349ad462 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -412,6 +412,11 @@ message AddErasureCodingPolicyResponseProto { optional string errorMsg = 3; } +message ECTopologyVerifierResultProto { + required string resultMessage = 1; + required bool isSupported = 2; +} + /** * Placeholder type for consistent HDFS operations. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java index e0432f5e7eee6..3454db9308b28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java @@ -73,7 +73,8 @@ public class TestReadOnly { "getEditsFromTxid", "getQuotaUsage", "msync", - "getHAServiceState" + "getHAServiceState", + "getECTopologyResultForPolicies" ) ); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java index 480b232ca422b..5e6fa27ca961a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; @@ -170,6 +171,27 @@ public void unsetErasureCodingPolicy(String src) throws IOException { rpcClient.invokeSequential(locations, remoteMethod, null, null); } + public ECTopologyVerifierResult getECTopologyResultForPolicies( + String[] policyNames) throws IOException { + RemoteMethod method = new RemoteMethod("getECTopologyResultForPolicies", + new Class[] {String[].class}, new Object[] {policyNames}); + Set nss = namenodeResolver.getNamespaces(); + if (nss.isEmpty()) { + throw new IOException("No namespace availaible."); + } + Map ret = rpcClient + .invokeConcurrent(nss, method, true, false, + ECTopologyVerifierResult.class); + for (Map.Entry entry : ret + .entrySet()) { + if (!entry.getValue().isSupported()) { + return entry.getValue(); + } + } + // If no negative result, return the result from the first namespace. + return ret.get(nss.iterator().next()); + } + public ECBlockGroupStats getECBlockGroupStats() throws IOException { rpcServer.checkOperation(OperationCategory.READ); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index 57f0584487458..8455f4b1a046b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; @@ -1507,6 +1508,13 @@ public void unsetErasureCodingPolicy(String src) throws IOException { erasureCoding.unsetErasureCodingPolicy(src); } + @Override + public ECTopologyVerifierResult getECTopologyResultForPolicies( + String... policyNames) throws IOException { + rpcServer.checkOperation(NameNode.OperationCategory.UNCHECKED, true); + return erasureCoding.getECTopologyResultForPolicies(policyNames); + } + @Override public ECBlockGroupStats getECBlockGroupStats() throws IOException { return erasureCoding.getECBlockGroupStats(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index c7b4b5819b396..ad9b18739dbae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; @@ -1169,6 +1170,12 @@ public void unsetErasureCodingPolicy(String src) throws IOException { clientProto.unsetErasureCodingPolicy(src); } + @Override + public ECTopologyVerifierResult getECTopologyResultForPolicies( + String... policyNames) throws IOException { + return clientProto.getECTopologyResultForPolicies(policyNames); + } + @Override // ClientProtocol public ECBlockGroupStats getECBlockGroupStats() throws IOException { return clientProto.getECBlockGroupStats(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index e34713d665afe..7b59e3c5bea9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -570,6 +570,17 @@ public void setNumDatanodesPerNameservice(int num) { this.numDatanodesPerNameservice = num; } + /** Racks for datanodes. */ + private String[] racks = null; + + /** + * Set racks for each datanode. If racks is uninitialized or passed null then + * default is used. + */ + public void setRacks(String[] racks) { + this.racks = racks; + } + /** * Set the DNs to belong to only one subcluster. */ @@ -723,6 +734,7 @@ public void startCluster(Configuration overrideConf) { .numDataNodes(numDNs) .nnTopology(topology) .dataNodeConfOverlays(dnConfs) + .racks(racks) .build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMultiRack.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMultiRack.java new file mode 100644 index 0000000000000..540a1230e5fbc --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMultiRack.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.*; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test class with clusters having multiple racks. + */ +public class TestRouterMultiRack { + private static StateStoreDFSCluster cluster; + private static RouterContext routerContext; + private static DistributedFileSystem routerFs; + private static NamenodeContext nnContext0; + private static NamenodeContext nnContext1; + private static DistributedFileSystem nnFs0; + private static DistributedFileSystem nnFs1; + + @BeforeClass + public static void setUp() throws Exception { + + // Build and start a federated cluster + cluster = new StateStoreDFSCluster(false, 2, + MultipleDestinationMountTableResolver.class); + Configuration routerConf = + new RouterConfigBuilder().stateStore().admin().quota().rpc().build(); + Configuration hdfsConf = new Configuration(false); + cluster.addNamenodeOverrides(hdfsConf); + cluster.addRouterOverrides(routerConf); + cluster.setNumDatanodesPerNameservice(9); + cluster.setIndependentDNs(); + cluster.setRacks( + new String[] {"/rack1", "/rack1", "/rack1", "/rack2", "/rack2", + "/rack2", "/rack3", "/rack3", "/rack3", "/rack4", "/rack4", + "/rack4", "/rack5", "/rack5", "/rack5", "/rack6", "/rack6", + "/rack6"}); + cluster.startCluster(); + cluster.startRouters(); + cluster.waitClusterUp(); + + routerContext = cluster.getRandomRouter(); + routerFs = (DistributedFileSystem) routerContext.getFileSystem(); + nnContext0 = cluster.getNamenode("ns0", null); + nnContext1 = cluster.getNamenode("ns1", null); + nnFs0 = (DistributedFileSystem) nnContext0.getFileSystem(); + nnFs1 = (DistributedFileSystem) nnContext1.getFileSystem(); + } + + @AfterClass + public static void tearDown() { + if (cluster != null) { + cluster.stopRouter(routerContext); + cluster.shutdown(); + cluster = null; + } + } + + @Test + public void testGetECTopologyResultForPolicies() throws IOException { + routerFs.enableErasureCodingPolicy("RS-6-3-1024k"); + // No policies specified should return result for the enabled policy. + ECTopologyVerifierResult result = routerFs.getECTopologyResultForPolicies(); + assertTrue(result.isSupported()); + // Specified policy requiring more datanodes than present in + // the actual cluster. + result = routerFs.getECTopologyResultForPolicies("RS-10-4-1024k"); + assertFalse(result.isSupported()); + // Specify multiple policies with one policy requiring more datanodes than + // present in the actual cluster + result = routerFs + .getECTopologyResultForPolicies("RS-10-4-1024k", "RS-3-2-1024k"); + assertFalse(result.isSupported()); + // Specify multiple policies that require datanodes equal or less then + // present in the actual cluster + result = routerFs + .getECTopologyResultForPolicies("XOR-2-1-1024k", "RS-3-2-1024k"); + assertTrue(result.isSupported()); + // Specify multiple policies with one policy requiring more datanodes than + // present in the actual cluster + result = routerFs + .getECTopologyResultForPolicies("RS-10-4-1024k", "RS-3-2-1024k"); + assertFalse(result.isSupported()); + // Enable a policy requiring more datanodes than present in + // the actual cluster. + routerFs.enableErasureCodingPolicy("RS-10-4-1024k"); + result = routerFs.getECTopologyResultForPolicies(); + assertFalse(result.isSupported()); + // Check without specifying any policy, with one cluster having + // all supported, but one cluster having one unsupported policy. The + nnFs0.disableErasureCodingPolicy("RS-10-4-1024k"); + nnFs1.enableErasureCodingPolicy("RS-10-4-1024k"); + result = routerFs.getECTopologyResultForPolicies(); + assertFalse(result.isSupported()); + nnFs1.disableErasureCodingPolicy("RS-10-4-1024k"); + nnFs0.enableErasureCodingPolicy("RS-10-4-1024k"); + result = routerFs.getECTopologyResultForPolicies(); + assertFalse(result.isSupported()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index abeec1c60ccf6..c6871d6976be0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; @@ -262,6 +263,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECTopologyResultForPoliciesRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECTopologyResultForPoliciesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.*; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; @@ -1621,6 +1624,24 @@ public UnsetErasureCodingPolicyResponseProto unsetErasureCodingPolicy( } } + @Override + public GetECTopologyResultForPoliciesResponseProto getECTopologyResultForPolicies( + RpcController controller, GetECTopologyResultForPoliciesRequestProto req) + throws ServiceException { + try { + List policies = req.getPoliciesList(); + ECTopologyVerifierResult result = server.getECTopologyResultForPolicies( + policies.toArray(policies.toArray(new String[policies.size()]))); + GetECTopologyResultForPoliciesResponseProto.Builder builder = + GetECTopologyResultForPoliciesResponseProto.newBuilder(); + builder + .setResponse(PBHelperClient.convertECTopologyVerifierResult(result)); + return builder.build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + @Override public SetXAttrResponseProto setXAttr(RpcController controller, SetXAttrRequestProto req) throws ServiceException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java index 66c7c4ed2b191..621ebff557a78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java @@ -18,12 +18,12 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.server.namenode.ECTopologyVerifierResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -52,7 +52,8 @@ private ECTopologyVerifier() {} * @return the status of the verification */ public static ECTopologyVerifierResult getECTopologyVerifierResult( - final DatanodeInfo[] report, final ErasureCodingPolicy... policies) { + final DatanodeInfo[] report, + final Collection policies) { final int numOfRacks = getNumberOfRacks(report); return getECTopologyVerifierResult(numOfRacks, report.length, policies); } @@ -60,14 +61,14 @@ public static ECTopologyVerifierResult getECTopologyVerifierResult( /** * Verifies whether the cluster setup can support all enabled EC policies. * - * @param policies erasure coding policies to verify * @param numOfRacks number of racks * @param numOfDataNodes number of data nodes + * @param policies erasure coding policies to verify * @return the status of the verification */ public static ECTopologyVerifierResult getECTopologyVerifierResult( final int numOfRacks, final int numOfDataNodes, - final ErasureCodingPolicy... policies) { + final Collection policies) { int minDN = 0; int minRack = 0; for (ErasureCodingPolicy policy: policies) { @@ -126,10 +127,8 @@ private static int getNumberOfRacks(DatanodeInfo[] report) { } private static String getReadablePolicies( - final ErasureCodingPolicy... policies) { - return Arrays.asList(policies) - .stream() - .map(policyInfo -> policyInfo.getName()) - .collect(Collectors.joining(", ")); + final Collection policies) { + return policies.stream().map(policyInfo -> policyInfo.getName()) + .collect(Collectors.joining(", ")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index b0bc5e40ebe06..011c72ea49c6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -68,7 +68,7 @@ private FSDirErasureCodingOp() {} * @return an erasure coding policy if ecPolicyName is valid and enabled * @throws IOException */ - static ErasureCodingPolicy getErasureCodingPolicyByName( + static ErasureCodingPolicy getEnabledErasureCodingPolicyByName( final FSNamesystem fsn, final String ecPolicyName) throws IOException { assert fsn.hasReadLock(); ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager() @@ -92,6 +92,27 @@ static ErasureCodingPolicy getErasureCodingPolicyByName( return ecPolicy; } + /** + * Check if the ecPolicyName is valid, return the corresponding + * EC policy if is, including the REPLICATION EC policy. + * @param fsn namespace + * @param ecPolicyName name of EC policy to be checked + * @return an erasure coding policy if ecPolicyName is valid + * @throws IOException + */ + static ErasureCodingPolicy getErasureCodingPolicyByName( + final FSNamesystem fsn, final String ecPolicyName) throws IOException { + assert fsn.hasReadLock(); + ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager() + .getErasureCodingPolicyByName(ecPolicyName); + if (ecPolicy == null) { + throw new HadoopIllegalArgumentException( + "The given erasure coding " + "policy " + ecPolicyName + + " does not exist."); + } + return ecPolicy; + } + /** * Set an erasure coding policy on the given path. * @@ -118,7 +139,7 @@ static FileStatus setErasureCodingPolicy(final FSNamesystem fsn, List xAttrs; fsd.writeLock(); try { - ErasureCodingPolicy ecPolicy = getErasureCodingPolicyByName(fsn, + ErasureCodingPolicy ecPolicy = getEnabledErasureCodingPolicyByName(fsn, ecPolicyName); iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); // Write access is required to set erasure coding policy @@ -374,7 +395,7 @@ static ErasureCodingPolicy getErasureCodingPolicy(FSNamesystem fsn, String ecPolicyName, INodesInPath iip) throws IOException { ErasureCodingPolicy ecPolicy; if (!StringUtils.isEmpty(ecPolicyName)) { - ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName( + ecPolicy = FSDirErasureCodingOp.getEnabledErasureCodingPolicyByName( fsn, ecPolicyName); } else { ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ff8866344a64d..a93799849478b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -91,6 +91,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT_DEFAULT; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; @@ -7767,6 +7768,48 @@ void unsetErasureCodingPolicy(final String srcArg, } } + /** + * Verifies if the given policies are supported in the given cluster setup. + * If not policy is specified checks for all enabled policies. + * @param policyNames name of policies. + * @return the result if the given policies are supported in the cluster setup + * @throws IOException + */ + public ECTopologyVerifierResult getECTopologyResultForPolicies( + String[] policyNames) throws IOException { + String operationName = "getECTopologyResultForPolicies"; + checkSuperuserPrivilege(operationName); + checkOperation(OperationCategory.UNCHECKED); + ECTopologyVerifierResult result; + readLock(); + try { + checkOperation(OperationCategory.UNCHECKED); + // If no policy name is specified return the result + // for all enabled policies. + if (policyNames == null || policyNames.length == 0) { + result = getEcTopologyVerifierResultForEnabledPolicies(); + } else { + Collection policies = + new ArrayList(); + for (int i = 0; i < policyNames.length; i++) { + policies.add(FSDirErasureCodingOp + .getErasureCodingPolicyByName(this, policyNames[i])); + } + int numOfDataNodes = + getBlockManager().getDatanodeManager().getNumOfDataNodes(); + int numOfRacks = + getBlockManager().getDatanodeManager().getNetworkTopology() + .getNumOfRacks(); + result = ECTopologyVerifier + .getECTopologyVerifierResult(numOfRacks, numOfDataNodes, policies); + } + } finally { + readUnlock(); + } + logAuditEvent(true, operationName, null); + return result; + } + /** * Get the erasure coding policy information for specified path */ @@ -8194,15 +8237,8 @@ public int getNumEnteringMaintenanceDataNodes() { @Override // NameNodeMXBean public String getVerifyECWithTopologyResult() { - int numOfDataNodes = getBlockManager().getDatanodeManager() - .getNumOfDataNodes(); - int numOfRacks = getBlockManager().getDatanodeManager() - .getNetworkTopology().getNumOfRacks(); - ErasureCodingPolicy[] enabledEcPolicies = - getErasureCodingPolicyManager().getCopyOfEnabledPolicies(); ECTopologyVerifierResult result = - ECTopologyVerifier.getECTopologyVerifierResult( - numOfRacks, numOfDataNodes, enabledEcPolicies); + getEcTopologyVerifierResultForEnabledPolicies(); Map resultMap = new HashMap(); resultMap.put("isSupported", Boolean.toString(result.isSupported())); @@ -8210,6 +8246,18 @@ public String getVerifyECWithTopologyResult() { return JSON.toString(resultMap); } + private ECTopologyVerifierResult getEcTopologyVerifierResultForEnabledPolicies() { + int numOfDataNodes = + getBlockManager().getDatanodeManager().getNumOfDataNodes(); + int numOfRacks = getBlockManager().getDatanodeManager().getNetworkTopology() + .getNumOfRacks(); + ErasureCodingPolicy[] enabledEcPolicies = + getErasureCodingPolicyManager().getCopyOfEnabledPolicies(); + return ECTopologyVerifier + .getECTopologyVerifierResult(numOfRacks, numOfDataNodes, + Arrays.asList(enabledEcPolicies)); + } + // This method logs operatoinName without super user privilege. // It should be called without holding FSN lock. void checkSuperuserPrivilege(String operationName) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index ff1fea1d32f71..3a8cc40e929ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -101,6 +101,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; @@ -2484,6 +2485,12 @@ public void unsetErasureCodingPolicy(String src) throws IOException { } } + @Override + public ECTopologyVerifierResult getECTopologyResultForPolicies( + String... policyNames) throws IOException { + return namesystem.getECTopologyResultForPolicies(policyNames); + } + @Override public AddErasureCodingPolicyResponse[] addErasureCodingPolicies( ErasureCodingPolicy[] policies) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 80d1b6f40170a..e499799481efa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -26,15 +26,13 @@ import org.apache.hadoop.fs.shell.CommandFormat; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; -import org.apache.hadoop.hdfs.server.common.ECTopologyVerifier; -import org.apache.hadoop.hdfs.server.namenode.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.util.ECPolicyLoader; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.tools.TableListing; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; @@ -536,14 +534,14 @@ public int run(Configuration conf, List args) throws IOException { final DistributedFileSystem dfs = AdminHelper.getDFS(conf); try { dfs.enableErasureCodingPolicy(ecPolicyName); - System.out.println("Erasure coding policy " + ecPolicyName + - " is enabled"); + System.out + .println("Erasure coding policy " + ecPolicyName + " is enabled"); ECTopologyVerifierResult result = - getECTopologyVerifierResultForPolicy(dfs, ecPolicyName); + dfs.getECTopologyResultForPolicies(ecPolicyName); if (!result.isSupported()) { - System.err.println("Warning: The cluster setup does not support " + - "EC policy " + ecPolicyName + ". Reason: " + - result.getResultMessage()); + System.err.println( + "Warning: The cluster setup does not support " + "EC policy " + + ecPolicyName + ". Reason: " + result.getResultMessage()); } } catch (IOException e) { System.err.println(AdminHelper.prettifyException(e)); @@ -630,14 +628,21 @@ public String getLongUsage() { public int run(Configuration conf, List args) throws IOException { boolean isPolicyOption = StringUtils.popOption("-policy", args); final DistributedFileSystem dfs = AdminHelper.getDFS(conf); - ECTopologyVerifierResult result; + ECTopologyVerifierResult result = null; if (isPolicyOption) { CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE); c.parse(args); String[] parameters = args.toArray(new String[args.size()]); - result = getECTopologyResultForPolicies(dfs, parameters); + try { + result = dfs.getECTopologyResultForPolicies(parameters); + } catch (RemoteException e) { + if (e.getClassName().contains("HadoopIllegalArgumentException")) { + throw new HadoopIllegalArgumentException(e.getMessage()); + } + throw e; + } } else { - result = getECTopologyVerifierResult(dfs); + result = dfs.getECTopologyResultForPolicies(); } System.out.println(result.getResultMessage()); if (result.isSupported()) { @@ -647,62 +652,6 @@ public int run(Configuration conf, List args) throws IOException { } } - private static ECTopologyVerifierResult getECTopologyVerifierResult( - final DistributedFileSystem dfs) throws IOException { - final ErasureCodingPolicyInfo[] policies = - dfs.getClient().getNamenode().getErasureCodingPolicies(); - final DatanodeInfo[] report = dfs.getClient().getNamenode() - .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); - - return ECTopologyVerifier.getECTopologyVerifierResult(report, - getEnabledPolicies(policies)); - } - - private static ECTopologyVerifierResult getECTopologyResultForPolicies( - final DistributedFileSystem dfs, final String... policyNames) - throws IOException { - ErasureCodingPolicy[] policies = - new ErasureCodingPolicy[policyNames.length]; - for (int i = 0; i < policyNames.length; i++) { - policies[i] = - getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(), - policyNames[i]); - } - final DatanodeInfo[] report = dfs.getClient().getNamenode() - .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); - return ECTopologyVerifier.getECTopologyVerifierResult(report, policies); - } - - private static ECTopologyVerifierResult getECTopologyVerifierResultForPolicy( - final DistributedFileSystem dfs, final String policyName) - throws IOException { - final ErasureCodingPolicy policy = - getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(), - policyName); - final DatanodeInfo[] report = dfs.getClient().getNamenode() - .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); - return ECTopologyVerifier.getECTopologyVerifierResult(report, policy); - } - - private static ErasureCodingPolicy getPolicy( - final ErasureCodingPolicyInfo[] policies, final String policyName) { - for (ErasureCodingPolicyInfo policy : policies) { - if (policyName.equals(policy.getPolicy().getName())) { - return policy.getPolicy(); - } - } - throw new HadoopIllegalArgumentException("The given erasure coding " + - "policy " + policyName + " does not exist."); - } - - private static ErasureCodingPolicy[] getEnabledPolicies( - final ErasureCodingPolicyInfo[] policies) { - return Arrays.asList(policies).stream() - .filter(policyInfo -> policyInfo.isEnabled()) - .map(ErasureCodingPolicyInfo::getPolicy) - .toArray(ErasureCodingPolicy[]::new); - } - private static final AdminHelper.Command[] COMMANDS = { new ListECPoliciesCommand(), new AddECPoliciesCommand(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 8bef655ce39b8..598cbf8f7778b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -89,6 +89,7 @@ import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -2016,4 +2017,35 @@ public void testStorageFavouredNodes() assertEquals("Number of SSD should be 1 but was : " + numSSD, 1, numSSD); } } + + @Test + public void testGetECTopologyResultForPolicies() throws Exception { + Configuration conf = new HdfsConfiguration(); + try (MiniDFSCluster cluster = DFSTestUtil.setupCluster(conf, 9, 3, 0)) { + DistributedFileSystem dfs = cluster.getFileSystem(); + dfs.enableErasureCodingPolicy("RS-6-3-1024k"); + // No policies specified should return result for the enabled policy. + ECTopologyVerifierResult result = dfs.getECTopologyResultForPolicies(); + assertTrue(result.isSupported()); + // Specified policy requiring more datanodes than present in + // the actual cluster. + result = dfs.getECTopologyResultForPolicies("RS-10-4-1024k"); + assertFalse(result.isSupported()); + // Specify multiple policies that require datanodes equlal or less then + // present in the actual cluster + result = + dfs.getECTopologyResultForPolicies("XOR-2-1-1024k", "RS-3-2-1024k"); + assertTrue(result.isSupported()); + // Specify multiple policies with one policy requiring more datanodes than + // present in the actual cluster + result = + dfs.getECTopologyResultForPolicies("RS-10-4-1024k", "RS-3-2-1024k"); + assertFalse(result.isSupported()); + // Enable a policy requiring more datanodes than present in + // the actual cluster. + dfs.enableErasureCodingPolicy("RS-10-4-1024k"); + result = dfs.getECTopologyResultForPolicies(); + assertFalse(result.isSupported()); + } + } }