From 5187408accc41fc5e6ac8695ec801ce77309c827 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 28 May 2021 23:00:56 +0800 Subject: [PATCH 1/2] HBASE-25939 Move more tests code for StochasticLoadBalancer to hbase-balancer module --- .../master/balancer/LoadBalancerFactory.java | 0 .../HeterogeneousCostRulesTestHelper.java | 67 +++++++++ .../LoadBalancerPerformanceEvaluation.java | 0 .../master/balancer/TestDoubleArrayCost.java | 0 ...ochasticLoadBalancerHeterogeneousCost.java | 93 ++++++------- ...ticLoadBalancerHeterogeneousCostRules.java | 128 +++++------------- ...cerHeterogeneousCostRulesLoadFromHDFS.java | 81 +++++++++++ 7 files changed, 232 insertions(+), 137 deletions(-) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java (100%) create mode 100644 hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/HeterogeneousCostRulesTestHelper.java rename {hbase-server => hbase-balancer}/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java (100%) rename {hbase-server => hbase-balancer}/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDoubleArrayCost.java (100%) rename {hbase-server => hbase-balancer}/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java (78%) rename {hbase-server => hbase-balancer}/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRules.java (53%) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/HeterogeneousCostRulesTestHelper.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/HeterogeneousCostRulesTestHelper.java new file mode 100644 index 000000000000..46300c6e3dfa --- /dev/null +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/HeterogeneousCostRulesTestHelper.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class HeterogeneousCostRulesTestHelper { + + private static final Logger LOG = LoggerFactory.getLogger(HeterogeneousCostRulesTestHelper.class); + + static final String DEFAULT_RULES_FILE_NAME = "hbase-balancer.rules"; + + private HeterogeneousCostRulesTestHelper() { + } + + /** + * Create rule file with the given rules. + * @param file Name of file to write rules into. + * @return Full file name of the rules file which is dir + DEFAULT_RULES_FILE_NAME. + */ + static String createRulesFile(String file, final List rules) throws IOException { + cleanup(file); + Path path = Files.createFile(FileSystems.getDefault().getPath(file)); + return Files.write(path, rules, StandardCharsets.UTF_8).toString(); + } + + /** + * Create rule file with empty rules. + * @param file Name of file to write rules into. + * @return Full file name of the rules file which is dir + DEFAULT_RULES_FILE_NAME. + */ + static String createRulesFile(String file) throws IOException { + return createRulesFile(file, Collections.emptyList()); + } + + static void cleanup(String file) throws IOException { + try { + Files.delete(FileSystems.getDefault().getPath(file)); + } catch (NoSuchFileException nsfe) { + LOG.warn("FileNotFoundException for {}", file, nsfe); + } + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java rename to hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDoubleArrayCost.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDoubleArrayCost.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDoubleArrayCost.java rename to hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDoubleArrayCost.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java similarity index 78% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java rename to hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java index 66a46ca6317f..64c149673c19 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java @@ -1,22 +1,27 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hbase.master.balancer; -import static junit.framework.TestCase.assertNotNull; -import static junit.framework.TestCase.assertTrue; +import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.DEFAULT_RULES_FILE_NAME; +import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.createRulesFile; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Arrays; @@ -30,7 +35,7 @@ import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -48,14 +53,15 @@ @Category({ MasterTests.class, MediumTests.class }) public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalancerTestBase { + @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCost.class); + HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCost.class); private static final Logger LOG = - LoggerFactory.getLogger(TestStochasticLoadBalancerHeterogeneousCost.class); + LoggerFactory.getLogger(TestStochasticLoadBalancerHeterogeneousCost.class); private static final double ALLOWED_WINDOW = 1.20; - private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); + private static final HBaseCommonTestingUtility HTU = new HBaseCommonTestingUtility(); private static String RULES_FILE; @BeforeClass @@ -69,10 +75,8 @@ public static void beforeAllTests() throws IOException { HeterogeneousRegionCountCostFunction.class.getName()); // Need to ensure test dir has been created. assertTrue(FileSystem.get(HTU.getConfiguration()).mkdirs(HTU.getDataTestDir())); - RULES_FILE = HTU.getDataTestDir( - TestStochasticLoadBalancerHeterogeneousCostRules.DEFAULT_RULES_FILE_NAME).toString(); - conf.set( - HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE, + RULES_FILE = HTU.getDataTestDir(DEFAULT_RULES_FILE_NAME).toString(); + conf.set(HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE, RULES_FILE); loadBalancer = new StochasticLoadBalancer(); loadBalancer.setClusterInfoProvider(new DummyClusterInfoProvider(conf)); @@ -146,28 +150,28 @@ public void testOverloaded() throws IOException { final int numRegions = 120; final int numRegionsPerServer = 60; - TestStochasticLoadBalancerHeterogeneousCostRules.createRulesFile(RULES_FILE); + createRulesFile(RULES_FILE); final Map> serverMap = - this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1); + this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1); final List plans = - loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); + loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); // As we disabled all the other cost functions, balancing only according to // the heterogeneous cost function should return nothing. assertNull(plans); } private void testHeterogeneousWithCluster(final int numNodes, final int numRegions, - final int numRegionsPerServer, final List rules) throws IOException { + final int numRegionsPerServer, final List rules) throws IOException { - TestStochasticLoadBalancerHeterogeneousCostRules.createRulesFile(RULES_FILE, rules); + createRulesFile(RULES_FILE, rules); final Map> serverMap = - this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1); + this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1); this.testWithCluster(serverMap, null, true, false); } protected void testWithCluster(final Map> serverMap, - final RackManager rackManager, final boolean assertFullyBalanced, - final boolean assertFullyBalancedForReplicas) { + final RackManager rackManager, final boolean assertFullyBalanced, + final boolean assertFullyBalancedForReplicas) { final List list = this.convertToList(serverMap); LOG.info("Mock Cluster : " + this.printMock(list) + " " + this.printStats(list)); @@ -175,7 +179,7 @@ protected void testWithCluster(final Map> serverMap // Run the balancer. final List plans = - loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); + loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); assertNotNull(plans); // Check to see that this actually got to a stable place. @@ -188,16 +192,15 @@ protected void testWithCluster(final Map> serverMap if (assertFullyBalanced) { final List secondPlans = - loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); + loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap); assertNull(secondPlans); // create external cost function to retrieve limit // for each RS final HeterogeneousRegionCountCostFunction cf = - new HeterogeneousRegionCountCostFunction(conf); + new HeterogeneousRegionCountCostFunction(conf); assertNotNull(cf); - BalancerClusterState cluster = - new BalancerClusterState(serverMap, null, null, null); + BalancerClusterState cluster = new BalancerClusterState(serverMap, null, null, null); cf.prepare(cluster); // checking that we all hosts have a number of regions below their limit @@ -212,10 +215,9 @@ protected void testWithCluster(final Map> serverMap // as the balancer is stochastic, we cannot check exactly the result of the balancing, // hence the allowedWindow parameter - assertTrue("Host " + sn.getHostname() + " should be below " - + cf.overallUsage * ALLOWED_WINDOW * 100 + "%; " + cf.overallUsage + - ", " + usage + ", " + numberRegions + ", " + limit, - usage <= cf.overallUsage * ALLOWED_WINDOW); + assertTrue("Host " + sn.getHostname() + " should be below " + + cf.overallUsage * ALLOWED_WINDOW * 100 + "%; " + cf.overallUsage + ", " + usage + ", " + + numberRegions + ", " + limit, usage <= cf.overallUsage * ALLOWED_WINDOW); } } @@ -227,7 +229,7 @@ protected void testWithCluster(final Map> serverMap @Override protected Map> createServerMap(int numNodes, int numRegions, - int numRegionsPerServer, int replication, int numTables) { + int numRegionsPerServer, int replication, int numTables) { // construct a cluster of numNodes, having a total of numRegions. Each RS will hold // numRegionsPerServer many regions except for the last one, which will host all the // remaining regions @@ -254,7 +256,7 @@ protected Map> createServerMap(int numNodes, int nu @Override protected TreeMap> mockClusterServers(int[] mockCluster, - int numTables) { + int numTables) { int numServers = mockCluster.length; TreeMap> servers = new TreeMap<>(); for (int i = 0; i < numServers; i++) { @@ -280,12 +282,11 @@ private ServerAndLoad createServer(final String host) { return new ServerAndLoad(sn, 0); } - static class FairRandomCandidateGenerator extends - RandomCandidateGenerator { + static class FairRandomCandidateGenerator extends RandomCandidateGenerator { @Override - public BalanceAction pickRandomRegions(BalancerClusterState cluster, - int thisServer, int otherServer) { + public BalanceAction pickRandomRegions(BalancerClusterState cluster, int thisServer, + int otherServer) { if (thisServer < 0 || otherServer < 0) { return BalanceAction.NULL_ACTION; } @@ -301,4 +302,4 @@ BalanceAction generate(BalancerClusterState cluster) { return super.generate(cluster); } } -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRules.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRules.java similarity index 53% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRules.java rename to hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRules.java index 1b64f195aac7..b7c66ba9be4a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRules.java +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRules.java @@ -1,36 +1,34 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hbase.master.balancer; +import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.DEFAULT_RULES_FILE_NAME; +import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.cleanup; +import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.createRulesFile; +import static org.junit.Assert.assertEquals; + import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.FileSystems; -import java.nio.file.NoSuchFileException; import java.util.Arrays; import java.util.Collections; -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -41,15 +39,15 @@ @Category({ MasterTests.class, MediumTests.class }) public class TestStochasticLoadBalancerHeterogeneousCostRules extends StochasticBalancerTestBase { + @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCostRules.class); @Rule public TestName name = new TestName(); - static final String DEFAULT_RULES_FILE_NAME = "hbase-balancer.rules"; private HeterogeneousRegionCountCostFunction costFunction; - private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); + private static final HBaseCommonTestingUtility HTU = new HBaseCommonTestingUtility(); /** * Make a file for rules that is inside a temporary test dir named for the method so it doesn't @@ -60,47 +58,22 @@ public class TestStochasticLoadBalancerHeterogeneousCostRules extends Stochastic @BeforeClass public static void beforeClass() throws IOException { // Ensure test dir is created - HTU.getTestFileSystem().mkdirs(HTU.getDataTestDir()); + HTU.getDataTestDir().getFileSystem(HTU.getConfiguration()).mkdirs(HTU.getDataTestDir()); } @Before public void before() throws IOException { // New rules file name per test. - this.rulesFilename = HTU.getDataTestDir( - this.name.getMethodName() + "." + DEFAULT_RULES_FILE_NAME).toString(); + this.rulesFilename = HTU + .getDataTestDir( + this.name.getMethodName() + "." + DEFAULT_RULES_FILE_NAME) + .toString(); // Set the created rules filename into the configuration. HTU.getConfiguration().set( HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE, this.rulesFilename); } - /** - * @param file Name of file to write rules into. - * @return Full file name of the rules file which is dir + DEFAULT_RULES_FILE_NAME. - */ - static String createRulesFile(String file, final List lines) throws IOException { - cleanup(file); - java.nio.file.Path path = - java.nio.file.Files.createFile(FileSystems.getDefault().getPath(file)); - return java.nio.file.Files.write(path, lines, Charset.forName("UTF-8")).toString(); - } - - /** - * @param file Name of file to write rules into. - * @return Full file name of the rules file which is dir + DEFAULT_RULES_FILE_NAME. - */ - static String createRulesFile(String file) throws IOException { - return createRulesFile(file, Collections.emptyList()); - } - - private static void cleanup(String file) throws IOException { - try { - java.nio.file.Files.delete(FileSystems.getDefault().getPath(file)); - } catch (NoSuchFileException nsfe) { - System.out.println("FileNotFoundException for " + file); - } - } - @Test public void testNoRules() throws IOException { // Override what is in the configuration with the name of a non-existent file! @@ -109,7 +82,7 @@ public void testNoRules() throws IOException { "non-existent-file!"); this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration()); this.costFunction.loadRules(); - Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); + assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); } @Test @@ -119,18 +92,18 @@ public void testBadFormatInRules() throws IOException { // in the configuration. this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration()); this.costFunction.loadRules(); - Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); + assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); createRulesFile(this.rulesFilename, Collections.singletonList("bad rules format")); this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration()); this.costFunction.loadRules(); - Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); + assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); createRulesFile(this.rulesFilename, Arrays.asList("srv[1-2] 10", "bad_rules format", "a")); this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration()); this.costFunction.loadRules(); - Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded()); + assertEquals(1, this.costFunction.getNumberOfRulesLoaded()); } @Test @@ -144,7 +117,7 @@ public void testTwoRules() throws IOException { createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21")); this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration()); this.costFunction.loadRules(); - Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded()); + assertEquals(2, this.costFunction.getNumberOfRulesLoaded()); } @Test @@ -158,7 +131,7 @@ public void testBadRegexp() throws IOException { createRulesFile(this.rulesFilename, Collections.singletonList("server[ 1")); this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration()); this.costFunction.loadRules(); - Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); + assertEquals(0, this.costFunction.getNumberOfRulesLoaded()); } @Test @@ -169,38 +142,11 @@ public void testNoOverride() throws IOException { createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21")); this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration()); this.costFunction.loadRules(); - Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded()); + assertEquals(2, this.costFunction.getNumberOfRulesLoaded()); // loading malformed configuration does not overload current cleanup(this.rulesFilename); this.costFunction.loadRules(); - Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded()); - } - - @Test - public void testLoadingFomHDFS() throws Exception { - HTU.startMiniDFSCluster(3); - try { - MiniDFSCluster cluster = HTU.getDFSCluster(); - DistributedFileSystem fs = cluster.getFileSystem(); - // Writing file - Path path = new Path(fs.getHomeDirectory(), DEFAULT_RULES_FILE_NAME); - FSDataOutputStream stream = fs.create(path); - stream.write("server1 10".getBytes()); - stream.flush(); - stream.close(); - - Configuration configuration = HTU.getConfiguration(); - - // start costFunction - configuration.set( - HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE, - path.toString()); - this.costFunction = new HeterogeneousRegionCountCostFunction(configuration); - this.costFunction.loadRules(); - Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded()); - } finally { - HTU.shutdownMiniCluster(); - } + assertEquals(2, this.costFunction.getNumberOfRulesLoaded()); } -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS.java new file mode 100644 index 000000000000..7d52f5afed0d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.DEFAULT_RULES_FILE_NAME; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS + extends StochasticBalancerTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS.class); + + private HeterogeneousRegionCountCostFunction costFunction; + private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); + + @Before + public void setUp() throws Exception { + HTU.startMiniCluster(1); + } + + @After + public void tearDown() throws IOException { + HTU.shutdownMiniCluster(); + } + + @Test + public void testLoadingFomHDFS() throws Exception { + MiniDFSCluster cluster = HTU.getDFSCluster(); + DistributedFileSystem fs = cluster.getFileSystem(); + // Writing file + Path path = new Path(fs.getHomeDirectory(), DEFAULT_RULES_FILE_NAME); + FSDataOutputStream stream = fs.create(path); + stream.write("server1 10".getBytes()); + stream.flush(); + stream.close(); + + Configuration configuration = HTU.getConfiguration(); + + // start costFunction + configuration.set( + HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE, + path.toString()); + this.costFunction = new HeterogeneousRegionCountCostFunction(configuration); + this.costFunction.loadRules(); + assertEquals(1, this.costFunction.getNumberOfRulesLoaded()); + } +} \ No newline at end of file From 6c8ca01ec704c12d86e9b3a3cfd2593145feaf08 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sat, 29 May 2021 09:39:38 +0800 Subject: [PATCH 2/2] fix error prone and checkstyle warnings --- .../hadoop/hbase/master/balancer/LoadBalancerFactory.java | 5 ++++- .../master/balancer/LoadBalancerPerformanceEvaluation.java | 3 ++- .../TestStochasticLoadBalancerHeterogeneousCost.java | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java index 5af0180cf145..a43fdc88f148 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerFactory.java @@ -27,7 +27,10 @@ * The class that creates a load balancer from a conf. */ @InterfaceAudience.Private -public class LoadBalancerFactory { +public final class LoadBalancerFactory { + + private LoadBalancerFactory() { + } /** * The default {@link LoadBalancer} class. diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java index 38e19e2a1bc2..1dd092dea286 100644 --- a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java @@ -87,7 +87,8 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool { // Non-default configurations. private void setupConf() { - conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, loadBalancerClazz, LoadBalancer.class); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, loadBalancerClazz, + LoadBalancer.class); loadBalancer = LoadBalancerFactory.getLoadBalancer(conf); } diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java index 64c149673c19..18d7d05212aa 100644 --- a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java @@ -24,9 +24,9 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; @@ -169,6 +169,7 @@ private void testHeterogeneousWithCluster(final int numNodes, final int numRegio this.testWithCluster(serverMap, null, true, false); } + @Override protected void testWithCluster(final Map> serverMap, final RackManager rackManager, final boolean assertFullyBalanced, final boolean assertFullyBalancedForReplicas) { @@ -268,7 +269,7 @@ protected TreeMap> mockClusterServers(int[] mockClu return servers; } - private Queue serverQueue = new LinkedList<>(); + private Queue serverQueue = new ArrayDeque<>(); private ServerAndLoad createServer(final String host) { if (!this.serverQueue.isEmpty()) {