Skip to content
Permalink
Browse files
Add assigns/unassigns operations. Add tests.
  • Loading branch information
saintstack committed Sep 18, 2018
1 parent c6af272 commit 0aef9d8c17035c2091ec88ebf9993d1160dbcbdb
Showing 4 changed files with 107 additions and 73 deletions.
@@ -11,3 +11,11 @@ hbase-2.x cluster. It may do damage. While _hbck1_ is still bundled inside hbase
output) -- it's write-facility has been removed. It can report on the state of an
hbase-2.x cluster but its assessments may be inaccurate.

## Running _HBCK2_
`org.apache.hbase.HBCK2` is the name of the main class. Running the below
will dump out the HBCK2 usage:

~~~~
$ HBASE_CLASSPATH_PREFIX=/tmp/hbase-hbck2-1.0.0-SNAPSHOT.jar ./bin/hbase org.apache.hbase.HBCK2
~~~~

@@ -57,6 +57,17 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>org.apache.hbase.HBCK2</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<!--Used packaging a fat jar-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -109,22 +120,24 @@
<artifactId>log4j-core</artifactId>
<version>2.11.1</version>
</dependency>
<!--Try to use shaded client in hbck2.-->
<!-- https://mvnrepository.com/artifact/org.apache.hbase.thirdparty/hbase-shaded-miscellaneous -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
<version>${hbase.version}</version>
<scope>test</scope>
</dependency>
<!--We want to use the shaded client but for testing, we need to rely on hbase-server.
HBASE-15666 is about how shaded-client and hbase-server won't work.
TODO: Come back and move tests out to separate module so can use shaded client
only in here.-->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
<version>${hbase.version}</version>
<scope>test</scope>
</dependency>
</dependencies>


<profiles>
<!-- Needs to make the profile in apache parent pom -->
<profile>
@@ -25,13 +25,16 @@
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Hbck;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -54,36 +57,41 @@
// + Add test of Master version to ensure it supports hbck functionality.
// + Doc how we just take pointer to zk ensemble... If want to do more exotic config. on client,
// then add a hbase-site.xml onto CLASSPATH for this tool to pick up.
public class HBCK2 {
// + Kerberized cluster
public class HBCK2 extends Configured implements Tool {
private static final Logger LOG = LogManager.getLogger(HBCK2.class);
public static final int EXIT_SUCCESS = 0;
public static final int EXIT_FAILURE = 1;
// Commands
private static final String SET_TABLE_STATE = "setTableState";
private static final String ASSIGN = "assign";
private static final String UNASSIGN = "unassign";
private static final String ASSIGNS = "assigns";
private static final String UNASSIGNS = "unassigns";
private Configuration conf;

static TableState setTableState(Configuration conf, TableName tableName, TableState.State state)
TableState setTableState(TableName tableName, TableState.State state)
throws IOException {
try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
try (ClusterConnection conn =
(ClusterConnection)ConnectionFactory.createConnection(getConf())) {
try (Hbck hbck = conn.getHbck()) {
return hbck.setTableStateInMeta(new TableState(tableName, state));
}
}
}

static List<Long> assigns(Configuration conf, List<String> encodedRegionNames)
List<Long> assigns(List<String> encodedRegionNames)
throws IOException {
try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
try (ClusterConnection conn =
(ClusterConnection)ConnectionFactory.createConnection(getConf())) {
try (Hbck hbck = conn.getHbck()) {
return hbck.assigns(encodedRegionNames);
}
}
}

static List<Long> unassigns(Configuration conf, List<String> encodedRegionNames)
List<Long> unassigns(List<String> encodedRegionNames)
throws IOException {
try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) {
try (ClusterConnection conn =
(ClusterConnection)ConnectionFactory.createConnection(getConf())) {
try (Hbck hbck = conn.getHbck()) {
return hbck.unassigns(encodedRegionNames);
}
@@ -105,7 +113,7 @@ private static final String getCommandUsage() {
writer.println(" $ HBCK2 setTableState users ENABLED");
writer.println(" Returns whatever the previous table state was.");
writer.println();
writer.println(" " + ASSIGN + " <ENCODED_REGIONNAME> ...");
writer.println(" " + ASSIGNS + " <ENCODED_REGIONNAME> ...");
writer.println(" A 'raw' assign that can be used even during Master initialization.");
writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
writer.println(" e.g. 1588230740 is hard-coded encoding for hbase:meta region and");
@@ -114,7 +122,7 @@ private static final String getCommandUsage() {
writer.println(" $ HBCK2 assign 1588230740 de00010733901a05f5a2a3a382e27dd4");
writer.println(" Returns the pid of the created AssignProcedure or -1 if none.");
writer.println();
writer.println(" " + UNASSIGN + " <ENCODED_REGIONNAME> ...");
writer.println(" " + UNASSIGNS + " <ENCODED_REGIONNAME> ...");
writer.println(" A 'raw' unassign that can be used even during Master initialization.");
writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
@@ -139,25 +147,32 @@ static void usage(Options options, String error) {
"\nOptions:", options, getCommandUsage());
}

@Override
public void setConf(Configuration configuration) {
this.conf = configuration;
}

@Override
public Configuration getConf() {
return this.conf;
}

/**
* @return Return what to exit with.
*/
static int doWork(String [] args) throws IOException {
@Override
public int run(String [] args) throws IOException {
// Configure Options. The below article was more helpful than the commons-cli doc:
// https://dzone.com/articles/java-command-line-interfaces-part-1-apache-commons
Options options = new Options();
Option help = Option.builder("h").longOpt("help").desc("output this help message").build();
options.addOption(help);
Option debug = Option.builder("d").longOpt("debug").desc("run with debug output").build();
options.addOption(debug);
Option quorum = Option.builder().longOpt("hbase.zookeeper.quorum").
Option quorum = Option.builder("q").longOpt(HConstants.ZOOKEEPER_QUORUM).hasArg().
desc("ensemble of target hbase").build();
options.addOption(quorum);
Option parent = Option.builder().longOpt("zookeeper.znode.parent").
Option parent = Option.builder("z").longOpt(HConstants.ZOOKEEPER_ZNODE_PARENT).
desc("parent znode of target hbase").build();
options.addOption(parent);
Option peerPort = Option.builder().longOpt("hbase.zookeeper.peerport").
Option peerPort = Option.builder("p").longOpt(HConstants.ZOOKEEPER_CLIENT_PORT).
desc("peerport of target hbase ensemble").type(Integer.class).build();
options.addOption(peerPort);

@@ -181,18 +196,19 @@ static int doWork(String [] args) throws IOException {
}

// Build up Configuration for client to use connecting to hbase zk ensemble.
Configuration conf = HBaseConfiguration.create();
if (options.hasOption(quorum.getOpt())) {
conf.set(HConstants.ZOOKEEPER_QUORUM, commandLine.getOptionValue(quorum.getOpt()));
if (commandLine.hasOption(quorum.getOpt())) {
getConf().set(HConstants.ZOOKEEPER_QUORUM, commandLine.getOptionValue(quorum.getOpt()));
}
if (options.hasOption(peerPort.getOpt())) {
conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT,
if (commandLine.hasOption(peerPort.getOpt())) {
getConf().setInt(HConstants.ZOOKEEPER_CLIENT_PORT,
Integer.valueOf(commandLine.getOptionValue(peerPort.getOpt())));
}
if (options.hasOption(parent.getOpt())) {
conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, commandLine.getOptionValue(parent.getOpt()));
if (commandLine.hasOption(parent.getOpt())) {
getConf().set(HConstants.ZOOKEEPER_ZNODE_PARENT, commandLine.getOptionValue(parent.getOpt()));
}

System.out.println("HERE: " + getConf().get("hbase.master.kerberos.principal", "NOTHING"));

// Now process commands.
String [] commands = commandLine.getArgs();
String command = commands[0];
@@ -202,31 +218,31 @@ static int doWork(String [] args) throws IOException {
usage(options, command + " takes tablename and state arguments: e.g. user ENABLED");
return EXIT_FAILURE;
}
System.out.println(setTableState(conf,
TableName.valueOf(commands[1]), TableState.State.valueOf(commands[2])));
System.out.println(setTableState(TableName.valueOf(commands[1]),
TableState.State.valueOf(commands[2])));
break;

case ASSIGN:
case ASSIGNS:
if (commands.length < 2) {
usage(options, command + " takes one or more encoded region names");
return EXIT_FAILURE;
}
System.out.println(pidsToString(
assigns(conf, Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
System.out.println(
pidsToString(assigns(Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
break;

case UNASSIGN:
case UNASSIGNS:
if (commands.length < 2) {
usage(options, command + " takes one or more encoded region names");
return EXIT_FAILURE;
}
System.out.println(pidsToString(
unassigns(conf, Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
unassigns(Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
break;

default:
usage(options, "Unsupported command: " + command);
System.exit(1);
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
@@ -235,10 +251,15 @@ private static String pidsToString(List<Long> pids) {
return pids.stream().map(i -> i.toString()).collect(Collectors.joining(", "));
}

public static void main(String [] args) throws IOException {
int exitCode = doWork(args);
if (exitCode != 0) {
System.exit(exitCode);
HBCK2(Configuration conf) {
super(conf);
}

public static void main(String [] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
int errCode = ToolRunner.run(new HBCK2(conf), args);
if (errCode != 0) {
System.exit(errCode);
}
return;
}
@@ -46,32 +46,28 @@

public class TestHBCK2 {
private static final org.apache.logging.log4j.Logger LOG = LogManager.getLogger(TestHBCK2.class);
// private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final TableName TABLE_NAME = TableName.valueOf(TestHBCK2.class.getSimpleName());

@BeforeClass
public static void setUpBeforeClass() throws Exception {
/*
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.createMultiRegionTable(TABLE_NAME, Bytes.toBytes("family1"), 5);
*/
}

@AfterClass
public static void tearDownAfterClass() throws Exception {
/*
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
*/
}

@Test
public void testHelp() throws ParseException, IOException {
// TODO
ByteArrayOutputStream os = new ByteArrayOutputStream();
PrintStream stream = new PrintStream(os);
PrintStream oldOut = System.out;
System.setOut(stream);
HBCK2.doWork(new String [] {"-h"});
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
hbck.run(new String [] {"-h"});
stream.close();
os.close();
System.setOut(oldOut);
@@ -81,26 +77,25 @@ public void testHelp() throws ParseException, IOException {

@Test
public void testSetTableStateInMeta() throws IOException {
/*
TableState state =
HBCK2.setTableState(TEST_UTIL.getConfiguration(), TABLE_NAME, TableState.State.DISABLED);
TestCase.assertTrue("Found=" + state.getState(), state.isDisabled());
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
TableState state = hbck.setTableState(TABLE_NAME, TableState.State.DISABLED);
TestCase.assertTrue("Found=" + state.getState(), state.isEnabled());
// Restore the state.
HBCK2.setTableState(TEST_UTIL.getConfiguration(), TABLE_NAME, state.getState());
*/
state = hbck.setTableState(TABLE_NAME, state.getState());
TestCase.assertTrue("Found=" + state.getState(), state.isDisabled());
}

@Test
public void testAssigns() throws IOException {
/*
try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
List<RegionInfo> regions = admin.getRegions(TABLE_NAME);
for (RegionInfo ri: regions) {
RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
getRegionStates().getRegionState(ri.getEncodedName());
LOG.info("RS: {}", rs.toString());
}
List<Long> pids = HBCK2.unassigns(TEST_UTIL.getConfiguration(),
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
List<Long> pids = hbck.unassigns(
regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
waitOnPids(pids);
for (RegionInfo ri: regions) {
@@ -109,7 +104,7 @@ public void testAssigns() throws IOException {
LOG.info("RS: {}", rs.toString());
TestCase.assertTrue(rs.toString(), rs.isClosed());
}
pids = HBCK2.assigns(TEST_UTIL.getConfiguration(),
pids = hbck.assigns(
regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList()));
waitOnPids(pids);
for (RegionInfo ri: regions) {
@@ -119,23 +114,20 @@ public void testAssigns() throws IOException {
TestCase.assertTrue(rs.toString(), rs.isOpened());
}
// What happens if crappy region list passed?
pids = HBCK2.assigns(TEST_UTIL.getConfiguration(),
pids = hbck.assigns(
Arrays.stream(new String [] {"a", "some rubbish name"}).collect(Collectors.toList()));
for (long pid: pids) {
assertEquals(org.apache.hadoop.hbase.procedure2.Procedure.NO_PROC_ID, pid);
}
}
*/
}

private void waitOnPids(List<Long> pids) {
/*
for (Long pid: pids) {
while (!TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor().
isFinished(pid)) {
Threads.sleep(100);
}
}
*/
}
}

0 comments on commit 0aef9d8

Please sign in to comment.