Skip to content
Permalink
Browse files
HBASE-21210 Add bypassProcedure() API to HBCK2
Adds a 'bypass' command that takes --force and
--waitTime options. First command that itself
has options.
  • Loading branch information
saintstack committed Sep 19, 2018
1 parent 0aef9d8 commit bf22da3e92ad9cf06ab7211af4dd3209ad12b47a
Showing 5 changed files with 156 additions and 29 deletions.
@@ -121,9 +121,8 @@
<version>2.11.1</version>
</dependency>
<!--We want to use the shaded client but for testing, we need to rely on hbase-server.
HBASE-15666 is about how shaded-client and hbase-server won't work.
TODO: Come back and move tests out to separate module so can use shaded client
only in here.-->
HBASE-15666 is about how shaded-client and hbase-server won't work together.
TODO: Fix.-->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
@@ -66,6 +66,7 @@ public class HBCK2 extends Configured implements Tool {
private static final String SET_TABLE_STATE = "setTableState";
private static final String ASSIGNS = "assigns";
private static final String UNASSIGNS = "unassigns";
private static final String BYPASS = "bypass";
private Configuration conf;

TableState setTableState(TableName tableName, TableState.State state)
@@ -98,6 +99,47 @@ List<Long> unassigns(List<String> encodedRegionNames)
}
}

/**
* @return List of results OR null if failed to run.
*/
List<Boolean> bypass(List<String> args)
throws IOException {
// Bypass has two options....
Options options = new Options();
Option force = Option.builder("f").longOpt("force").
desc("'force' the procedure finish").build();
options.addOption(force);
Option wait = Option.builder("w").longOpt("waitTime").hasArg().
desc("time to wait on entity lock before giving up").type(Integer.class).build();
options.addOption(wait);
// Parse command-line.
CommandLineParser parser = new DefaultParser();
CommandLine commandLine = null;
try {
commandLine = parser.parse(options, args.toArray(new String [] {}), false);
} catch (ParseException e) {
usage(options, e.getMessage());
return null;
}
boolean b = commandLine.hasOption(force.getOpt());
long waitTime = 0;
if (commandLine.hasOption(wait.getOpt())) {
waitTime = Integer.valueOf(commandLine.getOptionValue(wait.getOpt()));
waitTime *= 1000; // Because time is in seconds.
}
String [] pidStrs = commandLine.getArgs();
if (pidStrs == null || pidStrs.length <= 0) {
usage(options, "No pids supplied.");
return null;
}
List<Long> pids = Arrays.stream(pidStrs).map(i -> Long.valueOf(i)).collect(Collectors.toList());
try (ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection(getConf())) {
try (Hbck hbck = c.getHbck()) {
return hbck.bypassProcedure(pids, waitTime, b);
}
}
}

private static final String getCommandUsage() {
StringWriter sw = new StringWriter();
PrintWriter writer = new PrintWriter(sw);
@@ -113,7 +155,7 @@ private static final String getCommandUsage() {
writer.println(" $ HBCK2 setTableState users ENABLED");
writer.println(" Returns whatever the previous table state was.");
writer.println();
writer.println(" " + ASSIGNS + " <ENCODED_REGIONNAME> ...");
writer.println(" " + ASSIGNS + " <ENCODED_REGIONNAME>...");
writer.println(" A 'raw' assign that can be used even during Master initialization.");
writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
writer.println(" e.g. 1588230740 is hard-coded encoding for hbase:meta region and");
@@ -122,14 +164,23 @@ private static final String getCommandUsage() {
writer.println(" $ HBCK2 assign 1588230740 de00010733901a05f5a2a3a382e27dd4");
writer.println(" Returns the pid of the created AssignProcedure or -1 if none.");
writer.println();
writer.println(" " + UNASSIGNS + " <ENCODED_REGIONNAME> ...");
writer.println(" " + BYPASS + " [OPTIONS] <PID>...");
writer.println(" Pass one (or more) procedure 'pid's to skip to the procedure finish.");
writer.println(" Parent procedures will also skip to their finish. Entities will be");
writer.println(" left in an inconsistent state and will require manual fixup.");
writer.println(" Pass --force to break any outstanding locks.");
writer.println(" Pass --waitTime=<seconds> to wait on entity lock before giving up.");
writer.println(" Default: force=false and waitTime=0.");
writer.println();
writer.println(" " + UNASSIGNS + " <ENCODED_REGIONNAME>...");
writer.println(" A 'raw' unassign that can be used even during Master initialization.");
writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:");
writer.println(" de00010733901a05f5a2a3a382e27dd4 is an example of what a random");
writer.println(" user-space encoded Region name looks like. For example:");
writer.println(" $ HBCK2 unassign 1588230740 de00010733901a05f5a2a3a382e27dd4");
writer.println(" Returns the pid of the created UnassignProcedure or -1 if none.");

writer.close();
return sw.toString();
}
@@ -143,7 +194,7 @@ static void usage(Options options, String error) {
System.out.println("ERROR: " + error);
}
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp( "HBCK2 <OPTIONS> COMMAND [<ARGS>]",
formatter.printHelp( "HBCK2 [OPTIONS] COMMAND <ARGS>",
"\nOptions:", options, getCommandUsage());
}

@@ -180,8 +231,9 @@ public int run(String [] args) throws IOException {
CommandLineParser parser = new DefaultParser();
CommandLine commandLine = null;
try {
commandLine = parser.parse(options, args);
commandLine = parser.parse(options, args, true);
} catch (ParseException e) {
LOG.info("", e);
usage(options, e.getMessage());
return EXIT_FAILURE;
}
@@ -207,8 +259,6 @@ public int run(String [] args) throws IOException {
getConf().set(HConstants.ZOOKEEPER_ZNODE_PARENT, commandLine.getOptionValue(parent.getOpt()));
}

System.out.println("HERE: " + getConf().get("hbase.master.kerberos.principal", "NOTHING"));

// Now process commands.
String [] commands = commandLine.getArgs();
String command = commands[0];
@@ -228,15 +278,28 @@ public int run(String [] args) throws IOException {
return EXIT_FAILURE;
}
System.out.println(
pidsToString(assigns(Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
toString(bypass(Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
break;

case BYPASS:
if (commands.length < 2) {
usage(options, command + " takes one or more pids");
return EXIT_FAILURE;
}
List<Boolean> bs = bypass(Arrays.stream(commands).skip(1).collect(Collectors.toList()));
if (bs == null) {
// Something went wrong w/ the parse and command didn't run.
return EXIT_FAILURE;
}
System.out.println(toString(bs));
break;

case UNASSIGNS:
if (commands.length < 2) {
usage(options, command + " takes one or more encoded region names");
return EXIT_FAILURE;
}
System.out.println(pidsToString(
System.out.println(toString(
unassigns(Arrays.stream(commands).skip(1).collect(Collectors.toList()))));
break;

@@ -247,8 +310,8 @@ public int run(String [] args) throws IOException {
return EXIT_SUCCESS;
}

private static String pidsToString(List<Long> pids) {
return pids.stream().map(i -> i.toString()).collect(Collectors.joining(", "));
private static String toString(List<?> things) {
return things.stream().map(i -> i.toString()).collect(Collectors.joining(", "));
}

HBCK2(Configuration conf) {
@@ -35,7 +35,6 @@

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.List;
@@ -44,6 +43,10 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

/**
* Tests commands. For command-line parsing, see adjacent test.
* @see TestHBCKCommandLineParsing
*/
public class TestHBCK2 {
private static final org.apache.logging.log4j.Logger LOG = LogManager.getLogger(TestHBCK2.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -60,21 +63,6 @@ public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

@Test
public void testHelp() throws ParseException, IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
PrintStream stream = new PrintStream(os);
PrintStream oldOut = System.out;
System.setOut(stream);
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
hbck.run(new String [] {"-h"});
stream.close();
os.close();
System.setOut(oldOut);
String output = os.toString();
assertTrue(output, output.startsWith("usage: HBCK2"));
}

@Test
public void testSetTableStateInMeta() throws IOException {
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
@@ -0,0 +1,76 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hbase;

import junit.framework.TestCase;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.logging.log4j.LogManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

/**
* Does command-line parsing tests. No clusters.
* @see TestHBCK2 for cluster-tests.
*/
public class TestHBCKCommandLineParsing {
private static final org.apache.logging.log4j.Logger LOG =
LogManager.getLogger(TestHBCKCommandLineParsing.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

@Test
public void testHelp() throws ParseException, IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
PrintStream stream = new PrintStream(os);
PrintStream oldOut = System.out;
System.setOut(stream);
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
hbck.run(new String[]{"-h"});
stream.close();
os.close();
System.setOut(oldOut);
String output = os.toString();
assertTrue(output, output.startsWith("usage: HBCK2"));
}

@Test (expected=NumberFormatException.class)
public void testCommandWithOptions() throws IOException {
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
// The 'x' below should cause the NumberFormatException. The Options should all be good.
hbck.run(new String[]{"bypass", "--waitTime=3", "--force", "x"});
}
}

@@ -127,6 +127,7 @@
<maven.compiler.version>3.6.1</maven.compiler.version>
<surefire.version>2.21.0</surefire.version>
<surefire.provider>surefire-junit47</surefire.provider>
<test.output.tofile>true</test.output.tofile>
</properties>
<dependencyManagement>
<dependencies>

0 comments on commit bf22da3

Please sign in to comment.