Skip to content
Permalink
Browse files
HBASE-26656 Utility to correct corrupt RegionInfo's in hbase:meta
A standalone utility which corrects hbase:meta given the problem
described by HBASE-23328. Includes the ability to both "report" corrupt
regions as well as correct them. This tool will ensure that other
HBCK2 utilities continue to work without additional modification.

Signed-off-by: Peter Somogyi <psomogyi@apache.org>

Closes #102
  • Loading branch information
joshelser authored and Josh Elser committed Jan 12, 2022
1 parent b9b50ac commit e3a8f96d0b9e9b985d7f6f2952aaa929047c5b08
Showing 4 changed files with 689 additions and 0 deletions.
@@ -107,6 +107,7 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
private static final String RECOVER_UNKNOWN = "recoverUnknown";
private static final String GENERATE_TABLE_INFO = "generateMissingTableDescriptorFile";
private static final String FIX_META = "fixMeta";
private static final String REGIONINFO_MISMATCH = "regionInfoMismatch";
// TODO update this map in case of the name of a method changes in Hbck interface
// in org.apache.hadoop.hbase.client package. Or a new command is added and the hbck command
// does not equals to the method name in Hbck interface.
@@ -422,6 +423,23 @@ List<Long> recoverUnknown(Hbck hbck) throws IOException {
return hbck.scheduleSCPsForUnknownServers();
}

/**
* Runs the RegionInfoMismatchTool using CLI options.
*/
void regionInfoMismatch(String[] args) throws Exception {
// CLI Options
Options options = new Options();
Option dryRunOption = Option.builder("f").longOpt("fix").hasArg(false).build();
options.addOption(dryRunOption);
// Parse command-line.
CommandLineParser parser = new DefaultParser();
CommandLine commandLine = parser.parse(options, args, false);
final boolean fix = commandLine.hasOption(dryRunOption.getOpt());
try (ClusterConnection connection = connect()) {
new RegionInfoMismatchTool(connection).run(fix);
}
}

private HBaseProtos.ServerName parseServerName(String serverName) {
ServerName sn = ServerName.parseServerName(serverName);
return HBaseProtos.ServerName.newBuilder().setHostName(sn.getHostname()).
@@ -472,6 +490,8 @@ private static String getCommandUsage() {
writer.println();
usageUnassigns(writer);
writer.println();
usageRegioninfoMismatch(writer);
writer.println();
writer.close();
return sw.toString();
}
@@ -728,6 +748,27 @@ private static void usageUnassigns(PrintWriter writer) {
writer.println(" hbase:meta tool. See the HBCK2 README for how to use.");
}

private static void usageRegioninfoMismatch(PrintWriter writer) {
writer.println(" " + REGIONINFO_MISMATCH);
writer.println(" Options:");
writer.println(" -f,--fix Update hbase:meta with the corrections");
writer.println(" It is recommended to first run this utility without the fix");
writer.println(" option to ensure that the utility is generating the correct");
writer.println(" serialized RegionInfo data structures. Inspect the output to");
writer.println(" confirm that the hbase:meta rowkey matches the new RegionInfo.");
writer.println();
writer.println(" This tool will read hbase:meta and report any regions whose rowkey");
writer.println(" and cell value differ in their encoded region name. HBASE-23328 ");
writer.println(" illustrates a problem for read-replica enabled tables in which ");
writer.println(" the encoded region name (the MD5 hash) does not match between ");
writer.println(" the rowkey and the value. This problem is generally harmless ");
writer.println(" for normal operation, but can break other HBCK2 tools.");
writer.println();
writer.println(" Run this command to determine if any regions are affected by ");
writer.println(" this bug and use the -f/--fix option to then correct any");
writer.println(" affected regions.");
}

static void showErrorMessage(String error) {
if (error != null) {
System.out.println("ERROR: " + error);
@@ -1034,6 +1075,20 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc
tableInfoGenerator.generateTableDescriptorFileIfMissing(commands[1].trim());
break;

case REGIONINFO_MISMATCH:
// `commands` includes the `regionInfoMismatch` argument.
if (commands.length > 2) {
showErrorMessage(command + " takes one optional argument, got more than one.");
return EXIT_FAILURE;
}
try {
regionInfoMismatch(commands);
} catch (Exception e) {
e.printStackTrace();
return EXIT_FAILURE;
}
break;

default:
showErrorMessage("Unsupported command: " + command);
return EXIT_FAILURE;
@@ -0,0 +1,150 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hbase;


import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.util.ByteArrayHashKey;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.HashKey;
import org.apache.hadoop.hbase.util.JenkinsHash;

/**
* A copy of utilities from {@link org.apache.hadoop.hbase.client.RegionInfo} to
* copy internal methods into HBCK2 for stability to avoid using Private methods.
*/
public final class HBCKRegionInfo {

/**
* Separator used to demarcate the encodedName in a region name
* in the new format. See description on new format above.
*/
static final int ENC_SEPARATOR = '.';

static final int MD5_HEX_LENGTH = 32;

static final int DEFAULT_REPLICA_ID = 0;

static final byte REPLICA_ID_DELIMITER = (byte)'_';

private HBCKRegionInfo() {}

/**
* Does region name contain its encoded name?
* @param regionName region name
* @return boolean indicating if this a new format region
* name which contains its encoded name.
*/
public static boolean hasEncodedName(final byte[] regionName) {
// check if region name ends in ENC_SEPARATOR
return (regionName.length >= 1) &&
(regionName[regionName.length - 1] == RegionInfo.ENC_SEPARATOR);
}

/**
* @return the encodedName
*/
public static String encodeRegionName(final byte [] regionName) {
String encodedName;
if (hasEncodedName(regionName)) {
// region is in new format:
// <tableName>,<startKey>,<regionIdTimeStamp>/encodedName/
encodedName = Bytes.toString(regionName,
regionName.length - MD5_HEX_LENGTH - 1,
MD5_HEX_LENGTH);
} else {
// old format region name. First hbase:meta region also
// use this format.EncodedName is the JenkinsHash value.
HashKey<byte[]> key = new ByteArrayHashKey(regionName, 0, regionName.length);
int hashVal = Math.abs(JenkinsHash.getInstance().hash(key, 0));
encodedName = String.valueOf(hashVal);
}
return encodedName;
}

/**
* Separate elements of a regionName.
* Region name is of the format:
* <code>tablename,startkey,regionIdTimestamp[_replicaId][.encodedName.]</code>.
* Startkey can contain the delimiter (',') so we parse from the start and then parse from
* the end.
* @return Array of byte[] containing tableName, startKey and id OR null if not parseable
* as a region name.
*/
public static byte [][] parseRegionNameOrReturnNull(final byte[] regionName) {
int offset = -1;
for (int i = 0; i < regionName.length; i++) {
if (regionName[i] == HConstants.DELIMITER) {
offset = i;
break;
}
}
if (offset == -1) {
return null;
}
byte[] tableName = new byte[offset];
System.arraycopy(regionName, 0, tableName, 0, offset);
offset = -1;

int endOffset = regionName.length;
// check whether regionName contains encodedName
if (regionName.length > MD5_HEX_LENGTH + 2 &&
regionName[regionName.length-1] == ENC_SEPARATOR &&
regionName[regionName.length-MD5_HEX_LENGTH-2] == ENC_SEPARATOR) {
endOffset = endOffset - MD5_HEX_LENGTH - 2;
}

// parse from end
byte[] replicaId = null;
int idEndOffset = endOffset;
for (int i = endOffset - 1; i > 0; i--) {
if (regionName[i] == REPLICA_ID_DELIMITER) { //replicaId may or may not be present
replicaId = new byte[endOffset - i - 1];
System.arraycopy(regionName, i + 1, replicaId, 0,
endOffset - i - 1);
idEndOffset = i;
// do not break, continue to search for id
}
if (regionName[i] == HConstants.DELIMITER) {
offset = i;
break;
}
}
if (offset == -1) {
return null;
}
byte [] startKey = HConstants.EMPTY_BYTE_ARRAY;
if(offset != tableName.length + 1) {
startKey = new byte[offset - tableName.length - 1];
System.arraycopy(regionName, tableName.length + 1, startKey, 0,
offset - tableName.length - 1);
}
byte [] id = new byte[idEndOffset - offset - 1];
System.arraycopy(regionName, offset + 1, id, 0,
idEndOffset - offset - 1);
byte [][] elements = new byte[replicaId == null ? 3 : 4][];
elements[0] = tableName;
elements[1] = startKey;
elements[2] = id;
if (replicaId != null) {
elements[3] = replicaId;
}
return elements;
}
}

0 comments on commit e3a8f96

Please sign in to comment.