Skip to content

Commit

Permalink
HBASE-12987 Pare repeated hbck output and increase verbosity in long-…
Browse files Browse the repository at this point in the history 
…running tasks.
  • Loading branch information
@joshelser @ndimiduk
joshelser authored and ndimiduk committed Apr 15, 2015
1 parent 14261bc commit 682a29a
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 8 deletions.
55 changes: 53 additions & 2 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
View file
Expand Up @@ -66,6 +66,7 @@
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
import org.apache.hadoop.hbase.regionserver.HRegion;
Expand Down Expand Up @@ -1545,6 +1546,28 @@ public static FileSystem getCurrentFileSystem(Configuration conf)
*/
public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
final FileSystem fs, final Path hbaseRootDir, TableName tableName)
throws IOException {
return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null);
}

/**
* Runs through the HBase rootdir/tablename and creates a reverse lookup map for
* table StoreFile names to the full Path.
* <br>
* Example...<br>
* Key = 3944417774205889744 <br>
* Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
*
* @param map map to add values. If null, this method will create and populate one to return
* @param fs The file system to use.
* @param hbaseRootDir The root directory to scan.
* @param tableName name of the table to scan.
* @param errors ErrorReporter instance or null
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
*/
public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
final FileSystem fs, final Path hbaseRootDir, TableName tableName, ErrorReporter errors)
throws IOException {
if (map == null) {
map = new HashMap<String, Path>();
Expand All @@ -1557,10 +1580,16 @@ public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
PathFilter familyFilter = new FamilyDirFilter(fs);
FileStatus[] regionDirs = fs.listStatus(tableDir, new RegionDirFilter(fs));
for (FileStatus regionDir : regionDirs) {
if (null != errors) {
errors.progress();
}
Path dd = regionDir.getPath();
// else its a region name, now look in region for families
FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
for (FileStatus familyDir : familyDirs) {
if (null != errors) {
errors.progress();
}
Path family = familyDir.getPath();
if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) {
continue;
Expand All @@ -1569,6 +1598,9 @@ public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
// put in map
FileStatus[] familyStatus = fs.listStatus(family);
for (FileStatus sfStatus : familyStatus) {
if (null != errors) {
errors.progress();
}
Path sf = sfStatus.getPath();
map.put( sf.getName(), sf);
}
Expand All @@ -1589,7 +1621,6 @@ public static int getRegionReferenceFileCount(final FileSystem fs, final Path p)
return result;
}


/**
* Runs through the HBase rootdir and creates a reverse lookup map for
* table StoreFile names to the full Path.
Expand All @@ -1605,6 +1636,26 @@ public static int getRegionReferenceFileCount(final FileSystem fs, final Path p)
*/
public static Map<String, Path> getTableStoreFilePathMap(
final FileSystem fs, final Path hbaseRootDir)
throws IOException {
return getTableStoreFilePathMap(fs, hbaseRootDir, null);
}

/**
* Runs through the HBase rootdir and creates a reverse lookup map for
* table StoreFile names to the full Path.
* <br>
* Example...<br>
* Key = 3944417774205889744 <br>
* Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
*
* @param fs The file system to use.
* @param hbaseRootDir The root directory to scan.
* @param errors ErrorReporter instance or null
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
*/
public static Map<String, Path> getTableStoreFilePathMap(
final FileSystem fs, final Path hbaseRootDir, ErrorReporter errors)
throws IOException {
Map<String, Path> map = new HashMap<String, Path>();

Expand All @@ -1614,7 +1665,7 @@ public static Map<String, Path> getTableStoreFilePathMap(
// only include the directory paths to tables
for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
getTableStoreFilePathMap(map, fs, hbaseRootDir,
FSUtils.getTableName(tableDir));
FSUtils.getTableName(tableDir), errors);
}
return map;
}
Expand Down
38 changes: 32 additions & 6 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
View file
Expand Up @@ -641,13 +641,17 @@ public int onlineConsistencyRepair() throws IOException, KeeperException,

// load regiondirs and regioninfos from HDFS
if (shouldCheckHdfs()) {
LOG.info("Loading region directories from HDFS");
loadHdfsRegionDirs();
LOG.info("Loading region information from HDFS");
loadHdfsRegionInfos();
}

// fix the orphan tables
fixOrphanTables();

LOG.info("Checking and fixing region consistency");

// Check and fix consistency
checkAndFixConsistency();

Expand Down Expand Up @@ -970,7 +974,10 @@ private void offlineReferenceFileRepair() throws IOException {
Configuration conf = getConf();
Path hbaseRoot = FSUtils.getRootDir(conf);
FileSystem fs = hbaseRoot.getFileSystem(conf);
Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
LOG.info("Computing mapping of all store files");
Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
errors.print("");
LOG.info("Validating mapping using HDFS state");
for (Path path: allFiles.values()) {
boolean isReference = false;
try {
Expand Down Expand Up @@ -1168,6 +1175,7 @@ private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
}

loadTableInfosForTablesWithNoRegion();
errors.print("");

return tablesInfo;
}
Expand Down Expand Up @@ -1358,6 +1366,7 @@ private ArrayList<Put> generatePuts(
*/
private void suggestFixes(
SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
logParallelMerge();
for (TableInfo tInfo : tablesInfo.values()) {
TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
tInfo.checkRegionChain(handler);
Expand Down Expand Up @@ -1431,9 +1440,23 @@ public boolean rebuildMeta(boolean fix) throws IOException,
return true;
}

/**
* Log an appropriate message about whether or not overlapping merges are computed in parallel.
*/
private void logParallelMerge() {
if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
" false to run serially.");
} else {
LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" +
" true to run in parallel.");
}
}

private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
boolean fixOverlaps) throws IOException {
LOG.info("Checking HBase region split map from HDFS data...");
logParallelMerge();
for (TableInfo tInfo : tablesInfo.values()) {
TableIntegrityErrorHandler handler;
if (fixHoles || fixOverlaps) {
Expand Down Expand Up @@ -1662,6 +1685,7 @@ public void loadHdfsRegionDirs() throws IOException, InterruptedException {
LOG.warn("Could not load region dir " , e.getCause());
}
}
errors.print("");
}

/**
Expand Down Expand Up @@ -2395,6 +2419,7 @@ SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {

loadTableInfosForTablesWithNoRegion();

logParallelMerge();
for (TableInfo tInfo : tablesInfo.values()) {
TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
if (!tInfo.checkRegionChain(handler)) {
Expand Down Expand Up @@ -3011,15 +3036,11 @@ public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOExc

// TODO fold this into the TableIntegrityHandler
if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
" false to run serially.");
boolean ok = handleOverlapsParallel(handler, prevKey);
if (!ok) {
return false;
}
} else {
LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" +
" true to run in parallel.");
for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
handler.handleOverlapGroup(overlap);
}
Expand Down Expand Up @@ -3745,6 +3766,8 @@ void reportError(
static class PrintingErrorReporter implements ErrorReporter {
public int errorCount = 0;
private int showProgress;
// How frequently calls to progress() will create output
private static final int progressThreshold = 100;

Set<TableInfo> errorTables = new HashSet<TableInfo>();

Expand Down Expand Up @@ -3859,7 +3882,7 @@ public synchronized void detail(String message) {

@Override
public synchronized void progress() {
if (showProgress++ == 10) {
if (showProgress++ == progressThreshold) {
if (!summary) {
System.out.print(".");
}
Expand Down Expand Up @@ -3956,6 +3979,7 @@ public synchronized Void call() throws IOException {
// level 2: <HBASE_DIR>/<table>/*
FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
for (FileStatus regionDir : regionDirs) {
errors.progress();
String encodedName = regionDir.getPath().getName();
// ignore directories that aren't hexadecimal
if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
Expand Down Expand Up @@ -3983,6 +4007,7 @@ public synchronized Void call() throws IOException {
FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
for (FileStatus subDir : subDirs) {
errors.progress();
String sdName = subDir.getPath().getName();
if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
he.hdfsOnlyEdits = false;
Expand Down Expand Up @@ -4023,6 +4048,7 @@ public synchronized Void call() throws IOException {
// only load entries that haven't been loaded yet.
if (hbi.getHdfsHRI() == null) {
try {
errors.progress();
hbck.loadHdfsRegioninfo(hbi);
} catch (IOException ioe) {
String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
Expand Down

0 comments on commit 682a29a

Please sign in to comment.