diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index b938ff240fce..56c1d6c6443a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.master.SnapshotSentinel; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; +import org.apache.hadoop.hbase.procedure.Procedure; import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs; import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs; @@ -322,7 +323,15 @@ public boolean isSnapshotDone(SnapshotDescription expected) throws IOException { try { handler.rethrowException(); } catch (ForeignException e) { - throw new HBaseSnapshotException("Snapshot " + ssString + " had an error from RS", e, + // Give some procedure info on an exception. + String status; + Procedure p = coordinator.getProcedure(expected.getName()); + if (p != null) { + status = p.getStatus(); + } else { + status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames(); + } + throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e, expected); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java index a7891502e23d..7cfb006d69d5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java @@ -164,6 +164,19 @@ public String getName() { return procName; } + /** + * Returns a copy of the procedure members still trying to enter the barrier. + * @return + */ + public String getStatus() { + String waiting, done; + synchronized (joinBarrierLock) { + waiting = acquiringMembers.toString(); + done = inBarrierMembers.toString(); + } + return "Procedure " + procName + " { waiting=" + waiting + " done="+ done + " }"; + } + /** * Get the ExternalErrorDispatcher * @return the Procedure's monitor. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java index e6d6fbead731..dfacfd5de69f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureCoordinator.java @@ -19,7 +19,9 @@ import java.io.IOException; import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -246,4 +248,21 @@ void memberFinishedBarrier(String procName, final String member) { ProcedureCoordinatorRpcs getRpcs() { return rpcs; } + + /** + * Returns the procedure. This Procedure is a live instance so should not be modified but can + * be inspected. + * @param name Name of the procedure + * @return Procedure or null if not present any more + */ + public Procedure getProcedure(String name) { + return procedures.get(name); + } + + /** + * @return Return set of all procedure names. + */ + public Set getProcedureNames() { + return new HashSet(procedures.keySet()); + } } \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java index 28ebe38adc4a..37dd09f2c9b6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureMember.java @@ -76,7 +76,7 @@ public static ThreadPoolExecutor defaultPool(long wakeFrequency, long keepAlive, int procThreads, String memberName) { return new ThreadPoolExecutor(1, procThreads, keepAlive, TimeUnit.SECONDS, new SynchronousQueue(), - new DaemonThreadFactory("( member-" + memberName + ") subprocedure-pool")); + new DaemonThreadFactory("member: '" + memberName + "' subprocedure-pool")); } /**