track and handle errors during nodetool bootstrap resume properly

patch by Leonard Ma; reviewed by Caleb Rackliffe and Aleksey Yeschenko for CASSANDRA-16491
apache · Oct 28, 2022 · 8ec0436 · 8ec0436
1 parent 39a4702
commit 8ec0436
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 1 deletion.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,4 +1,5 @@
 4.2
+ * Nodetool bootstrap resume will now return an error if the operation fails (CASSANDRA-16491)
  * Disable resumable bootstrap by default (CASSANDRA-17679)
  * Include Git SHA in --verbose flag for nodetool version (CASSANDRA-17753)
  * Update Byteman to 4.0.20 and Jacoco to 0.8.8 (CASSANDRA-16413)

diff --git a/src/java/org/apache/cassandra/tools/BootstrapMonitor.java b/src/java/org/apache/cassandra/tools/BootstrapMonitor.java
@@ -34,6 +34,7 @@ public class BootstrapMonitor extends JMXNotificationProgressListener
     private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
     private final PrintStream out;
     private final Condition condition = newOneTimeCondition();
+    private volatile Exception error;
 
     public BootstrapMonitor(PrintStream out)
     {
@@ -66,7 +67,7 @@ public void handleConnectionClosed(long timestamp, String message)
     @Override
     public void handleConnectionFailed(long timestamp, String message)
     {
-        Exception error = new IOException(String.format("[%s] JMX connection closed. (%s)",
+        error = new IOException(String.format("[%s] JMX connection closed. (%s)",
                                               format.format(timestamp), message));
         out.println(error.getMessage());
         condition.signalAll();
@@ -82,9 +83,19 @@ public void progress(String tag, ProgressEvent event)
             message = message + " (progress: " + (int)event.getProgressPercentage() + "%)";
         }
         out.println(message);
+        if (type == ProgressEventType.ERROR)
+        {
+            error = new RuntimeException(String.format("Bootstrap resume has failed with error: %s", message));
+            condition.signalAll();
+        }
         if (type == ProgressEventType.COMPLETE)
         {
             condition.signalAll();
         }
     }
+
+    public Exception getError()
+    {
+        return error;
+    }
 }
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -2005,6 +2005,8 @@ public void resumeBootstrap(PrintStream out) throws IOException
             {
                 out.println("Resuming bootstrap");
                 monitor.awaitCompletion();
+                if (monitor.getError() != null)
+                    throw monitor.getError();
             }
             else
             {

diff --git a/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java b/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java
@@ -131,6 +131,7 @@ private static void bootstrap(Cluster cluster,
             .failure()
             .errorContains("Cannot join the ring until bootstrap completes");
 
+        node.nodetoolResult("bootstrap", "resume").asserts().failure();
         RewriteEnabled.disable();
         node.nodetoolResult("bootstrap", "resume").asserts().success();
         if (isWriteSurvey)