Stop renew retention leases when follow task fails (#65168)

If a shard follow-task hits a non-retryable error and stops, then we should also stop the retention-leases renewal process associated with that follow-task.
elastic · Nov 18, 2020 · 4bf4614 · 4bf4614
1 parent 767003c
commit 4bf4614
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 5 deletions.
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java
@@ -486,12 +486,18 @@ private void handleFailure(Exception e, AtomicInteger retryCounter, Runnable tas
                 scheduler.accept(TimeValue.timeValueMillis(delay), task);
             }
         } else {
-            setFatalException(e);
+            onFatalFailure(e);
         }
     }
 
-    void setFatalException(Exception e) {
-        fatalException = ExceptionsHelper.convertToElastic(e);
+    final void onFatalFailure(Exception e) {
+        synchronized (this) {
+            this.fatalException = ExceptionsHelper.convertToElastic(e);
+            if (this.renewable != null) {
+                this.renewable.cancel();
+                this.renewable = null;
+            }
+        }
         LOGGER.warn("shard follow task encounter non-retryable error", e);
     }
 

diff --git a/...plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/...plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
@@ -405,7 +405,7 @@ protected void nodeOperation(final AllocatedPersistentTask task, final ShardFoll
                     shardFollowNodeTask), e);
                 threadPool.schedule(() -> nodeOperation(task, params, state), params.getMaxRetryDelay(), Ccr.CCR_THREAD_POOL_NAME);
             } else {
-                shardFollowNodeTask.setFatalException(e);
+                shardFollowNodeTask.onFatalFailure(e);
             }
         };
 

diff --git a/...plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskTests.java b/...plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskTests.java
@@ -924,7 +924,7 @@ public void testRetryableError() {
         assertThat(status.followerGlobalCheckpoint(), equalTo(-1L));
     }
 
-    public void testNonRetryableError() {
+    public void testNonRetryableError() throws Exception {
         ShardFollowTaskParams params = new ShardFollowTaskParams();
         params.maxReadRequestOperationCount = 64;
         params.maxOutstandingReadRequests = 1;
@@ -948,6 +948,7 @@ public void testNonRetryableError() {
         ShardFollowNodeTaskStatus status = task.getStatus();
         assertThat(status.outstandingWriteRequests(), equalTo(1));
         assertThat(status.followerGlobalCheckpoint(), equalTo(-1L));
+        assertBusy(() -> assertNull(task.getRenewable()));
     }
 
     public void testMaxWriteRequestSize() {