From 5f195468ab19b7eeb884f9a4ec95d4e99aa3ac73 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 29 Aug 2019 11:10:09 +0100
Subject: [PATCH] Repro of #45956

---
 .../indices/recovery/RecoverySourceHandler.java      |  9 +++++++++
 .../indices/recovery/IndexRecoveryIT.java            | 12 ++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index e5ab1d7890eb44..84dd649260c6db 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -405,10 +405,12 @@ private Releasable acquireStore(Store store) {
             threadPool.generic().execute(new ActionRunnable<>(future) {
                 @Override
                 protected void doRun() {
+                    logger.trace("calling store.decRef");
                     store.decRef();
                     listener.onResponse(null);
                 }
             });
+            logger.trace("waiting for store.decRef");
             FutureUtils.get(future);
         });
     }
@@ -913,6 +915,7 @@ public void close() throws IOException {
 
     private void cleanFiles(Store store, Store.MetadataSnapshot sourceMetadata, IntSupplier translogOps,
                             long globalCheckpoint, ActionListener<Void> listener) {
+        logger.trace("waiting before cleanFiles");
         // Send the CLEAN_FILES request, which takes all of the files that
         // were transferred and renames them from their temporary file
         // names to the actual file names. It also writes checksums for
@@ -921,6 +924,12 @@ private void cleanFiles(Store store, Store.MetadataSnapshot sourceMetadata, IntS
         // Once the files have been renamed, any other files that are not
         // related to this recovery (out of date segments, for example)
         // are deleted
+        try {
+            Thread.sleep(1000); // NOCOMMIT
+        } catch (InterruptedException e) {
+            throw new AssertionError(e);
+        }
+        logger.trace("calling cleanFiles");
         cancellableThreads.execute(() -> recoveryTarget.cleanFiles(translogOps.getAsInt(), globalCheckpoint, sourceMetadata,
             ActionListener.delegateResponse(listener, (l, e) -> ActionListener.completeWith(l, () -> {
                 StoreFileMetaData[] mds = StreamSupport.stream(sourceMetadata.spliterator(), false).toArray(StoreFileMetaData[]::new);
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index 995e0a53ec234e..0582cd6ccc83ce 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -93,6 +93,7 @@
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.engine.MockEngineSupport;
 import org.elasticsearch.test.junit.annotations.TestIssueLogging;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.test.transport.StubbableTransport;
@@ -857,6 +858,17 @@ public void sendRequest(Transport.Connection connection, long requestId, String
         }
     }
 
+    @TestLogging(reason = "nocommit", value = "org.elasticsearch.indices.recovery:TRACE")
+    public void testCloseNodeDuringRecovery() throws Exception {
+        internalCluster().startNode();
+        assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)));
+        ensureYellow("test");
+        internalCluster().startNode();
+        internalCluster().fullRestart();
+    }
+
     @TestIssueLogging(value = "org.elasticsearch:DEBUG", issueUrl = "https://github.com/elastic/elasticsearch/issues/45953")
     public void testHistoryRetention() throws Exception {
         internalCluster().startNodes(3);