Restart ES after keystore upgrade in postinst (#84224)

Closes #82433. If the environment variable `RESTART_ON_UPGRADE` is true,
then ensure that we delay restarting Elasticseach until after the
keystore is upgraded, or else we can run into permissions problems.

distribution/packages/src/common/scripts/postinst

@@ -122,15 +122,6 @@ if [ "x$IS_UPGRADE" != "xtrue" ]; then
         echo " sudo systemctl start elasticsearch.service"
     fi
     set -e
-
-elif [ "$RESTART_ON_UPGRADE" = "true" ]; then
-
-    echo -n "Restarting elasticsearch service..."
-    if command -v systemctl >/dev/null; then
-        systemctl daemon-reload
-        systemctl restart elasticsearch.service || true
-    fi
-    echo " OK"
 fi
 
 # the equivalent code for rpm is in posttrans
@@ -149,4 +140,14 @@ if [ "$PACKAGE" = "deb" ]; then
     fi
 fi
 
+if [ "$RESTART_ON_UPGRADE" = "true" ]; then
+
+    echo -n "Restarting elasticsearch service..."
+    if command -v systemctl >/dev/null; then
+        systemctl daemon-reload
+        systemctl restart elasticsearch.service || true
+    fi
+    echo " OK"
+fi
+
 @scripts.footer@

docs/changelog/84224.yaml

@@ -0,0 +1,6 @@
+pr: 84224
+summary: Restart ES after keystore upgrade in postinst
+area: Packaging
+type: bug
+issues:
+ - 82433

qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java

@@ -9,6 +9,8 @@
 package org.elasticsearch.packaging.test;
 
 import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.Packages;
+import org.elasticsearch.packaging.util.ServerUtils;
 import org.junit.BeforeClass;
 
 import java.nio.file.Paths;
@@ -83,5 +85,27 @@ public void test30Purge() throws Exception {
         assertPathsDoNotExist(installation.config, installation.envFile);
 
         assertThat(packageStatus(distribution()).exitCode(), is(1));
+
+        installation = null;
+    }
+
+    /**
+     * Check that restarting on upgrade doesn't run into a problem where the keystore
+     * upgrade is attempted as the wrong user i.e. the restart happens at the correct
+     * point. See #82433.
+     */
+    public void test40RestartOnUpgrade() throws Exception {
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution());
+        assertInstalled(distribution());
+
+        // Ensure ES is started
+        Packages.runElasticsearchStartCommand(sh);
+        ServerUtils.waitForElasticsearch(installation);
+
+        sh.getEnv().put("RESTART_ON_UPGRADE", "true");
+        installation = installPackage(sh, distribution());
+
+        ServerUtils.waitForElasticsearch(installation);
     }
 }