[9.1] Set coredump filter for Elasticsearch (#137300)

qa/packaging/src/test/java/org/elasticsearch/packaging/test/LinuxSystemTests.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.ProcessInfo;
+import org.junit.BeforeClass;
+
+import java.util.List;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue;
+import static org.hamcrest.Matchers.equalTo;
+
+// tests for how the linux distro interacts with the OS
+public class LinuxSystemTests extends PackagingTestCase {
+
+    @BeforeClass
+    public static void ensureLinux() {
+        assumeTrue(Platforms.LINUX);
+    }
+
+    public void test10Install() throws Exception {
+        install();
+    }
+
+    public void test20CoredumpFilter() throws Exception {
+        startElasticsearch();
+
+        // find the Elasticsearch process
+        int esPid = -1;
+        List<ProcessInfo> procs = ProcessInfo.getProcessInfo(sh, "java");
+        for (ProcessInfo proc : procs) {
+            if (proc.commandLine().contains("org.elasticsearch.bootstrap.Elasticsearch")) {
+                esPid = proc.pid();
+            }
+        }
+        if (esPid == -1) {
+            fail("Could not find Elasticsearch process, existing processes:\n" + procs);
+        }
+
+        // check the coredump filter was set correctly
+        String coredumpFilter = sh.run("cat /proc/" + esPid + "/coredump_filter").stdout().trim();
+        assertThat(coredumpFilter, equalTo("00000023"));
+    }
+
+}

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ProcessInfo.java

@@ -22,7 +22,7 @@
  * works in Linux containers. At the moment that isn't a problem, because we only publish Docker images
  * for Linux.
  */
-public record ProcessInfo(int pid, int uid, int gid, String username, String group) {
+public record ProcessInfo(int pid, int uid, int gid, String username, String group, String commandLine) {
 
     /**
      * Fetches process information about <code>command</code>, using <code>sh</code> to execute commands.
@@ -53,7 +53,9 @@ public static List<ProcessInfo> getProcessInfo(Shell sh, String command) {
             final String username = sh.run("getent passwd " + uid + " | cut -f1 -d:").stdout().trim();
             final String group = sh.run("getent group " + gid + " | cut -f1 -d:").stdout().trim();
 
-            infos.add(new ProcessInfo(pid, uid, gid, username, group));
+            final String commandLine = sh.run("cat /proc/" + pid + "/cmdline").stdout().trim();
+
+            infos.add(new ProcessInfo(pid, uid, gid, username, group, commandLine));
         }
         return Collections.unmodifiableList(infos);
     }

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -484,6 +484,10 @@ static void initializeNatives(final Path tmpFile, final boolean mlockAll, final
             }
         }
 
+        if (IOUtils.LINUX) {
+            setCoredumpFilter();
+        }
+
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();
     }
@@ -598,6 +602,20 @@ static Map<String, Set<String>> findPluginsWithNativeAccess(Map<String, Policy>
         return pluginsWithNativeAccess;
     }
 
+    @SuppressForbidden(reason = "access proc filesystem")
+    private static void setCoredumpFilter() {
+        // The coredump filter determines which types of memory are added to core dumps. By default, Java
+        // includes memory mapped files, bits 2 and 3. Here we disable those bits. Note that the VM
+        // has special options to disable these, but the filter is then inherited from the parent process
+        // which is the server CLI, which is also a JVM so it has these bits set. Thus, we set it explicitly.
+        // See https://man7.org/linux/man-pages/man5/core.5.html for more info on the relevant bits of the filter
+        try {
+            Files.writeString(Path.of("/proc/self/coredump_filter"), "0x23");
+        } catch (IOException e) {
+            throw new RuntimeException("Could not set coredump filter", e);
+        }
+    }
+
     // -- instance
 
     private static volatile Elasticsearch INSTANCE;