Introduce jvm.options.d for customizing JVM options

distribution/archives/build.gradle

@@ -44,6 +44,11 @@ task createPluginsDir(type: EmptyDirTask) {
   dir = "${pluginsDir}"
   dirMode = 0755
 }
+ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d')
+task createJvmOptionsDir(type: EmptyDirTask) {
+  dir = "${jvmOptionsDir}"
+  dirMode = 0750
+}
 
 CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
   return copySpec {
@@ -55,6 +60,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
         dirMode 0750
         fileMode 0660
         with configFiles(distributionType, oss, jdk)
+        from {
+          dirMode 0750
+          jvmOptionsDir.getParent()
+        }
       }
       into('bin') {
         with binFiles(distributionType, oss, jdk)
@@ -94,7 +103,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 
 // common config across all zip/tar
 tasks.withType(AbstractArchiveTask) {
-  dependsOn createLogsDir, createPluginsDir
+  dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir
   String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1)
   destinationDir = file("${subdir}/build/distributions")
   baseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}"

distribution/docker/src/docker/Dockerfile

@@ -30,8 +30,8 @@ ${source_elasticsearch}
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
 RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \
     && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
-RUN mkdir -p config data logs
-RUN chmod 0775 config data logs
+RUN mkdir -p config config/jvm.options.d data logs
+RUN chmod 0775 config config/jvm.options.d data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
 RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 

distribution/packages/build.gradle

@@ -79,8 +79,9 @@ void addProcessFilesTask(String type, boolean oss, boolean jdk) {
       mkdir "${packagingFiles}/var/lib/elasticsearch"
       mkdir "${packagingFiles}/usr/share/elasticsearch/plugins"
 
-      // bare empty dir for /etc/elasticsearch
+      // bare empty dir for /etc/elasticsearch and /etc/elasticsearch/jvm.options.d
       mkdir "${packagingFiles}/elasticsearch"
+      mkdir "${packagingFiles}/elasticsearch/jvm.options.d"
     }
   }
 }
@@ -197,6 +198,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
       includeEmptyDirs true
       createDirectoryEntry true
       include("elasticsearch") // empty dir, just to add directory entry
+      include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry
     }
     from("${packagingFiles}/etc/elasticsearch") {
       into('/etc/elasticsearch')

distribution/packages/src/common/scripts/postrm

@@ -17,6 +17,7 @@ else
 fi
 
 REMOVE_DIRS=false
+REMOVE_JVM_OPTIONS_DIRECTORY=false
 REMOVE_USER_AND_GROUP=false
 
 case "$1" in
@@ -27,6 +28,8 @@ case "$1" in
     ;;
 
     purge)
+        REMOVE_DIRS=true
+        REMOVE_JVM_OPTIONS_DIRECTORY=true
         REMOVE_USER_AND_GROUP=true
     ;;
     failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear)
@@ -80,6 +83,20 @@ if [ "$REMOVE_DIRS" = "true" ]; then
         rmdir --ignore-fail-on-non-empty /var/lib/elasticsearch
     fi
 
+    # delete the jvm.options.d directory if and only if empty
+    if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
+        rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}/jvm.options.d"
+    fi
+
+    # delete the jvm.options.d directory if we are purging
+    if [ "$REMOVE_JVM_OPTIONS_DIRECTORY" = "true" ]; then
+      if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
+        echo -n "Deleting jvm.options.d directory..."
+        rm -rf "${ES_PATH_CONF}/jvm.options.d"
+        echo " OK"
+      fi
+    fi
+
     # delete the conf directory if and only if empty
     if [ -d "${ES_PATH_CONF}" ]; then
         rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}"

distribution/packages/src/deb/lintian/elasticsearch

@@ -16,6 +16,7 @@ missing-dep-on-jarwrapper
 # we prefer to not make our config and log files world readable
 non-standard-file-perm etc/default/elasticsearch 0660 != 0644
 non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
+non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
 non-standard-file-perm etc/elasticsearch/*
 non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
 non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755

distribution/src/bin/elasticsearch

@@ -46,8 +46,7 @@ then
   fi
 fi
 
-ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
-ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"`
+ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
 
 # manual parsing to find out, if process should be detached
 if [[ $DAEMONIZE = false ]]; then

distribution/src/bin/elasticsearch-service.bat

@@ -115,7 +115,7 @@ set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
 
 @setlocal
-for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
@@ -167,15 +167,15 @@ for %%a in ("%ES_JAVA_OPTS:;=","%") do (
 @endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS% & set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS%
 
 if "%JVM_MS%" == "" (
-  echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo minimum heap size not set; configure using -Xms via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 if "%JVM_MX%" == "" (
-  echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo maximum heap size not set; configure using -Xmx via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 if "%JVM_SS%" == "" (
-  echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo thread stack size not set; configure using -Xss via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS:"=%

distribution/src/bin/elasticsearch.bat

@@ -72,9 +72,8 @@ if not defined ES_TMPDIR (
   for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set  ES_TMPDIR=%%a
 )
 
-set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 @setlocal
-for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -27,7 +27,9 @@
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -43,6 +45,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 /**
  * Parses JVM options from a file and prints a single line with all JVM options to standard output.
@@ -53,82 +56,90 @@ final class JvmOptionsParser {
      * The main entry point. The exit code is 0 if the JVM options were successfully parsed, otherwise the exit code is 1. If an improperly
      * formatted line is discovered, the line is output to standard error.
      *
-     * @param args the args to the program which should consist of a single option, the path to the JVM options
+     * @param args the args to the program which should consist of a single option, the path to ES_PATH_CONF
      */
     public static void main(final String[] args) throws InterruptedException, IOException {
         if (args.length != 1) {
-            throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args));
-        }
-        final List<String> jvmOptions = new ArrayList<>();
-        final SortedMap<Integer, String> invalidLines = new TreeMap<>();
-        try (
-            InputStream is = Files.newInputStream(Paths.get(args[0]));
-            Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
-            BufferedReader br = new BufferedReader(reader)
-        ) {
-            parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, new JvmOptionConsumer() {
-                @Override
-                public void accept(final String jvmOption) {
-                    jvmOptions.add(jvmOption);
-                }
-            }, new InvalidLineConsumer() {
-                @Override
-                public void accept(final int lineNumber, final String line) {
-                    invalidLines.put(lineNumber, line);
-                }
-            });
+            throw new IllegalArgumentException("expected one argument specifying path to ES_PATH_CONF but was " + Arrays.toString(args));
         }
 
-        if (invalidLines.isEmpty()) {
-            // now append the JVM options from ES_JAVA_OPTS
-            final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
-            if (environmentJvmOptions != null) {
-                jvmOptions.addAll(
-                    Arrays.stream(environmentJvmOptions.split("\\s+"))
-                        .filter(Predicate.not(String::isBlank))
-                        .collect(Collectors.toUnmodifiableList())
-                );
+        final ArrayList<Path> jvmOptionsFiles = new ArrayList<>();
+        jvmOptionsFiles.add(Paths.get(args[0], "jvm.options"));
+
+        final Path jvmOptionsDirectory = Paths.get(args[0], "jvm.options.d");
+
+        if (Files.isDirectory(jvmOptionsDirectory)) {
+            try (
+                DirectoryStream<Path> jvmOptionsDirectoryStream = Files.newDirectoryStream(Paths.get(args[0], "jvm.options.d"), "*.options")
+            ) {
+                // collect the matching JVM options files after sorting them by Path::compareTo
+                StreamSupport.stream(jvmOptionsDirectoryStream.spliterator(), false).sorted().forEach(jvmOptionsFiles::add);
             }
-            final Map<String, String> substitutions = new HashMap<>();
-            substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
-            if (null != System.getenv("ES_PATH_CONF")) {
-                substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
+        }
+
+        final List<String> jvmOptions = new ArrayList<>();
+
+        for (final Path jvmOptionsFile : jvmOptionsFiles) {
+            final SortedMap<Integer, String> invalidLines = new TreeMap<>();
+            try (
+                InputStream is = Files.newInputStream(jvmOptionsFile);
+                Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
+                BufferedReader br = new BufferedReader(reader)
+            ) {
+                parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, jvmOptions::add, invalidLines::put);
             }
-            final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
-            final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
-            final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
-            final List<String> finalJvmOptions = new ArrayList<>(
-                systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
-            );
-            finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
-            finalJvmOptions.addAll(substitutedJvmOptions);
-            finalJvmOptions.addAll(ergonomicJvmOptions);
-            final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
-            Launchers.outPrintln(spaceDelimitedJvmOptions);
-            Launchers.exit(0);
-        } else {
-            final String errorMessage = String.format(
-                Locale.ROOT,
-                "encountered [%d] error%s parsing [%s]",
-                invalidLines.size(),
-                invalidLines.size() == 1 ? "" : "s",
-                args[0]
-            );
-            Launchers.errPrintln(errorMessage);
-            int count = 0;
-            for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
-                count++;
-                final String message = String.format(
+            if (invalidLines.isEmpty() == false) {
+                final String errorMessage = String.format(
                     Locale.ROOT,
-                    "[%d]: encountered improperly formatted JVM option line [%s] on line number [%d]",
-                    count,
-                    entry.getValue(),
-                    entry.getKey()
+                    "encountered [%d] error%s parsing [%s]",
+                    invalidLines.size(),
+                    invalidLines.size() == 1 ? "" : "s",
+                    jvmOptionsFile
                 );
-                Launchers.errPrintln(message);
+                Launchers.errPrintln(errorMessage);
+                int count = 0;
+                for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
+                    count++;
+                    final String message = String.format(
+                        Locale.ROOT,
+                        "[%d]: encountered improperly formatted JVM option in [%s] on line number [%d]: [%s]",
+                        count,
+                        jvmOptionsFile,
+                        entry.getKey(),
+                        entry.getValue()
+                    );
+                    Launchers.errPrintln(message);
+                }
+                Launchers.exit(1);
             }
-            Launchers.exit(1);
         }
+
+        // now append the JVM options from ES_JAVA_OPTS
+        final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
+        if (environmentJvmOptions != null) {
+            jvmOptions.addAll(
+                Arrays.stream(environmentJvmOptions.split("\\s+"))
+                    .filter(Predicate.not(String::isBlank))
+                    .collect(Collectors.toUnmodifiableList())
+            );
+        }
+        final Map<String, String> substitutions = new HashMap<>();
+        substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
+        if (null != System.getenv("ES_PATH_CONF")) {
+            substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
+        }
+        final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
+        final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
+        final List<String> finalJvmOptions = new ArrayList<>(
+            systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
+        );
+        finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
+        finalJvmOptions.addAll(substitutedJvmOptions);
+        finalJvmOptions.addAll(ergonomicJvmOptions);
+        final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
+        Launchers.outPrintln(spaceDelimitedJvmOptions);
+        Launchers.exit(0);
     }
 
     static List<String> substitutePlaceholders(final List<String> jvmOptions, final Map<String, String> substitutions) {

docs/reference/setup/important-settings/heap-size.asciidoc

@@ -49,7 +49,7 @@ caches, but the less memory it leaves available for the operating system to use
 for the filesystem cache. Also, larger heaps can cause longer garbage
 collection pauses.
 
-Here are examples of how to set the heap size via the jvm.options file:
+Here is an example of how to set the heap size via a `jvm.options.d/` file:
 
 [source,txt]
 ------------------
@@ -60,8 +60,7 @@ Here are examples of how to set the heap size via the jvm.options file:
 <2> Set the maximum heap size to 2g.
 
 It is also possible to set the heap size via an environment variable. This can
-be done by commenting out the `Xms` and `Xmx` settings in the
-<<jvm-options,`jvm.options`>> file and setting these values via `ES_JAVA_OPTS`:
+be done by setting these values via `ES_JAVA_OPTS`:
 
 [source,sh]
 ------------------

docs/reference/setup/install/docker.asciidoc

@@ -266,22 +266,23 @@ unless you are pinning one container per host.
 [[docker-set-heap-size]]
 ===== Set the heap size
 
-Use the `ES_JAVA_OPTS` environment variable to set the heap size.
-For example, to use 16GB, specify `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with
-`docker run`. Note that while the default configuration file `jvm.options`
-sets a default heap of 1GB, any value you set in `ES_JAVA_OPTS` will
-override it.
+To configure the heap size, you can bind mount a <<jvm-options,JVM options>>
+file under `/usr/share/elasticsearch/config/jvm.options.d` that includes your
+desired <<heap-size,heap size>> settings. Note that while the default root
+`jvm.options` file sets a default heap of 1 GB, any value you set in a
+bind-mounted JVM options file will override it.
+
+While setting the heap size via bind-mounted JVM options is the recommended
+method, you can also configure this by using the `ES_JAVA_OPTS` environment
+variable to set the heap size. For example, to use 16 GB, specify
+`-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`. Note that while the
+default root `jvm.options` file sets a default heap of 1 GB, any value you set
+in `ES_JAVA_OPTS` will override it.
 
 IMPORTANT: You must <<heap-size,configure the heap size>> even if you are
 https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory[limiting
 memory access] to the container.
 
-While setting the heap size via an environment variable is the recommended
-method, you can also configure this by bind-mounting your own `jvm.options`
-file under `/usr/share/elasticsearch/config/`. The file that {es} provides
-contains some important settings, so you should start by taking a copy of
-`jvm.options` from an {es} container and editing it as you require.
-
 ===== Pin deployments to a specific image version
 
 Pin your deployments to a specific version of the {es} Docker image. For