remove option to configure custom config file via CONF_FILE or -Des.d…

…efault.conf

It is rarely used and was not consistently handled by different distributions anyway.
This commit also adds a test for specifying CONF_DIR when installing plugins and
starting elasticsearch.

relates to #12712 and #12954
closes #5329
closes #13715

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.cli.CliTool;
 import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.CreationException;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.logging.ESLogger;
@@ -249,13 +248,13 @@ static void init(String[] args) throws Throwable {
 
         Environment environment = initialSettings(foreground);
         Settings settings = environment.settings();
+        setupLogging(settings, environment);
+        checkForCustomConfFile();
 
         if (environment.pidFile() != null) {
             PidFile.create(environment.pidFile(), true);
         }
 
-        setupLogging(settings, environment);
-
         if (System.getProperty("es.max-open-files", "false").equals("true")) {
             ESLogger logger = Loggers.getLogger(Bootstrap.class);
             logger.info("max_open_files [{}]", ProcessProbe.getInstance().getMaxFileDescriptorCount());
@@ -330,4 +329,21 @@ private static void sysError(String line, boolean flush) {
             System.err.flush();
         }
     }
+
+    private static void checkForCustomConfFile() {
+        String confFileSetting = System.getProperty("es.default.config");
+        checkUnsetAndMaybeExit(confFileSetting, "es.default.config");
+        confFileSetting = System.getProperty("es.config");
+        checkUnsetAndMaybeExit(confFileSetting, "es.config");
+        confFileSetting = System.getProperty("elasticsearch.config");
+        checkUnsetAndMaybeExit(confFileSetting, "elasticsearch.config");
+    }
+
+    private static void checkUnsetAndMaybeExit(String confFileSetting, String settingName) {
+        if (confFileSetting != null && confFileSetting.isEmpty() == false) {
+            ESLogger logger = Loggers.getLogger(Bootstrap.class);
+            logger.info("{} is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed.", settingName);
+            System.exit(1);
+        }
+    }
 }

core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java

@@ -83,42 +83,20 @@ public static Environment prepareEnvironment(Settings input, Terminal terminal)
         initializeSettings(output, input, true);
         Environment environment = new Environment(output.build());
 
-        // TODO: can we simplify all of this and have a single filename, which is looked up in the config dir?
-        boolean loadFromEnv = true;
-        if (useSystemProperties(input)) {
-            // if its default, then load it, but also load form env
-            if (Strings.hasText(System.getProperty("es.default.config"))) {
-                // TODO: we don't allow multiple config files, but having loadFromEnv true here allows just that
-                loadFromEnv = true;
-                output.loadFromPath(environment.configFile().resolve(System.getProperty("es.default.config")));
-            }
-            // TODO: these should be elseifs so that multiple files cannot be loaded
-            // if explicit, just load it and don't load from env
-            if (Strings.hasText(System.getProperty("es.config"))) {
-                loadFromEnv = false;
-                output.loadFromPath(environment.configFile().resolve(System.getProperty("es.config")));
-            }
-            if (Strings.hasText(System.getProperty("elasticsearch.config"))) {
-                loadFromEnv = false;
-                output.loadFromPath(environment.configFile().resolve(System.getProperty("elasticsearch.config")));
-            }
-        }
-        if (loadFromEnv) {
-            boolean settingsFileFound = false;
-            Set<String> foundSuffixes = new HashSet<>();
-            for (String allowedSuffix : ALLOWED_SUFFIXES) {
-                Path path = environment.configFile().resolve("elasticsearch" + allowedSuffix);
-                if (Files.exists(path)) {
-                    if (!settingsFileFound) {
-                        output.loadFromPath(path);
-                    }
-                    settingsFileFound = true;
-                    foundSuffixes.add(allowedSuffix);
+        boolean settingsFileFound = false;
+        Set<String> foundSuffixes = new HashSet<>();
+        for (String allowedSuffix : ALLOWED_SUFFIXES) {
+            Path path = environment.configFile().resolve("elasticsearch" + allowedSuffix);
+            if (Files.exists(path)) {
+                if (!settingsFileFound) {
+                    output.loadFromPath(path);
                 }
+                settingsFileFound = true;
+                foundSuffixes.add(allowedSuffix);
             }
-            if (foundSuffixes.size() > 1) {
-                throw new SettingsException("multiple settings files found with suffixes: " + Strings.collectionToDelimitedString(foundSuffixes, ","));
-            }
+        }
+        if (foundSuffixes.size() > 1) {
+            throw new SettingsException("multiple settings files found with suffixes: " + Strings.collectionToDelimitedString(foundSuffixes, ","));
         }
 
         // re-initialize settings now that the config file has been loaded

distribution/deb/pom.xml

@@ -76,6 +76,7 @@
                                         <include>bin/elasticsearch</include>
                                         <include>bin/elasticsearch.in.sh</include>
                                         <include>bin/plugin</include>
+                                        <include>bin/elasticsearch-systemd-pre-exec</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -110,7 +111,7 @@
                                 <data>
                                     <src>${project.build.directory}/generated-packaging/deb/bin</src>
                                     <type>directory</type>
-                                    <includes>elasticsearch,elasticsearch.in.sh,plugin</includes>
+                                    <includes>elasticsearch,elasticsearch.in.sh,plugin,elasticsearch-systemd-pre-exec</includes>
                                     <mapper>
                                         <type>perm</type>
                                         <prefix>${packaging.elasticsearch.bin.dir}</prefix>

distribution/deb/src/main/packaging/init.d/elasticsearch

@@ -74,9 +74,6 @@ DATA_DIR=/var/lib/$NAME
 # Elasticsearch configuration directory
 CONF_DIR=/etc/$NAME
 
-# Elasticsearch configuration file (elasticsearch.yml)
-CONF_FILE=$CONF_DIR/elasticsearch.yml
-
 # Maximum number of VMA (Virtual Memory Areas) a process can own
 MAX_MAP_COUNT=262144
 
@@ -93,10 +90,16 @@ if [ -f "$DEFAULT" ]; then
 	. "$DEFAULT"
 fi
 
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON=$ES_HOME/bin/elasticsearch
-DAEMON_OPTS="-d -p $PID_FILE --default.config=$CONF_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
+DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
 
 export ES_HEAP_SIZE
 export ES_HEAP_NEWSIZE

distribution/deb/src/main/packaging/packaging.properties

@@ -6,7 +6,6 @@ packaging.env.file=/etc/default/elasticsearch
 
 # Default configuration directory and file to use in bin/plugin script
 packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
-packaging.plugin.default.config.file=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
 
 # Simple marker to check that properties are correctly overridden
 packaging.type=deb

distribution/rpm/pom.xml

@@ -79,6 +79,7 @@
                                         <include>bin/elasticsearch</include>
                                         <include>bin/elasticsearch.in.sh</include>
                                         <include>bin/plugin</include>
+                                        <include>bin/elasticsearch-systemd-pre-exec</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -127,6 +128,7 @@
                                         <include>elasticsearch</include>
                                         <include>elasticsearch.in.sh</include>
                                         <include>plugin</include>
+                                        <include>elasticsearch-systemd-pre-exec</include>
                                     </includes>
                                 </source>
                             </sources>

distribution/rpm/src/main/packaging/init.d/elasticsearch

@@ -40,7 +40,7 @@ MAX_MAP_COUNT=${packaging.os.max.map.count}
 LOG_DIR="${packaging.elasticsearch.log.dir}"
 DATA_DIR="${packaging.elasticsearch.data.dir}"
 CONF_DIR="${packaging.elasticsearch.conf.dir}"
-CONF_FILE="${packaging.elasticsearch.conf.dir}/elasticsearch.yml"
+
 PID_DIR="${packaging.elasticsearch.pid.dir}"
 
 # Source the default env file
@@ -49,6 +49,12 @@ if [ -f "$ES_ENV_FILE" ]; then
     . "$ES_ENV_FILE"
 fi
 
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
 exec="$ES_HOME/bin/elasticsearch"
 prog="elasticsearch"
 pidfile="$PID_DIR/${prog}.pid"
@@ -83,7 +89,6 @@ checkJava() {
 start() {
     checkJava
     [ -x $exec ] || exit 5
-    [ -f $CONF_FILE ] || exit 6
     if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
         echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
         return 7

distribution/rpm/src/main/packaging/packaging.properties

@@ -6,7 +6,6 @@ packaging.env.file=/etc/sysconfig/elasticsearch
 
 # Default configuration directory and file to use in bin/plugin script
 packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
-packaging.plugin.default.config.file=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
 
 # Simple marker to check that properties are correctly overridden
 packaging.type=rpm

distribution/src/main/packaging/env/elasticsearch

@@ -8,9 +8,6 @@
 # Elasticsearch configuration directory
 #CONF_DIR=${packaging.elasticsearch.conf.dir}
 
-# Elasticsearch configuration file
-#CONF_FILE=$CONF_DIR/elasticsearch.yml
-
 # Elasticsearch data directory
 #DATA_DIR=${packaging.elasticsearch.data.dir}
 

distribution/src/main/packaging/packaging.properties

@@ -8,7 +8,6 @@ packaging.env.file=
 
 # Default configuration directory and file to use in bin/plugin script
 packaging.plugin.default.config.dir=$ES_HOME/config
-packaging.plugin.default.config.file=$ES_HOME/config/elasticsearch.yml
 
 # Default values for min/max heap memory allocated to elasticsearch java process
 packaging.elasticsearch.heap.min=256m

distribution/src/main/packaging/systemd/elasticsearch.service

@@ -7,7 +7,6 @@ After=network-online.target
 [Service]
 Environment=ES_HOME=${packaging.elasticsearch.home.dir}
 Environment=CONF_DIR=${packaging.elasticsearch.conf.dir}
-Environment=CONF_FILE=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
 Environment=DATA_DIR=${packaging.elasticsearch.data.dir}
 Environment=LOG_DIR=${packaging.elasticsearch.log.dir}
 Environment=PID_DIR=${packaging.elasticsearch.pid.dir}
@@ -18,12 +17,13 @@ WorkingDirectory=${packaging.elasticsearch.home.dir}
 User=${packaging.elasticsearch.user}
 Group=${packaging.elasticsearch.group}
 
+ExecStartPre=${packaging.elasticsearch.bin.dir}/elasticsearch-systemd-pre-exec
+
 ExecStart=${packaging.elasticsearch.bin.dir}/elasticsearch \
                                                 -Des.pidfile=${PID_DIR}/elasticsearch.pid \
                                                 -Des.default.path.home=${ES_HOME} \
                                                 -Des.default.path.logs=${LOG_DIR} \
                                                 -Des.default.path.data=${DATA_DIR} \
-                                                -Des.default.config=${CONF_FILE} \
                                                 -Des.default.path.conf=${CONF_DIR}
 
 # Connects standard output to /dev/null

distribution/src/main/resources/bin/elasticsearch

@@ -42,10 +42,10 @@
 # Be aware that you will be entirely responsible for populating the needed
 # environment variables.
 
-
 # Maven will replace the project.name with elasticsearch below. If that
 # hasn't been done, we assume that this is not a packaged version and the
 # user has forgotten to run Maven to create a package.
+
 IS_PACKAGED_VERSION='${project.parent.artifactId}'
 if [ "$IS_PACKAGED_VERSION" != "distributions" ]; then
     cat >&2 << EOF

distribution/src/main/resources/bin/elasticsearch-systemd-pre-exec

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi

distribution/src/main/resources/bin/plugin

@@ -1,5 +1,6 @@
 #!/bin/sh
 
+
 CDPATH=""
 SCRIPT="$0"
 
@@ -21,17 +22,10 @@ ES_HOME=`dirname "$SCRIPT"`/..
 # make ELASTICSEARCH_HOME absolute
 ES_HOME=`cd "$ES_HOME"; pwd`
 
+
 # Sets the default values for elasticsearch variables used in this script
 if [ -z "$CONF_DIR" ]; then
   CONF_DIR="${packaging.plugin.default.config.dir}"
-
-  if [ -z "$CONF_FILE" ]; then
-    CONF_FILE="$CONF_DIR/elasticsearch.yml"
-  fi
-fi
-
-if [ -z "$CONF_FILE" ]; then
-  CONF_FILE="${packaging.plugin.default.config.file}"
 fi
 
 # The default env file is defined at building/packaging time.
@@ -66,6 +60,12 @@ if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then
     unset JAVA_TOOL_OPTIONS
 fi
 
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
 if [ -x "$JAVA_HOME/bin/java" ]; then
     JAVA=$JAVA_HOME/bin/java
 else
@@ -105,16 +105,6 @@ if [ -e "$CONF_DIR" ]; then
   esac
 fi
 
-if [ -e "$CONF_FILE" ]; then
-  case "$properties" in
-    *-Des.default.config=*|*-Des.config=*)
-    ;;
-    *)
-      properties="$properties -Des.default.config=\"$CONF_FILE\""
-    ;;
-  esac
-fi
-
 # full hostname passed through cut for portability on systems that do not support hostname -s
 # export on separate line for shells that do not support combining definition and export
 HOSTNAME=`hostname | cut -d. -f1`

distribution/src/main/resources/bin/service.bat

@@ -5,6 +5,8 @@ TITLE Elasticsearch Service ${project.version}
 
 if NOT DEFINED JAVA_HOME goto err
 
+if not "%CONF_FILE%" == "" goto conffileset
+
 set SCRIPT_DIR=%~dp0
 for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
 
@@ -147,9 +149,7 @@ if "%DATA_DIR%" == "" set DATA_DIR=%ES_HOME%\data
 
 if "%CONF_DIR%" == "" set CONF_DIR=%ES_HOME%\config
 
-if "%CONF_FILE%" == "" set CONF_FILE=%ES_HOME%\config\elasticsearch.yml
-
-set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.config="%CONF_FILE%";-Des.default.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
+set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
 
 set JVM_OPTS=%JAVA_OPTS: =;%
 
@@ -207,4 +207,8 @@ set /a conv=%conv% * 1024
 set "%~2=%conv%"
 goto:eof
 
+:conffileset
+echo CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed.
+goto:eof
+
 ENDLOCAL

docs/reference/setup/as-a-service.asciidoc

@@ -22,7 +22,6 @@ Each package features a configuration file, which allows you to set the followin
 `LOG_DIR`::               Log directory, defaults to `/var/log/elasticsearch`
 `DATA_DIR`::              Data directory, defaults to `/var/lib/elasticsearch`
 `CONF_DIR`::              Configuration file directory (which needs to include `elasticsearch.yml` and `logging.yml` files), defaults to `/etc/elasticsearch`
-`CONF_FILE`::             Path to configuration file, defaults to `/etc/elasticsearch/elasticsearch.yml`
 `ES_JAVA_OPTS`::          Any additional java options you may want to apply. This may be useful, if you need to set the `node.name` property, but do not want to change the `elasticsearch.yml` configuration file, because it is distributed via a provisioning system like puppet or chef. Example: `ES_JAVA_OPTS="-Des.node.name=search-01"`
 `RESTART_ON_UPGRADE`::    Configure restart on package upgrade, defaults to `false`. This means you will have to restart your elasticsearch instance after installing a package manually. The reason for this is to ensure, that upgrades in a cluster do not result in a continuous shard reallocation resulting in high network traffic and reducing the response times of your cluster.
 `ES_GC_LOG_FILE` ::       The absolute log file path for creating a garbage collection logfile, which is done by the JVM. Note that this logfile can grow pretty quick and thus is disabled by default.
@@ -72,9 +71,9 @@ sudo service elasticsearch start
 
 
 [float]
-===== Using systemd
+==== Using systemd
 
-Distributions like SUSE do not use the `chkconfig` tool to register services, but rather `systemd` and its command `/bin/systemctl` to start and stop services (at least in newer versions, otherwise use the `chkconfig` commands above). The configuration file is also placed at `/etc/sysconfig/elasticsearch`. After installing the RPM, you have to change the systemd configuration and then start up elasticsearch
+Distributions like SUSE do not use the `chkconfig` tool to register services, but rather `systemd` and its command `/bin/systemctl` to start and stop services (at least in newer versions, otherwise use the `chkconfig` commands above). The configuration file is also placed at `/etc/sysconfig/elasticsearch` if the system is rpm based and `/etc/default/elasticsearch` if it is deb. After installing the RPM, you have to change the systemd configuration and then start up elasticsearch
 
 [source,sh]
 --------------------------------------------------

docs/reference/setup/configuration.asciidoc

@@ -298,14 +298,6 @@ Enter value for [node.name]:
 NOTE: Elasticsearch will not start if `${prompt.text}` or `${prompt.secret}`
 is used in the settings and the process is run as a service or in the background.
 
-The location of the configuration file can be set externally using a
-system property:
-
-[source,sh]
---------------------------------------------------
-$ elasticsearch -Des.config=/path/to/config/file
---------------------------------------------------
-
 [float]
 [[configuration-index-settings]]
 === Index Settings