Merge pull request #5682 from Icinga/feature/cert-migration

Implement support for migrating certificates to /var/lib/icinga2/certs

doc/02-getting-started.md

@@ -175,11 +175,11 @@ By default Icinga 2 uses the following files and directories:
   /usr/lib\*/icinga2				| Libraries and the Icinga 2 binary (use `find /usr -type f -name icinga2` to locate the binary path).
   /usr/share/doc/icinga2              		| Documentation files that come with Icinga 2.
   /usr/share/icinga2/include          		| The Icinga Template Library and plugin command configuration.
+  /var/lib/icinga2                    		| Icinga 2 state file, cluster log, master CA, node certificates and configuration files (cluster, api).
   /var/run/icinga2                    		| PID file.
   /var/run/icinga2/cmd                		| Command pipe and Livestatus socket.
   /var/cache/icinga2                  		| status.dat/objects.cache, icinga2.debug files
   /var/spool/icinga2                  		| Used for performance data spool files.
-  /var/lib/icinga2                    		| Icinga 2 state file, cluster log, local CA and configuration files (cluster, api).
   /var/log/icinga2                    		| Log file location and compat/ directory for the CompatLogger feature.
 
 FreeBSD uses slightly different paths:
@@ -194,11 +194,11 @@ By default Icinga 2 uses the following files and directories:
   /usr/local/lib/icinga2              | Libraries and the Icinga 2 binary.
   /usr/local/share/doc/icinga2        | Documentation files that come with Icinga 2.
   /usr/local/share/icinga2/include    | The Icinga Template Library and plugin command configuration.
+  /var/lib/icinga2                    | Icinga 2 state file, cluster log, master CA, node certificates and configuration files (cluster, api).
   /var/run/icinga2                    | PID file.
   /var/run/icinga2/cmd                | Command pipe and Livestatus socket.
   /var/cache/icinga2                  | status.dat/objects.cache, icinga2.debug files
   /var/spool/icinga2                  | Used for performance data spool files.
-  /var/lib/icinga2                    | Icinga 2 state file, cluster log, local CA and configuration files (cluster, api).
   /var/log/icinga2                    | Log file location and compat/ directory for the CompatLogger feature.
 
 ## Setting up Check Plugins <a id="setting-up-check-plugins"></a>
@@ -877,5 +877,6 @@ popular addons is available in the
 Ensure to include the following in your backups:
 
 * Configuration files in `/etc/icinga2`
-* Runtime files in `/var/lib/icinga2` (the master's CA is stored here as well)
+* Certificate files in `/var/lib/icinga2/ca` (Master CA key pair) and `/var/lib/icinga2/certs` (node certificates)
+* Runtime files in `/var/lib/icinga2`
 * Optional: IDO database backup

doc/06-distributed-monitoring.md

@@ -553,6 +553,11 @@ The setup wizard will ensure that the following steps are taken:
 
 You can verify that the certificate files are stored in the `/var/lib/icinga2/certs` directory.
 
+> **Note**
+>
+> The certificate location changed in v2.8 to `/var/lib/icinga2/certs`. Please read the [upgrading chapter](16-upgrading-icinga-2.md#upgrading-to-2-8-certificate-paths)
+> for more details.
+
 > **Note**
 >
 > If the client is not directly connected to the certificate signing master,
@@ -2358,6 +2363,11 @@ Sign the CSR with the previously created CA:
 
     [root@icinga2-master1.localdomain /root]# icinga2 pki sign-csr --csr icinga2-master1.localdomain.csr --cert icinga2-master1.localdomain
 
+> **Note**
+>
+> The certificate location changed in v2.8 to `/var/lib/icinga2/certs`. Please read the [upgrading chapter](16-upgrading-icinga-2.md#upgrading-to-2-8-certificate-paths)
+> for more details.
+
 Copy the host's certificate files and the public CA certificate to `/var/lib/icinga2/certs`:
 
     [root@icinga2-master1.localdomain /root]# mkdir -p /var/lib/icinga2/certs
@@ -2444,7 +2454,12 @@ host/port you can specify it like this:
 
 #### Node Setup with Satellites/Clients <a id="distributed-monitoring-automation-cli-node-setup-satellite-client"></a>
 
-Make sure that the `/var/lib/icinga2/certs` exists and is owned by the `icinga`
+> **Note**
+>
+> The certificate location changed in v2.8 to `/var/lib/icinga2/certs`. Please read the [upgrading chapter](16-upgrading-icinga-2.md#upgrading-to-2-8-certificate-paths)
+> for more details.
+
+Make sure that the `/var/lib/icinga2/certs` directory exists and is owned by the `icinga`
 user (or the user Icinga 2 is running as).
 
     [root@icinga2-client1.localdomain /]# mkdir -p /var/lib/icinga2/certs
@@ -2499,7 +2514,7 @@ Pass the following details to the `node setup` CLI command:
   Accept config       | **Optional.** Whether this node accepts configuration sync from the master node (required for [config sync mode](06-distributed-monitoring.md#distributed-monitoring-top-down-config-sync)).
   Accept commands     | **Optional.** Whether this node accepts command execution messages from the master node (required for [command endpoint mode](06-distributed-monitoring.md#distributed-monitoring-top-down-command-endpoint)).
 
-Example:
+Example for Icinga 2 v2.8:
 
     [root@icinga2-client1.localdomain /]# icinga2 node setup --ticket ead2d570e18c78abf285d6b85524970a0f69c22d \
     --cn icinga2-client1.localdomain \

doc/09-object-types.md

@@ -41,9 +41,8 @@ Example:
 
 ```
 object ApiListener "api" {
-  cert_path = LocalStateDir + "/lib/icinga2/certs/" + NodeName + ".crt"
-  key_path = LocalStateDir + "/lib/icinga2/certs/" + NodeName + ".key"
-  ca_path = LocalStateDir + "/lib/icinga2/certs/ca.crt"
+  accept_commands = true
+  accept_config = true
 
   ticket_salt = TicketSalt
 }
@@ -53,9 +52,9 @@ Configuration Attributes:
 
   Name                                  | Type                  | Description
   --------------------------------------|-----------------------|----------------------------------
-  cert\_path                            | String                | **Required.** Path to the public key.
-  key\_path                             | String                | **Required.** Path to the private key.
-  ca\_path                              | String                | **Required.** Path to the CA certificate file.
+  cert\_path                            | String                | **Deprecated.** Path to the public key.
+  key\_path                             | String                | **Deprecated.** Path to the private key.
+  ca\_path                              | String                | **Deprecated.** Path to the CA certificate file.
   ticket\_salt                          | String                | **Optional.** Private key for [CSR auto-signing](06-distributed-monitoring.md#distributed-monitoring-setup-csr-auto-signing). **Required** for a signing master instance.
   crl\_path                             | String                | **Optional.** Path to the CRL file.
   bind\_host                            | String                | **Optional.** The IP address the api listener should be bound to. Defaults to `0.0.0.0`.
@@ -69,6 +68,20 @@ Configuration Attributes:
   access\_control\_allow\_headers       | String                | **Optional.** Used in response to a preflight request to indicate which HTTP headers can be used when making the actual request. Defaults to `Authorization`. [(MDN docs)](https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS#Access-Control-Allow-Headers)
   access\_control\_allow\_methods       | String                | **Optional.** Used in response to a preflight request to indicate which HTTP methods can be used when making the actual request. Defaults to `GET, POST, PUT, DELETE`. [(MDN docs)](https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS#Access-Control-Allow-Methods)
 
+The ApiListener type expects its certificate files to be in the following locations:
+
+  Type                 | Location
+  ---------------------|-------------------------------------
+  Private key          | `LocalStateDir + "/lib/icinga2/certs/" + NodeName + ".key"`
+  Certificate file     | `LocalStateDir + "/lib/icinga2/certs/" + NodeName + ".crt"`
+  CA certificate file  | `LocalStateDir + "/lib/icinga2/certs/ca.crt"`
+
+If the deprecated attributes `cert_path`, `key_path` and/or `ca_path` are specified Icinga 2
+copies those files to the new location in `LocalStateDir + "/lib/icinga2/certs"` unless the
+file(s) there are newer.
+
+Please check the [upgrading chapter](16-upgrading-icinga-2.md#upgrading-to-2-8-certificate-paths) for more details.
+
 ## ApiUser <a id="objecttype-apiuser"></a>
 
 ApiUser objects are used for authentication against the [Icinga 2 API](12-icinga2-api.md#icinga2-api-authentication).

doc/16-upgrading-icinga-2.md

@@ -10,19 +10,103 @@ are scheme updates for the IDO database.
 The default certificate path was changed from `/etc/icinga2/pki` to
 `/var/lib/icinga2/certs`.
 
+  Old Path                                           | New Path
+  ---------------------------------------------------|---------------------------------------------------
+  `/etc/icinga2/pki/icinga2-client1.localdomain.crt` | `/var/lib/icinga2/certs/icinga2-client1.localdomain.crt`
+  `/etc/icinga2/pki/icinga2-client1.localdomain.key` | `/var/lib/icinga2/certs/icinga2-client1.localdomain.key`
+  `/etc/icinga2/pki/ca.crt`                          | `/var/lib/icinga2/certs/ca.crt`
+
 This applies to Windows clients in the same way: `%ProgramData%\etc\icinga2\pki`
-was moved to `%ProgramData%`\var\lib\icinga2\certs`.
+was moved to `%ProgramData%\var\lib\icinga2\certs`.
+
+  Old Path                                                        | New Path
+  ----------------------------------------------------------------|----------------------------------------------------------------
+  `%ProgramData%\etc\icinga2\pki\icinga2-client1.localdomain.crt` | `%ProgramData%\var\lib\icinga2\certs\icinga2-client1.localdomain.crt`
+  `%ProgramData%\etc\icinga2\pki\icinga2-client1.localdomain.key` | `%ProgramData%\var\lib\icinga2\certs\icinga2-client1.localdomain.key`
+  `%ProgramData%\etc\icinga2\pki\ca.crt`                          | `%ProgramData%\var\lib\icinga2\certs\ca.crt`
+
+
+> **Note**
+>
+> The default expected path for client certificates is `/var/lib/icinga2/certs/ + NodeName + {.crt,.key}`.
+> The `NodeName` constant is usually the FQDN and certificate common name (CN). Check the [conventions](06-distributed-monitoring.md#distributed-monitoring-conventions)
+> section inside the Distributed Monitoring chapter.
 
 The [setup CLI commands](06-distributed-monitoring.md#distributed-monitoring-setup-master) and the
 default [ApiListener configuration](06-distributed-monitoring.md#distributed-monitoring-apilistener)
 have been adjusted to these paths too.
 
+The [ApiListener](09-object-types.md#objecttype-apilistener) object attributes `cert_path`, `key_path`
+and `ca_path` have been deprecated and removed from the example configuration.
+
+#### Migration Path <a id="upgrading-to-2-8-certificate-paths-migration-path"></a>
+
+> **Note**
+>
+> Icinga 2 automatically migrates the certificates to the new default location if they
+> are configured and detected in `/etc/icinga2/pki`.
+
+During startup, the migration kicks in and ensures to copy the certificates to the new
+location. This will also happen if someone updates the certificate files in `/etc/icinga2/pki`
+to ensure that the new certificate location always has the latest files.
+
+This has been implemented in the Icinga 2 binary to ensure it works on both Linux/Unix
+and the Windows platform.
+
+If you are not using the built-in CLI commands and setup wizards to deploy the client certificates,
+please ensure to update your deployment tools/scripts. This mainly affects
+
+* Puppet modules
+* Ansible playbooks
+* Chef cookbooks
+* Salt recipes
+* Custom scripts, e.g. Windows Powershell or self-made implementations
+
+In order to support a smooth migration between versions older than 2.8 and future releases,
+the built-in certificate migration path is planned to exist as long as the deprecated
+`ApiListener` object attributes exist.
+
+You are safe to use the existing configuration paths inside the `api` feature. If you plan your migration,
+look at the following example taken from the Director Linux deployment script for clients.
+
+* Ensure that the default certificate path is changed from `/etc/icinga2/pki` to `/var/lib/icinga2/certs`.
+
+```
+-ICINGA2_SSL_DIR="${ICINGA2_CONF_DIR}/pki"
++ICINGA2_SSL_DIR="${ICINGA2_STATE_DIR}/lib/icinga2/certs"
+```
+
+* Remove the ApiListener configuration attributes.
+
+```
+object ApiListener "api" {
+-  cert_path = SysconfDir + "/icinga2/pki/${ICINGA2_NODENAME}.crt"
+-  key_path = SysconfDir + "/icinga2/pki/${ICINGA2_NODENAME}.key"
+-  ca_path = SysconfDir + "/icinga2/pki/ca.crt"
+  accept_commands = true
+  accept_config = true
+}
+```
+
+Test the script with a fresh client installation before putting it into production.
+
+> **Tip**
+>
+> Please support module and script developers in their migration. If you find
+> any project which would require these changes, create an issue or a patchset in a PR
+> and help them out. Thanks in advance!
+
+
 ### Removed Bottom Up Client Mode <a id="upgrading-to-2-8-removed-bottom-up-client-mode"></a>
 
 This client mode was deprecated in 2.6 and was removed in 2.8.
 
 The node CLI command does not provide `list` or `update-config` anymore.
 
+> **Note**
+>
+> The old migration guide can be found on [GitHub](https://github.com/Icinga/icinga2/blob/v2.7.0/doc/06-distributed-monitoring.md#bottom-up-migration-to-top-down-).
+
 The clients don't need to have a local `conf.d` directory included.
 The setup wizards for Linux and Windows attempt to disable this by default.
 
@@ -31,6 +115,7 @@ You are advised to [migrate](https://github.com/Icinga/icinga2/issues/4798)
 any existing configuration to the "top down" mode with the help of the
 Icinga Director or config management tools such as Puppet, Ansible, etc.
 
+
 ### Removed Classic UI Config Package <a id="upgrading-to-2-8-removed-classicui-config-package"></a>
 
 The config meta package `classicui-config` and the configuration files

etc/icinga2/features-available/api.conf

@@ -3,9 +3,8 @@
  */
 
 object ApiListener "api" {
-  cert_path = LocalStateDir + "/lib/icinga2/certs/" + NodeName + ".crt"
-  key_path = LocalStateDir + "/lib/icinga2/certs/" + NodeName + ".key"
-  ca_path = LocalStateDir + "/lib/icinga2/certs/ca.crt"
+  //accept_config = false
+  //accept_commands = false
 
   ticket_salt = TicketSalt
 }

lib/base/function.ti

@@ -28,7 +28,7 @@ abstract class Function
 {
 	[config] String "name";
 	[config] bool side_effect_free;
-	[config] bool deprecated;
+	[config] bool "deprecated";
 	[config] Array::Ptr arguments;
 };
 

lib/base/type.hpp

@@ -39,7 +39,8 @@ enum FieldAttribute
 	FARequired = 256,
 	FANavigation = 512,
 	FANoUserModify = 1024,
-	FANoUserView = 2048
+	FANoUserView = 2048,
+	FADeprecated = 4096,
 };
 
 class Type;

lib/base/utility.cpp

@@ -816,6 +816,10 @@ void Utility::CollectPaths(const String& path, std::vector<String>& paths)
 	paths.push_back(path);
 }
 
+/*
+ * Copies a source file to a target location.
+ * Caller must ensure that the target's base directory exists and is writable.
+ */
 void Utility::CopyFile(const String& source, const String& target)
 {
 	std::ifstream ifs(source.CStr(), std::ios::binary);

lib/cli/nodesetupcommand.cpp

@@ -173,10 +173,7 @@ int NodeSetupCommand::SetupMaster(const boost::program_options::variables_map& v
 	fp << "/**\n"
 	    << " * The API listener is used for distributed monitoring setups.\n"
 	    << " */\n"
-	    << "object ApiListener \"api\" {\n"
-	    << "  cert_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".crt\"\n"
-	    << "  key_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".key\"\n"
-	    << "  ca_path = LocalStateDir + \"/lib/icinga2/certs/ca.crt\"\n";
+	    << "object ApiListener \"api\" {\n";
 
 	if (vm.count("listen")) {
 		std::vector<String> tokens;
@@ -378,10 +375,7 @@ int NodeSetupCommand::SetupNode(const boost::program_options::variables_map& vm,
 	fp << "/**\n"
 	    << " * The API listener is used for distributed monitoring setups.\n"
 	    << " */\n"
-	    << "object ApiListener \"api\" {\n"
-	    << "  cert_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".crt\"\n"
-	    << "  key_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".key\"\n"
-	    << "  ca_path = LocalStateDir + \"/lib/icinga2/certs/ca.crt\"\n";
+	    << "object ApiListener \"api\" {\n";
 
 	if (vm.count("listen")) {
 		std::vector<String> tokens;

lib/cli/nodewizardcommand.cpp

@@ -472,10 +472,6 @@ int NodeWizardCommand::ClientSetup(void) const
 	    << " * The API listener is used for distributed monitoring setups.\n"
 	    << " */\n"
 	    << "object ApiListener \"api\" {\n"
-	    << "  cert_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".crt\"\n"
-	    << "  key_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".key\"\n"
-	    << "  ca_path = LocalStateDir + \"/lib/icinga2/certs/ca.crt\"\n"
-	    << "\n"
 	    << "  accept_config = " << acceptConfig << "\n"
 	    << "  accept_commands = " << acceptCommands << "\n";
 
@@ -629,10 +625,7 @@ int NodeWizardCommand::MasterSetup(void) const
 	fp << "/**\n"
 	    << " * The API listener is used for distributed monitoring setups.\n"
 	    << " */\n"
-	    << "object ApiListener \"api\" {\n"
-	    << "  cert_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".crt\"\n"
-	    << "  key_path = LocalStateDir + \"/lib/icinga2/certs/\" + NodeName + \".key\"\n"
-	    << "  ca_path = LocalStateDir + \"/lib/icinga2/certs/ca.crt\"\n";
+	    << "object ApiListener \"api\" {\n";
 
 	if (!bindHost.IsEmpty())
 		fp << "  bind_host = \"" << bindHost << "\"\n";