[FEATURE] Enhance LDAP/SMTP TLS Configuration and Unify Them (#1557)

* add new directive in the global scope `certificates_directory` which is used to bulk load certs and trust them in Authelia
* this is in ADDITION to system certs and are trusted by both LDAP and SMTP
* added a shared TLSConfig struct to be used by both SMTP and LDAP, and anything else in the future that requires tuning the TLS
* remove usage of deprecated LDAP funcs Dial and DialTLS in favor of DialURL which is also easier to use
* use the server name from LDAP URL or SMTP host when validating the certificate unless otherwise defined in the TLS section
* added temporary translations from the old names to the new ones for all deprecated options
* added docs
* updated example configuration
* final deprecations to be done in 4.28.0
* doc updates
* fix misc linting issues
* uniform deprecation notices for ease of final removal
* added additional tests covering previously uncovered areas and the new configuration options
* add non-fatal to certificate loading when system certs could not be loaded
* adjust timeout of Suite ShortTimeouts
* add warnings pusher for the StructValidator
* make the schema suites uninform
* utilize the warnings in the StructValidator
* fix test suite usage for skip_verify
* extract LDAP filter parsing into it's own function to make it possible to test
* test LDAP filter parsing
* update ErrorContainer interface
* add tests to the StructValidator
* add NewTLSConfig test
* move baseDN for users/groups into parsed values
* add tests to cover many of the outstanding areas in LDAP
* add explicit deferred LDAP conn close to UpdatePassword
* add some basic testing to SMTP notifier
* suggestions from code review

BREAKING.md

@@ -6,10 +6,19 @@ recommended not to use the 'latest' Docker image tag blindly but pick a version
 and read this documentation before upgrading. This is where you will get information about
 breaking changes and about what you should do to overcome those changes.
 
+## Breaking in v4.25.0
+
+### Deprecation Notice(s)
+* All of these deprecations will be fully removed in release 4.28.0
+* The SMTP notifiers `trusted_cert` option has been deprecated (replaced by global certificates_directory)
+* The SMTP notifiers `disable_verify_cert` option has been deprecated (replaced by `notifier.smtp.tls.skip_verify`)
+* The LDAP authentication backends `skip_verify` option has been deprecated (replaced by `authentication_backend.ldap.tls.skip_verify`)
+* The LDAP authentication backends `minimum_tls_version` option has been deprecated (replaced by `authentication_backend.ldap.tls.minimum_version`)
+
 ## Breaking in v4.24.0
 
 ### Deprecation Notice(s)
-* LDAP User Provider Filters (final removal in 4.27.0):
+* LDAP User Provider Filters (final removal in 4.28.0):
   * User Filters containing `{0}` are being deprecated and will generate warnings. Replaced with `{input}`.
   * Group Filters containing `{0}` or `{1}` are being deprecated and will generate warnings. 
   Replaced with `{input}` and `{username}` respectively.

cmd/authelia/main.go

@@ -35,6 +35,21 @@ func startServer() {
 		os.Exit(1)
 	}
 
+	autheliaCertPool, errs, nonFatalErrs := utils.NewX509CertPool(config.CertificatesDirectory, config)
+	if len(errs) > 0 {
+		for _, err := range errs {
+			logging.Logger().Error(err)
+		}
+
+		os.Exit(2)
+	}
+
+	if len(nonFatalErrs) > 0 {
+		for _, err := range nonFatalErrs {
+			logging.Logger().Warn(err)
+		}
+	}
+
 	if err := logging.InitializeLogger(config.LogFormat, config.LogFilePath); err != nil {
 		logging.Logger().Fatalf("Cannot initialize logger: %v", err)
 	}
@@ -55,17 +70,6 @@ func startServer() {
 		logging.Logger().Info("===> Authelia is running in development mode. <===")
 	}
 
-	var userProvider authentication.UserProvider
-
-	switch {
-	case config.AuthenticationBackend.File != nil:
-		userProvider = authentication.NewFileUserProvider(config.AuthenticationBackend.File)
-	case config.AuthenticationBackend.Ldap != nil:
-		userProvider = authentication.NewLDAPUserProvider(*config.AuthenticationBackend.Ldap)
-	default:
-		logging.Logger().Fatalf("Unrecognized authentication backend")
-	}
-
 	var storageProvider storage.Provider
 
 	switch {
@@ -79,11 +83,22 @@ func startServer() {
 		logging.Logger().Fatalf("Unrecognized storage backend")
 	}
 
+	var userProvider authentication.UserProvider
+
+	switch {
+	case config.AuthenticationBackend.File != nil:
+		userProvider = authentication.NewFileUserProvider(config.AuthenticationBackend.File)
+	case config.AuthenticationBackend.Ldap != nil:
+		userProvider = authentication.NewLDAPUserProvider(*config.AuthenticationBackend.Ldap, autheliaCertPool)
+	default:
+		logging.Logger().Fatalf("Unrecognized authentication backend")
+	}
+
 	var notifier notification.Notifier
 
 	switch {
 	case config.Notifier.SMTP != nil:
-		notifier = notification.NewSMTPNotifier(*config.Notifier.SMTP)
+		notifier = notification.NewSMTPNotifier(*config.Notifier.SMTP, autheliaCertPool)
 	case config.Notifier.FileSystem != nil:
 		notifier = notification.NewFileNotifier(*config.Notifier.FileSystem)
 	default:

config.template.yml

@@ -107,14 +107,18 @@ authentication_backend:
     # The url to the ldap server. Scheme can be ldap or ldaps in the format (port optional) <scheme>://<address>[:<port>].
     url: ldap://127.0.0.1
 
-    # Skip verifying the server certificate (to allow a self-signed certificate).
-    skip_verify: false
-
     # Use StartTLS with the LDAP connection.
     start_tls: false
 
-    # Minimum TLS version for either Secure LDAP or LDAP StartTLS.
-    minimum_tls_version: TLS1.2
+    tls:
+      # Server Name for certificate validation (in case it's not set correctly in the URL).
+      # server_name: ldap.example.com
+
+      # Skip verifying the server certificate (to allow a self-signed certificate).
+      skip_verify: false
+
+      # Minimum TLS version for either Secure LDAP or LDAP StartTLS.
+      minimum_version: TLS1.2
 
     # The base dn for every entries.
     base_dn: dc=example,dc=com
@@ -406,13 +410,7 @@ notifier:
   # [Security] By default Authelia will:
   #   - force all SMTP connections over TLS including unauthenticated connections
   #      - use the disable_require_tls boolean value to disable this requirement (only works for unauthenticated connections)
-  #   - validate the SMTP server x509 certificate during the TLS handshake against the hosts trusted certificates
-  #     - trusted_cert option:
-  #       - this is a string value, that may specify the path of a PEM format cert, it is completely optional
-  #       - if it is not set, a blank string, or an invalid path; will still trust the host machine/containers cert store
-  #     - defaults to the host machine (or docker container's) trusted certificate chain for validation
-  #     - use the trusted_cert string value to specify the path of a PEM format public cert to trust in addition to the hosts trusted certificates
-  #     - use the disable_verify_cert boolean value to disable the validation (prefer the trusted_cert option as it's more secure)
+  #   - validate the SMTP server x509 certificate during the TLS handshake against the hosts trusted certificates (configure in tls section)
   smtp:
     username: test
     # Password can also be set using a secret: https://docs.authelia.com/configuration/secrets.html
@@ -427,11 +425,19 @@ notifier:
     subject: "[Authelia] {title}"
     # This address is used during the startup check to verify the email configuration is correct. It's not important what it is except if your email server only allows local delivery.
     startup_check_address: test@authelia.com
-    trusted_cert: ""
     disable_require_tls: false
-    disable_verify_cert: false
     disable_html_emails: false
 
+    tls:
+      # Server Name for certificate validation (in case you are using the IP or non-FQDN in the host option).
+      # server_name: smtp.example.com
+
+      # Skip verifying the server certificate (to allow a self-signed certificate).
+      skip_verify: false
+
+      # Minimum TLS version for either StartTLS or SMTPS.
+      minimum_version: TLS1.2
+
   # Sending an email using a Gmail account is as simple as the next section.
   # You need to create an app password by following: https://support.google.com/accounts/answer/185833?hl=en
   ## smtp:

docs/configuration/authentication/ldap.md

@@ -52,15 +52,19 @@ authentication_backend:
 
     # The url to the ldap server. Scheme can be ldap or ldaps in the format (port optional) <scheme>://<address>[:<port>].
     url: ldap://127.0.0.1
-
-    # Skip verifying the server certificate (to allow a self-signed certificate).
-    skip_verify: false
 
     # Use StartTLS with the LDAP connection.
     start_tls: false
 
-    # Minimum TLS version for either Secure LDAP or LDAP StartTLS.
-    minimum_tls_version: TLS1.2
+    tls:
+      # Server Name for certificate validation (in case it's not set correctly in the URL).
+      # server_name: ldap.example.com
+
+      # Skip verifying the server certificate (to allow a self-signed certificate).
+      skip_verify: false
+
+      # Minimum TLS version for either Secure LDAP or LDAP StartTLS.
+      minimum_version: TLS1.2
 
     # The base dn for every entries.
     base_dn: dc=example,dc=com
@@ -139,24 +143,15 @@ url: ldap://[fd00:1111:2222:3333::1]
 
 ## TLS Settings
 
-### Skip Verify
-
-The key `skip_verify` disables checking the authenticity of the TLS certificate. You should not disable this, instead
-you should add the certificate that signed the certificate of your LDAP server to the machines certificate PKI trust.
-For docker you can just add this to the hosts trusted store.
-
 ### Start TLS
 
 The key `start_tls` enables use of the LDAP StartTLS process which is not commonly used. You should only configure this
 if you know you need it. The initial connection will be over plain text, and Authelia will try to upgrade it with the
 LDAP server. LDAPS URL's are slightly more secure.
 
-### Minimum TLS Version
+### TLS (section)
 
-The key `minimum_tls_version` controls the minimum TLS version Authelia will use when opening LDAP connections.
-The possible values are `TLS1.3`, `TLS1.2`, `TLS1.1`, `TLS1.0`. Anything other than `TLS1.3` or `TLS1.2`
-are very old and deprecated. You should avoid using these and upgrade your LDAP solution instead of decreasing
-this value. 
+The key `tls` is a map of options for tuning TLS options. You can see how to configure the tls section [here](../index.md#tls-configuration).
 
 ## Implementation
 

docs/configuration/index.md

@@ -54,4 +54,26 @@ denoting the unit of time measurement. The table below describes the units of ti
 Examples:
 * 1 hour and 30 minutes: 90m
 * 1 day: 1d
-* 10 hours: 10h
+* 10 hours: 10h
+
+## TLS Configuration
+
+Various sections of the configuration use a uniform configuration section called TLS. Notably LDAP and SMTP.
+This section documents the usage.
+
+### Server Name
+
+The key `server_name` overrides the name checked against the certificate in the verification process. Useful if you
+require to use a direct IP address for the address of the backend service but want to verify a specific SNI.
+
+### Skip Verify
+
+The key `skip_verify` completely negates validating the certificate of the backend service. This is not recommended,
+instead you should tweak the `server_name` option, and the global option [certificates_directory](./miscellaneous.md#certificates-directory).
+
+### Minimum Version
+
+The key `minimum_version` controls the minimum TLS version Authelia will use when opening TLS connections.
+The possible values are `TLS1.3`, `TLS1.2`, `TLS1.1`, `TLS1.0`. Anything other than `TLS1.3` or `TLS1.2`
+are very old and deprecated. You should avoid using these and upgrade your backend service instead of decreasing
+this value.

docs/configuration/miscellaneous.md

@@ -38,6 +38,16 @@ tls_key: /config/ssl/key.pem
 tls_cert: /config/ssl/cert.pem
 ```
 
+## Certificates Directory
+
+`optional: true`
+
+This option defines the location of additional certificates to load into the trust chain specifically for Authelia.
+This currently affects both the SMTP notifier and the LDAP authentication backend. The certificates should all be in the
+PEM format and end with the extension `.pem` or `.crt`. You can either add the individual certificates public key 
+or the CA public key which signed them (don't add the private key).
+
+
 ## Log
 
 ### Log level

docs/configuration/notifier/smtp.md

@@ -7,7 +7,6 @@ nav_order: 2
 ---
 
 # SMTP
-
 **Authelia** can send emails to users through an SMTP server.
 It can be configured as described below.
 
@@ -29,13 +28,7 @@ notifier:
   # [Security] By default Authelia will:
   #   - force all SMTP connections over TLS including unauthenticated connections
   #      - use the disable_require_tls boolean value to disable this requirement (only works for unauthenticated connections)
-  #   - validate the SMTP server x509 certificate during the TLS handshake against the hosts trusted certificates
-  #     - trusted_cert option:
-  #       - this is a string value, that may specify the path of a PEM format cert, it is completely optional
-  #       - if it is not set, a blank string, or an invalid path; will still trust the host machine/containers cert store
-  #     - defaults to the host machine (or docker container's) trusted certificate chain for validation
-  #     - use the trusted_cert string value to specify the path of a PEM format public cert to trust in addition to the hosts trusted certificates
-  #     - use the disable_verify_cert boolean value to disable the validation (prefer the trusted_cert option as it's more secure)
+  #   - validate the SMTP server x509 certificate during the TLS handshake against the hosts trusted certificates (configure in tls section)
   smtp:
     username: test
     # Password can also be set using a secret: https://docs.authelia.com/configuration/secrets.html
@@ -50,19 +43,37 @@ notifier:
     subject: "[Authelia] {title}"
     # This address is used during the startup check to verify the email configuration is correct. It's not important what it is except if your email server only allows local delivery.
     startup_check_address: test@authelia.com
-    trusted_cert: ""
     disable_require_tls: false
-    disable_verify_cert: false
     disable_html_emails: false
+
+    tls:
+      # Server Name for certificate validation (in case you are using the IP or non-FQDN in the host option).
+      # server_name: smtp.example.com
+
+      # Skip verifying the server certificate (to allow a self-signed certificate).
+      skip_verify: false
+
+      # Minimum TLS version for either StartTLS or SMTPS.
+      minimum_version: TLS1.2
+
+  # Sending an email using a Gmail account is as simple as the next section.
+  # You need to create an app password by following: https://support.google.com/accounts/answer/185833?hl=en
+  ## smtp:
+  ##   username: myaccount@gmail.com
+  ##   # Password can also be set using a secret: https://docs.authelia.com/configuration/secrets.html
+  ##   password: yourapppassword
+  ##   sender: admin@example.com
+  ##   host: smtp.gmail.com
+  ##   port: 587
 ```
 
 ## Configuration options
-
 Most configuration options are self-explanatory, however here is an explanation of the ones that may not
 be as obvious.
 
 ### host
 If utilising an IPv6 literal address it must be enclosed by square brackets and quoted:
+
 ```yaml
 host: "[fd00:1111:2222:3333::1]"
 ```
@@ -79,21 +90,14 @@ be included in all emails as it is the internal descriptor for the contents of t
 For security reasons the default settings for Authelia require the SMTP connection is encrypted by TLS. See [security] for
 more information. This option disables this measure (not recommended).
 
-###  disable_verify_cert
-For security reasons Authelia only trusts certificates valid according to the OS's PKI chain. See [security] for more information.
-This option disables this measure (not recommended).
-
 ### disable_html_emails
 This option forces Authelia to only send plain text email via the notifier. This is the default for the file based 
 notifier, but some users may wish to use plain text for security reasons.
 
-### trusted_cert
-This option allows you to specify the file path to a public key portion of a X509 certificate in order to trust it, or 
-certificates signed with the private key portion of the X509 certificate. This is an alternative to `disable_verify_cert`
-that is much more secure. This is not required if your certificate is trusted by the operating system PKI. 
+### TLS (section)
+The key `tls` is a map of options for tuning TLS options. You can see how to configure the tls section [here](../index.md#tls-configuration).
 
 ## Using Gmail
-
 You need to generate an app password in order to use Gmail SMTP servers. The process is
 described [here](https://support.google.com/accounts/answer/185833?hl=en)
 
@@ -109,7 +113,6 @@ notifier:
 ```
 
 ## Loading a password from a secret instead of inside the configuration
-
 Password can also be defined using a [secret](../secrets.md).
 
 [security]: ../../security/measures.md#notifier-security-measures-smtp

internal/authentication/ldap_connection_factory.go

@@ -57,8 +57,7 @@ func (lc *LDAPConnectionImpl) StartTLS(config *tls.Config) error {
 
 // LDAPConnectionFactory an interface of factory of ldap connections.
 type LDAPConnectionFactory interface {
-	DialTLS(network, addr string, config *tls.Config) (LDAPConnection, error)
-	Dial(network, addr string) (LDAPConnection, error)
+	DialURL(addr string, opts ldap.DialOpt) (LDAPConnection, error)
 }
 
 // LDAPConnectionFactoryImpl the production implementation of an ldap connection factory.
@@ -69,19 +68,9 @@ func NewLDAPConnectionFactoryImpl() *LDAPConnectionFactoryImpl {
 	return &LDAPConnectionFactoryImpl{}
 }
 
-// DialTLS contact ldap server over TLS.
-func (lcf *LDAPConnectionFactoryImpl) DialTLS(network, addr string, config *tls.Config) (LDAPConnection, error) {
-	conn, err := ldap.DialTLS(network, addr, config)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewLDAPConnectionImpl(conn), nil
-}
-
-// Dial contact ldap server over raw tcp.
-func (lcf *LDAPConnectionFactoryImpl) Dial(network, addr string) (LDAPConnection, error) {
-	conn, err := ldap.Dial(network, addr)
+// DialURL creates a connection from an LDAP URL when successful.
+func (lcf *LDAPConnectionFactoryImpl) DialURL(addr string, opts ldap.DialOpt) (LDAPConnection, error) {
+	conn, err := ldap.DialURL(addr, opts)
 	if err != nil {
 		return nil, err
 	}