Add TLS key-pair reload.

Implementation is based heavily on similar functionality in Nomad.

hashicorp/nomad#3479

agent/agent.go

@@ -874,6 +874,61 @@ func (a *Agent) reloadWatches(cfg *config.RuntimeConfig) error {
 	return nil
 }
 
+// reloadTLSConfig performs TLS config reloads. Must be called from HandleReload.
+// This currently only handles changes to the key pair (CertFile and KeyFile).
+func (a *Agent) reloadTLSConfig(newCfg *config.RuntimeConfig) error {
+	haveNewPair := newCfg.CertFile != "" && newCfg.KeyFile != ""
+	haveOldPair := a.config.CertFile != "" && a.config.KeyFile != ""
+
+	if haveNewPair != haveOldPair {
+		// unimplemented: enabling or disabling TLS while running
+		if haveNewPair {
+			return fmt.Errorf("Cannot enable TLS while running")
+		} else {
+			return fmt.Errorf("Cannot disable TLS while running")
+		}
+	}
+
+	if newCfg.VerifyIncoming != a.config.VerifyIncoming {
+		return fmt.Errorf("Cannot modify verify-incoming while running")
+	}
+	if newCfg.VerifyOutgoing != a.config.VerifyOutgoing {
+		return fmt.Errorf("Cannot modify verify-outgoing while running")
+	}
+
+	// TODO(ag) : Support reloading CA cert.
+	if newCfg.CAFile != a.config.CAFile || newCfg.CAPath != a.config.CAPath {
+		return fmt.Errorf("Cannot modify CA certs while running")
+	}
+
+	// TODO(ag) : Support reloading node/server name and TLS cipher config.
+	if newCfg.NodeName != a.config.NodeName {
+		return fmt.Errorf("Cannot modify node name while running")
+	}
+	if newCfg.ServerName != a.config.ServerName {
+		return fmt.Errorf("Cannot modify server name while running")
+	}
+	minverMismatch := newCfg.TLSMinVersion != a.config.TLSMinVersion
+	// cipher compare is not 100% correct, but should be fine for catching changes
+	cipherMismatch := len(newCfg.TLSCipherSuites) != len(a.config.TLSCipherSuites)
+	preferMismatch := newCfg.TLSPreferServerCipherSuites != a.config.TLSPreferServerCipherSuites
+	if minverMismatch || cipherMismatch || preferMismatch {
+		return fmt.Errorf("Cannot modify TLS cipher suites while running")
+	}
+
+	// Phew, everything is compatible, so reload key pair
+	_, err := a.config.GetKeyLoader().LoadKeyPair(newCfg.CertFile, newCfg.KeyFile)
+	if err != nil {
+		return err
+	}
+
+	// Successfully reloaded key pair, so update local config to reflect that
+	a.config.CertFile = newCfg.CertFile
+	a.config.KeyFile = newCfg.KeyFile
+
+	return nil
+}
+
 // consulConfig is used to return a consul configuration
 func (a *Agent) consulConfig() (*consul.Config, error) {
 	// Start with the provided config or default config
@@ -1056,6 +1111,7 @@ func (a *Agent) consulConfig() (*consul.Config, error) {
 	base.CAPath = a.config.CAPath
 	base.CertFile = a.config.CertFile
 	base.KeyFile = a.config.KeyFile
+	base.KeyLoader = a.config.GetKeyLoader()
 	base.ServerName = a.config.ServerName
 	base.Domain = a.config.DNSDomain
 	base.TLSMinVersion = a.config.TLSMinVersion
@@ -2103,7 +2159,7 @@ func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *structs.CheckType,
 				chkType.Interval = checks.MinInterval
 			}
 
-			tlsClientConfig, err := a.setupTLSClientConfig(chkType.TLSSkipVerify)
+			tlsClientConfig, err := a.setupCheckTLSClientConfig(chkType.TLSSkipVerify)
 			if err != nil {
 				return fmt.Errorf("Failed to set up TLS: %v", err)
 			}
@@ -2158,7 +2214,7 @@ func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *structs.CheckType,
 			var tlsClientConfig *tls.Config
 			if chkType.GRPCUseTLS {
 				var err error
-				tlsClientConfig, err = a.setupTLSClientConfig(chkType.TLSSkipVerify)
+				tlsClientConfig, err = a.setupCheckTLSClientConfig(chkType.TLSSkipVerify)
 				if err != nil {
 					return fmt.Errorf("Failed to set up TLS: %v", err)
 				}
@@ -2296,21 +2352,44 @@ func (a *Agent) AddCheck(check *structs.HealthCheck, chkType *structs.CheckType,
 	return nil
 }
 
-func (a *Agent) setupTLSClientConfig(skipVerify bool) (tlsClientConfig *tls.Config, err error) {
-	// We re-use the API client's TLS structure since it
-	// closely aligns with Consul's internal configuration.
-	tlsConfig := &api.TLSConfig{
-		InsecureSkipVerify: skipVerify,
+// setupAgentTLSClientConfig returns TLS client configuration for connecting
+// to agents.
+func (a *Agent) setupAgentTLSClientConfig(skipVerify bool) (*tls.Config, error) {
+	// Start with the outgoing TLS config for connecting to agents
+	tlscfg, err := a.config.OutgoingAgentTLSConfig()
+	if err != nil {
+		return tlscfg, err
 	}
-	if a.config.EnableAgentTLSForChecks {
-		tlsConfig.Address = a.config.ServerName
-		tlsConfig.KeyFile = a.config.KeyFile
-		tlsConfig.CertFile = a.config.CertFile
-		tlsConfig.CAFile = a.config.CAFile
-		tlsConfig.CAPath = a.config.CAPath
+
+	if tlscfg == nil {
+		// Agent TLS disabled, so no TLS config
+		return nil, nil
 	}
-	tlsClientConfig, err = api.SetupTLSConfig(tlsConfig)
-	return
+
+	// Modify it for skipVerify
+	tlscfg.InsecureSkipVerify = skipVerify
+
+	return tlscfg, err
+}
+
+// setupCheckTLSClientConfig returns TLS client configuration for performing
+// checks.
+func (a *Agent) setupCheckTLSClientConfig(skipVerify bool) (*tls.Config, error) {
+	// Start with the outgoing TLS config for connecting for checks
+	tlscfg, err := a.config.OutgoingCheckTLSConfig()
+	if err != nil {
+		return tlscfg, err
+	}
+
+	if tlscfg == nil {
+		// Check TLS disabled, so no TLS config
+		return nil, nil
+	}
+
+	// Modify it for skipVerify
+	tlscfg.InsecureSkipVerify = skipVerify
+
+	return tlscfg, err
 }
 
 // RemoveCheck is used to remove a health check.
@@ -3396,6 +3475,11 @@ func (a *Agent) ReloadConfig(newCfg *config.RuntimeConfig) error {
 		return err
 	}
 
+	// Reload TLS configuration.
+	if err := a.reloadTLSConfig(newCfg); err != nil {
+		return fmt.Errorf("Failed reloading TLS configuration: %s", err)
+	}
+
 	// Update filtered metrics
 	metrics.UpdateFilter(newCfg.Telemetry.AllowedPrefixes,
 		newCfg.Telemetry.BlockedPrefixes)

agent/config/runtime.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"reflect"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/hashicorp/consul/agent/structs"
@@ -781,6 +782,11 @@ type RuntimeConfig struct {
 	// hcl: key_file = string
 	KeyFile string
 
+	// KeyLoader dynamically reloads TLS configuration.
+	KeyLoader *tlsutil.KeyLoader
+
+	keyloaderLock sync.Mutex
+
 	// LeaveDrainTime is used to wait after a server has left the LAN Serf
 	// pool for RPCs to drain and new requests to be sent to other servers.
 	//
@@ -1430,25 +1436,6 @@ type RuntimeConfig struct {
 	Watches []map[string]interface{}
 }
 
-// IncomingHTTPSConfig returns the TLS configuration for HTTPS
-// connections to consul.
-func (c *RuntimeConfig) IncomingHTTPSConfig() (*tls.Config, error) {
-	tc := &tlsutil.Config{
-		VerifyIncoming:           c.VerifyIncoming || c.VerifyIncomingHTTPS,
-		VerifyOutgoing:           c.VerifyOutgoing,
-		CAFile:                   c.CAFile,
-		CAPath:                   c.CAPath,
-		CertFile:                 c.CertFile,
-		KeyFile:                  c.KeyFile,
-		NodeName:                 c.NodeName,
-		ServerName:               c.ServerName,
-		TLSMinVersion:            c.TLSMinVersion,
-		CipherSuites:             c.TLSCipherSuites,
-		PreferServerCipherSuites: c.TLSPreferServerCipherSuites,
-	}
-	return tc.IncomingTLSConfig()
-}
-
 func (c *RuntimeConfig) apiAddresses(maxPerType int) (unixAddrs, httpAddrs, httpsAddrs []string) {
 	if len(c.HTTPSAddrs) > 0 {
 		for i, addr := range c.HTTPSAddrs {
@@ -1579,6 +1566,93 @@ func (c *RuntimeConfig) APIConfig(includeClientCerts bool) (*api.Config, error)
 	return cfg, nil
 }
 
+// TLSConfig returns the full TLS configuration for both incoming and
+// outgoing connections.
+func (c *RuntimeConfig) TLSConfig() (*tlsutil.Config, error) {
+	tc := &tlsutil.Config{
+		VerifyIncoming:           c.VerifyIncoming || c.VerifyIncomingHTTPS,
+		VerifyOutgoing:           c.VerifyOutgoing,
+		CAFile:                   c.CAFile,
+		CAPath:                   c.CAPath,
+		CertFile:                 c.CertFile,
+		KeyFile:                  c.KeyFile,
+		KeyLoader:                c.GetKeyLoader(),
+		NodeName:                 c.NodeName,
+		ServerName:               c.ServerName,
+		TLSMinVersion:            c.TLSMinVersion,
+		CipherSuites:             c.TLSCipherSuites,
+		PreferServerCipherSuites: c.TLSPreferServerCipherSuites,
+	}
+	return tc, nil
+}
+
+// IncomingHTTPSConfig returns the TLS configuration for HTTPS
+// connections to consul.
+func (c *RuntimeConfig) IncomingHTTPSConfig() (*tls.Config, error) {
+	tc, err := c.TLSConfig()
+
+	if err != nil {
+		return nil, err
+	}
+	if tc == nil {
+		return nil, fmt.Errorf("no TLS configuration available")
+	}
+
+	return tc.IncomingTLSConfig()
+}
+
+// OutgoingTLSConfig returns the TLS configuration for connections to agents.
+func (c *RuntimeConfig) OutgoingAgentTLSConfig() (*tls.Config, error) {
+	tc, err := c.TLSConfig()
+
+	if err != nil {
+		return nil, err
+	}
+	if tc == nil {
+		return nil, fmt.Errorf("no TLS configuration available")
+	}
+
+	return tc.OutgoingTLSConfig()
+}
+
+// OutgoingCheckTLSConfig returns the TLS configuration for check
+// connections from consul.
+func (c *RuntimeConfig) OutgoingCheckTLSConfig() (*tls.Config, error) {
+	// Start with the agent TLS information
+	tc, err := c.TLSConfig()
+
+	if err != nil {
+		return nil, err
+	}
+	if tc == nil {
+		return nil, fmt.Errorf("no TLS configuration available")
+	}
+
+	// Then remove key information if TLS is disabled for checks
+	if !c.EnableAgentTLSForChecks {
+		tc.VerifyOutgoing = false
+		tc.CAFile = ""
+		tc.CAPath = ""
+		tc.CertFile = ""
+		tc.KeyFile = ""
+	}
+
+	return tc.OutgoingTLSConfig()
+}
+
+// GetKeyLoader returns the keyloader for a RuntimeConfig object. If the
+// keyloader has not been initialized, it will first do so.
+func (c *RuntimeConfig) GetKeyLoader() *tlsutil.KeyLoader {
+	c.keyloaderLock.Lock()
+	defer c.keyloaderLock.Unlock()
+
+	// If the keyloader has not yet been initialized, do it here
+	if c.KeyLoader == nil {
+		c.KeyLoader = &tlsutil.KeyLoader{}
+	}
+	return c.KeyLoader
+}
+
 // Sanitized returns a JSON/HCL compatible representation of the runtime
 // configuration where all fields with potential secrets had their
 // values replaced by 'hidden'. In addition, network addresses and

agent/consul/config.go

@@ -193,6 +193,9 @@ type Config struct {
 	// Must be provided to serve TLS connections.
 	KeyFile string
 
+	// KeyLoader dynamically reloads TLS configuration.
+	KeyLoader *tlsutil.KeyLoader
+
 	// ServerName is used with the TLS certificate to ensure the name we
 	// provide matches the certificate
 	ServerName string
@@ -512,6 +515,7 @@ func (c *Config) tlsConfig() *tlsutil.Config {
 		CAPath:                   c.CAPath,
 		CertFile:                 c.CertFile,
 		KeyFile:                  c.KeyFile,
+		KeyLoader:                c.KeyLoader,
 		NodeName:                 c.NodeName,
 		ServerName:               c.ServerName,
 		Domain:                   c.Domain,