Plumbing for dynamic apiserver serving certificates

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -233,9 +233,10 @@ type SecureServingInfo struct {
 
 	// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
 	// allowed to be in SNICerts.
-	Cert *tls.Certificate
+	Cert dynamiccertificates.CertKeyContentProvider
 
 	// SNICerts are the TLS certificates by name used for SNI.
+	// todo: use dynamic certificates
 	SNICerts map[string]*tls.Certificate
 
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "cert_key.go",
         "client_ca.go",
         "static_content.go",
         "tlsconfig.go",
@@ -40,6 +41,7 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = [
+        "cert_key_test.go",
         "client_ca_test.go",
         "tlsconfig_test.go",
     ],

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// CertKeyContentProvider provides a certificate and matching private key
+type CertKeyContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCertKeyContent provides cert and key byte content
+	CurrentCertKeyContent() ([]byte, []byte)
+}
+
+// caBundleContent holds the content for the cert and key
+type certKeyContent struct {
+	cert []byte
+	key  []byte
+}
+
+func (c *certKeyContent) Equal(rhs *certKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.key, rhs.key) && bytes.Equal(c.cert, rhs.cert)
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key_test.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import "testing"
+
+func TestCertKeyContentEquals(t *testing.T) {
+	tests := []struct {
+		name     string
+		lhs      *certKeyContent
+		rhs      *certKeyContent
+		expected bool
+	}{
+		{
+			name:     "both nil",
+			expected: true,
+		},
+		{
+			name:     "lhs nil",
+			rhs:      &certKeyContent{},
+			expected: false,
+		},
+		{
+			name:     "rhs nil",
+			lhs:      &certKeyContent{},
+			expected: false,
+		},
+		{
+			name:     "same",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			expected: true,
+		},
+		{
+			name:     "different cert",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("bar"), key: []byte("baz")},
+			expected: false,
+		},
+		{
+			name:     "different key",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("foo"), key: []byte("qux")},
+			expected: false,
+		},
+		{
+			name:     "different cert and key",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("bar"), key: []byte("qux")},
+			expected: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := test.lhs.Equal(test.rhs)
+			if actual != test.expected {
+				t.Error(actual)
+			}
+		})
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -30,10 +30,10 @@ type CAContentProvider interface {
 }
 
 // dynamicCertificateContent holds the content that overrides the baseTLSConfig
-// TODO add the serving certs to this struct
 type dynamicCertificateContent struct {
 	// clientCA holds the content for the clientCA bundle
-	clientCA caBundleContent
+	clientCA    caBundleContent
+	servingCert certKeyContent
 }
 
 // caBundleContent holds the content for the clientCA bundle.  Wrapping the bytes makes the Equals work nicely with the
@@ -51,6 +51,10 @@ func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
 		return false
 	}
 
+	if !c.servingCert.Equal(&rhs.servingCert) {
+		return false
+	}
+
 	return true
 }
 

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca_test.go

@@ -59,6 +59,42 @@ func TestDynamicCertificateContentEquals(t *testing.T) {
 			},
 			expected: false,
 		},
+		{
+			name: "same with serving",
+			lhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			expected: true,
+		},
+		{
+			name: "different serving cert",
+			lhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("bar"), key: []byte("foo")},
+			},
+			expected: false,
+		},
+		{
+			name: "different serving key",
+			lhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("bar")},
+			},
+			expected: false,
+		},
 	}
 
 	for _, test := range tests {

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -17,6 +17,7 @@ limitations under the License.
 package dynamiccertificates
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 )
@@ -56,3 +57,55 @@ func (c *staticCAContent) Name() string {
 func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
 	return c.caBundle
 }
+
+type staticCertKeyContent struct {
+	name string
+	cert []byte
+	key  []byte
+}
+
+// NewStaticCertKeyContentFromFiles returns a CertKeyContentProvider based on a filename
+func NewStaticCertKeyContentFromFiles(certFile, keyFile string) (CertKeyContentProvider, error) {
+	if len(certFile) == 0 {
+		return nil, fmt.Errorf("missing filename for certificate")
+	}
+	if len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for key")
+	}
+
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewStaticCertKeyContent(fmt.Sprintf("cert: %s, key: %s", certFile, keyFile), certPEMBlock, keyPEMBlock)
+}
+
+// NewStaticCertKeyContent returns a CertKeyContentProvider that always returns the same value
+func NewStaticCertKeyContent(name string, cert, key []byte) (CertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCertKeyContent{
+		name: name,
+		cert: cert,
+		key:  key,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCertKeyContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides cert and key content
+func (c *staticCertKeyContent) CurrentCertKeyContent() ([]byte, []byte) {
+	return c.cert, c.key
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -44,6 +44,8 @@ type DynamicServingCertificateController struct {
 
 	// clientCA provides the very latest content of the ca bundle
 	clientCA CAContentProvider
+	// servingCert provides the very latest content of the default serving certificate
+	servingCert CertKeyContentProvider
 
 	// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
 	// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
@@ -60,11 +62,13 @@ type DynamicServingCertificateController struct {
 func NewDynamicServingCertificateController(
 	baseTLSConfig tls.Config,
 	clientCA CAContentProvider,
+	servingCert CertKeyContentProvider,
 	eventRecorder events.EventRecorder,
 ) *DynamicServingCertificateController {
 	c := &DynamicServingCertificateController{
 		baseTLSConfig: baseTLSConfig,
 		clientCA:      clientCA,
+		servingCert:   servingCert,
 
 		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
 		eventRecorder: eventRecorder,
@@ -91,13 +95,23 @@ func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tl
 func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
 	newContent := &dynamicCertificateContent{}
 
-	currClientCABundle := c.clientCA.CurrentCABundleContent()
-	// don't remove all content.  The value was configured at one time, so continue using that.
-	// Errors reading content can be reported by lower level controllers.
-	if len(currClientCABundle) == 0 {
-		return nil, fmt.Errorf("not loading an empty client ca bundle from %q", c.clientCA.Name())
+	if c.clientCA != nil {
+		currClientCABundle := c.clientCA.CurrentCABundleContent()
+		// don't remove all content.  The value was configured at one time, so continue using that.
+		if len(currClientCABundle) == 0 {
+			return nil, fmt.Errorf("not loading an empty client ca bundle from %q", c.clientCA.Name())
+		}
+		newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
+	}
+
+	if c.servingCert != nil {
+		currServingCert, currServingKey := c.servingCert.CurrentCertKeyContent()
+		if len(currServingCert) == 0 || len(currServingKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty serving certificate from %q", c.servingCert.Name())
+		}
+
+		newContent.servingCert = certKeyContent{cert: currServingCert, key: currServingKey}
 	}
-	newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
 
 	return newContent, nil
 }
@@ -115,9 +129,12 @@ func (c *DynamicServingCertificateController) syncCerts() error {
 		return nil
 	}
 
+	// make a shallow copy and override the dynamic pieces which have changed.
+	newTLSConfigCopy := c.baseTLSConfig.Clone()
+
 	// parse new content to add to TLSConfig
-	newClientCAPool := x509.NewCertPool()
 	if len(newContent.clientCA.caBundle) > 0 {
+		newClientCAPool := x509.NewCertPool()
 		newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
 		if err != nil {
 			return fmt.Errorf("unable to load client CA file: %v", err)
@@ -130,11 +147,36 @@ func (c *DynamicServingCertificateController) syncCerts() error {
 
 			newClientCAPool.AddCert(cert)
 		}
+
+		newTLSConfigCopy.ClientCAs = newClientCAPool
 	}
 
-	// make a copy and override the dynamic pieces which have changed.
-	newTLSConfigCopy := c.baseTLSConfig.Clone()
-	newTLSConfigCopy.ClientCAs = newClientCAPool
+	if len(newContent.servingCert.cert) > 0 && len(newContent.servingCert.key) > 0 {
+		cert, err := tls.X509KeyPair(newContent.servingCert.cert, newContent.servingCert.key)
+		if err != nil {
+			return fmt.Errorf("invalid serving cert keypair: %v", err)
+		}
+
+		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("invalid serving cert: %v", err)
+		}
+
+		klog.V(2).Infof("loaded serving cert [%q]: %s", c.servingCert.Name(), GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "ServingCertificateReload", "loaded serving cert [%q]: %s", c.clientCA.Name(), GetHumanCertDetail(x509Cert))
+		}
+
+		newTLSConfigCopy.Certificates = []tls.Certificate{cert}
+
+		// append all named certs. Otherwise, the go tls stack will think no SNI processing
+		// is necessary because there is only one cert anyway.
+		// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
+		// cert will become the default cert. That's what we expect anyway.
+		for _, c := range newTLSConfigCopy.NameToCertificate {
+			newTLSConfigCopy.Certificates = append(newTLSConfigCopy.Certificates, *c)
+		}
+	}
 
 	// store new values of content for serving.
 	c.currentServingTLSConfig.Store(newTLSConfigCopy)