add ability to authenticators for dynamic update of certs for delegated authn

cmd/kube-apiserver/app/server.go

@@ -525,7 +525,10 @@ func buildGenericConfig(
 
 // BuildAuthenticator constructs the authenticator
 func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
-	authenticatorConfig := s.Authentication.ToAuthenticationConfig()
+	authenticatorConfig, err := s.Authentication.ToAuthenticationConfig()
+	if err != nil {
+		return nil, nil, err
+	}
 	if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
 		authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
 			extclient,

cmd/kubelet/app/BUILD

@@ -118,6 +118,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",

cmd/kubelet/app/auth.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	clientset "k8s.io/client-go/kubernetes"
@@ -63,10 +64,15 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
 
 // BuildAuthn creates an authenticator compatible with the kubelet's needs
 func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) {
+	clientCertVerifier, err := x509.NewStaticVerifierFromFile(authn.X509.ClientCAFile)
+	if err != nil {
+		return nil, err
+	}
+
 	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
-		Anonymous:    authn.Anonymous.Enabled,
-		CacheTTL:     authn.Webhook.CacheTTL.Duration,
-		ClientCAFile: authn.X509.ClientCAFile,
+		Anonymous:            authn.Anonymous.Enabled,
+		CacheTTL:             authn.Webhook.CacheTTL.Duration,
+		ClientVerifyOptionFn: clientCertVerifier,
 	}
 
 	if authn.Webhook.Enabled {

pkg/kubeapiserver/authenticator/BUILD

@@ -30,7 +30,6 @@ go_library(
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc:go_default_library",
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library",
         "//staging/src/k8s.io/client-go/plugin/pkg/client/auth:go_default_library",
-        "//staging/src/k8s.io/client-go/util/cert:go_default_library",
         "//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
         "//vendor/github.com/go-openapi/spec:go_default_library",
     ],

pkg/kubeapiserver/authenticator/config.go

@@ -41,18 +41,17 @@ import (
 
 	// Initialize all known client auth plugins.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
 // Config contains the data on how to authenticate a request to the Kube API Server
 type Config struct {
-	Anonymous                   bool
-	BasicAuthFile               string
-	BootstrapToken              bool
-	ClientCAFile                string
+	Anonymous      bool
+	BasicAuthFile  string
+	BootstrapToken bool
+
 	TokenAuthFile               string
 	OIDCIssuerURL               string
 	OIDCClientID                string
@@ -78,6 +77,10 @@ type Config struct {
 	// TODO, this is the only non-serializable part of the entire config.  Factor it out into a clientconfig
 	ServiceAccountTokenGetter   serviceaccount.ServiceAccountTokenGetter
 	BootstrapTokenAuthenticator authenticator.Token
+	// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// Generally this is the CA bundle file used to authenticate client certificates
+	// If this value is nil, then mutual TLS is disabled.
+	ClientVerifyOptionFn x509.VerifyOptionFunc
 }
 
 // New returns an authenticator.Request or an error that supports the standard
@@ -90,8 +93,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 	// front-proxy, BasicAuth methods, local first, then remote
 	// Add the front proxy authenticator if requested
 	if config.RequestHeaderConfig != nil {
-		requestHeaderAuthenticator, err := headerrequest.NewSecure(
-			config.RequestHeaderConfig.ClientCA,
+		requestHeaderAuthenticator, err := headerrequest.NewDynamicVerifyOptionsSecure(
+			config.RequestHeaderConfig.VerifyOptionFn,
 			config.RequestHeaderConfig.AllowedClientNames,
 			config.RequestHeaderConfig.UsernameHeaders,
 			config.RequestHeaderConfig.GroupHeaders,
@@ -120,11 +123,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 	}
 
 	// X509 methods
-	if len(config.ClientCAFile) > 0 {
-		certAuth, err := newAuthenticatorFromClientCAFile(config.ClientCAFile)
-		if err != nil {
-			return nil, nil, err
-		}
+	if config.ClientVerifyOptionFn != nil {
+		certAuth := x509.NewDynamic(config.ClientVerifyOptionFn, x509.CommonNameUserConversion)
 		authenticators = append(authenticators, certAuth)
 	}
 
@@ -307,19 +307,6 @@ func newServiceAccountAuthenticator(iss string, keyfiles []string, apiAudiences
 	return tokenAuthenticator, nil
 }
 
-// newAuthenticatorFromClientCAFile returns an authenticator.Request or an error
-func newAuthenticatorFromClientCAFile(clientCAFile string) (authenticator.Request, error) {
-	roots, err := certutil.NewPool(clientCAFile)
-	if err != nil {
-		return nil, err
-	}
-
-	opts := x509.DefaultVerifyOptions()
-	opts.Roots = roots
-
-	return x509.New(opts, x509.CommonNameUserConversion), nil
-}
-
 func newWebhookTokenAuthenticator(webhookConfigFile string, ttl time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) {
 	webhookTokenAuthenticator, err := webhook.New(webhookConfigFile, implicitAuds)
 	if err != nil {

pkg/kubeapiserver/options/BUILD

@@ -89,6 +89,12 @@ go_test(
         "authentication_test.go",
         "authorization_test.go",
     ],
+    data = [
+        "testdata/client-expired.pem",
+        "testdata/client-valid.pem",
+        "testdata/intermediate.pem",
+        "testdata/root.pem",
+    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/kubeapiserver/authenticator:go_default_library",
@@ -97,5 +103,6 @@ go_test(
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//vendor/github.com/google/go-cmp/cmp:go_default_library",
     ],
 )

pkg/kubeapiserver/options/authentication.go

@@ -308,7 +308,7 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 	}
 }
 
-func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticator.Config {
+func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticator.Config, error) {
 	ret := kubeauthenticator.Config{
 		TokenSuccessCacheTTL: s.TokenSuccessCacheTTL,
 		TokenFailureCacheTTL: s.TokenFailureCacheTTL,
@@ -323,7 +323,11 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticato
 	}
 
 	if s.ClientCert != nil {
-		ret.ClientCAFile = s.ClientCert.ClientCA
+		var err error
+		ret.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
+		if err != nil {
+			return kubeauthenticator.Config{}, err
+		}
 	}
 
 	if s.OIDC != nil {
@@ -343,7 +347,11 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticato
 	}
 
 	if s.RequestHeader != nil {
-		ret.RequestHeaderConfig = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
+		var err error
+		ret.RequestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
+		if err != nil {
+			return kubeauthenticator.Config{}, err
+		}
 	}
 
 	ret.APIAudiences = s.APIAudiences
@@ -374,7 +382,7 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticato
 		}
 	}
 
-	return ret
+	return ret, nil
 }
 
 func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error {

pkg/kubeapiserver/options/authentication_test.go

@@ -22,6 +22,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
@@ -101,7 +103,7 @@ func TestToAuthenticationConfig(t *testing.T) {
 			Allow: false,
 		},
 		ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
-			ClientCA: "/client-ca",
+			ClientCA: "testdata/root.pem",
 		},
 		WebHook: &WebHookAuthenticationOptions{
 			CacheTTL:   180000000000,
@@ -124,7 +126,7 @@ func TestToAuthenticationConfig(t *testing.T) {
 			UsernameHeaders:     []string{"x-remote-user"},
 			GroupHeaders:        []string{"x-remote-group"},
 			ExtraHeaderPrefixes: []string{"x-remote-extra-"},
-			ClientCAFile:        "/testClientCAFile",
+			ClientCAFile:        "testdata/root.pem",
 			AllowedNames:        []string{"kube-aggregator"},
 		},
 		ServiceAccounts: &ServiceAccountAuthenticationOptions{
@@ -143,7 +145,7 @@ func TestToAuthenticationConfig(t *testing.T) {
 		Anonymous:                   false,
 		BasicAuthFile:               "/testBasicAuthFile",
 		BootstrapToken:              false,
-		ClientCAFile:                "/client-ca",
+		ClientVerifyOptionFn:        nil, // this is nil because you can't compare functions
 		TokenAuthFile:               "/testTokenFile",
 		OIDCIssuerURL:               "testIssuerURL",
 		OIDCClientID:                "testClientID",
@@ -162,13 +164,27 @@ func TestToAuthenticationConfig(t *testing.T) {
 			UsernameHeaders:     []string{"x-remote-user"},
 			GroupHeaders:        []string{"x-remote-group"},
 			ExtraHeaderPrefixes: []string{"x-remote-extra-"},
-			ClientCA:            "/testClientCAFile",
+			VerifyOptionFn:      nil, // this is nil because you can't compare functions
 			AllowedClientNames:  []string{"kube-aggregator"},
 		},
 	}
 
-	resultConfig := testOptions.ToAuthenticationConfig()
+	resultConfig, err := testOptions.ToAuthenticationConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// nil these out because you cannot compare pointers.  Ensure they are non-nil first
+	if resultConfig.ClientVerifyOptionFn == nil {
+		t.Error("missing client verify")
+	}
+	if resultConfig.RequestHeaderConfig.VerifyOptionFn == nil {
+		t.Error("missing requestheader verify")
+	}
+	resultConfig.ClientVerifyOptionFn = nil
+	resultConfig.RequestHeaderConfig.VerifyOptionFn = nil
+
 	if !reflect.DeepEqual(resultConfig, expectConfig) {
-		t.Errorf("Got AuthenticationConfig:\n\t%v\nExpected AuthenticationConfig:\n\t%v", resultConfig, expectConfig)
+		t.Error(cmp.Diff(resultConfig, expectConfig))
 	}
 }

pkg/kubeapiserver/options/testdata/client-expired.pem

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBpTCCAUugAwIBAgIUPV4LAC5KK8YWY1FegyTuhkGUr3EwCgYIKoZIzj0EAwIw
+GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMB4XDTkwMTIzMTIzNTkwMFoXDTkw
+MTIzMTIzNTkwMFowFDESMBAGA1UEAxMJTXkgQ2xpZW50MFkwEwYHKoZIzj0CAQYI
+KoZIzj0DAQcDQgAEyYUnseNUN87rfHgekrfZu5sj4wlt5LYr3JYZZkfSbsb+BW3/
+RzX02ifjp+8w7mI4qUGg6y6J7oXHGFT3uj9kj6N1MHMwDgYDVR0PAQH/BAQDAgWg
+MBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFKsX
+EnXwDg8j2LIEM1QzmFrE6537MB8GA1UdIwQYMBaAFF+p0JcY31pz+mjNZnjv0Gum
+92vZMAoGCCqGSM49BAMCA0gAMEUCIG4FBcb57oqOCoaFiJ+Yx6S0zkaash7bTv3V
+CIy9JvFdAiEAy8bf2S9EkvZyURZ6ycgEMnekll57Ebze6rjlPx8+B1Y=
+-----END CERTIFICATE-----

pkg/kubeapiserver/options/testdata/client-valid.pem

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
+GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
+MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
+BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
+KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
+BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
+K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
+a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
+MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
+-----END CERTIFICATE-----

pkg/kubeapiserver/options/testdata/client.config.json

@@ -0,0 +1,24 @@
+{
+    "signing": {
+        "profiles": {
+            "valid": {
+                "expiry": "876000h",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "client auth"
+                ]
+            },
+            "expired": {
+                "expiry": "1h",
+                "not_before": "1990-12-31T23:59:00Z",
+                "not_after": "1990-12-31T23:59:00Z",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "client auth"
+                ]
+            }
+        }
+    }
+}

pkg/kubeapiserver/options/testdata/client.csr.json

@@ -0,0 +1,3 @@
+{
+    "CN": "My Client"
+}

pkg/kubeapiserver/options/testdata/generate.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cfssl gencert -initca root.csr.json | cfssljson -bare root
+
+cfssl gencert -initca intermediate.csr.json | cfssljson -bare intermediate
+cfssl sign -ca root.pem -ca-key root-key.pem -config intermediate.config.json intermediate.csr | cfssljson -bare intermediate
+
+cfssl gencert -ca intermediate.pem -ca-key intermediate-key.pem -config client.config.json --profile=valid   client.csr.json | cfssljson -bare client-valid
+cfssl gencert -ca intermediate.pem -ca-key intermediate-key.pem -config client.config.json --profile=expired client.csr.json | cfssljson -bare client-expired
+

pkg/kubeapiserver/options/testdata/intermediate.config.json

@@ -0,0 +1,18 @@
+{
+    "signing": {
+        "default": {
+            "usages": [
+                "digital signature",
+                "cert sign",
+                "crl sign",
+                "signing",
+                "key encipherment",
+                "client auth"
+            ],
+            "expiry": "876000h",
+            "ca_constraint": {
+                "is_ca": true
+            }
+        }
+    }
+}

pkg/kubeapiserver/options/testdata/intermediate.csr.json

@@ -0,0 +1,6 @@
+{
+    "CN": "Intermediate-CA",
+    "ca": {
+        "expiry": "876000h"
+    }
+}

pkg/kubeapiserver/options/testdata/intermediate.pem

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBqDCCAU6gAwIBAgIUfqZtjoFgczZ+oQZbEC/BDSS2J6wwCgYIKoZIzj0EAwIw
+EjEQMA4GA1UEAxMHUm9vdC1DQTAgFw0xNjEwMTEwNTA2MDBaGA8yMTE2MDkxNzA1
+MDYwMFowGjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMFkwEwYHKoZIzj0CAQYI
+KoZIzj0DAQcDQgAEyWHEMMCctJg8Xa5YWLqaCPbk3MjB+uvXac42JM9pj4k9jedD
+kpUJRkWIPzgJI8Zk/3cSzluUTixP6JBSDKtwwaN4MHYwDgYDVR0PAQH/BAQDAgGm
+MBMGA1UdJQQMMAoGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
+FF+p0JcY31pz+mjNZnjv0Gum92vZMB8GA1UdIwQYMBaAFB7P6+i4/pfNjqZgJv/b
+dgA7Fe4tMAoGCCqGSM49BAMCA0gAMEUCIQCTT1YWQZaAqfQ2oBxzOkJE2BqLFxhz
+3smQlrZ5gCHddwIgcvT7puhYOzAgcvMn9+SZ1JOyZ7edODjshCVCRnuHK2c=
+-----END CERTIFICATE-----

pkg/kubeapiserver/options/testdata/root.csr.json

@@ -0,0 +1,6 @@
+{
+    "CN": "Root-CA",
+    "ca": {
+        "expiry": "876000h"
+    }
+}