Merge pull request #41281 from ericchiang/bootstrap-token-authenticator

Automatic merge from submit-queue (batch tested with PRs 41812, 41665, 40007, 41281, 41771)

kube-apiserver: add a bootstrap token authenticator for TLS bootstrapping

Follows up on #36101

Still needs:

* More tests.
* To be hooked up to the API server.
  - Do I have to do that in a separate PR after k8s.io/apiserver is synced?
* Docs (kubernetes.io PR).
* Figure out caching strategy.
* Release notes.

cc @kubernetes/sig-auth-api-reviews @liggitt @luxas @jbeda

```release-notes
Added a new secret type "bootstrap.kubernetes.io/token" for dynamically creating TLS bootstrapping bearer tokens.
```

cmd/kube-apiserver/app/BUILD

@@ -20,6 +20,7 @@ go_library(
         "//pkg/apis/batch:go_default_library",
         "//pkg/capabilities:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
+        "//pkg/client/informers/informers_generated/internalversion:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers:go_default_library",
         "//pkg/controller/informers:go_default_library",
@@ -51,10 +52,12 @@ go_library(
         "//plugin/pkg/admission/securitycontext/scdeny:go_default_library",
         "//plugin/pkg/admission/serviceaccount:go_default_library",
         "//plugin/pkg/admission/storageclass/default:go_default_library",
+        "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
         "//vendor:github.com/go-openapi/spec",
         "//vendor:github.com/golang/glog",
         "//vendor:github.com/spf13/cobra",
         "//vendor:github.com/spf13/pflag",
+        "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
         "//vendor:k8s.io/apimachinery/pkg/openapi",
         "//vendor:k8s.io/apimachinery/pkg/runtime/schema",
         "//vendor:k8s.io/apimachinery/pkg/util/errors",

cmd/kube-apiserver/app/server.go

@@ -35,6 +35,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/openapi"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -50,6 +51,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/batch"
 	"k8s.io/kubernetes/pkg/capabilities"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	"k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller/informers"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
@@ -61,6 +63,7 @@ import (
 	"k8s.io/kubernetes/pkg/master/tunneler"
 	"k8s.io/kubernetes/pkg/registry/cachesize"
 	"k8s.io/kubernetes/pkg/version"
+	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
 )
 
 // NewAPIServerCommand creates a *cobra.Command object with default parameters
@@ -253,11 +256,6 @@ func Run(s *options.ServerRunOptions) error {
 		authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
 	}
 
-	apiAuthenticator, securityDefinitions, err := authenticatorConfig.New()
-	if err != nil {
-		return fmt.Errorf("invalid Authentication Config: %v", err)
-	}
-
 	client, err := internalclientset.NewForConfig(genericConfig.LoopbackClientConfig)
 	if err != nil {
 		kubeAPIVersions := os.Getenv("KUBE_API_VERSIONS")
@@ -267,10 +265,29 @@ func Run(s *options.ServerRunOptions) error {
 
 		// KUBE_API_VERSIONS is used in test-update-storage-objects.sh, disabling a number of API
 		// groups. This leads to a nil client above and undefined behaviour further down.
+		//
 		// TODO: get rid of KUBE_API_VERSIONS or define sane behaviour if set
 		glog.Errorf("Failed to create clientset with KUBE_API_VERSIONS=%q. KUBE_API_VERSIONS is only for testing. Things will break.", kubeAPIVersions)
 	}
+
+	// TODO: Internal informers should switch to using 'pkg/client/informers/informers_generated',
+	// the second informer created here. Refactor clients which take the former to accept the latter.
 	sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)
+	internalSharedInformers := internalversion.NewSharedInformerFactory(client, 10*time.Minute)
+
+	if client == nil {
+		// TODO: Remove check once client can never be nil.
+		glog.Errorf("Failed to setup bootstrap token authenticator because the loopback clientset was not setup properly.")
+	} else {
+		authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(
+			internalSharedInformers.Core().InternalVersion().Secrets().Lister().Secrets(v1.NamespaceSystem),
+		)
+	}
+
+	apiAuthenticator, securityDefinitions, err := authenticatorConfig.New()
+	if err != nil {
+		return fmt.Errorf("invalid authentication config: %v", err)
+	}
 
 	authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
 	apiAuthorizer, err := authorizationConfig.New()
@@ -350,6 +367,7 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	sharedInformers.Start(wait.NeverStop)
+	internalSharedInformers.Start(wait.NeverStop)
 	m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
 	return nil
 }

hack/.linted_packages

@@ -267,6 +267,7 @@ plugin/pkg/admission/resourcequota/apis/resourcequota/install
 plugin/pkg/admission/resourcequota/apis/resourcequota/validation
 plugin/pkg/admission/securitycontext/scdeny
 plugin/pkg/auth
+plugin/pkg/auth/authenticator/token/bootstrap
 plugin/pkg/auth/authorizer
 plugin/pkg/auth/authorizer/rbac/bootstrappolicy
 staging/src/k8s.io/apimachinery/pkg/api/equality

hack/verify-flags/known-flags.txt

@@ -205,6 +205,7 @@ executor-suicide-timeout
 exit-on-lock-contention
 experimental-allowed-unsafe-sysctls
 experimental-bootstrap-kubeconfig
+experimental-bootstrap-token-auth
 experimental-keystone-url
 experimental-keystone-ca-file
 experimental-mounter-path

pkg/bootstrap/api/types.go

@@ -33,12 +33,12 @@ const (
 	SecretTypeBootstrapToken v1.SecretType = "bootstrap.kubernetes.io/token"
 
 	// BootstrapTokenIDKey is the id of this token. This can be transmitted in the
-	// clear and encoded in the name of the secret. It should be a random 6
-	// character string. Required
+	// clear and encoded in the name of the secret. It must be a random 6 character
+	// string that matches the regexp `^([a-z0-9]{6})$`. Required.
 	BootstrapTokenIDKey = "token-id"
 
-	// BootstrapTokenSecretKey is the actual secret. Typically this is a random 16
-	// character string. Required.
+	// BootstrapTokenSecretKey is the actual secret. It must be a random 16 character
+	// string that matches the regexp `^([a-z0-9]{16})$`. Required.
 	BootstrapTokenSecretKey = "token-secret"
 
 	// BootstrapTokenExpirationKey is when this token should be expired and no
@@ -52,6 +52,13 @@ const (
 	// other value is assumed to be false. Optional.
 	BootstrapTokenUsageSigningKey = "usage-bootstrap-signing"
 
+	// BootstrapTokenUsageAuthentication signals that this token should be used
+	// as a bearer token to authenticate against the Kubernetes API. The bearer
+	// token takes the form "<token-id>.<token-secret>" and authenticates as the
+	// user "system:bootstrap:<token-id>" in the group "system:bootstrappers".
+	// Value must be "true". Any other value is assumed to be false. Optional.
+	BootstrapTokenUsageAuthentication = "usage-bootstrap-authentication"
+
 	// ConfigMapClusterInfo defines the name for the ConfigMap where the information how to connect and trust the cluster exist
 	ConfigMapClusterInfo = "cluster-info"
 
@@ -60,4 +67,11 @@ const (
 
 	// JWSSignatureKeyPrefix defines what key prefix the JWS-signed tokens have
 	JWSSignatureKeyPrefix = "jws-kubeconfig-"
+
+	// BootstrapUserPrefix is the username prefix bootstrapping bearer tokens
+	// authenticate as. The full username given is "system:bootstrap:<token-id>".
+	BootstrapUserPrefix = "system:bootstrap:"
+
+	// BootstrapGroup is the group bootstrapping bearer tokens authenticate in.
+	BootstrapGroup = "system:bootstrappers"
 )

pkg/kubeapiserver/authenticator/config.go

@@ -49,6 +49,7 @@ type AuthenticatorConfig struct {
 	Anonymous                   bool
 	AnyToken                    bool
 	BasicAuthFile               string
+	BootstrapToken              bool
 	ClientCAFile                string
 	TokenAuthFile               string
 	OIDCIssuerURL               string
@@ -66,7 +67,8 @@ type AuthenticatorConfig struct {
 	RequestHeaderConfig *authenticatorfactory.RequestHeaderConfig
 
 	// TODO, this is the only non-serializable part of the entire config.  Factor it out into a clientconfig
-	ServiceAccountTokenGetter serviceaccount.ServiceAccountTokenGetter
+	ServiceAccountTokenGetter   serviceaccount.ServiceAccountTokenGetter
+	BootstrapTokenAuthenticator authenticator.Token
 }
 
 // New returns an authenticator.Request or an error that supports the standard
@@ -136,6 +138,13 @@ func (config AuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDe
 		authenticators = append(authenticators, serviceAccountAuth)
 		hasTokenAuth = true
 	}
+	if config.BootstrapToken {
+		if config.BootstrapTokenAuthenticator != nil {
+			// TODO: This can sometimes be nil because of
+			authenticators = append(authenticators, bearertoken.New(config.BootstrapTokenAuthenticator))
+			hasTokenAuth = true
+		}
+	}
 	// NOTE(ericchiang): Keep the OpenID Connect after Service Accounts.
 	//
 	// Because both plugins verify JWTs whichever comes first in the union experiences

pkg/kubeapiserver/options/authentication.go

@@ -33,6 +33,7 @@ import (
 type BuiltInAuthenticationOptions struct {
 	Anonymous       *AnonymousAuthenticationOptions
 	AnyToken        *AnyTokenAuthenticationOptions
+	BootstrapToken  *BootstrapTokenAuthenticationOptions
 	ClientCert      *genericoptions.ClientCertAuthenticationOptions
 	Keystone        *KeystoneAuthenticationOptions
 	OIDC            *OIDCAuthenticationOptions
@@ -51,6 +52,10 @@ type AnonymousAuthenticationOptions struct {
 	Allow bool
 }
 
+type BootstrapTokenAuthenticationOptions struct {
+	Allow bool
+}
+
 type KeystoneAuthenticationOptions struct {
 	URL    string
 	CAFile string
@@ -90,6 +95,7 @@ func (s *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
 	return s.
 		WithAnyonymous().
 		WithAnyToken().
+		WithBootstrapToken().
 		WithClientCert().
 		WithKeystone().
 		WithOIDC().
@@ -110,6 +116,11 @@ func (s *BuiltInAuthenticationOptions) WithAnyToken() *BuiltInAuthenticationOpti
 	return s
 }
 
+func (s *BuiltInAuthenticationOptions) WithBootstrapToken() *BuiltInAuthenticationOptions {
+	s.BootstrapToken = &BootstrapTokenAuthenticationOptions{}
+	return s
+}
+
 func (s *BuiltInAuthenticationOptions) WithClientCert() *BuiltInAuthenticationOptions {
 	s.ClientCert = &genericoptions.ClientCertAuthenticationOptions{}
 	return s
@@ -172,6 +183,12 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 
 	}
 
+	if s.BootstrapToken != nil {
+		fs.BoolVar(&s.BootstrapToken.Allow, "experimental-bootstrap-token-auth", s.BootstrapToken.Allow, ""+
+			"Enable to allow secrets of type 'bootstrap.kubernetes.io/token' in the 'kube-system' "+
+			"namespace to be used for TLS bootstrapping authentication.")
+	}
+
 	if s.ClientCert != nil {
 		s.ClientCert.AddFlags(fs)
 	}
@@ -255,6 +272,10 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() authenticator.Au
 		ret.AnyToken = s.AnyToken.Allow
 	}
 
+	if s.BootstrapToken != nil {
+		ret.BootstrapToken = s.BootstrapToken.Allow
+	}
+
 	if s.ClientCert != nil {
 		ret.ClientCAFile = s.ClientCert.ClientCA
 	}

plugin/pkg/auth/BUILD

@@ -24,6 +24,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//plugin/pkg/auth/authenticator/token/bootstrap:all-srcs",
         "//plugin/pkg/auth/authorizer:all-srcs",
     ],
     tags = ["automanaged"],

plugin/pkg/auth/authenticator/token/bootstrap/BUILD

@@ -0,0 +1,52 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["bootstrap_test.go"],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/bootstrap/api:go_default_library",
+        "//vendor:k8s.io/apimachinery/pkg/api/errors",
+        "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
+        "//vendor:k8s.io/apimachinery/pkg/labels",
+        "//vendor:k8s.io/apimachinery/pkg/runtime/schema",
+        "//vendor:k8s.io/apiserver/pkg/authentication/user",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["bootstrap.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/bootstrap/api:go_default_library",
+        "//pkg/client/listers/core/internalversion:go_default_library",
+        "//vendor:github.com/golang/glog",
+        "//vendor:k8s.io/apimachinery/pkg/api/errors",
+        "//vendor:k8s.io/apiserver/pkg/authentication/user",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)