Skip to content

Commit

Permalink
Add dynamic reload support for authentication configuration
Browse files Browse the repository at this point in the history 
Signed-off-by: Monis Khan <mok@microsoft.com>
  • Loading branch information
@enj
enj committed Mar 9, 2024
1 parent 28c4d00 commit b4935d9
Show file tree
Hide file tree
Showing 9 changed files with 855 additions and 105 deletions.
6 changes: 4 additions & 2 deletions pkg/controlplane/apiserver/config.go
View file
Expand Up @@ -144,14 +144,16 @@ func BuildGenericConfig(
return
}

ctx := wait.ContextForChannel(genericConfig.DrainedNotify())

// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers); lastErr != nil {
if lastErr = s.Authentication.ApplyTo(ctx, &genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers); lastErr != nil {
return
}

var enablesRBAC bool
genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, enablesRBAC, err = BuildAuthorizer(
wait.ContextForChannel(genericConfig.ShutdownInitiatedNotify()),
ctx,
s,
genericConfig.EgressSelector,
genericConfig.APIServerID,
Expand Down
148 changes: 121 additions & 27 deletions pkg/kubeapiserver/authenticator/config.go
View file
Expand Up @@ -17,10 +17,13 @@ limitations under the License.
package authenticator

import (
"context"
"errors"
"fmt"
"sync/atomic"
"time"

utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/apis/apiserver"
Expand Down Expand Up @@ -57,6 +60,7 @@ type Config struct {

TokenAuthFile string
AuthenticationConfig *apiserver.AuthenticationConfiguration
AuthenticationConfigData string
OIDCSigningAlgs []string
ServiceAccountKeyFiles []string
ServiceAccountLookup bool
Expand Down Expand Up @@ -90,7 +94,7 @@ type Config struct {

// New returns an authenticator.Request or an error that supports the standard
// Kubernetes authentication mechanisms.
func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, spec3.SecuritySchemes, error) {
func (config Config) New(serverLifecycle context.Context) (authenticator.Request, func(context.Context, *apiserver.AuthenticationConfiguration) error, *spec.SecurityDefinitions, spec3.SecuritySchemes, error) {
var authenticators []authenticator.Request
var tokenAuthenticators []authenticator.Token
securityDefinitionsV2 := spec.SecurityDefinitions{}
Expand Down Expand Up @@ -119,21 +123,21 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp
if len(config.TokenAuthFile) > 0 {
tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
if err != nil {
return nil, nil, nil, err
return nil, nil, nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
}
if len(config.ServiceAccountKeyFiles) > 0 {
serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter, config.SecretsWriter)
if err != nil {
return nil, nil, nil, err
return nil, nil, nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
}
if len(config.ServiceAccountIssuers) > 0 {
serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuers, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter)
if err != nil {
return nil, nil, nil, err
return nil, nil, nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
}
Expand All @@ -148,33 +152,33 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp
// cache misses for all requests using the other. While the service account plugin
// simply returns an error, the OpenID Connect plugin may query the provider to
// update the keys, causing performance hits.
var updateAuthenticationConfig func(context.Context, *apiserver.AuthenticationConfiguration) error
if config.AuthenticationConfig != nil {
for _, jwtAuthenticator := range config.AuthenticationConfig.JWT {
var oidcCAContent oidc.CAContentProvider
if len(jwtAuthenticator.Issuer.CertificateAuthority) > 0 {
var oidcCAError error
oidcCAContent, oidcCAError = dynamiccertificates.NewStaticCAContent("oidc-authenticator", []byte(jwtAuthenticator.Issuer.CertificateAuthority))
if oidcCAError != nil {
return nil, nil, nil, oidcCAError
}
}
oidcAuth, err := oidc.New(oidc.Options{
JWTAuthenticator: jwtAuthenticator,
CAContentProvider: oidcCAContent,
SupportedSigningAlgs: config.OIDCSigningAlgs,
DisallowedIssuers: config.ServiceAccountIssuers,
})
if err != nil {
return nil, nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, oidcAuth))
initialJWTAuthenticator, err := newJWTAuthenticator(serverLifecycle, config.AuthenticationConfig, config.OIDCSigningAlgs, config.APIAudiences, config.ServiceAccountIssuers)
if err != nil {
return nil, nil, nil, nil, err
}

jwtAuthenticatorPtr := &atomic.Pointer[jwtAuthenticatorWithCancel]{}
jwtAuthenticatorPtr.Store(initialJWTAuthenticator)

updateAuthenticationConfig = (&authenticationConfigUpdater{
serverLifecycle: serverLifecycle,
config: config,
jwtAuthenticatorPtr: jwtAuthenticatorPtr,
}).updateAuthenticationConfig

tokenAuthenticators = append(tokenAuthenticators,
authenticator.TokenFunc(func(ctx context.Context, token string) (*authenticator.Response, bool, error) {
return jwtAuthenticatorPtr.Load().jwtAuthenticator.AuthenticateToken(ctx, token)
}),
)
}

if len(config.WebhookTokenAuthnConfigFile) > 0 {
webhookTokenAuth, err := newWebhookTokenAuthenticator(config)
if err != nil {
return nil, nil, nil, err
return nil, nil, nil, nil, err
}

tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)
Expand Down Expand Up @@ -209,9 +213,9 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp

if len(authenticators) == 0 {
if config.Anonymous {
return anonymous.NewAuthenticator(), &securityDefinitionsV2, securitySchemesV3, nil
return anonymous.NewAuthenticator(), nil, &securityDefinitionsV2, securitySchemesV3, nil
}
return nil, &securityDefinitionsV2, securitySchemesV3, nil
return nil, nil, &securityDefinitionsV2, securitySchemesV3, nil
}

authenticator := union.New(authenticators...)
Expand All @@ -224,7 +228,97 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp
authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
}

return authenticator, &securityDefinitionsV2, securitySchemesV3, nil
return authenticator, updateAuthenticationConfig, &securityDefinitionsV2, securitySchemesV3, nil
}

type jwtAuthenticatorWithCancel struct {
jwtAuthenticator authenticator.Token
healthCheck func() error
cancel func()
}

func newJWTAuthenticator(serverLifecycle context.Context, config *apiserver.AuthenticationConfiguration, oidcSigningAlgs []string, apiAudiences authenticator.Audiences, disallowedIssuers []string) (_ *jwtAuthenticatorWithCancel, buildErr error) {
ctx, cancel := context.WithCancel(serverLifecycle)

defer func() {
if buildErr != nil {
cancel()
}
}()
var jwtAuthenticators []authenticator.Token
var healthChecks []func() error
for _, jwtAuthenticator := range config.JWT {
// TODO remove this CAContentProvider indirection
var oidcCAContent oidc.CAContentProvider
if len(jwtAuthenticator.Issuer.CertificateAuthority) > 0 {
var oidcCAError error
oidcCAContent, oidcCAError = dynamiccertificates.NewStaticCAContent("oidc-authenticator", []byte(jwtAuthenticator.Issuer.CertificateAuthority))
if oidcCAError != nil {
return nil, oidcCAError
}
}
oidcAuth, err := oidc.New(ctx, oidc.Options{
JWTAuthenticator: jwtAuthenticator,
CAContentProvider: oidcCAContent,
SupportedSigningAlgs: oidcSigningAlgs,
DisallowedIssuers: disallowedIssuers,
})
if err != nil {
return nil, err
}
jwtAuthenticators = append(jwtAuthenticators, oidcAuth)
healthChecks = append(healthChecks, oidcAuth.HealthCheck)
}
return &jwtAuthenticatorWithCancel{
jwtAuthenticator: authenticator.WrapAudienceAgnosticToken(apiAudiences, tokenunion.NewFailOnError(jwtAuthenticators...)), // this handles the empty jwtAuthenticators slice case correctly
healthCheck: func() error {
var errs []error
for _, check := range healthChecks {
if err := check(); err != nil {
errs = append(errs, err)
}
}
return utilerrors.NewAggregate(errs)
},
cancel: cancel,
}, nil
}

type authenticationConfigUpdater struct {
serverLifecycle context.Context
config Config
jwtAuthenticatorPtr *atomic.Pointer[jwtAuthenticatorWithCancel]
}

// the input ctx controls the timeout for updateAuthenticationConfig to return, not the lifetime of the constructed authenticators.
func (c *authenticationConfigUpdater) updateAuthenticationConfig(ctx context.Context, authConfig *apiserver.AuthenticationConfiguration) error {
updatedJWTAuthenticator, err := newJWTAuthenticator(c.serverLifecycle, authConfig, c.config.OIDCSigningAlgs, c.config.APIAudiences, c.config.ServiceAccountIssuers)
if err != nil {
return err
}

var lastErr error
if waitErr := wait.PollUntilContextCancel(ctx, 10*time.Second, true, func(_ context.Context) (done bool, err error) {
lastErr = updatedJWTAuthenticator.healthCheck()
return lastErr == nil, nil
}); lastErr != nil || waitErr != nil {
updatedJWTAuthenticator.cancel()
return utilerrors.NewAggregate([]error{lastErr, waitErr}) // filters out nil errors
}

oldJWTAuthenticator := c.jwtAuthenticatorPtr.Swap(updatedJWTAuthenticator)
go func() {
t := time.NewTimer(time.Minute)
defer t.Stop()
select {
case <-c.serverLifecycle.Done():
case <-t.C:
}
// TODO maybe track requests so we know when this is safe to do
oldJWTAuthenticator.cancel()
}()

return nil
}

// IsValidServiceAccountKeyFile returns true if a valid public RSA key can be read from the given file
Expand Down

0 comments on commit b4935d9

Please sign in to comment.