Skip to content

Commit

Permalink
use service accounts as clients for controllers
Browse files Browse the repository at this point in the history
  • Loading branch information
deads2k committed Sep 22, 2016
1 parent 4ab5a76 commit 1f0f79e
Show file tree
Hide file tree
Showing 2 changed files with 204 additions and 34 deletions.
156 changes: 156 additions & 0 deletions cmd/kube-controller-manager/app/client_builder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app implements a server that runs a set of active
// components. This includes replication controllers, service endpoints and
// nodes.
//
// CAUTION: If you update code in this file, you may need to also update code
// in contrib/mesos/pkg/controllermanager/controllermanager.go
package app

import (
"fmt"
"time"

"k8s.io/kubernetes/pkg/api"
apierrors "k8s.io/kubernetes/pkg/api/errors"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned"
"k8s.io/kubernetes/pkg/client/restclient"
"k8s.io/kubernetes/pkg/fields"
"k8s.io/kubernetes/pkg/watch"

"github.com/golang/glog"
)

// ControllerClientBuilder allow syou to get clients and configs for controllers
type ControllerClientBuilder interface {
Config(name string) (*restclient.Config, error)
Client(name string) (clientset.Interface, error)
ClientOrDie(name string) clientset.Interface
}

// SimpleControllerClientBuilder returns a fixed client with different user agents
type SimpleControllerClientBuilder struct {
// ClientConfig is a skeleton config to clone and use as the basis for each controller client
ClientConfig *restclient.Config
}

func (b SimpleControllerClientBuilder) Config(name string) (*restclient.Config, error) {
clientConfig := *b.ClientConfig
return &clientConfig, nil
}

func (b SimpleControllerClientBuilder) Client(name string) (clientset.Interface, error) {
clientConfig, err := b.Config(name)
if err != nil {
return nil, err
}
return clientset.NewForConfig(restclient.AddUserAgent(clientConfig, name))
}

func (b SimpleControllerClientBuilder) ClientOrDie(name string) clientset.Interface {
client, err := b.Client(name)
if err != nil {
glog.Fatal(err)
}
return client
}

// SAControllerClientBuilder is a ControllerClientBuilder that returns clients identifying as
// service accounts
type SAControllerClientBuilder struct {
// ClientConfig is a skeleton config to clone and use as the basis for each controller client
ClientConfig *restclient.Config

// CoreClient is used to provision service accounts if needed and watch for their associated tokens
// to construct a controller client
CoreClient unversionedcore.CoreInterface

// Namespace is the namespace used to host the service accounts that will back the
// controllers. It must be highly privileged namespace which normal users cannot inspect.
Namespace string
}

// config returns a complete clientConfig for constructing clients. This is separate in anticipation of composition
// which means that not all clientsets are known here
func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, error) {
clientConfig := *b.ClientConfig

// we need the SA UID to find a matching SA token
sa, err := b.CoreClient.ServiceAccounts(b.Namespace).Get(name)
if err != nil && !apierrors.IsNotFound(err) {
return nil, err
} else if apierrors.IsNotFound(err) {
sa, err = b.CoreClient.ServiceAccounts(b.Namespace).Create(
&api.ServiceAccount{ObjectMeta: api.ObjectMeta{Namespace: b.Namespace, Name: name}})
if err != nil {
return nil, err
}
}

watcher, err := b.CoreClient.Secrets(b.Namespace).Watch(
api.ListOptions{
ResourceVersion: "0",
FieldSelector: fields.SelectorFromSet(map[string]string{api.SecretTypeField: string(api.SecretTypeServiceAccountToken)}),
})
if err != nil {
return nil, err
}
_, err = watch.Until(30*time.Second, watcher,
func(event watch.Event) (bool, error) {
switch event.Type {
case watch.Deleted:
return false, nil
case watch.Error:
return false, fmt.Errorf("error watching")
case watch.Added, watch.Modified:
secret := event.Object.(*api.Secret)
if secret.Annotations[api.ServiceAccountNameKey] != sa.Name ||
secret.Annotations[api.ServiceAccountUIDKey] != string(sa.UID) ||
len(secret.Data[api.ServiceAccountTokenKey]) == 0 {
return false, nil
}
clientConfig.BearerToken = string(secret.Data[api.ServiceAccountTokenKey])
return true, nil

default:
return false, fmt.Errorf("unexpected event type: %v", event.Type)
}
})
if err != nil {
return nil, fmt.Errorf("unable to get token for service account: %v", err)
}

return &clientConfig, nil
}

func (b SAControllerClientBuilder) Client(name string) (clientset.Interface, error) {
clientConfig, err := b.Config(name)
if err != nil {
return nil, err
}
return clientset.NewForConfig(clientConfig)
}

func (b SAControllerClientBuilder) ClientOrDie(name string) clientset.Interface {
client, err := b.Client(name)
if err != nil {
glog.Fatal(err)
}
return client
}
82 changes: 48 additions & 34 deletions cmd/kube-controller-manager/app/controllermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,21 @@ func Run(s *options.CMServer) error {
recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "controller-manager"})

run := func(stop <-chan struct{}) {
err := StartControllers(s, kubeconfig, stop, recorder)
rootClientBuilder := SimpleControllerClientBuilder{
ClientConfig: kubeconfig,
}
var clientBuilder ControllerClientBuilder
if len(s.ServiceAccountKeyFile) > 0 {
clientBuilder = SAControllerClientBuilder{
ClientConfig: kubeconfig,
CoreClient: kubeClient.Core(),
Namespace: "kube-system",
}
} else {
clientBuilder = rootClientBuilder
}

err := StartControllers(s, kubeconfig, rootClientBuilder, clientBuilder, stop, recorder)
glog.Fatalf("error running controllers: %v", err)
panic("unreachable")
}
Expand Down Expand Up @@ -198,20 +212,50 @@ func Run(s *options.CMServer) error {
panic("unreachable")
}

func StartControllers(s *options.CMServer, kubeconfig *restclient.Config, stop <-chan struct{}, recorder record.EventRecorder) error {
func StartControllers(s *options.CMServer, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder) error {
client := func(userAgent string) clientset.Interface {
return clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, userAgent))
return rootClientBuilder.ClientOrDie(userAgent)
}
discoveryClient := client("controller-discovery").Discovery()
sharedInformers := informers.NewSharedInformerFactory(client("shared-informers"), ResyncPeriod(s)())

// always start the SA token controller first using a full-power client, since it needs to mint tokens for the rest
if len(s.ServiceAccountKeyFile) > 0 {
privateKey, err := serviceaccount.ReadPrivateKey(s.ServiceAccountKeyFile)
if err != nil {
return fmt.Errorf("Error reading key for service account token controller: %v", err)
} else {
var rootCA []byte
if s.RootCAFile != "" {
rootCA, err = ioutil.ReadFile(s.RootCAFile)
if err != nil {
return fmt.Errorf("error reading root-ca-file at %s: %v", s.RootCAFile, err)
}
if _, err := certutil.ParseCertsPEM(rootCA); err != nil {
return fmt.Errorf("error parsing root-ca-file at %s: %v", s.RootCAFile, err)
}
} else {
rootCA = kubeconfig.CAData
}

go serviceaccountcontroller.NewTokensController(
rootClientBuilder.ClientOrDie("tokens-controller"),
serviceaccountcontroller.TokensControllerOptions{
TokenGenerator: serviceaccount.JWTTokenGenerator(privateKey),
RootCA: rootCA,
},
).Run(int(s.ConcurrentSATokenSyncs), wait.NeverStop)
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
}
}

go endpointcontroller.NewEndpointController(sharedInformers.Pods().Informer(), client("endpoint-controller")).
Run(int(s.ConcurrentEndpointSyncs), wait.NeverStop)
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))

go replicationcontroller.NewReplicationManager(
sharedInformers.Pods().Informer(),
client("replication-controller"),
clientBuilder.ClientOrDie("replication-controller"),
ResyncPeriod(s),
replicationcontroller.BurstReplicas,
int(s.LookupCacheSizeForRC),
Expand Down Expand Up @@ -477,36 +521,6 @@ func StartControllers(s *options.CMServer, kubeconfig *restclient.Config, stop <
}
}

var rootCA []byte

if s.RootCAFile != "" {
rootCA, err = ioutil.ReadFile(s.RootCAFile)
if err != nil {
return fmt.Errorf("error reading root-ca-file at %s: %v", s.RootCAFile, err)
}
if _, err := certutil.ParseCertsPEM(rootCA); err != nil {
return fmt.Errorf("error parsing root-ca-file at %s: %v", s.RootCAFile, err)
}
} else {
rootCA = kubeconfig.CAData
}

if len(s.ServiceAccountKeyFile) > 0 {
privateKey, err := serviceaccount.ReadPrivateKey(s.ServiceAccountKeyFile)
if err != nil {
glog.Errorf("Error reading key for service account token controller: %v", err)
} else {
go serviceaccountcontroller.NewTokensController(
client("tokens-controller"),
serviceaccountcontroller.TokensControllerOptions{
TokenGenerator: serviceaccount.JWTTokenGenerator(privateKey),
RootCA: rootCA,
},
).Run(int(s.ConcurrentSATokenSyncs), wait.NeverStop)
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
}
}

serviceaccountcontroller.NewServiceAccountsController(
client("service-account-controller"),
serviceaccountcontroller.DefaultServiceAccountsControllerOptions(),
Expand Down

0 comments on commit 1f0f79e

Please sign in to comment.