From 3ae5413f6325a2ef3d8f3cb2cb7c0ee6be3ddfd9 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 15 Jul 2021 14:04:48 -0700 Subject: [PATCH 1/2] Use keypair IDs for non-kops-controller-issued worker node certs --- nodeup/pkg/model/context.go | 42 +++++++++++----------- nodeup/pkg/model/networking/kube_router.go | 2 +- upup/pkg/fi/cloudup/apply_cluster.go | 6 ++++ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 6f3c84f64cbab..bba0105043392 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -289,7 +289,26 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model return kubeConfig.GetConfig(), nil } else { - cert, key, err := c.GetPrimaryKeypair(name) + keyset, err := c.KeyStore.FindKeyset(name) + if err != nil { + return nil, fmt.Errorf("error fetching keyset: %v from keystore: %v", name, err) + } + + keypairID := c.NodeupConfig.KeypairIDs[name] + if keypairID == "" { + return nil, fmt.Errorf("keypairID for %s missing from NodeupConfig", name) + } + item := keyset.Items[keypairID] + if item == nil { + return nil, fmt.Errorf("keypairID %s missing from %s keyset", keypairID, name) + } + + cert, err := item.Certificate.AsBytes() + if err != nil { + return nil, err + } + + key, err := item.PrivateKey.AsBytes() if err != nil { return nil, err } @@ -632,27 +651,6 @@ func EvaluateHostnameOverride(hostnameOverride string) (string, error) { return *(result.Reservations[0].Instances[0].PrivateDnsName), nil } -// GetPrimaryKeypair is a helper method to retrieve a primary keypair from the store. -// TODO: Use the KeysetID in NodeupConfig instead of the Primary keypair. -func (c *NodeupModelContext) GetPrimaryKeypair(name string) (cert []byte, key []byte, err error) { - keyset, err := c.KeyStore.FindKeyset(name) - if err != nil { - return nil, nil, fmt.Errorf("error fetching keyset: %v from keystore: %v", name, err) - } - - cert, err = keyset.Primary.Certificate.AsBytes() - if err != nil { - return nil, nil, err - } - - key, err = keyset.Primary.PrivateKey.AsBytes() - if err != nil { - return nil, nil, err - } - - return cert, key, nil -} - func (b *NodeupModelContext) AddCNIBinAssets(c *fi.ModelBuilderContext, assetNames []string) error { for _, assetName := range assetNames { re, err := regexp.Compile(fmt.Sprintf("^%s$", regexp.QuoteMeta(assetName))) diff --git a/nodeup/pkg/model/networking/kube_router.go b/nodeup/pkg/model/networking/kube_router.go index c000df7092f04..3b8c8662cd72e 100644 --- a/nodeup/pkg/model/networking/kube_router.go +++ b/nodeup/pkg/model/networking/kube_router.go @@ -41,7 +41,7 @@ func (b *KuberouterBuilder) Build(c *fi.ModelBuilderContext) error { var kubeconfig fi.Resource var err error - if b.IsMaster { + if b.HasAPIServer { kubeconfig = b.BuildIssuedKubeconfig("kube-router", nodetasks.PKIXName{CommonName: rbac.KubeRouter}, c) } else { kubeconfig, err = b.BuildBootstrapKubeconfig("kube-router", c) diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 417e08f840957..9424a9102d880 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -1349,6 +1349,12 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit if err != nil { return nil, nil, fmt.Errorf("encoding service-account keys: %w", err) } + } else { + for _, key := range []string{"kubelet", "kube-proxy", "kube-router"} { + if caTasks[key] != nil { + config.KeypairIDs[key] = caTasks[key].Keyset().Primary.Id + } + } } if isMaster || useGossip { From 1d3bdf54af48cf47683a05468b928831f01aa301 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 15 Jul 2021 14:05:08 -0700 Subject: [PATCH 2/2] Update keypair rotation documentation --- docs/operations/rotate-secrets.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/operations/rotate-secrets.md b/docs/operations/rotate-secrets.md index 68e958c5714ff..c80ec317ff938 100644 --- a/docs/operations/rotate-secrets.md +++ b/docs/operations/rotate-secrets.md @@ -1,3 +1,4 @@ + # How to rotate all secrets / credentials There are two types of credentials managed by kOps: @@ -71,11 +72,15 @@ Promote the new keypairs to primary with: ```shell kops promote keypair all kops update cluster --yes -kops rolling-update cluster --force --yes +kops rolling-update cluster --yes ``` -As of the writing of this document, rolling-update will not necessarily identify all -relevant nodes as needing update, so should be invoked with the `--force` flag. +On cloud providers, such as AWS, that use kops-controller to bootstrap worker nodes, after +the `kops update cluster --yes` step there is a temporary impediment to node scale-up. +Instances using the new launch template will not be able to bootstrap off of old kops-controllers. +Similarly, instances using the old launch template and which have not yet bootstrapped will not +be able to bootstrap off of new kops-controllers. The subsequent rolling update will eventually +replace all instances using the old launch template. #### Rollback procedure @@ -89,7 +94,7 @@ To roll back this change: most likely by identifying the issue dates. * Then use `kops promote keypair` to promote each of them by keyset and ID. * Then use `kops update cluster --yes` -* Then use `kops rolling-update cluster --force --yes` +* Then use `kops rolling-update cluster --yes` ### Export and distribute new kubeconfig admin credentials