Verify CA keypair IDs for kops-controller-issued certs

cmd/kops-controller/pkg/server/keystore.go

@@ -22,6 +22,7 @@ import (
 	"path"
 
 	"k8s.io/kops/pkg/pki"
+	"sigs.k8s.io/yaml"
 )
 
 type keystore struct {
@@ -43,27 +44,27 @@ func (k keystore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.Privat
 	return entry.certificate, entry.key, nil
 }
 
-func newKeystore(basePath string, cas []string) (pki.Keystore, error) {
+func newKeystore(basePath string, cas []string) (pki.Keystore, map[string]string, error) {
 	keystore := &keystore{
 		keys: map[string]keystoreEntry{},
 	}
 	for _, name := range cas {
 		certBytes, err := ioutil.ReadFile(path.Join(basePath, name+".crt"))
 		if err != nil {
-			return nil, fmt.Errorf("reading %q certificate: %v", name, err)
+			return nil, nil, fmt.Errorf("reading %q certificate: %v", name, err)
 		}
 		certificate, err := pki.ParsePEMCertificate(certBytes)
 		if err != nil {
-			return nil, fmt.Errorf("parsing %q certificate: %v", name, err)
+			return nil, nil, fmt.Errorf("parsing %q certificate: %v", name, err)
 		}
 
 		keyBytes, err := ioutil.ReadFile(path.Join(basePath, name+".key"))
 		if err != nil {
-			return nil, fmt.Errorf("reading %q key: %v", name, err)
+			return nil, nil, fmt.Errorf("reading %q key: %v", name, err)
 		}
 		key, err := pki.ParsePEMPrivateKey(keyBytes)
 		if err != nil {
-			return nil, fmt.Errorf("parsing %q key: %v", name, err)
+			return nil, nil, fmt.Errorf("parsing %q key: %v", name, err)
 		}
 
 		keystore.keys[name] = keystoreEntry{
@@ -72,5 +73,15 @@ func newKeystore(basePath string, cas []string) (pki.Keystore, error) {
 		}
 	}
 
-	return keystore, nil
+	var keypairIDs map[string]string
+	keypairIDsBytes, err := ioutil.ReadFile(path.Join(basePath, "keypair-ids.yaml"))
+	if err != nil {
+		return nil, nil, fmt.Errorf("reading keypair-ids.yaml")
+	}
+	err = yaml.Unmarshal(keypairIDsBytes, &keypairIDs)
+	if err != nil {
+		return nil, nil, fmt.Errorf("parsing keypair-ids.yaml")
+	}
+
+	return keystore, keypairIDs, nil
 }

cmd/kops-controller/pkg/server/server.go

@@ -40,11 +40,12 @@ import (
 )
 
 type Server struct {
-	opt       *config.Options
-	certNames sets.String
-	server    *http.Server
-	verifier  fi.Verifier
-	keystore  pki.Keystore
+	opt        *config.Options
+	certNames  sets.String
+	keypairIDs map[string]string
+	server     *http.Server
+	verifier   fi.Verifier
+	keystore   pki.Keystore
 
 	// configBase is the base of the configuration storage.
 	configBase vfs.Path
@@ -81,7 +82,7 @@ func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) {
 
 func (s *Server) Start() error {
 	var err error
-	s.keystore, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs)
+	s.keystore, s.keypairIDs, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs)
 	if err != nil {
 		return err
 	}
@@ -152,7 +153,7 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) {
 	validHours := (455 * 24) + (hash.Sum32() % (30 * 24))
 
 	for name, pubKey := range req.Certs {
-		cert, err := s.issueCert(name, pubKey, id, validHours)
+		cert, err := s.issueCert(name, pubKey, id, validHours, req.KeypairIDs)
 		if err != nil {
 			klog.Infof("bootstrap %s cert %q issue err: %v", r.RemoteAddr, name, err)
 			w.WriteHeader(http.StatusBadRequest)
@@ -167,7 +168,7 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) {
 	klog.Infof("bootstrap %s %s success", r.RemoteAddr, id.NodeName)
 }
 
-func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32) (string, error) {
+func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32, keypairIDs map[string]string) (string, error) {
 	block, _ := pem.Decode([]byte(pubKey))
 	if block.Type != "RSA PUBLIC KEY" {
 		return "", fmt.Errorf("unexpected key type %q", block.Type)
@@ -216,6 +217,13 @@ func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, vali
 		return "", fmt.Errorf("unexpected key name")
 	}
 
+	// This field was added to the protocol in kOps 1.22.
+	if len(keypairIDs) > 0 {
+		if keypairIDs[issueReq.Signer] != s.keypairIDs[issueReq.Signer] {
+			return "", fmt.Errorf("request's keypair ID %q for %s didn't match server's %q", keypairIDs[issueReq.Signer], issueReq.Signer, s.keypairIDs[issueReq.Signer])
+		}
+	}
+
 	cert, _, _, err := pki.IssueCert(issueReq, s.keystore)
 	if err != nil {
 		return "", fmt.Errorf("issuing certificate: %v", err)

nodeup/pkg/model/bootstrap_client.go

@@ -64,8 +64,9 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	bootstrapClientTask := &nodetasks.BootstrapClientTask{
-		Client: bootstrapClient,
-		Certs:  b.bootstrapCerts,
+		Client:     bootstrapClient,
+		Certs:      b.bootstrapCerts,
+		KeypairIDs: b.bootstrapKeypairIDs,
 	}
 
 	for _, cert := range b.bootstrapCerts {

nodeup/pkg/model/context.go

@@ -65,8 +65,9 @@ type NodeupModelContext struct {
 	// HasAPIServer is true if the InstanceGroup has a role of master or apiserver (pupulated by Init)
 	HasAPIServer bool
 
-	kubernetesVersion semver.Version
-	bootstrapCerts    map[string]*nodetasks.BootstrapCert
+	kubernetesVersion   semver.Version
+	bootstrapCerts      map[string]*nodetasks.BootstrapCert
+	bootstrapKeypairIDs map[string]string
 
 	// ConfigurationMode determines if we are prewarming an instance or running it live
 	ConfigurationMode string
@@ -81,6 +82,7 @@ func (c *NodeupModelContext) Init() error {
 	}
 	c.kubernetesVersion = *k8sVersion
 	c.bootstrapCerts = map[string]*nodetasks.BootstrapCert{}
+	c.bootstrapKeypairIDs = map[string]string{}
 
 	role := c.BootConfig.InstanceGroupRole
 
@@ -240,7 +242,7 @@ func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetask
 }
 
 // GetBootstrapCert requests a certificate keypair from kops-controller.
-func (c *NodeupModelContext) GetBootstrapCert(name string) (cert, key fi.Resource) {
+func (c *NodeupModelContext) GetBootstrapCert(name string, signer string) (cert, key fi.Resource, err error) {
 	if c.IsMaster {
 		panic("control plane nodes can't get certs from kops-controller")
 	}
@@ -252,13 +254,20 @@ func (c *NodeupModelContext) GetBootstrapCert(name string) (cert, key fi.Resourc
 		}
 		c.bootstrapCerts[name] = b
 	}
-	return b.Cert, b.Key
+	c.bootstrapKeypairIDs[signer] = c.NodeupConfig.KeypairIDs[signer]
+	if c.bootstrapKeypairIDs[signer] == "" {
+		return nil, nil, fmt.Errorf("no keypairID for %q", signer)
+	}
+	return b.Cert, b.Key, nil
 }
 
 // BuildBootstrapKubeconfig generates a kubeconfig with a client certificate from either kops-controller or the state store.
 func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.ModelBuilderContext) (fi.Resource, error) {
 	if c.UseKopsControllerForNodeBootstrap() {
-		cert, key := c.GetBootstrapCert(name)
+		cert, key, err := c.GetBootstrapCert(name, fi.CertificateIDCA)
+		if err != nil {
+			return nil, err
+		}
 
 		kubeConfig := &nodetasks.KubeConfig{
 			Name: name,
@@ -273,7 +282,7 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model
 			kubeConfig.ServerURL = "https://" + c.Cluster.Spec.MasterInternalName
 		}
 
-		err := ctx.EnsureTask(kubeConfig)
+		err = ctx.EnsureTask(kubeConfig)
 		if err != nil {
 			return nil, err
 		}

nodeup/pkg/model/kops_controller.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/kops/pkg/wellknownusers"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+	"sigs.k8s.io/yaml"
 )
 
 // KopsControllerBuilder installs the keys for a kops-controller.
@@ -91,5 +92,17 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 	}
 
+	keypairIDs, err := yaml.Marshal(b.NodeupConfig.KeypairIDs)
+	if err != nil {
+		return err
+	}
+	c.AddTask(&nodetasks.File{
+		Path:     filepath.Join(pkiDir, "keypair-ids.yaml"),
+		Contents: fi.NewBytesResource(keypairIDs),
+		Type:     nodetasks.FileType_File,
+		Mode:     s("0600"),
+		Owner:    s(wellknownusers.KopsControllerName),
+	})
+
 	return nil
 }

nodeup/pkg/model/kubelet.go

@@ -559,7 +559,10 @@ func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.ModelBuilderContex
 		}
 
 		if !b.HasAPIServer {
-			cert, key := b.GetBootstrapCert(name)
+			cert, key, err := b.GetBootstrapCert(name, fi.CertificateIDCA)
+			if err != nil {
+				return err
+			}
 
 			c.AddTask(&nodetasks.File{
 				Path:           filepath.Join(dir, name+".crt"),

nodeup/pkg/model/networking/cilium.go

@@ -123,7 +123,10 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error
 		return issueCert.AddFileTasks(c, dir, name, "", nil)
 	} else {
 		if b.UseKopsControllerForNodeBootstrap() {
-			cert, key := b.GetBootstrapCert(name)
+			cert, key, err := b.GetBootstrapCert(name, signer)
+			if err != nil {
+				return err
+			}
 
 			c.AddTask(&nodetasks.File{
 				Path:           filepath.Join(dir, name+".crt"),

nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml

@@ -2,6 +2,14 @@ mode: "0755"
 path: /etc/kubernetes/kops-controller
 type: directory
 ---
+contents: |
+  kubernetes-ca: "3"
+  service-account: "2"
+mode: "0600"
+owner: kops-controller
+path: /etc/kubernetes/kops-controller/keypair-ids.yaml
+type: file
+---
 contents:
   task:
     Name: kops-controller

pkg/apis/nodeup/bootstrap.go

@@ -24,6 +24,8 @@ type BootstrapRequest struct {
 	APIVersion string `json:"apiVersion"`
 	// Certs are the requested certificates and their respective public keys.
 	Certs map[string]string `json:"certs"`
+	// KeypairIDs are the keypair IDs of the CAs to use for issuing certificates.
+	KeypairIDs map[string]string `json:"keypairIDs"`
 
 	// IncludeNodeConfig controls whether the cluster & instance group configuration should be returned.
 	// This allows for nodes without access to the kops state store.

tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml

@@ -163,7 +163,7 @@ Resources.AWSEC2LaunchTemplateapiserverapiserversminimalexamplecom.Properties.La
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: apiserver
   InstanceGroupRole: APIServer
-  NodeupConfigHash: m42Jgc8PoZhVAC89vuntMvXrH++DSM8mwoTImuyRjzc=
+  NodeupConfigHash: iWrD5bIyJd86oWdSRDFwjcezKpalA0SeCY3CTvmTQjM=
 
   __EOF_KUBE_ENV
 
@@ -587,7 +587,7 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: nodes
   InstanceGroupRole: Node
-  NodeupConfigHash: tkKsjnWeOxwXl+xmlcu9fg6LL9i/1GpYYZbbNEUSOgk=
+  NodeupConfigHash: A0AyiJo03pbqluaXrVtbacjofP1NmBexAl0w2y4oS5o=
 
   __EOF_KUBE_ENV
 

tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_nodes.minimal.example.com_user_data

@@ -162,7 +162,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: tkKsjnWeOxwXl+xmlcu9fg6LL9i/1GpYYZbbNEUSOgk=
+NodeupConfigHash: A0AyiJo03pbqluaXrVtbacjofP1NmBexAl0w2y4oS5o=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_bucket_object_nodeupconfig-nodes_content

@@ -35,7 +35,8 @@ ClusterName: minimal.example.com
 Hooks:
 - null
 - null
-KeypairIDs: {}
+KeypairIDs:
+  kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd

tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data

@@ -171,7 +171,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/bastionuserdata.example.com
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: v3vAGhTxpuP2+WEcyx3spjODFbRipxYbiD0PoMfW/1c=
+NodeupConfigHash: XE9IpKZd9oCrCv/x+HIIxysnlj82y2WpjG3iuknb0Mk=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_bucket_object_nodeupconfig-bastion_content

@@ -35,7 +35,8 @@ ClusterName: bastionuserdata.example.com
 Hooks:
 - null
 - null
-KeypairIDs: {}
+KeypairIDs:
+  kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_bucket_object_nodeupconfig-nodes_content

@@ -35,7 +35,8 @@ ClusterName: bastionuserdata.example.com
 Hooks:
 - null
 - null
-KeypairIDs: {}
+KeypairIDs:
+  kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd

tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml

@@ -451,7 +451,7 @@ Resources.AWSEC2LaunchTemplatenodescomplexexamplecom.Properties.LaunchTemplateDa
   ConfigBase: memfs://clusters.example.com/complex.example.com
   InstanceGroupName: nodes
   InstanceGroupRole: Node
-  NodeupConfigHash: W/7jMLzECfR0XTWLf7Hg6HMBQwKn1xRYsr0w3UsEUoY=
+  NodeupConfigHash: YhMDAtYXeVayMfXL+lAMRKJ5B1N2nmXjIEp3zx2O3SY=
 
   __EOF_KUBE_ENV
 

tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data

@@ -171,7 +171,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/complex.example.com
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: W/7jMLzECfR0XTWLf7Hg6HMBQwKn1xRYsr0w3UsEUoY=
+NodeupConfigHash: YhMDAtYXeVayMfXL+lAMRKJ5B1N2nmXjIEp3zx2O3SY=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/complex/data/aws_s3_bucket_object_nodeupconfig-nodes_content

@@ -35,7 +35,8 @@ ClusterName: complex.example.com
 Hooks:
 - null
 - null
-KeypairIDs: {}
+KeypairIDs:
+  kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd

tests/integration/update_cluster/compress/data/aws_launch_template_nodes.compress.example.com_user_data

@@ -133,7 +133,7 @@ ensure-install-dir
 
 echo "H4sIAAAAAAAA/3yS0W7bPAyF7/0UvCl601+2i/7BJvRiW7KhAZYuSJ6AsRhXiEymouQsbz/YTZplA3Zl+Bzah/zIJkh2U+Gtb20BgAf9+mU9Xc9n0fcUBwmAGDeBnIUtBqUCoEPGltZJIrY0DahKaiHFTEUjnNAzxVXm5Duy8K64i+mGHwdpv1NPwYLnrRQAPUX1whZq82AmhZNm99aC7vx+zpowhFPMLm9oGeXncbCbkDVRnM5nKwt1VZmPE1OZqqzrwdznFb1m0jR6XQHwIpoYO/rRU4zekYXbT3jQ2wLAd9iShd0HNW0TjZdySPpvP0b1tbmvTXXV+v3YSqA00mPhYydZP+f0csHVtFHy/kQU9KiJOveur0SShXKcQ7JbRum9GwrxoJfhZs/rt9kmD6YydfWbIx16tudXE6TBUJy3NqNNblvP7ROyCxTPewKg3jfJCz9hdBY66iQeDfbow/DdY11VC3/H4mirV/LNWfTDU79Fosf/b+5Gcn+VntWr2n+vYODZjAe5xIFi2WMsg9+UJ9DlpeCPTQAwpYPE3TLk1vMzjufHfjCEF6ivmSI6uhzKCLMqR5x7cQtkvyVNp2BKzRgWmRJp2Z1cLX4BAAD//wEAAP//GWuHvzUDAAA=" | base64 -d | gzip -d > conf/cluster_spec.yaml
 
-echo "H4sIAAAAAAAA/1SNwU6EMBQA7/2KfgEoJkiaeBCMiiZI1JC9Nu1jYWn7mj5K+fzNZk9c5jBzmMZg1H3AbdYQBJeJWINunM+1JBDcgh1J5LkykVYIlMEurTeQKbS5QusD0EGy1tEqnYKPgNF30oLgDjXQMfyiAcE71MBuiP4+/ZQ0CV49PG3FV5re96TK6vsU3Pi8LPXj5W94bfe3/7Ie+mJb3fKTXtgVAAD//wEAAP//5xLuGcEAAAA=" | base64 -d | gzip -d > conf/kube_env.yaml
+echo "H4sIAAAAAAAA/1SNwUrEMBQA7/mK3IUGxSIGvFiktWKVghWPIXmt3SZ52bykW/brl2VPvcxh5jCVxWy+I66zgSi5OhGr0I/z9KoIJHfgRpJCaJspQaQCNuWChUKjExpdiEA7yd49JeU11BFz6JQDyT0aoH3o0YLkHRpgV+RwmzaK/iV/m5qtPD/V83N32I5N36ZyEHf+cfn6yOn351Mv93lYH3Ir/l7YBQAA//8BAAD//w/PoYLBAAAA" | base64 -d | gzip -d > conf/kube_env.yaml
 
 download-release
 echo "== nodeup node config done =="

tests/integration/update_cluster/compress/data/aws_s3_bucket_object_nodeupconfig-nodes_content

@@ -35,7 +35,8 @@ ClusterName: compress.example.com
 Hooks:
 - null
 - null
-KeypairIDs: {}
+KeypairIDs:
+  kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd

tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml

@@ -432,7 +432,7 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat
   ConfigBase: memfs://clusters.example.com/containerd.example.com
   InstanceGroupName: nodes
   InstanceGroupRole: Node
-  NodeupConfigHash: mQ3AvnQERQ6PiYxBIkJ2VkeEAL1WvRLVxQeb5hi5WBc=
+  NodeupConfigHash: ZYCe88bkLCotK0J/VSldr8XpzuA7qmdkQKAw8+Lxqt0=
 
   __EOF_KUBE_ENV