Refactor apiserver-aggregator-ca

cmd/kops/create_keypair.go

@@ -79,6 +79,7 @@ type CreateKeypairOptions struct {
 }
 
 var rotatableKeysets = sets.NewString(
+	"apiserver-aggregator-ca",
 	"ca",
 	"etcd-clients-ca-cilium",
 	"service-account",

cmd/kops/integration_test.go

@@ -734,6 +734,12 @@ func (i *integrationTest) setupCluster(t *testing.T, inputYAML string, ctx conte
 		secondaryKey:         "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----\n",
 		secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26\nA6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG\nSIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq\n9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==\n-----END CERTIFICATE-----",
 	})
+	storeKeyset(t, keyStore, "apiserver-aggregator-ca", &testingKeyset{
+		primaryKey:           "-----BEGIN RSA PRIVATE KEY-----\nMIIBOwIBAAJBAMshO9QDlN4KOVxXoC0On4nSNC4YTMews6U84dsVinB1H2zSO4rY\nCbwv/hpchuVvgxeVe22tCCYkC7Bb3tKC3XsCAwEAAQJAe4xCLGjlQcvsKYsuZFlR\nle0hSawD/y0thuIp6SwH4O92AOsfrWDdiWIVCP6S47oBv351BOcoPbOjxfMTN+f6\naQIhAPIfBCHL/GecX1IVyitI1ueG1z0n5DDOKQAxmxTg82SnAiEA1sYK+vXMIV/e\nCl/CHxKwu7f+ufh1bV0OFyd+eI2+Vw0CICs6eG1kUzNYivhH5ammvp/lxkYn+ijw\nlgdv0+V9aFdfAiEAsTUytiK8zQTGthSQnQbU3+5OtK82ZIgVKjGh/mIlnLkCIQC1\neG3yBXM7/cxw1doWZ7AzMncufx9R8Q2Hblm80UrpaQ==\n-----END RSA PRIVATE KEY-----",
+		primaryCertificate:   "-----BEGIN CERTIFICATE-----\nMIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe\nBgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX\nDTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It\nY2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM\nx7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB\no0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU\nALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX\nWM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk\nCzMeMdr4\n-----END CERTIFICATE-----",
+		secondaryKey:         "-----BEGIN RSA PRIVATE KEY-----\nMIIBOwIBAAJBAMshO9QDlN4KOVxXoC0On4nSNC4YTMews6U84dsVinB1H2zSO4rY\nCbwv/hpchuVvgxeVe22tCCYkC7Bb3tKC3XsCAwEAAQJAe4xCLGjlQcvsKYsuZFlR\nle0hSawD/y0thuIp6SwH4O92AOsfrWDdiWIVCP6S47oBv351BOcoPbOjxfMTN+f6\naQIhAPIfBCHL/GecX1IVyitI1ueG1z0n5DDOKQAxmxTg82SnAiEA1sYK+vXMIV/e\nCl/CHxKwu7f+ufh1bV0OFyd+eI2+Vw0CICs6eG1kUzNYivhH5ammvp/lxkYn+ijw\nlgdv0+V9aFdfAiEAsTUytiK8zQTGthSQnQbU3+5OtK82ZIgVKjGh/mIlnLkCIQC1\neG3yBXM7/cxw1doWZ7AzMncufx9R8Q2Hblm80UrpaQ==\n-----END RSA PRIVATE KEY-----",
+		secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe\nBgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX\nDTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It\nY2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM\nx7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB\no0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU\nALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN\nQdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW\nHLtkTXH8\n-----END CERTIFICATE-----",
+	})
 	storeKeyset(t, keyStore, "etcd-clients-ca", &testingKeyset{
 		primaryKey:           "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----",
 		primaryCertificate:   "-----BEGIN CERTIFICATE-----\nMIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW\nBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx\nNjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB\nAQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep\nuPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE\nAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s\nx+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY\nqCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY=\n-----END CERTIFICATE-----",

nodeup/pkg/model/kube_apiserver.go

@@ -95,7 +95,7 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 		Mode:     s("0600"),
 	})
 
-	if b.UseEtcdManager() && b.UseEtcdTLS() {
+	if b.UseEtcdManager() {
 		c.AddTask(&nodetasks.File{
 			Path:     filepath.Join(pathSrvKAPI, "etcd-ca.crt"),
 			Contents: fi.NewStringResource(b.NodeupConfig.CAs["etcd-clients-ca"]),
@@ -124,6 +124,31 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 		kubeAPIServer.EtcdKeyFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client-key.pem")
 	}
 
+	{
+		c.AddTask(&nodetasks.File{
+			Path:     filepath.Join(pathSrvKAPI, "apiserver-aggregator-ca.crt"),
+			Contents: fi.NewStringResource(b.NodeupConfig.CAs["apiserver-aggregator-ca"]),
+			Type:     nodetasks.FileType_File,
+			Mode:     fi.String("0644"),
+		})
+		kubeAPIServer.RequestheaderClientCAFile = filepath.Join(pathSrvKAPI, "apiserver-aggregator-ca.crt")
+
+		issueCert := &nodetasks.IssueCert{
+			Name:   "apiserver-aggregator",
+			Signer: "apiserver-aggregator-ca",
+			Type:   "client",
+			// Must match RequestheaderAllowedNames
+			Subject: nodetasks.PKIXName{CommonName: "aggregator"},
+		}
+		c.AddTask(issueCert)
+		err := issueCert.AddFileTasks(c, pathSrvKAPI, "apiserver-aggregator", "", nil)
+		if err != nil {
+			return err
+		}
+		kubeAPIServer.ProxyClientCertFile = fi.String(filepath.Join(pathSrvKAPI, "apiserver-aggregator.crt"))
+		kubeAPIServer.ProxyClientKeyFile = fi.String(filepath.Join(pathSrvKAPI, "apiserver-aggregator.key"))
+	}
+
 	{
 		pod, err := b.buildPod(&kubeAPIServer)
 		if err != nil {
@@ -371,26 +396,6 @@ func (b *KubeAPIServerBuilder) buildPod(kubeAPIServer *kops.KubeAPIServerConfig)
 	kubeAPIServer.KubeletClientCertificate = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.crt")
 	kubeAPIServer.KubeletClientKey = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.key")
 
-	{
-		certPath := filepath.Join(b.PathSrvKubernetes(), "apiserver-aggregator.crt")
-		kubeAPIServer.ProxyClientCertFile = &certPath
-		keyPath := filepath.Join(b.PathSrvKubernetes(), "apiserver-aggregator.key")
-		kubeAPIServer.ProxyClientKeyFile = &keyPath
-	}
-
-	// APIServer aggregation options
-	{
-		cert, err := b.KeyStore.FindCert("apiserver-aggregator-ca")
-		if err != nil {
-			return nil, fmt.Errorf("apiserver aggregator CA cert lookup failed: %v", err.Error())
-		}
-
-		if cert != nil {
-			certPath := filepath.Join(b.PathSrvKubernetes(), "apiserver-aggregator-ca.crt")
-			kubeAPIServer.RequestheaderClientCAFile = certPath
-		}
-	}
-
 	// @fixup: the admission controller migrated from --admission-control to --enable-admission-plugins, but
 	// most people will still have c.Spec.KubeAPIServer.AdmissionControl references into their configuration we need
 	// to fix up. A PR https://github.com/kubernetes/kops/pull/5221/ introduced the issue and since the command line

nodeup/pkg/model/secrets.go

@@ -134,21 +134,6 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 	}
 
-	{
-		issueCert := &nodetasks.IssueCert{
-			Name:   "apiserver-aggregator",
-			Signer: "apiserver-aggregator-ca",
-			Type:   "client",
-			// Must match RequestheaderAllowedNames
-			Subject: nodetasks.PKIXName{CommonName: "aggregator"},
-		}
-		c.AddTask(issueCert)
-		err := issueCert.AddFileTasks(c, b.PathSrvKubernetes(), "apiserver-aggregator", "apiserver-aggregator-ca", nil)
-		if err != nil {
-			return err
-		}
-	}
-
 	if err := b.BuildPrivateKeyTask(c, "service-account", "service-account.key", nil); err != nil {
 		return err
 	}

nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml

@@ -54,10 +54,10 @@ contents: |
       - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
-      - --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.crt
-      - --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key
+      - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+      - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key
       - --requestheader-allowed-names=aggregator
-      - --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.crt
+      - --requestheader-client-ca-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
       - --requestheader-extra-headers-prefix=X-Remote-Extra-
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
@@ -232,6 +232,33 @@ type: directory
 ---
 contents: ""
 mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.key
+type: file
+---
+contents: ""
+mode: "0644"
 path: /srv/kubernetes/kube-apiserver/etcd-ca.crt
 type: file
 ---
@@ -298,6 +325,12 @@ mode: "0400"
 path: /var/log/kube-apiserver.log
 type: file
 ---
+Name: apiserver-aggregator
+signer: apiserver-aggregator-ca
+subject:
+  CommonName: aggregator
+type: client
+---
 Name: aws-iam-authenticator
 alternateNames:
 - localhost

nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml

@@ -32,10 +32,10 @@ contents: |
       - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
-      - --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.crt
-      - --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key
+      - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+      - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key
       - --requestheader-allowed-names=aggregator
-      - --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.crt
+      - --requestheader-client-ca-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
       - --requestheader-extra-headers-prefix=X-Remote-Extra-
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
@@ -172,6 +172,33 @@ type: directory
 ---
 contents: ""
 mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.key
+type: file
+---
+contents: ""
+mode: "0644"
 path: /srv/kubernetes/kube-apiserver/etcd-ca.crt
 type: file
 ---
@@ -238,6 +265,12 @@ mode: "0400"
 path: /var/log/kube-apiserver.log
 type: file
 ---
+Name: apiserver-aggregator
+signer: apiserver-aggregator-ca
+subject:
+  CommonName: aggregator
+type: client
+---
 Name: etcd-client
 signer: etcd-clients-ca
 subject:

nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml

@@ -32,10 +32,10 @@ contents: |
       - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
-      - --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.crt
-      - --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key
+      - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+      - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key
       - --requestheader-allowed-names=aggregator
-      - --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.crt
+      - --requestheader-client-ca-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
       - --requestheader-extra-headers-prefix=X-Remote-Extra-
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
@@ -172,6 +172,33 @@ type: directory
 ---
 contents: ""
 mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.key
+type: file
+---
+contents: ""
+mode: "0644"
 path: /srv/kubernetes/kube-apiserver/etcd-ca.crt
 type: file
 ---
@@ -238,6 +265,12 @@ mode: "0400"
 path: /var/log/kube-apiserver.log
 type: file
 ---
+Name: apiserver-aggregator
+signer: apiserver-aggregator-ca
+subject:
+  CommonName: aggregator
+type: client
+---
 Name: etcd-client
 signer: etcd-clients-ca
 subject:

nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml

@@ -2,39 +2,6 @@ mode: "0755"
 path: /srv/kubernetes
 type: directory
 ---
-contents:
-  task:
-    Name: apiserver-aggregator
-    signer: apiserver-aggregator-ca
-    subject:
-      CommonName: aggregator
-    type: client
-mode: "0644"
-path: /srv/kubernetes/apiserver-aggregator-ca.crt
-type: file
----
-contents:
-  task:
-    Name: apiserver-aggregator
-    signer: apiserver-aggregator-ca
-    subject:
-      CommonName: aggregator
-    type: client
-mode: "0644"
-path: /srv/kubernetes/apiserver-aggregator.crt
-type: file
----
-contents:
-  task:
-    Name: apiserver-aggregator
-    signer: apiserver-aggregator-ca
-    subject:
-      CommonName: aggregator
-    type: client
-mode: "0600"
-path: /srv/kubernetes/apiserver-aggregator.key
-type: file
----
 contents: |
   -----BEGIN CERTIFICATE-----
   MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw
@@ -124,12 +91,6 @@ mode: "0600"
 path: /srv/kubernetes/service-account.key
 type: file
 ---
-Name: apiserver-aggregator
-signer: apiserver-aggregator-ca
-subject:
-  CommonName: aggregator
-type: client
----
 Name: master
 alternateNames:
 - kubernetes