kubernetes · k8s-ci-robot · Jul 11, 2021 · Jul 11, 2021 · Jul 11, 2021 · Jul 11, 2021
diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go
@@ -802,7 +802,7 @@ type testingKeyset struct {
 	secondaryCertificate string
 }
 
-func storeKeyset(t *testing.T, keyStore fi.CAStore, name string, testingKeyset *testingKeyset) {
+func storeKeyset(t *testing.T, keyStore fi.Keystore, name string, testingKeyset *testingKeyset) {
 	{
 		privateKey, err := pki.ParsePEMPrivateKey([]byte(testingKeyset.primaryKey))
 		if err != nil {

diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go
@@ -54,7 +54,7 @@ type NodeupModelContext struct {
 	Cluster      *kops.Cluster
 	ConfigBase   vfs.Path
 	Distribution distributions.Distribution
-	KeyStore     fi.CAStore
+	KeyStore     fi.Keystore
 	BootConfig   *nodeup.BootConfig
 	NodeupConfig *nodeup.Config
 	SecretStore  fi.SecretStore
@@ -214,10 +214,11 @@ func (c *NodeupModelContext) KubeletKubeConfig() string {
 // BuildIssuedKubeconfig generates a kubeconfig with a locally issued client certificate.
 func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetasks.PKIXName, ctx *fi.ModelBuilderContext) *fi.TaskDependentResource {
 	issueCert := &nodetasks.IssueCert{
-		Name:    name,
-		Signer:  fi.CertificateIDCA,
-		Type:    "client",
-		Subject: subject,
+		Name:      name,
+		Signer:    fi.CertificateIDCA,
+		KeypairID: c.NodeupConfig.KeypairIDs[fi.CertificateIDCA],
+		Type:      "client",
+		Subject:   subject,
 	}
 	ctx.AddTask(issueCert)
 	certResource, keyResource, caResource := issueCert.GetResources()
@@ -485,16 +486,16 @@ func (c *NodeupModelContext) buildCertificatePairTask(ctx *fi.ModelBuilderContex
 
 // BuildCertificateTask builds a task to create a certificate file.
 func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error {
-	cert, err := c.KeyStore.FindCert(name)
+	keyset, err := c.KeyStore.FindKeyset(name)
 	if err != nil {
 		return err
 	}
 
-	if cert == nil {
-		return fmt.Errorf("certificate %q not found", name)
+	if keyset == nil {
+		return fmt.Errorf("keyset %q not found", name)
 	}
 
-	serialized, err := cert.AsString()
+	serialized, err := keyset.Primary.Certificate.AsString()
 	if err != nil {
 		return err
 	}
@@ -516,16 +517,16 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n
 
 // BuildLegacyPrivateKeyTask builds a task to create a private key file.
 func (c *NodeupModelContext) BuildLegacyPrivateKeyTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error {
-	cert, err := c.KeyStore.FindPrivateKey(name)
+	keyset, err := c.KeyStore.FindKeyset(name)
 	if err != nil {
 		return err
 	}
 
-	if cert == nil {
-		return fmt.Errorf("private key %q not found", name)
+	if keyset == nil {
+		return fmt.Errorf("keyset %q not found", name)
 	}
 
-	serialized, err := cert.AsString()
+	serialized, err := keyset.Primary.PrivateKey.AsString()
 	if err != nil {
 		return err
 	}
@@ -622,25 +623,20 @@ func EvaluateHostnameOverride(hostnameOverride string) (string, error) {
 	return *(result.Reservations[0].Instances[0].PrivateDnsName), nil
 }
 
-// GetPrimaryKeypair is a helper method to retrieve a primary keypair from the store
+// GetPrimaryKeypair is a helper method to retrieve a primary keypair from the store.
+// TODO: Use the KeysetID in NodeupConfig instead of the Primary keypair.
 func (c *NodeupModelContext) GetPrimaryKeypair(name string) (cert []byte, key []byte, err error) {
-	certificate, privateKey, err := c.KeyStore.FindPrimaryKeypair(name)
+	keyset, err := c.KeyStore.FindKeyset(name)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error fetching certificate: %v from keystore: %v", name, err)
-	}
-	if certificate == nil {
-		return nil, nil, fmt.Errorf("unable to find certificate: %s", name)
-	}
-	if privateKey == nil {
-		return nil, nil, fmt.Errorf("unable to find key: %s", name)
+		return nil, nil, fmt.Errorf("error fetching keyset: %v from keystore: %v", name, err)
 	}
 
-	cert, err = certificate.AsBytes()
+	cert, err = keyset.Primary.Certificate.AsBytes()
 	if err != nil {
 		return nil, nil, err
 	}
 
-	key, err = privateKey.AsBytes()
+	key, err = keyset.Primary.PrivateKey.AsBytes()
 	if err != nil {
 		return nil, nil, err
 	}

diff --git a/nodeup/pkg/model/etcd_manager_tls.go b/nodeup/pkg/model/etcd_manager_tls.go
@@ -17,11 +17,7 @@ limitations under the License.
 package model
 
 import (
-	"fmt"
-	"path/filepath"
-
 	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 )
 
 // EtcdManagerTLSBuilder configures TLS support for etcd-manager
@@ -54,22 +50,7 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
 		}
 
 		for fileName, keystoreName := range keys {
-			cert, err := b.KeyStore.FindCert(keystoreName)
-			if err != nil {
-				return err
-			}
-			if cert == nil {
-				return fmt.Errorf("keypair %q not found", keystoreName)
-			}
-
-			ctx.AddTask(&nodetasks.File{
-				Path:     filepath.Join(d, fileName+".crt"),
-				Contents: fi.NewStringResource(b.NodeupConfig.CAs[keystoreName]),
-				Type:     nodetasks.FileType_File,
-				Mode:     fi.String("0600"),
-			})
-
-			if err := b.BuildPrivateKeyTask(ctx, keystoreName, d, fileName, nil, nil); err != nil {
+			if err := b.buildCertificatePairTask(ctx, keystoreName, d, fileName, nil, nil, true); err != nil {
 				return err
 			}
 		}

diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go
@@ -27,16 +27,15 @@ import (
 	"k8s.io/kops/util/pkg/vfs"
 )
 
-// fakeCAStore mocks out some of fi.CAStore, for our tests.
-type fakeCAStore struct {
+// fakeKeystore mocks out some of fi.Keystore, for our tests.
+type fakeKeystore struct {
 	T              *testing.T
 	privateKeysets map[string]*kops.Keyset
-	certs          map[string]*pki.Certificate
 }
 
-var _ fi.CAStore = &fakeCAStore{}
+var _ fi.Keystore = &fakeKeystore{}
 
-func (k fakeCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
+func (k fakeKeystore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
 	keyset, err := k.FindKeyset(name)
 	if err != nil {
 		return nil, nil, err
@@ -45,7 +44,7 @@ func (k fakeCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.Pri
 	return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil
 }
 
-func (k fakeCAStore) FindKeyset(name string) (*fi.Keyset, error) {
+func (k fakeKeystore) FindKeyset(name string) (*fi.Keyset, error) {
 	kopsKeyset := k.privateKeysets[name]
 	if kopsKeyset == nil {
 		return nil, nil
@@ -83,32 +82,14 @@ func (k fakeCAStore) FindKeyset(name string) (*fi.Keyset, error) {
 	return keyset, nil
 }
 
-func (k fakeCAStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
-	panic("fakeCAStore does not implement CreateKeypair")
+func (k fakeKeystore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
+	panic("fakeKeystore does not implement CreateKeypair")
 }
 
-func (k fakeCAStore) StoreKeyset(name string, keyset *fi.Keyset) error {
-	panic("fakeCAStore does not implement StoreKeyset")
+func (k fakeKeystore) StoreKeyset(name string, keyset *fi.Keyset) error {
+	panic("fakeKeystore does not implement StoreKeyset")
 }
 
-func (k fakeCAStore) MirrorTo(basedir vfs.Path) error {
-	panic("fakeCAStore does not implement MirrorTo")
-}
-
-func (k fakeCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) {
-	primaryId := k.privateKeysets[name].Spec.PrimaryId
-	for _, item := range k.privateKeysets[name].Spec.Keys {
-		if item.Id == primaryId {
-			return pki.ParsePEMPrivateKey(item.PrivateMaterial)
-		}
-	}
-	return nil, nil
-}
-
-func (k fakeCAStore) FindCert(name string) (*pki.Certificate, error) {
-	return k.certs[name], nil
-}
-
-func (k fakeCAStore) ListKeysets() (map[string]*fi.Keyset, error) {
-	panic("fakeCAStore does not implement ListKeysets")
+func (k fakeKeystore) MirrorTo(basedir vfs.Path) error {
+	panic("fakeKeystore does not implement MirrorTo")
 }
diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go
@@ -56,6 +56,7 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error {
 	issueCert := &nodetasks.IssueCert{
 		Name:           "kops-controller",
 		Signer:         fi.CertificateIDCA,
+		KeypairID:      b.NodeupConfig.KeypairIDs[fi.CertificateIDCA],
 		Type:           "server",
 		Subject:        nodetasks.PKIXName{CommonName: "kops-controller"},
 		AlternateNames: []string{"kops-controller.internal." + b.Cluster.ObjectMeta.Name},

diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go
@@ -116,9 +116,10 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 		kubeAPIServer.EtcdCAFile = filepath.Join(pathSrvKAPI, "etcd-ca.crt")
 
 		issueCert := &nodetasks.IssueCert{
-			Name:   "etcd-client",
-			Signer: "etcd-clients-ca",
-			Type:   "client",
+			Name:      "etcd-client",
+			Signer:    "etcd-clients-ca",
+			KeypairID: b.NodeupConfig.KeypairIDs["etcd-clients-ca"],
+			Type:      "client",
 			Subject: nodetasks.PKIXName{
 				CommonName: "kube-apiserver",
 			},
@@ -145,9 +146,10 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 		kubeAPIServer.RequestheaderClientCAFile = filepath.Join(pathSrvKAPI, "apiserver-aggregator-ca.crt")
 
 		issueCert := &nodetasks.IssueCert{
-			Name:   "apiserver-aggregator",
-			Signer: "apiserver-aggregator-ca",
-			Type:   "client",
+			Name:      "apiserver-aggregator",
+			Signer:    "apiserver-aggregator-ca",
+			KeypairID: b.NodeupConfig.KeypairIDs["apiserver-aggregator-ca"],
+			Type:      "client",
 			// Must match RequestheaderAllowedNames
 			Subject: nodetasks.PKIXName{CommonName: "aggregator"},
 		}
@@ -186,10 +188,11 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	issueCert := &nodetasks.IssueCert{
-		Name:    "kubelet-api",
-		Signer:  fi.CertificateIDCA,
-		Type:    "client",
-		Subject: nodetasks.PKIXName{CommonName: "kubelet-api"},
+		Name:      "kubelet-api",
+		Signer:    fi.CertificateIDCA,
+		KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA],
+		Type:      "client",
+		Subject:   nodetasks.PKIXName{CommonName: "kubelet-api"},
 	}
 	c.AddTask(issueCert)
 	err := issueCert.AddFileTasks(c, b.PathSrvKubernetes(), "kubelet-api", "", nil)
@@ -258,28 +261,15 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte
 		kubeAPIServer.AuthenticationTokenWebhookConfigFile = fi.String(PathAuthnConfig)
 
 		{
-			caCertificate, _, err := b.NodeupModelContext.KeyStore.FindPrimaryKeypair(fi.CertificateIDCA)
-			if err != nil {
-				return fmt.Errorf("error fetching AWS IAM Authentication CA certificate from keystore: %v", err)
-			}
-			if caCertificate == nil {
-				return fmt.Errorf("AWS IAM  Authentication CA certificate %q not found", fi.CertificateIDCA)
-			}
-
 			cluster := kubeconfig.KubectlCluster{
-				Server: "https://127.0.0.1:21362/authenticate",
+				Server:                   "https://127.0.0.1:21362/authenticate",
+				CertificateAuthorityData: []byte(b.NodeupConfig.CAs[fi.CertificateIDCA]),
 			}
 			context := kubeconfig.KubectlContext{
 				Cluster: "aws-iam-authenticator",
 				User:    "kube-apiserver",
 			}
 
-			// Since we're talking to localhost, we don't need the entire certificate bundle.
-			cluster.CertificateAuthorityData, err = caCertificate.AsBytes()
-			if err != nil {
-				return fmt.Errorf("error encoding AWS IAM Authentication CA certificate: %v", err)
-			}
-
 			config := kubeconfig.KubectlConfig{}
 			config.Clusters = append(config.Clusters, &kubeconfig.KubectlClusterWithName{
 				Name:    "aws-iam-authenticator",
@@ -321,10 +311,11 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte
 
 		{
 			issueCert := &nodetasks.IssueCert{
-				Name:    id,
-				Signer:  fi.CertificateIDCA,
-				Type:    "server",
-				Subject: nodetasks.PKIXName{CommonName: id},
+				Name:      id,
+				Signer:    fi.CertificateIDCA,
+				KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA],
+				Type:      "server",
+				Subject:   nodetasks.PKIXName{CommonName: id},
 				AlternateNames: []string{
 					"localhost",
 					"127.0.0.1",

diff --git a/nodeup/pkg/model/kube_apiserver_healthcheck.go b/nodeup/pkg/model/kube_apiserver_healthcheck.go
@@ -82,9 +82,10 @@ func (b *KubeAPIServerBuilder) addHealthcheckSidecarTasks(c *fi.ModelBuilderCont
 	}
 
 	issueCert := &nodetasks.IssueCert{
-		Name:   id,
-		Signer: fi.CertificateIDCA,
-		Type:   "client",
+		Name:      id,
+		Signer:    fi.CertificateIDCA,
+		KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA],
+		Type:      "client",
 		Subject: nodetasks.PKIXName{
 			CommonName: id,
 		},

diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go
@@ -552,7 +552,6 @@ func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.ModelBuilderContex
 	if b.UseKopsControllerForNodeBootstrap() {
 		name := "kubelet-server"
 		dir := b.PathSrvKubernetes()
-		signer := fi.CertificateIDCA
 
 		nodeName, err := b.NodeName()
 		if err != nil {
@@ -580,9 +579,10 @@ func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.ModelBuilderContex
 
 		} else {
 			issueCert := &nodetasks.IssueCert{
-				Name:   name,
-				Signer: signer,
-				Type:   "server",
+				Name:      name,
+				Signer:    fi.CertificateIDCA,
+				KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA],
+				Type:      "server",
 				Subject: nodetasks.PKIXName{
 					CommonName: nodeName,
 				},

diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go
@@ -376,7 +376,7 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu
 		t.Fatal(err)
 	}
 
-	keystore := &fakeCAStore{}
+	keystore := &fakeKeystore{}
 	keystore.T = t
 	saKeyset, _ := rotatingPrivateKeyset().ToAPIObject("service-account", true)
 	keystore.privateKeysets = map[string]*kops.Keyset{
@@ -387,13 +387,6 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu
 		"kube-scheduler":          simplePrivateKeyset(dummyCertificate, dummyKey),
 		"service-account":         saKeyset,
 	}
-	keystore.certs = map[string]*pki.Certificate{
-		"kubernetes-ca":           mustParseCertificate(dummyCertificate),
-		"apiserver-aggregator-ca": mustParseCertificate(dummyCertificate),
-		"kube-controller-manager": mustParseCertificate(dummyCertificate),
-		"kube-proxy":              mustParseCertificate(dummyCertificate),
-		"kube-scheduler":          mustParseCertificate(dummyCertificate),
-	}
 
 	nodeupModelContext, err := BuildNodeupModelContext(model)
 	if err != nil {

diff --git a/nodeup/pkg/model/networking/cilium.go b/nodeup/pkg/model/networking/cilium.go
@@ -109,11 +109,12 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error
 		Type:     nodetasks.FileType_File,
 		Mode:     fi.String("0600"),
 	})
-	if b.IsMaster {
+	if b.HasAPIServer {
 		issueCert := &nodetasks.IssueCert{
-			Name:   name,
-			Signer: signer,
-			Type:   "client",
+			Name:      name,
+			Signer:    signer,
+			KeypairID: b.NodeupConfig.KeypairIDs[signer],
+			Type:      "client",
 			Subject: nodetasks.PKIXName{
 				CommonName: "cilium",
 			},

diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go
@@ -119,6 +119,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 		issueCert := &nodetasks.IssueCert{
 			Name:           "master",
 			Signer:         fi.CertificateIDCA,
+			KeypairID:      b.NodeupConfig.KeypairIDs[fi.CertificateIDCA],
 			Type:           "server",
 			Subject:        nodetasks.PKIXName{CommonName: "kubernetes-master"},
 			AlternateNames: alternateNames,