Refactor etcd-client-cilium secrets

cmd/kops/integration_test.go

@@ -68,6 +68,7 @@ type integrationTest struct {
 	sshKey                           bool
 	jsonOutput                       bool
 	bastionUserData                  bool
+	ciliumEtcd                       bool
 	// nth is true if we should check for files created by nth queue processor add on
 	nth bool
 }
@@ -132,6 +133,11 @@ func (i *integrationTest) withBastionUserData() *integrationTest {
 	return i
 }
 
+func (i *integrationTest) withCiliumEtcd() *integrationTest {
+	i.ciliumEtcd = true
+	return i
+}
+
 func (i *integrationTest) withNTH() *integrationTest {
 	i.nth = true
 	return i
@@ -262,8 +268,14 @@ func TestPrivateCilium2(t *testing.T) {
 }
 
 func TestPrivateCiliumAdvanced(t *testing.T) {
-	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").withPrivate().runTestTerraformAWS(t)
-	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").withPrivate().runTestCloudformation(t)
+	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
+		withPrivate().
+		withCiliumEtcd().
+		runTestTerraformAWS(t)
+	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
+		withPrivate().
+		withCiliumEtcd().
+		runTestCloudformation(t)
 }
 
 // TestPrivateCanal runs the test on a configuration with private topology, canal networking
@@ -598,15 +610,39 @@ func (i *integrationTest) setupCluster(t *testing.T, inputYAML string, ctx conte
 		t.Fatalf("error getting keystore: %v", err)
 	}
 
+	storeKeyset(t, keyStore, fi.CertificateIDCA, &testingKeyset{
+		primaryKey:           "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----",
+		primaryCertificate:   "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD\nMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG\nSIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp\nVWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==\n-----END CERTIFICATE-----",
+		secondaryKey:         "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----\n",
+		secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26\nA6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG\nSIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq\n9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==\n-----END CERTIFICATE-----",
+	})
+	if i.ciliumEtcd {
+		storeKeyset(t, keyStore, "etcd-clients-ca-cilium", &testingKeyset{
+			primaryKey:           "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----",
+			primaryCertificate:   "-----BEGIN CERTIFICATE-----\nMIIBgDCCASqgAwIBAgIMFotPsR9PsbCKkTJsMA0GCSqGSIb3DQEBCwUAMCExHzAd\nBgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyMTUyWhcN\nMzEwNjIxMjAyMTUyWjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt\nMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm\nXVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC\nMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW\n3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EAR4UEW5ZK+NVtqm7s\nHF/JbSYPd+BhcNaJVOv8JP+/CGfCOXOmxjpZICSYQqe6UjjjP7fbJy8FANTpKTuJ\nUQC1kQ==\n-----END CERTIFICATE-----",
+			secondaryKey:         "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----",
+			secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBgDCCASqgAwIBAgIMFotP940EXpD3N1D7MA0GCSqGSIb3DQEBCwUAMCExHzAd\nBgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyNjU1WhcN\nMzEwNjIxMjAyNjU1WjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt\nMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm\nXVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC\nMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW\n3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EARXoKy6mExpD6tHFO\nCN3ZGNZ5BsHl5W5y+gwUuVskgC7xt/bgTuXm5hz8TLgnG5kYtG4uxjFg4yCvtNg2\nMQNfAQ==\n-----END CERTIFICATE-----",
+		})
+	}
+
+	return factory
+}
+
+type testingKeyset struct {
+	primaryKey           string
+	primaryCertificate   string
+	secondaryKey         string
+	secondaryCertificate string
+}
+
+func storeKeyset(t *testing.T, keyStore fi.CAStore, name string, testingKeyset *testingKeyset) {
 	{
-		caKey := "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----"
-		privateKey, err := pki.ParsePEMPrivateKey([]byte(caKey))
+		privateKey, err := pki.ParsePEMPrivateKey([]byte(testingKeyset.primaryKey))
 		if err != nil {
 			t.Fatalf("error loading private key %v", err)
 		}
 
-		caCertificate := "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD\nMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG\nSIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp\nVWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==\n-----END CERTIFICATE-----"
-		cert, err := pki.ParsePEMCertificate([]byte(caCertificate))
+		cert, err := pki.ParsePEMCertificate([]byte(testingKeyset.primaryCertificate))
 		if err != nil {
 			t.Fatalf("error loading certificate %v", err)
 		}
@@ -616,26 +652,22 @@ func (i *integrationTest) setupCluster(t *testing.T, inputYAML string, ctx conte
 			t.Fatalf("error creating keyset: %v", err)
 		}
 
-		caKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----\n"
-		privateKey, err = pki.ParsePEMPrivateKey([]byte(caKey))
+		privateKey, err = pki.ParsePEMPrivateKey([]byte(testingKeyset.secondaryKey))
 		if err != nil {
 			t.Fatalf("error loading private key %v", err)
 		}
 
-		caCertificate = "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26\nA6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG\nSIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq\n9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==\n-----END CERTIFICATE-----"
-		cert, err = pki.ParsePEMCertificate([]byte(caCertificate))
+		cert, err = pki.ParsePEMCertificate([]byte(testingKeyset.secondaryCertificate))
 		if err != nil {
 			t.Fatalf("error loading certificate %v", err)
 		}
 
 		_ = keyset.AddItem(cert, privateKey, false)
-		err = keyStore.StoreKeyset(fi.CertificateIDCA, keyset)
+		err = keyStore.StoreKeyset(name, keyset)
 		if err != nil {
 			t.Fatalf("error storing user provided keys: %v", err)
 		}
 	}
-
-	return factory
 }
 
 func (i *integrationTest) runTestTerraformAWS(t *testing.T) {

nodeup/pkg/model/context.go

@@ -240,6 +240,9 @@ func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetask
 
 // GetBootstrapCert requests a certificate keypair from kops-controller.
 func (c *NodeupModelContext) GetBootstrapCert(name string) (cert, key fi.Resource) {
+	if c.IsMaster {
+		panic("control plane nodes can't get certs from kops-controller")
+	}
 	b, ok := c.bootstrapCerts[name]
 	if !ok {
 		b = &nodetasks.BootstrapCert{
@@ -404,7 +407,7 @@ func (c *NodeupModelContext) KubectlPath() string {
 }
 
 // BuildCertificatePairTask creates the tasks to create the certificate and private key files.
-func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, name, path, filename string, owner *string) error {
+func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, name, path, filename string, owner *string, beforeServices []string) error {
 	p := filepath.Join(path, filename)
 	if !filepath.IsAbs(p) {
 		p = filepath.Join(c.PathSrvKubernetes(), p)
@@ -440,11 +443,12 @@ func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContex
 	}
 
 	ctx.AddTask(&nodetasks.File{
-		Path:     p + ".crt",
-		Contents: fi.NewStringResource(cert),
-		Type:     nodetasks.FileType_File,
-		Mode:     s("0600"),
-		Owner:    owner,
+		Path:           p + ".crt",
+		Contents:       fi.NewStringResource(cert),
+		Type:           nodetasks.FileType_File,
+		Mode:           s("0600"),
+		Owner:          owner,
+		BeforeServices: beforeServices,
 	})
 
 	privateKey := item.PrivateKey

nodeup/pkg/model/kops_controller.go

@@ -84,7 +84,7 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 	for _, cert := range caList {
 		owner := wellknownusers.KopsControllerName
-		err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner)
+		err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner, nil)
 		if err != nil {
 			return err
 		}

nodeup/pkg/model/kube_controller_manager.go

@@ -54,7 +54,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 
 	// Include the CA Key
 	// @TODO: use a per-machine key?  use KMS?
-	if err := b.BuildCertificatePairTask(c, fi.CertificateIDCA, pathSrvKCM, "ca", nil); err != nil {
+	if err := b.BuildCertificatePairTask(c, fi.CertificateIDCA, pathSrvKCM, "ca", nil, nil); err != nil {
 		return err
 	}
 

nodeup/pkg/model/networking/cilium.go

@@ -127,27 +127,13 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error
 	name := "etcd-client-cilium"
 	dir := "/etc/kubernetes/pki/cilium"
 	signer := "etcd-clients-ca-cilium"
-	if b.UseKopsControllerForNodeBootstrap() && !b.IsMaster {
-		cert, key := b.GetBootstrapCert(name)
-
-		c.AddTask(&nodetasks.File{
-			Path:           filepath.Join(dir, name+".crt"),
-			Contents:       cert,
-			Type:           nodetasks.FileType_File,
-			Mode:           fi.String("0644"),
-			BeforeServices: []string{"kubelet.service"},
-		})
-
-		c.AddTask(&nodetasks.File{
-			Path:           filepath.Join(dir, name+".key"),
-			Contents:       key,
-			Type:           nodetasks.FileType_File,
-			Mode:           fi.String("0400"),
-			BeforeServices: []string{"kubelet.service"},
-		})
-
-		return b.BuildCertificateTask(c, signer, filepath.Join(dir, "etcd-ca.crt"), nil)
-	} else {
+	c.AddTask(&nodetasks.File{
+		Path:     filepath.Join(dir, "etcd-ca.crt"),
+		Contents: fi.NewStringResource(b.NodeupConfig.CAs[signer]),
+		Type:     nodetasks.FileType_File,
+		Mode:     fi.String("0600"),
+	})
+	if b.IsMaster {
 		issueCert := &nodetasks.IssueCert{
 			Name:   name,
 			Signer: signer,
@@ -157,6 +143,30 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error
 			},
 		}
 		c.AddTask(issueCert)
-		return issueCert.AddFileTasks(c, dir, name, "etcd-ca", nil)
+		return issueCert.AddFileTasks(c, dir, name, "", nil)
+	} else {
+		if b.UseKopsControllerForNodeBootstrap() {
+			cert, key := b.GetBootstrapCert(name)
+
+			c.AddTask(&nodetasks.File{
+				Path:           filepath.Join(dir, name+".crt"),
+				Contents:       cert,
+				Type:           nodetasks.FileType_File,
+				Mode:           fi.String("0644"),
+				BeforeServices: []string{"kubelet.service"},
+			})
+
+			c.AddTask(&nodetasks.File{
+				Path:           filepath.Join(dir, name+".key"),
+				Contents:       key,
+				Type:           nodetasks.FileType_File,
+				Mode:           fi.String("0400"),
+				BeforeServices: []string{"kubelet.service"},
+			})
+
+			return nil
+		} else {
+			return b.BuildCertificatePairTask(c, name, dir, name, nil, []string{"kubelet.service"})
+		}
 	}
 }

nodeup/pkg/model/networking/cilium_test.go

@@ -22,6 +22,7 @@ import (
 
 	"k8s.io/kops/nodeup/pkg/model"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/upup/pkg/fi"
 )
@@ -48,6 +49,11 @@ func TestCiliumBuilder(t *testing.T) {
 				},
 			},
 		},
+		NodeupConfig: &nodeup.Config{
+			CAs: map[string]string{
+				"etcd-clients-ca-cilium": "-----BEGIN CERTIFICATE-----\nMIIBbjCCARigAwIBAgIMFnbWaYo6t3AwKQtWMA0GCSqGSIb3DQEBCwUAMBgxFjAU\nBgNVBAMTDWNuPWt1YmVybmV0ZXMwHhcNMjEwNDE2MDMzNDI0WhcNMzEwNDE2MDMz\nNDI0WjAYMRYwFAYDVQQDEw1jbj1rdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQAD\nSwAwSAJBANLVh1dSDxJ5EcCd36av7++6+sDKqEm2GAzKIwOlfvPsm+pT+pClr51s\nd1m7V16nhWE6lhWjtsiMF8Q32+P5XZkCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIaNS7TlHC6K0r8yWYM1wExengDq\nMA0GCSqGSIb3DQEBCwUAA0EAoxha8yD6JLJcog/EOMdc5BpVPupQ/0FyO38Mb3l9\n0N7uZle0Tz1FQuadRtouySj37iq9nIxEeTh03Q52hNcl3A==\n-----END CERTIFICATE-----\n",
+			},
+		},
 		HasAPIServer: true,
 		KeyStore:     &fakeKeyStore{},
 		IsMaster:     true,

pkg/model/awsmodel/autoscalinggroup_test.go

@@ -70,6 +70,11 @@ func TestRootVolumeOptimizationFlag(t *testing.T) {
 		},
 		BootstrapScriptBuilder: &model.BootstrapScriptBuilder{
 			Lifecycle: fi.LifecycleSync,
+			Cluster: &kops.Cluster{
+				Spec: kops.ClusterSpec{
+					Networking: &kops.NetworkingSpec{},
+				},
+			},
 		},
 		Cluster: cluster,
 	}
@@ -159,6 +164,7 @@ func TestAPIServerAdditionalSecurityGroupsWithNLB(t *testing.T) {
 		},
 		BootstrapScriptBuilder: &model.BootstrapScriptBuilder{
 			Lifecycle: fi.LifecycleSync,
+			Cluster:   cluster,
 		},
 		Cluster: cluster,
 	}

pkg/model/azuremodel/vmscaleset_test.go

@@ -35,6 +35,11 @@ func TestVMScaleSetModelBuilder_Build(t *testing.T) {
 		AzureModelContext: newTestAzureModelContext(),
 		BootstrapScriptBuilder: &model.BootstrapScriptBuilder{
 			Lifecycle: fi.LifecycleSync,
+			Cluster: &kops.Cluster{
+				Spec: kops.ClusterSpec{
+					Networking: &kops.NetworkingSpec{},
+				},
+			},
 		},
 	}
 	c := &fi.ModelBuilderContext{

pkg/model/bootstrapscript.go

@@ -29,6 +29,7 @@ import (
 	"text/template"
 
 	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/upup/pkg/fi/utils"
 	"sigs.k8s.io/yaml"
 
@@ -43,14 +44,15 @@ import (
 )
 
 type NodeUpConfigBuilder interface {
-	BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error)
+	BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTasks map[string]*fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error)
 }
 
 // BootstrapScriptBuilder creates the bootstrap script
 type BootstrapScriptBuilder struct {
 	Lifecycle           fi.Lifecycle
 	NodeUpAssets        map[architectures.Architecture]*mirrors.MirroredAsset
 	NodeUpConfigBuilder NodeUpConfigBuilder
+	Cluster             *kops.Cluster
 }
 
 type BootstrapScript struct {
@@ -62,8 +64,8 @@ type BootstrapScript struct {
 	// alternateNameTasks are tasks that contribute api-server IP addresses.
 	alternateNameTasks []fi.HasAddress
 
-	// caTask holds the CA task, for dependency analysis.
-	caTask *fitasks.Keypair
+	// caTasks hold the CA tasks, for dependency analysis.
+	caTasks map[string]*fitasks.Keypair
 
 	// nodeupConfig contains the nodeup config.
 	nodeupConfig fi.TaskDependentResource
@@ -91,7 +93,7 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context) (string
 	}
 
 	sort.Strings(alternateNames)
-	config, bootConfig, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames, b.caTask)
+	config, bootConfig, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames, b.caTasks)
 	if err != nil {
 		return "", err
 	}
@@ -209,11 +211,22 @@ func (b *BootstrapScript) buildEnvironmentVariables(cluster *kops.Cluster) (map[
 // ResourceNodeUp generates and returns a nodeup (bootstrap) script from a
 // template file, substituting in specific env vars & cluster spec configuration
 func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *kops.InstanceGroup) (fi.Resource, error) {
-	caTaskObject, found := c.Tasks["Keypair/ca"]
-	if !found {
-		return nil, fmt.Errorf("keypair/ca task not found")
+	keypairs := []string{"ca"}
+	if model.UseCiliumEtcd(b.Cluster) {
+		keypairs = append(keypairs, "etcd-clients-ca-cilium")
+		if !model.UseKopsControllerForNodeBootstrap(b.Cluster) {
+			keypairs = append(keypairs, "etcd-client-cilium")
+		}
+	}
+
+	caTasks := map[string]*fitasks.Keypair{}
+	for _, keypair := range keypairs {
+		caTaskObject, found := c.Tasks["Keypair/"+keypair]
+		if !found {
+			return nil, fmt.Errorf("keypair/%s task not found", keypair)
+		}
+		caTasks[keypair] = caTaskObject.(*fitasks.Keypair)
 	}
-	caTask := caTaskObject.(*fitasks.Keypair)
 
 	// Bastions can have AdditionalUserData, but if there isn't any skip this part
 	if ig.IsBastion() && len(ig.Spec.AdditionalUserData) == 0 {
@@ -229,7 +242,7 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k
 		Lifecycle: b.Lifecycle,
 		ig:        ig,
 		builder:   b,
-		caTask:    caTask,
+		caTasks:   caTasks,
 	}
 	task.resource.Task = task
 	task.nodeupConfig.Task = task
@@ -258,7 +271,9 @@ func (b *BootstrapScript) GetDependencies(tasks map[string]fi.Task) []fi.Task {
 		}
 	}
 
-	deps = append(deps, b.caTask)
+	for _, task := range b.caTasks {
+		deps = append(deps, task)
+	}
 
 	return deps
 }