diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index b220ac3c89a7e..4e05add0e8598 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -68,6 +68,7 @@ type integrationTest struct { sshKey bool jsonOutput bool bastionUserData bool + ciliumEtcd bool // nth is true if we should check for files created by nth queue processor add on nth bool } @@ -132,6 +133,11 @@ func (i *integrationTest) withBastionUserData() *integrationTest { return i } +func (i *integrationTest) withCiliumEtcd() *integrationTest { + i.ciliumEtcd = true + return i +} + func (i *integrationTest) withNTH() *integrationTest { i.nth = true return i @@ -262,8 +268,14 @@ func TestPrivateCilium2(t *testing.T) { } func TestPrivateCiliumAdvanced(t *testing.T) { - newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").withPrivate().runTestTerraformAWS(t) - newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").withPrivate().runTestCloudformation(t) + newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced"). + withPrivate(). + withCiliumEtcd(). + runTestTerraformAWS(t) + newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced"). + withPrivate(). + withCiliumEtcd(). + runTestCloudformation(t) } // TestPrivateCanal runs the test on a configuration with private topology, canal networking @@ -598,15 +610,39 @@ func (i *integrationTest) setupCluster(t *testing.T, inputYAML string, ctx conte t.Fatalf("error getting keystore: %v", err) } + storeKeyset(t, keyStore, fi.CertificateIDCA, &testingKeyset{ + primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----", + primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD\nMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG\nSIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp\nVWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==\n-----END CERTIFICATE-----", + secondaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----\n", + secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26\nA6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG\nSIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq\n9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==\n-----END CERTIFICATE-----", + }) + if i.ciliumEtcd { + storeKeyset(t, keyStore, "etcd-clients-ca-cilium", &testingKeyset{ + primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----", + primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBgDCCASqgAwIBAgIMFotPsR9PsbCKkTJsMA0GCSqGSIb3DQEBCwUAMCExHzAd\nBgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyMTUyWhcN\nMzEwNjIxMjAyMTUyWjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt\nMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm\nXVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC\nMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW\n3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EAR4UEW5ZK+NVtqm7s\nHF/JbSYPd+BhcNaJVOv8JP+/CGfCOXOmxjpZICSYQqe6UjjjP7fbJy8FANTpKTuJ\nUQC1kQ==\n-----END CERTIFICATE-----", + secondaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----", + secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBgDCCASqgAwIBAgIMFotP940EXpD3N1D7MA0GCSqGSIb3DQEBCwUAMCExHzAd\nBgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyNjU1WhcN\nMzEwNjIxMjAyNjU1WjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt\nMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm\nXVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC\nMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW\n3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EARXoKy6mExpD6tHFO\nCN3ZGNZ5BsHl5W5y+gwUuVskgC7xt/bgTuXm5hz8TLgnG5kYtG4uxjFg4yCvtNg2\nMQNfAQ==\n-----END CERTIFICATE-----", + }) + } + + return factory +} + +type testingKeyset struct { + primaryKey string + primaryCertificate string + secondaryKey string + secondaryCertificate string +} + +func storeKeyset(t *testing.T, keyStore fi.CAStore, name string, testingKeyset *testingKeyset) { { - caKey := "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----" - privateKey, err := pki.ParsePEMPrivateKey([]byte(caKey)) + privateKey, err := pki.ParsePEMPrivateKey([]byte(testingKeyset.primaryKey)) if err != nil { t.Fatalf("error loading private key %v", err) } - caCertificate := "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD\nMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG\nSIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp\nVWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==\n-----END CERTIFICATE-----" - cert, err := pki.ParsePEMCertificate([]byte(caCertificate)) + cert, err := pki.ParsePEMCertificate([]byte(testingKeyset.primaryCertificate)) if err != nil { t.Fatalf("error loading certificate %v", err) } @@ -616,26 +652,22 @@ func (i *integrationTest) setupCluster(t *testing.T, inputYAML string, ctx conte t.Fatalf("error creating keyset: %v", err) } - caKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----\n" - privateKey, err = pki.ParsePEMPrivateKey([]byte(caKey)) + privateKey, err = pki.ParsePEMPrivateKey([]byte(testingKeyset.secondaryKey)) if err != nil { t.Fatalf("error loading private key %v", err) } - caCertificate = "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26\nA6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG\nSIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq\n9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==\n-----END CERTIFICATE-----" - cert, err = pki.ParsePEMCertificate([]byte(caCertificate)) + cert, err = pki.ParsePEMCertificate([]byte(testingKeyset.secondaryCertificate)) if err != nil { t.Fatalf("error loading certificate %v", err) } _ = keyset.AddItem(cert, privateKey, false) - err = keyStore.StoreKeyset(fi.CertificateIDCA, keyset) + err = keyStore.StoreKeyset(name, keyset) if err != nil { t.Fatalf("error storing user provided keys: %v", err) } } - - return factory } func (i *integrationTest) runTestTerraformAWS(t *testing.T) { diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 2cacced47d29d..1ae93ea8e4350 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -240,6 +240,9 @@ func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetask // GetBootstrapCert requests a certificate keypair from kops-controller. func (c *NodeupModelContext) GetBootstrapCert(name string) (cert, key fi.Resource) { + if c.IsMaster { + panic("control plane nodes can't get certs from kops-controller") + } b, ok := c.bootstrapCerts[name] if !ok { b = &nodetasks.BootstrapCert{ @@ -404,7 +407,7 @@ func (c *NodeupModelContext) KubectlPath() string { } // BuildCertificatePairTask creates the tasks to create the certificate and private key files. -func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, name, path, filename string, owner *string) error { +func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, name, path, filename string, owner *string, beforeServices []string) error { p := filepath.Join(path, filename) if !filepath.IsAbs(p) { p = filepath.Join(c.PathSrvKubernetes(), p) @@ -440,11 +443,12 @@ func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContex } ctx.AddTask(&nodetasks.File{ - Path: p + ".crt", - Contents: fi.NewStringResource(cert), - Type: nodetasks.FileType_File, - Mode: s("0600"), - Owner: owner, + Path: p + ".crt", + Contents: fi.NewStringResource(cert), + Type: nodetasks.FileType_File, + Mode: s("0600"), + Owner: owner, + BeforeServices: beforeServices, }) privateKey := item.PrivateKey diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go index be8136772ecd6..8773e0bfce477 100644 --- a/nodeup/pkg/model/kops_controller.go +++ b/nodeup/pkg/model/kops_controller.go @@ -84,7 +84,7 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error { } for _, cert := range caList { owner := wellknownusers.KopsControllerName - err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner) + err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner, nil) if err != nil { return err } diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index e027eeb31b899..205f168749571 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -54,7 +54,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { // Include the CA Key // @TODO: use a per-machine key? use KMS? - if err := b.BuildCertificatePairTask(c, fi.CertificateIDCA, pathSrvKCM, "ca", nil); err != nil { + if err := b.BuildCertificatePairTask(c, fi.CertificateIDCA, pathSrvKCM, "ca", nil, nil); err != nil { return err } diff --git a/nodeup/pkg/model/networking/BUILD.bazel b/nodeup/pkg/model/networking/BUILD.bazel index 9ee113cc6941b..c4ffb90311606 100644 --- a/nodeup/pkg/model/networking/BUILD.bazel +++ b/nodeup/pkg/model/networking/BUILD.bazel @@ -35,6 +35,7 @@ go_test( deps = [ "//nodeup/pkg/model:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/nodeup:go_default_library", "//pkg/pki:go_default_library", "//upup/pkg/fi:go_default_library", ], diff --git a/nodeup/pkg/model/networking/cilium.go b/nodeup/pkg/model/networking/cilium.go index 6f1d05c15eb09..8d7b6418290a8 100644 --- a/nodeup/pkg/model/networking/cilium.go +++ b/nodeup/pkg/model/networking/cilium.go @@ -127,27 +127,13 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error name := "etcd-client-cilium" dir := "/etc/kubernetes/pki/cilium" signer := "etcd-clients-ca-cilium" - if b.UseKopsControllerForNodeBootstrap() && !b.IsMaster { - cert, key := b.GetBootstrapCert(name) - - c.AddTask(&nodetasks.File{ - Path: filepath.Join(dir, name+".crt"), - Contents: cert, - Type: nodetasks.FileType_File, - Mode: fi.String("0644"), - BeforeServices: []string{"kubelet.service"}, - }) - - c.AddTask(&nodetasks.File{ - Path: filepath.Join(dir, name+".key"), - Contents: key, - Type: nodetasks.FileType_File, - Mode: fi.String("0400"), - BeforeServices: []string{"kubelet.service"}, - }) - - return b.BuildCertificateTask(c, signer, filepath.Join(dir, "etcd-ca.crt"), nil) - } else { + c.AddTask(&nodetasks.File{ + Path: filepath.Join(dir, "etcd-ca.crt"), + Contents: fi.NewStringResource(b.NodeupConfig.CAs[signer]), + Type: nodetasks.FileType_File, + Mode: fi.String("0600"), + }) + if b.IsMaster { issueCert := &nodetasks.IssueCert{ Name: name, Signer: signer, @@ -157,6 +143,30 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error }, } c.AddTask(issueCert) - return issueCert.AddFileTasks(c, dir, name, "etcd-ca", nil) + return issueCert.AddFileTasks(c, dir, name, "", nil) + } else { + if b.UseKopsControllerForNodeBootstrap() { + cert, key := b.GetBootstrapCert(name) + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(dir, name+".crt"), + Contents: cert, + Type: nodetasks.FileType_File, + Mode: fi.String("0644"), + BeforeServices: []string{"kubelet.service"}, + }) + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(dir, name+".key"), + Contents: key, + Type: nodetasks.FileType_File, + Mode: fi.String("0400"), + BeforeServices: []string{"kubelet.service"}, + }) + + return nil + } else { + return b.BuildCertificatePairTask(c, name, dir, name, nil, []string{"kubelet.service"}) + } } } diff --git a/nodeup/pkg/model/networking/cilium_test.go b/nodeup/pkg/model/networking/cilium_test.go index 1ea7bec6ea0e9..971183792961f 100644 --- a/nodeup/pkg/model/networking/cilium_test.go +++ b/nodeup/pkg/model/networking/cilium_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/kops/nodeup/pkg/model" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/pki" "k8s.io/kops/upup/pkg/fi" ) @@ -48,6 +49,11 @@ func TestCiliumBuilder(t *testing.T) { }, }, }, + NodeupConfig: &nodeup.Config{ + CAs: map[string]string{ + "etcd-clients-ca-cilium": "-----BEGIN CERTIFICATE-----\nMIIBbjCCARigAwIBAgIMFnbWaYo6t3AwKQtWMA0GCSqGSIb3DQEBCwUAMBgxFjAU\nBgNVBAMTDWNuPWt1YmVybmV0ZXMwHhcNMjEwNDE2MDMzNDI0WhcNMzEwNDE2MDMz\nNDI0WjAYMRYwFAYDVQQDEw1jbj1rdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQAD\nSwAwSAJBANLVh1dSDxJ5EcCd36av7++6+sDKqEm2GAzKIwOlfvPsm+pT+pClr51s\nd1m7V16nhWE6lhWjtsiMF8Q32+P5XZkCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIaNS7TlHC6K0r8yWYM1wExengDq\nMA0GCSqGSIb3DQEBCwUAA0EAoxha8yD6JLJcog/EOMdc5BpVPupQ/0FyO38Mb3l9\n0N7uZle0Tz1FQuadRtouySj37iq9nIxEeTh03Q52hNcl3A==\n-----END CERTIFICATE-----\n", + }, + }, HasAPIServer: true, KeyStore: &fakeKeyStore{}, IsMaster: true, diff --git a/pkg/model/awsmodel/autoscalinggroup_test.go b/pkg/model/awsmodel/autoscalinggroup_test.go index 6070996ce18be..b2eea736cce1a 100644 --- a/pkg/model/awsmodel/autoscalinggroup_test.go +++ b/pkg/model/awsmodel/autoscalinggroup_test.go @@ -70,6 +70,11 @@ func TestRootVolumeOptimizationFlag(t *testing.T) { }, BootstrapScriptBuilder: &model.BootstrapScriptBuilder{ Lifecycle: fi.LifecycleSync, + Cluster: &kops.Cluster{ + Spec: kops.ClusterSpec{ + Networking: &kops.NetworkingSpec{}, + }, + }, }, Cluster: cluster, } @@ -159,6 +164,7 @@ func TestAPIServerAdditionalSecurityGroupsWithNLB(t *testing.T) { }, BootstrapScriptBuilder: &model.BootstrapScriptBuilder{ Lifecycle: fi.LifecycleSync, + Cluster: cluster, }, Cluster: cluster, } diff --git a/pkg/model/azuremodel/vmscaleset_test.go b/pkg/model/azuremodel/vmscaleset_test.go index 38f8cf86a1b6d..21f6adfa2d765 100644 --- a/pkg/model/azuremodel/vmscaleset_test.go +++ b/pkg/model/azuremodel/vmscaleset_test.go @@ -35,6 +35,11 @@ func TestVMScaleSetModelBuilder_Build(t *testing.T) { AzureModelContext: newTestAzureModelContext(), BootstrapScriptBuilder: &model.BootstrapScriptBuilder{ Lifecycle: fi.LifecycleSync, + Cluster: &kops.Cluster{ + Spec: kops.ClusterSpec{ + Networking: &kops.NetworkingSpec{}, + }, + }, }, } c := &fi.ModelBuilderContext{ diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 8f6267428253b..cf238763d6e9d 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -29,6 +29,7 @@ import ( "text/template" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/upup/pkg/fi/utils" "sigs.k8s.io/yaml" @@ -43,7 +44,7 @@ import ( ) type NodeUpConfigBuilder interface { - BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) + BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTasks map[string]*fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) } // BootstrapScriptBuilder creates the bootstrap script @@ -51,6 +52,7 @@ type BootstrapScriptBuilder struct { Lifecycle fi.Lifecycle NodeUpAssets map[architectures.Architecture]*mirrors.MirroredAsset NodeUpConfigBuilder NodeUpConfigBuilder + Cluster *kops.Cluster } type BootstrapScript struct { @@ -62,8 +64,8 @@ type BootstrapScript struct { // alternateNameTasks are tasks that contribute api-server IP addresses. alternateNameTasks []fi.HasAddress - // caTask holds the CA task, for dependency analysis. - caTask *fitasks.Keypair + // caTasks hold the CA tasks, for dependency analysis. + caTasks map[string]*fitasks.Keypair // nodeupConfig contains the nodeup config. nodeupConfig fi.TaskDependentResource @@ -91,7 +93,7 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context) (string } sort.Strings(alternateNames) - config, bootConfig, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames, b.caTask) + config, bootConfig, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames, b.caTasks) if err != nil { return "", err } @@ -209,11 +211,22 @@ func (b *BootstrapScript) buildEnvironmentVariables(cluster *kops.Cluster) (map[ // ResourceNodeUp generates and returns a nodeup (bootstrap) script from a // template file, substituting in specific env vars & cluster spec configuration func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *kops.InstanceGroup) (fi.Resource, error) { - caTaskObject, found := c.Tasks["Keypair/ca"] - if !found { - return nil, fmt.Errorf("keypair/ca task not found") + keypairs := []string{"ca"} + if model.UseCiliumEtcd(b.Cluster) { + keypairs = append(keypairs, "etcd-clients-ca-cilium") + if !model.UseKopsControllerForNodeBootstrap(b.Cluster) { + keypairs = append(keypairs, "etcd-client-cilium") + } + } + + caTasks := map[string]*fitasks.Keypair{} + for _, keypair := range keypairs { + caTaskObject, found := c.Tasks["Keypair/"+keypair] + if !found { + return nil, fmt.Errorf("keypair/%s task not found", keypair) + } + caTasks[keypair] = caTaskObject.(*fitasks.Keypair) } - caTask := caTaskObject.(*fitasks.Keypair) // Bastions can have AdditionalUserData, but if there isn't any skip this part if ig.IsBastion() && len(ig.Spec.AdditionalUserData) == 0 { @@ -229,7 +242,7 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k Lifecycle: b.Lifecycle, ig: ig, builder: b, - caTask: caTask, + caTasks: caTasks, } task.resource.Task = task task.nodeupConfig.Task = task @@ -258,7 +271,9 @@ func (b *BootstrapScript) GetDependencies(tasks map[string]fi.Task) []fi.Task { } } - deps = append(deps, b.caTask) + for _, task := range b.caTasks { + deps = append(deps, task) + } return deps } diff --git a/pkg/model/bootstrapscript_test.go b/pkg/model/bootstrapscript_test.go index 849b38052bb8a..f65a8d908fd06 100644 --- a/pkg/model/bootstrapscript_test.go +++ b/pkg/model/bootstrapscript_test.go @@ -64,7 +64,7 @@ type nodeupConfigBuilder struct { cluster *kops.Cluster } -func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) { +func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTasks map[string]*fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) { config, bootConfig := nodeup.NewConfig(n.cluster, ig) return config, bootConfig, nil } @@ -152,6 +152,7 @@ func TestBootstrapUserData(t *testing.T) { Hash: hashing.MustFromString("e525c28a65ff0ce4f95f9e730195b4e67fdcb15ceb1f36b5ad6921a8a4490c71"), }, }, + Cluster: cluster, } res, err := bs.ResourceNodeUp(c, group) @@ -256,6 +257,7 @@ func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole, fileAssetSpecRoles Port: 80, }, }, + Networking: &kops.NetworkingSpec{}, Hooks: []kops.HookSpec{ { ExecContainer: &kops.ExecContainerAction{ diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index 269f567b62e6f..f8b8411f23b67 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -142,12 +142,23 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error { } if etcdCluster.Name == "cilium" { - c.AddTask(&fitasks.Keypair{ + clientsCaCilium := &fitasks.Keypair{ Name: fi.String("etcd-clients-ca-cilium"), Lifecycle: b.Lifecycle, Subject: "cn=etcd-clients-ca-cilium", Type: "ca", - }) + } + c.AddTask(clientsCaCilium) + + if !b.UseKopsControllerForNodeBootstrap() { + c.AddTask(&fitasks.Keypair{ + Name: fi.String("etcd-client-cilium"), + Lifecycle: b.Lifecycle, + Subject: "cn=cilium", + Type: "client", + Signer: clientsCaCilium, + }) + } } } diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index d2d4deeabb0ae..4fcddc7d51cfe 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -623,7 +623,7 @@ func ReadableStatePaths(cluster *kops.Cluster, role Subject) ([]string, error) { // @check if cilium is enabled as the CNI provider and permit access to the cilium etc client TLS certificate by default // As long as the Cilium Etcd cluster exists, we should do this if networkingSpec.Cilium != nil && model.UseCiliumEtcd(cluster) { - paths = append(paths, "/pki/private/etcd-clients-ca-cilium/*") + paths = append(paths, "/pki/private/etcd-client-cilium/*") } } } diff --git a/pkg/model/openstackmodel/servergroup_test.go b/pkg/model/openstackmodel/servergroup_test.go index b76f0527da7a0..bd34d32d66acb 100644 --- a/pkg/model/openstackmodel/servergroup_test.go +++ b/pkg/model/openstackmodel/servergroup_test.go @@ -1012,7 +1012,7 @@ func createBuilderForCluster(cluster *kops.Cluster, instanceGroups []*kops.Insta type nodeupConfigBuilder struct { } -func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) { +func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTasks map[string]*fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) { return &nodeup.Config{}, &nodeup.BootConfig{}, nil } @@ -1031,6 +1031,9 @@ func RunGoldenTest(t *testing.T, basedir string, testCase serverGroupModelBuilde testutils.SetupMockOpenstack() clusterLifecycle := fi.LifecycleSync + if testCase.cluster.Spec.Networking == nil { + testCase.cluster.Spec.Networking = &kops.NetworkingSpec{} + } bootstrapScriptBuilder := &model.BootstrapScriptBuilder{ NodeUpConfigBuilder: &nodeupConfigBuilder{}, NodeUpAssets: map[architectures.Architecture]*mirrors.MirroredAsset{ @@ -1043,6 +1046,7 @@ func RunGoldenTest(t *testing.T, basedir string, testCase serverGroupModelBuilde Hash: hashing.MustFromString("e525c28a65ff0ce4f95f9e730195b4e67fdcb15ceb1f36b5ad6921a8a4490c71"), }, }, + Cluster: testCase.cluster, } builder := createBuilderForCluster(testCase.cluster, testCase.instanceGroups, clusterLifecycle, bootstrapScriptBuilder) diff --git a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml index 4be859bce0d18..c79bf211cc227 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml @@ -249,7 +249,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumadvancedexamplec ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master - NodeupConfigHash: bRfnIu1P/bOf6azf9b2o5Dl8hjdt+rlKL9GJwuTgmLs= + NodeupConfigHash: 58rDov0rcoNX88fBs1Bs0tVQEjTzd5sXhdXUXLnPc0Q= __EOF_KUBE_ENV @@ -421,7 +421,7 @@ Resources.AWSEC2LaunchTemplatenodesprivateciliumadvancedexamplecom.Properties.La ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com InstanceGroupName: nodes InstanceGroupRole: Node - NodeupConfigHash: TcEhf4AmLmWCRh+0r9RC1PxaJEXfff4jHU1Pv5/MKDA= + NodeupConfigHash: UoM0gz2kOOat/kWK4++dCsgM+K/HwYqKpbEFkuAO0vs= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data index 574f7cc06803f..659e5af83485a 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data @@ -247,7 +247,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master -NodeupConfigHash: bRfnIu1P/bOf6azf9b2o5Dl8hjdt+rlKL9GJwuTgmLs= +NodeupConfigHash: 58rDov0rcoNX88fBs1Bs0tVQEjTzd5sXhdXUXLnPc0Q= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data index faa3387e811d7..7277c276ec8e9 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data @@ -163,7 +163,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com InstanceGroupName: nodes InstanceGroupRole: Node -NodeupConfigHash: TcEhf4AmLmWCRh+0r9RC1PxaJEXfff4jHU1Pv5/MKDA= +NodeupConfigHash: UoM0gz2kOOat/kWK4++dCsgM+K/HwYqKpbEFkuAO0vs= __EOF_KUBE_ENV diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index e355d8bc5f2f1..0b50b88f310c8 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -495,6 +495,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { Lifecycle: clusterLifecycle, NodeUpConfigBuilder: configBuilder, NodeUpAssets: c.NodeUpAssets, + Cluster: cluster, } { @@ -1296,7 +1297,7 @@ func newNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui } // BuildConfig returns the NodeUp config and auxiliary config. -func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) { +func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTasks map[string]*fitasks.Keypair) (*nodeup.Config, *nodeup.BootConfig, error) { cluster := n.cluster if ig == nil { @@ -1321,15 +1322,26 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit } } - cas, err := fi.ResourceAsString(caTask.Certificates()) + err := getTasksCertificate(caTasks, fi.CertificateIDCA, config) if err != nil { - // CA task may not have run yet; we'll retry - return nil, nil, fmt.Errorf("failed to read CA certificates: %w", err) + return nil, nil, err + } + if caTasks["etcd-clients-ca-cilium"] != nil { + err := getTasksCertificate(caTasks, "etcd-clients-ca-cilium", config) + if err != nil { + return nil, nil, err + } } - config.CAs[fi.CertificateIDCA] = cas if isMaster { - config.KeypairIDs[fi.CertificateIDCA] = caTask.Keyset().Primary.Id + config.KeypairIDs[fi.CertificateIDCA] = caTasks[fi.CertificateIDCA].Keyset().Primary.Id + if caTasks["etcd-clients-ca-cilium"] != nil { + config.KeypairIDs["etcd-clients-ca-cilium"] = caTasks["etcd-clients-ca-cilium"].Keyset().Primary.Id + } + } else { + if caTasks["etcd-client-cilium"] != nil { + config.KeypairIDs["etcd-client-cilium"] = caTasks["etcd-client-cilium"].Keyset().Primary.Id + } } if isMaster || useGossip { @@ -1405,6 +1417,16 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit return config, bootConfig, nil } +func getTasksCertificate(caTasks map[string]*fitasks.Keypair, name string, config *nodeup.Config) error { + cas, err := fi.ResourceAsString(caTasks[name].Certificates()) + if err != nil { + // CA task may not have run yet; we'll retry + return fmt.Errorf("failed to read %s certificates: %w", name, err) + } + config.CAs[name] = cas + return nil +} + func buildContainerdConfig(cluster *kops.Cluster) string { if cluster.Spec.ContainerRuntime != "containerd" { return ""