Merge pull request #11809 from johngmyers/rotate-5

Include multiple cluster CAs in trust stores

cmd/kops-controller/pkg/server/keystore.go

@@ -48,7 +48,7 @@ func newKeystore(basePath string, cas []string) (pki.Keystore, error) {
 		keys: map[string]keystoreEntry{},
 	}
 	for _, name := range cas {
-		certBytes, err := ioutil.ReadFile(path.Join(basePath, name+".pem"))
+		certBytes, err := ioutil.ReadFile(path.Join(basePath, name+".crt"))
 		if err != nil {
 			return nil, fmt.Errorf("reading %q certificate: %v", name, err)
 		}
@@ -57,7 +57,7 @@ func newKeystore(basePath string, cas []string) (pki.Keystore, error) {
 			return nil, fmt.Errorf("parsing %q certificate: %v", name, err)
 		}
 
-		keyBytes, err := ioutil.ReadFile(path.Join(basePath, name+"-key.pem"))
+		keyBytes, err := ioutil.ReadFile(path.Join(basePath, name+".key"))
 		if err != nil {
 			return nil, fmt.Errorf("reading %q key: %v", name, err)
 		}

cmd/kops/integration_test.go

@@ -63,10 +63,8 @@ type integrationTest struct {
 	expectServiceAccountRolePolicies []string
 	lifecycleOverrides               []string
 	sshKey                           bool
-	// caKey is true if we should use a provided ca.crt & ca.key as our CA
-	caKey           bool
-	jsonOutput      bool
-	bastionUserData bool
+	jsonOutput                       bool
+	bastionUserData                  bool
 	// nth is true if we should check for files created by nth queue processor add on
 	nth bool
 }
@@ -82,12 +80,6 @@ func newIntegrationTest(clusterName, srcDir string) *integrationTest {
 	}
 }
 
-// withCAKey indicates that we should use a fixed ca.crt & ca.key from the source directory as our CA.
-// This is needed when the CA is exposed, for example when using AWS WebIdentity federation.
-func (i *integrationTest) withCAKey() *integrationTest {
-	i.caKey = true
-	return i
-}
 func (i *integrationTest) withVersion(version string) *integrationTest {
 	i.version = version
 	return i
@@ -312,7 +304,6 @@ func TestDiscoveryFeatureGate(t *testing.T) {
 
 	// We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
 	newIntegrationTest("minimal.example.com", "public-jwks-apiserver").
-		withCAKey().
 		withServiceAccountRole("dns-controller.kube-system", true).
 		runTestTerraformAWS(t)
 
@@ -322,7 +313,6 @@ func TestVFSServiceAccountIssuerDiscovery(t *testing.T) {
 
 	// We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
 	newIntegrationTest("minimal.example.com", "vfs-said").
-		withCAKey().
 		runTestTerraformAWS(t)
 
 }
@@ -337,7 +327,6 @@ func TestAWSLBController(t *testing.T) {
 
 	// We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
 	newIntegrationTest("minimal.example.com", "aws-lb-controller").
-		withCAKey().
 		withServiceAccountRole("dns-controller.kube-system", true).
 		withServiceAccountRole("aws-load-balancer-controller.kube-system", true).
 		runTestTerraformAWS(t)
@@ -485,18 +474,30 @@ func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarn
 		}
 	}
 
-	if i.caKey {
+	{
 		options := &CreateKeypairCaOptions{}
 		options.ClusterName = i.clusterName
-		options.PrivateKeyPath = path.Join(i.srcDir, "ca.key")
-		options.CertPath = path.Join(i.srcDir, "ca.crt")
+		options.PrivateKeyPath = path.Join(i.srcDir, "../ca.key")
+		options.CertPath = path.Join(i.srcDir, "../ca.crt")
 		options.Primary = true
 
 		err := RunCreateKeypairCa(ctx, factory, &stdout, options)
 		if err != nil {
 			t.Fatalf("error running %q create CA keypair: %v", inputYAML, err)
 		}
 	}
+	{
+		options := &CreateKeypairCaOptions{}
+		options.ClusterName = i.clusterName
+		options.PrivateKeyPath = path.Join(i.srcDir, "../ca-next.key")
+		options.CertPath = path.Join(i.srcDir, "../ca-next.crt")
+		options.Primary = false
+
+		err := RunCreateKeypairCa(ctx, factory, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running %q create next CA keypair: %v", inputYAML, err)
+		}
+	}
 
 	{
 		options := &UpdateClusterOptions{}
@@ -758,6 +759,31 @@ func (i *integrationTest) runTestCloudformation(t *testing.T) {
 		}
 	}
 
+	{
+		options := &CreateKeypairCaOptions{}
+		options.ClusterName = i.clusterName
+		options.PrivateKeyPath = path.Join(i.srcDir, "../ca.key")
+		options.CertPath = path.Join(i.srcDir, "../ca.crt")
+		options.Primary = true
+
+		err := RunCreateKeypairCa(ctx, factory, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running %q create CA keypair: %v", inputYAML, err)
+		}
+	}
+	{
+		options := &CreateKeypairCaOptions{}
+		options.ClusterName = i.clusterName
+		options.PrivateKeyPath = path.Join(i.srcDir, "../ca-next.key")
+		options.CertPath = path.Join(i.srcDir, "../ca-next.crt")
+		options.Primary = false
+
+		err := RunCreateKeypairCa(ctx, factory, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running %q create next CA keypair: %v", inputYAML, err)
+		}
+	}
+
 	{
 		options := &UpdateClusterOptions{}
 		options.InitDefaults()

nodeup/pkg/model/bootstrap_client.go

@@ -51,11 +51,6 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
-	cert, err := b.GetCert(fi.CertificateIDCA)
-	if err != nil {
-		return err
-	}
-
 	baseURL := url.URL{
 		Scheme: "https",
 		Host:   net.JoinHostPort("kops-controller.internal."+b.Cluster.ObjectMeta.Name, strconv.Itoa(wellknownports.KopsControllerPort)),
@@ -64,7 +59,7 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error {
 
 	bootstrapClient := &nodetasks.KopsBootstrapClient{
 		Authenticator: authenticator,
-		CA:            cert,
+		CAs:           []byte(b.NodeupConfig.CAs[fi.CertificateIDCA]),
 		BaseURL:       baseURL,
 	}
 

nodeup/pkg/model/context.go

@@ -256,16 +256,11 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model
 	if c.UseKopsControllerForNodeBootstrap() {
 		cert, key := c.GetBootstrapCert(name)
 
-		ca, err := c.GetCert(fi.CertificateIDCA)
-		if err != nil {
-			return nil, err
-		}
-
 		kubeConfig := &nodetasks.KubeConfig{
 			Name: name,
 			Cert: cert,
 			Key:  key,
-			CA:   fi.NewBytesResource(ca),
+			CA:   fi.NewStringResource(c.NodeupConfig.CAs[fi.CertificateIDCA]),
 		}
 		if c.HasAPIServer {
 			// @note: use https even for local connections, so we can turn off the insecure port
@@ -274,24 +269,14 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model
 			kubeConfig.ServerURL = "https://" + c.Cluster.Spec.MasterInternalName
 		}
 
-		err = ctx.EnsureTask(kubeConfig)
+		err := ctx.EnsureTask(kubeConfig)
 		if err != nil {
 			return nil, err
 		}
 
 		return kubeConfig.GetConfig(), nil
 	} else {
-		ca, err := c.GetCert(fi.CertificateIDCA)
-		if err != nil {
-			return nil, err
-		}
-
-		cert, err := c.GetCert(name)
-		if err != nil {
-			return nil, err
-		}
-
-		key, err := c.GetPrivateKey(name)
+		cert, key, err := c.GetPrimaryKeypair(name)
 		if err != nil {
 			return nil, err
 		}
@@ -300,7 +285,7 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model
 			Name: name,
 			Cert: fi.NewBytesResource(cert),
 			Key:  fi.NewBytesResource(key),
-			CA:   fi.NewBytesResource(ca),
+			CA:   fi.NewStringResource(c.NodeupConfig.CAs[fi.CertificateIDCA]),
 		}
 		if c.HasAPIServer {
 			// @note: use https even for local connections, so we can turn off the insecure port
@@ -419,15 +404,52 @@ func (c *NodeupModelContext) KubectlPath() string {
 }
 
 // BuildCertificatePairTask creates the tasks to create the certificate and private key files.
-func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string, owner *string) error {
-	certificateName := filepath.Join(path, filename+".pem")
-	keyName := filepath.Join(path, filename+"-key.pem")
+func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, name, path, filename string, owner *string) error {
+	p := filepath.Join(path, filename)
+	if !filepath.IsAbs(p) {
+		p = filepath.Join(c.PathSrvKubernetes(), p)
+	}
+
+	certificate, privateKey, err := c.KeyStore.FindPrimaryKeypair(name)
+	if err != nil {
+		return err
+	}
+
+	if certificate == nil {
+		return fmt.Errorf("certificate %q not found", name)
+	}
 
-	if err := c.BuildCertificateTask(ctx, key, certificateName, owner); err != nil {
+	cert, err := certificate.AsString()
+	if err != nil {
 		return err
 	}
 
-	return c.BuildPrivateKeyTask(ctx, key, keyName, owner)
+	ctx.AddTask(&nodetasks.File{
+		Path:     p + ".crt",
+		Contents: fi.NewStringResource(cert),
+		Type:     nodetasks.FileType_File,
+		Mode:     s("0600"),
+		Owner:    owner,
+	})
+
+	if privateKey == nil {
+		return fmt.Errorf("private key %q not found", name)
+	}
+
+	key, err := privateKey.AsString()
+	if err != nil {
+		return err
+	}
+
+	ctx.AddTask(&nodetasks.File{
+		Path:     p + ".key",
+		Contents: fi.NewStringResource(key),
+		Type:     nodetasks.FileType_File,
+		Mode:     s("0600"),
+		Owner:    owner,
+	})
+
+	return nil
 }
 
 // BuildCertificateTask builds a task to create a certificate file.
@@ -569,30 +591,30 @@ func EvaluateHostnameOverride(hostnameOverride string) (string, error) {
 	return *(result.Reservations[0].Instances[0].PrivateDnsName), nil
 }
 
-// GetCert is a helper method to retrieve a certificate from the store
-func (c *NodeupModelContext) GetCert(name string) ([]byte, error) {
-	cert, err := c.KeyStore.FindCert(name)
+// GetPrimaryKeypair is a helper method to retrieve a primary keypair from the store
+func (c *NodeupModelContext) GetPrimaryKeypair(name string) (cert []byte, key []byte, err error) {
+	certificate, privateKey, err := c.KeyStore.FindPrimaryKeypair(name)
 	if err != nil {
-		return []byte{}, fmt.Errorf("error fetching certificate: %v from keystore: %v", name, err)
+		return nil, nil, fmt.Errorf("error fetching certificate: %v from keystore: %v", name, err)
 	}
-	if cert == nil {
-		return []byte{}, fmt.Errorf("unable to find certificate: %s", name)
+	if certificate == nil {
+		return nil, nil, fmt.Errorf("unable to find certificate: %s", name)
+	}
+	if privateKey == nil {
+		return nil, nil, fmt.Errorf("unable to find key: %s", name)
 	}
 
-	return cert.AsBytes()
-}
-
-// GetPrivateKey is a helper method to retrieve a private key from the store
-func (c *NodeupModelContext) GetPrivateKey(name string) ([]byte, error) {
-	key, err := c.KeyStore.FindPrivateKey(name)
+	cert, err = certificate.AsBytes()
 	if err != nil {
-		return []byte{}, fmt.Errorf("error fetching private key: %v from keystore: %v", name, err)
+		return nil, nil, err
 	}
-	if key == nil {
-		return []byte{}, fmt.Errorf("unable to find private key: %s", name)
+
+	key, err = privateKey.AsBytes()
+	if err != nil {
+		return nil, nil, err
 	}
 
-	return key.AsBytes()
+	return cert, key, nil
 }
 
 func (b *NodeupModelContext) AddCNIBinAssets(c *fi.ModelBuilderContext, assetNames []string) error {

nodeup/pkg/model/fakes_test.go

@@ -37,7 +37,12 @@ type fakeCAStore struct {
 var _ fi.CAStore = &fakeCAStore{}
 
 func (k fakeCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
-	panic("fakeCAStore does not implement FindPrimaryKeypair")
+	keyset, err := k.FindKeyset(name)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil
 }
 
 func (k fakeCAStore) FindKeyset(name string) (*fi.Keyset, error) {
@@ -94,10 +99,6 @@ func (k fakeCAStore) FindCertificatePool(name string) (*fi.CertificatePool, erro
 	panic("fakeCAStore does not implement FindCertificatePool")
 }
 
-func (k fakeCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) {
-	panic("fakeCAStore does not implement FindCertificateKeyset")
-}
-
 func (k fakeCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) {
 	primaryId := k.privateKeysets[name].Spec.PrimaryId
 	for _, item := range k.privateKeysets[name].Spec.Keys {
@@ -108,10 +109,6 @@ func (k fakeCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) {
 	return nil, nil
 }
 
-func (k fakeCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) {
-	return k.privateKeysets[name], nil
-}
-
 func (k fakeCAStore) FindCert(name string) (*pki.Certificate, error) {
 	return k.certs[name], nil
 }