Bug 1877374: Render bootstrap certificates

bindata/bootkube/bootstrap-manifests/etcd-member-pod.yaml

@@ -6,58 +6,6 @@ metadata:
   labels:
     k8s-app: etcd
 spec:
-  initContainers:
-  - name: certs
-    image: "{{.Images.KubeClientAgent}}"
-    command:
-    - /bin/sh
-    - -c
-    - |
-      #!/bin/sh
-      set -euxo pipefail
-
-      [ -e /etc/ssl/etcd/system:etcd-server:{{ .Hostname }}.crt -a \
-        -e /etc/ssl/etcd/system:etcd-server:{{ .Hostname }}.key ] || \
-        kube-client-agent \
-          request \
-            --kubeconfig=/etc/kubernetes/kubeconfig \
-            --orgname=system:etcd-servers \
-            --assetsdir=/etc/ssl/etcd \
-            --dnsnames={{.EtcdServerCertDNSNames}} \
-            --commonname=system:etcd-server:{{ .Hostname }} \
-            --ipaddrs={{ .EtcdAddress.EscapedBootstrapIP }},{{ .EtcdAddress.LocalHost }} \
-
-      [ -e /etc/ssl/etcd/system:etcd-peer:{{ .Hostname }}.crt -a \
-        -e /etc/ssl/etcd/system:etcd-peer:{{ .Hostname }}.key ] || \
-        kube-client-agent \
-          request \
-            --kubeconfig=/etc/kubernetes/kubeconfig \
-            --orgname=system:etcd-peers \
-            --assetsdir=/etc/ssl/etcd \
-            --dnsnames={{.EtcdPeerCertDNSNames}} \
-            --commonname=system:etcd-peer:{{ .Hostname }} \
-            --ipaddrs={{ .EtcdAddress.EscapedBootstrapIP }} \
-
-      [ -e /etc/ssl/etcd/system:etcd-metric:{{ .Hostname }}.crt -a \
-        -e /etc/ssl/etcd/system:etcd-metric:{{ .Hostname }}.key ] || \
-        kube-client-agent \
-          request \
-            --kubeconfig=/etc/kubernetes/kubeconfig \
-            --orgname=system:etcd-metrics \
-            --assetsdir=/etc/ssl/etcd \
-            --dnsnames={{.EtcdServerCertDNSNames}} \
-            --commonname=system:etcd-metric:{{ .Hostname }} \
-            --ipaddrs={{ .EtcdAddress.EscapedBootstrapIP }} \
-    terminationMessagePolicy: FallbackToLogsOnError
-    securityContext:
-      privileged: true
-    volumeMounts:
-    - name: discovery
-      mountPath: /run/etcd/
-    - name: certs
-      mountPath: /etc/ssl/etcd/
-    - name: kubeconfig
-      mountPath: /etc/kubernetes/kubeconfig
   containers:
   - name: etcd-member
     image: {{ .Images.Etcd }}
@@ -70,13 +18,13 @@ spec:
 
       exec etcd \
         --initial-advertise-peer-urls=https://{{ .EtcdAddress.EscapedBootstrapIP }}:2380 \
-        --cert-file=/etc/ssl/etcd/system:etcd-server:{{ .Hostname }}.crt \
-        --key-file=/etc/ssl/etcd/system:etcd-server:{{ .Hostname }}.key \
-        --trusted-ca-file=/etc/ssl/etcd/ca.crt \
+        --cert-file=/etc/ssl/etcd/etcd-all-serving/etcd-serving-{{ .Hostname }}.crt \
+        --key-file=/etc/ssl/etcd/etcd-all-serving/etcd-serving-{{ .Hostname }}.key \
+        --trusted-ca-file=/etc/openshift-tls/etcd-ca-bundle.crt \
         --client-cert-auth=true \
-        --peer-cert-file=/etc/ssl/etcd/system:etcd-peer:{{ .Hostname }}.crt \
-        --peer-key-file=/etc/ssl/etcd/system:etcd-peer:{{ .Hostname }}.key \
-        --peer-trusted-ca-file=/etc/ssl/etcd/ca.crt \
+        --peer-cert-file=/etc/ssl/etcd/etcd-all-peer/etcd-peer-{{ .Hostname }}.crt \
+        --peer-key-file=/etc/ssl/etcd/etcd-all-peer/etcd-peer-{{ .Hostname }}.key \
+        --peer-trusted-ca-file=/etc/openshift-tls/etcd-ca-bundle.crt \
         --peer-client-cert-auth=true \
         --advertise-client-urls=https://{{ .EtcdAddress.EscapedBootstrapIP }}:2379 \
         --listen-client-urls=https://{{ .EtcdAddress.ListenClient }} \
@@ -94,6 +42,8 @@ spec:
       mountPath: /run/etcd/
     - name: certs
       mountPath: /etc/ssl/etcd/
+    - name: openshift-tls
+      mountPath: /etc/openshift-tls
     - name: data-dir
       mountPath: /var/lib/etcd/
     - name: conf
@@ -108,49 +58,20 @@ spec:
     - name: server
       containerPort: 2379
       protocol: TCP
-  - name: etcd-metrics
-    image: {{ .Images.Etcd }}
-    command:
-    - /bin/sh
-    - -c
-    - |
-      #!/bin/sh
-      set -euo pipefail
-
-      exec etcd grpc-proxy start \
-        --endpoints https://{{ .EtcdAddress.EscapedBootstrapIP }}:9978 \
-        --metrics-addr https://{{ .EtcdAddress.ListenMetricProxy }}\
-        --listen-addr {{ .EtcdAddress.LocalHost }}:9977 \
-        --key /etc/ssl/etcd/system:etcd-peer:{{ .Hostname }}.key \
-        --key-file /etc/ssl/etcd/system:etcd-metric:{{ .Hostname }}.key \
-        --cert /etc/ssl/etcd/system:etcd-peer:{{ .Hostname }}.crt \
-        --cert-file /etc/ssl/etcd/system:etcd-metric:{{ .Hostname }}.crt \
-        --cacert /etc/ssl/etcd/ca.crt \
-        --trusted-ca-file /etc/ssl/etcd/metric-ca.crt \
-    resources:
-      requests:
-        cpu: 5m
-    terminationMessagePolicy: FallbackToLogsOnError
-    securityContext:
-      privileged: true
-    volumeMounts:
-    - name: discovery
-      mountPath: /run/etcd/
-    - name: certs
-      mountPath: /etc/ssl/etcd/
-    ports:
-    - name: metric
-      containerPort: 9979
-      protocol: TCP
   hostNetwork: true
   priorityClassName: system-node-critical
   tolerations:
   - operator: "Exists"
   restartPolicy: Always
   volumes:
+  # TODO: update installer to copy the certs into /etc/kubernetes/static-pod-resources/etcd-member
   - name: certs
     hostPath:
-      path: /etc/kubernetes/static-pod-resources/etcd-member
+      path: /opt/openshift/etcd-bootstrap/bootstrap-manifests/secrets
+  # TODO: update installer to copy the ca into /etc/kubernetes/static-pod-resources/etcd-member
+  - name: openshift-tls
+    hostPath:
+      path: /opt/openshift/tls
   - name: kubeconfig
     hostPath:
       path: /etc/kubernetes/kubeconfig

pkg/cmd/render/render.go

@@ -1,19 +1,22 @@
 package render
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
 
 	"github.com/openshift/cluster-etcd-operator/pkg/dnshelpers"
 	"github.com/openshift/cluster-etcd-operator/pkg/operator/etcd_assets"
+	"github.com/openshift/cluster-etcd-operator/pkg/tlshelpers"
 
 	"github.com/ghodss/yaml"
 	configv1 "github.com/openshift/api/config/v1"
@@ -34,7 +37,8 @@ type renderOpts struct {
 
 	errOut                   io.Writer
 	etcdCAFile               string
-	etcdMetricCAFile         string
+	etcdCAKeyFile            string
+	unusedEtcdMetricCAFile   string
 	etcdDiscoveryDomain      string
 	etcdImage                string
 	clusterEtcdOperatorImage string
@@ -122,7 +126,9 @@ func (r *renderOpts) AddFlags(fs *pflag.FlagSet) {
 	r.generic.AddFlags(fs)
 
 	fs.StringVar(&r.etcdCAFile, "etcd-ca", r.etcdCAFile, "path to etcd CA certificate")
-	fs.StringVar(&r.etcdMetricCAFile, "etcd-metric-ca", r.etcdMetricCAFile, "path to etcd metric CA certificate")
+	fs.StringVar(&r.etcdCAKeyFile, "etcd-ca-key", "/assets/tls/etcd-signer.key", "path to etcd CA certificate key")
+	// deprecated
+	fs.StringVar(&r.unusedEtcdMetricCAFile, "etcd-metric-ca", r.unusedEtcdMetricCAFile, "path to etcd metric CA certificate")
 	fs.StringVar(&r.etcdImage, "manifest-etcd-image", r.etcdImage, "etcd manifest image")
 	fs.StringVar(&r.clusterEtcdOperatorImage, "manifest-cluster-etcd-operator-image", r.clusterEtcdOperatorImage, "cluster-etcd-operator manifest image")
 	fs.StringVar(&r.kubeClientAgentImage, "manifest-kube-client-agent-image", r.kubeClientAgentImage, "kube-client-agent manifest image")
@@ -146,9 +152,6 @@ func (r *renderOpts) Validate() error {
 	if len(r.etcdCAFile) == 0 {
 		return errors.New("missing required flag: --etcd-ca")
 	}
-	if len(r.etcdMetricCAFile) == 0 {
-		return errors.New("missing required flag: --etcd-metric-ca")
-	}
 	if len(r.etcdImage) == 0 {
 		return errors.New("missing required flag: --manifest-etcd-image")
 	}
@@ -320,9 +323,59 @@ func (r *renderOpts) Run() error {
 		return err
 	}
 
+	bootstrapManifestsDir := filepath.Join(r.generic.AssetOutputDir, "bootstrap-manifests")
+
+	caCertData, err := ioutil.ReadFile(r.etcdCAFile)
+	if err != nil {
+		return fmt.Errorf("failed to read CA cert: %w", err)
+	}
+
+	caKeyData, err := ioutil.ReadFile(r.etcdCAKeyFile)
+	if err != nil {
+		return fmt.Errorf("failed to read CA key: %w", err)
+	}
+
+	certData, keyData, err := tlshelpers.CreateServerCertKey(caCertData, caKeyData, []string{templateData.BootstrapIP})
+	if err != nil {
+		return err
+	}
+	err = writeCertKeyFiles(bootstrapManifestsDir, tlshelpers.EtcdAllServingSecretName, tlshelpers.GetServingSecretNameForNode(templateData.Hostname), certData, keyData)
+	if err != nil {
+		return err
+	}
+
+	certData, keyData, err = tlshelpers.CreatePeerCertKey(caCertData, caKeyData, []string{templateData.BootstrapIP})
+	if err != nil {
+		return err
+	}
+	err = writeCertKeyFiles(bootstrapManifestsDir, tlshelpers.EtcdAllPeerSecretName, tlshelpers.GetPeerClientSecretNameForNode(templateData.Hostname), certData, keyData)
+	if err != nil {
+		return err
+	}
+
 	return WriteFiles(&r.generic, &templateData.FileConfig, templateData)
 }
 
+func writeCertKeyFiles(outputDir, allSecretName, nodeSecretName string, certData, keyData *bytes.Buffer) error {
+	dir := path.Join(outputDir, "secrets", allSecretName)
+
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		return fmt.Errorf("failed to create %s directory: %w", allSecretName, err)
+	}
+
+	err = ioutil.WriteFile(path.Join(dir, nodeSecretName+".crt"), certData.Bytes(), 0600)
+	if err != nil {
+		return fmt.Errorf("failed to write %s cert: %w", allSecretName, err)
+	}
+	err = ioutil.WriteFile(path.Join(dir, nodeSecretName+".key"), keyData.Bytes(), 0600)
+	if err != nil {
+		return fmt.Errorf("failed to write %s key: %w", allSecretName, err)
+	}
+
+	return nil
+}
+
 func (t *TemplateData) setBootstrapIP(machineCIDR string, ipv6 bool, excludedIPs []string) error {
 	ip, err := defaultBootstrapIPLocator.getBootstrapIP(ipv6, machineCIDR, excludedIPs)
 	if err != nil {

pkg/cmd/render/render_test.go

@@ -2,12 +2,19 @@ package render
 
 import (
 	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
 	"io"
 	"io/ioutil"
+	"math/big"
 	"net"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/openshift/cluster-etcd-operator/pkg/cmd/render/options"
 )
@@ -243,6 +250,56 @@ func testRender(tc *testConfig) {
 		tc.t.Fatal(err)
 	}
 
+	ca := &x509.Certificate{
+		SerialNumber: big.NewInt(2020),
+		Subject: pkix.Name{
+			Organization: []string{"Red Hat"},
+			Country:      []string{"US"},
+			Province:     []string{""},
+			Locality:     []string{"Raleigh"},
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		IsCA:                  true,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		tc.t.Fatal(err)
+	}
+	caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		tc.t.Fatal(err)
+	}
+	caPEM := new(bytes.Buffer)
+	pem.Encode(caPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caBytes,
+	})
+	caPrivKeyPEM := new(bytes.Buffer)
+	pem.Encode(caPrivKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey),
+	})
+	caFile, err := ioutil.TempFile(dir, "etcd-ca.crt")
+	if err != nil {
+		tc.t.Fatal(err)
+	}
+	defer caFile.Close()
+	if err := writeFile(caPEM.String(), caFile); err != nil {
+		tc.t.Fatal(err)
+	}
+	keyFile, err := ioutil.TempFile(dir, "etcd-ca.key")
+	if err != nil {
+		tc.t.Fatal(err)
+	}
+	defer keyFile.Close()
+	if err := writeFile(caPrivKeyPEM.String(), keyFile); err != nil {
+		tc.t.Fatal(err)
+	}
+
 	generic := options.GenericOptions{
 		AssetInputDir:    dir,
 		AssetOutputDir:   dir,
@@ -257,6 +314,8 @@ func testRender(tc *testConfig) {
 		networkConfigFile:    clusterConfigFile.Name(),
 		infraConfigFile:      infraConfigFile.Name(),
 		clusterConfigMapFile: clusterConfigMapFile.Name(),
+		etcdCAFile:           caFile.Name(),
+		etcdCAKeyFile:        keyFile.Name(),
 	}
 
 	if err := render.Run(); err != nil {