Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1186 from tjungblu/installer
ETCD-517: run certsignercontroller for bootstrap render
- Loading branch information
Showing
14 changed files
with
383 additions
and
288 deletions.
There are no files selected for viewing
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
10 changes: 0 additions & 10 deletions
10
bindata/bootkube/manifests/etcd-metric-serving-ca-configmap.yaml
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,69 +1,138 @@ | ||
package render | ||
|
||
import ( | ||
"crypto/x509/pkix" | ||
"time" | ||
|
||
"github.com/google/uuid" | ||
"github.com/openshift/library-go/pkg/crypto" | ||
"k8s.io/apiserver/pkg/authentication/user" | ||
"context" | ||
"fmt" | ||
operatorv1 "github.com/openshift/api/operator/v1" | ||
"github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers" | ||
"github.com/openshift/cluster-etcd-operator/pkg/operator/etcdcertsigner" | ||
"github.com/openshift/cluster-etcd-operator/pkg/operator/health" | ||
"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" | ||
"github.com/openshift/cluster-etcd-operator/pkg/operator/resourcesynccontroller" | ||
"github.com/openshift/cluster-etcd-operator/pkg/tlshelpers" | ||
"github.com/openshift/library-go/pkg/controller/factory" | ||
"github.com/openshift/library-go/pkg/operator/events" | ||
"github.com/openshift/library-go/pkg/operator/v1helpers" | ||
corev1 "k8s.io/api/core/v1" | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
"k8s.io/apimachinery/pkg/runtime" | ||
"k8s.io/client-go/kubernetes/fake" | ||
) | ||
|
||
// keyMaterial simplifies handling of the key data produced by createKeyMaterial | ||
type keyMaterial struct { | ||
caCert []byte | ||
caKey []byte | ||
caBundle []byte | ||
clientCert []byte | ||
clientKey []byte | ||
} | ||
// createCertSecrets will run the etcdcertsigner.EtcdCertSignerController once and collect all respective certs created. | ||
// The secrets will contain all signers, peer, serving and client certs. The configmaps contain all bundles. | ||
func createCertSecrets(nodes []*corev1.Node) ([]corev1.Secret, []corev1.ConfigMap, error) { | ||
var fakeObjs []runtime.Object | ||
for _, node := range nodes { | ||
fakeObjs = append(fakeObjs, node) | ||
} | ||
|
||
fakeKubeClient := fake.NewSimpleClientset(fakeObjs...) | ||
fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient(&operatorv1.StaticPodOperatorSpec{ | ||
OperatorSpec: operatorv1.OperatorSpec{ | ||
ManagementState: operatorv1.Managed, | ||
}, | ||
}, &operatorv1.StaticPodOperatorStatus{ | ||
OperatorStatus: operatorv1.OperatorStatus{Conditions: []operatorv1.OperatorCondition{}}, | ||
NodeStatuses: []operatorv1.NodeStatus{}, | ||
}, nil, nil) | ||
|
||
kubeInformers := v1helpers.NewKubeInformersForNamespaces(fakeKubeClient, "", "kube-system", | ||
operatorclient.TargetNamespace, operatorclient.OperatorNamespace, operatorclient.GlobalUserSpecifiedConfigNamespace) | ||
secretInformer := kubeInformers.InformersFor(operatorclient.TargetNamespace).Core().V1().Secrets() | ||
secretLister := secretInformer.Lister() | ||
secretClient := v1helpers.CachedSecretGetter(fakeKubeClient.CoreV1(), kubeInformers) | ||
|
||
recorder := events.NewInMemoryRecorder("etcd") | ||
// create openshift-config signers first, they will remain in openshift-config and are needed for the controller sync loop to function | ||
// TODO(thomas): once the rotation process is in place, we can remove that special case | ||
etcdSignerCert := tlshelpers.CreateBootstrapSignerCert(secretInformer, secretLister, secretClient, recorder) | ||
_, err := etcdSignerCert.EnsureSigningCertKeyPair(context.Background()) | ||
if err != nil { | ||
return nil, nil, fmt.Errorf("could not create etcd signer certificate: %w", err) | ||
} | ||
metricsSignerCert := tlshelpers.CreateBootstrapMetricsSignerCert(secretInformer, secretLister, secretClient, recorder) | ||
_, err = metricsSignerCert.EnsureSigningCertKeyPair(context.Background()) | ||
if err != nil { | ||
return nil, nil, fmt.Errorf("could not create etcd metrics signer certificate: %w", err) | ||
} | ||
|
||
controller := etcdcertsigner.NewEtcdCertSignerController( | ||
health.NewMultiAlivenessChecker(), | ||
fakeKubeClient, | ||
fakeOperatorClient, | ||
kubeInformers, | ||
recorder, | ||
&ceohelpers.AlwaysSafeQuorumChecker{}) | ||
|
||
stopChan := make(chan struct{}) | ||
defer close(stopChan) | ||
|
||
kubeInformers.Start(stopChan) | ||
for ns := range kubeInformers.Namespaces() { | ||
kubeInformers.InformersFor(ns).WaitForCacheSync(stopChan) | ||
} | ||
|
||
// createKeyMaterial returns the key material for a self-signed CA, its CA bundle, | ||
// and a client cert signed by the CA. | ||
func createKeyMaterial(signerName, clientName string) (*keyMaterial, error) { | ||
// CA | ||
caSubject := pkix.Name{CommonName: signerName, OrganizationalUnit: []string{"openshift"}} | ||
caExpiryDays := 10 * 365 // 10 years | ||
signerCAConfig, err := crypto.MakeSelfSignedCAConfigForSubject(caSubject, caExpiryDays) | ||
err = controller.Sync(context.Background(), factory.NewSyncContext("createCertSecrets", recorder)) | ||
if err != nil { | ||
return nil, err | ||
return nil, nil, fmt.Errorf("could not run etcd cert signer control loop: %w", err) | ||
} | ||
caCert, caKey, err := signerCAConfig.GetPEMBytes() | ||
|
||
// to finalize, we need to copy a few certificates around, which is handled by the resourcesynccontroller | ||
syncController, err := resourcesynccontroller.NewResourceSyncController(fakeOperatorClient, kubeInformers, fakeKubeClient, recorder) | ||
if err != nil { | ||
return nil, err | ||
return nil, nil, fmt.Errorf("could not create syncController: %w", err) | ||
} | ||
|
||
// Bundle | ||
caBundle, err := crypto.EncodeCertificates(signerCAConfig.Certs[0]) | ||
err = syncController.Sync(context.Background(), factory.NewSyncContext("createCertSecrets", recorder)) | ||
if err != nil { | ||
return nil, err | ||
return nil, nil, fmt.Errorf("could not run resourcesynccontroller loop: %w", err) | ||
} | ||
|
||
// Client cert | ||
ca := &crypto.CA{ | ||
Config: signerCAConfig, | ||
SerialGenerator: &crypto.RandomSerialGenerator{}, | ||
openshiftEtcdSecrets, err := fakeKubeClient.CoreV1().Secrets(operatorclient.TargetNamespace).List(context.Background(), metav1.ListOptions{}) | ||
if err != nil { | ||
return nil, nil, fmt.Errorf("error while listing fake client secrets in %s: %w", operatorclient.TargetNamespace, err) | ||
} | ||
clientUser := &user.DefaultInfo{ | ||
Name: clientName, | ||
UID: uuid.New().String(), | ||
Groups: []string{clientName}, | ||
|
||
openshiftConfigSecrets, err := fakeKubeClient.CoreV1().Secrets(operatorclient.GlobalUserSpecifiedConfigNamespace).List(context.Background(), metav1.ListOptions{}) | ||
if err != nil { | ||
return nil, nil, fmt.Errorf("error while listing fake client secrets in %s: %w", operatorclient.GlobalUserSpecifiedConfigNamespace, err) | ||
} | ||
|
||
var secrets []corev1.Secret | ||
// we have to add some extra information that the fake apiserver doesn't add for a valid k8s resource | ||
for _, s := range append(openshiftEtcdSecrets.Items, openshiftConfigSecrets.Items...) { | ||
s.APIVersion = "v1" | ||
s.Kind = "Secret" | ||
secrets = append(secrets, s) | ||
} | ||
clientCertDuration := time.Hour * 24 * 365 * 10 // 10 years | ||
clientCertConfig, err := ca.MakeClientCertificateForDuration(clientUser, clientCertDuration) | ||
|
||
openshiftEtcdBundles, err := fakeKubeClient.CoreV1().ConfigMaps(operatorclient.TargetNamespace).List(context.Background(), metav1.ListOptions{}) | ||
if err != nil { | ||
return nil, err | ||
return nil, nil, fmt.Errorf("error while listing fake client configmaps in %s: %w", operatorclient.TargetNamespace, err) | ||
} | ||
clientCert, clientKey, err := clientCertConfig.GetPEMBytes() | ||
|
||
openshiftConfigBundles, err := fakeKubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).List(context.Background(), metav1.ListOptions{}) | ||
if err != nil { | ||
return nil, err | ||
return nil, nil, fmt.Errorf("error while listing fake client configmaps in %s: %w", operatorclient.GlobalUserSpecifiedConfigNamespace, err) | ||
} | ||
|
||
var bundles []corev1.ConfigMap | ||
// we have to add some extra information that the fake apiserver doesn't add for a valid k8s resource | ||
for _, s := range append(openshiftEtcdBundles.Items, openshiftConfigBundles.Items...) { | ||
s.APIVersion = "v1" | ||
s.Kind = "ConfigMap" | ||
bundles = append(bundles, s) | ||
} | ||
|
||
return &keyMaterial{ | ||
caCert: caCert, | ||
caKey: caKey, | ||
caBundle: caBundle, | ||
clientCert: clientCert, | ||
clientKey: clientKey, | ||
}, nil | ||
return secrets, bundles, nil | ||
} | ||
|
||
func createBootstrapCertSecrets(hostName string, ipAddress string) ([]corev1.Secret, []corev1.ConfigMap, error) { | ||
return createCertSecrets([]*corev1.Node{ | ||
{ | ||
ObjectMeta: metav1.ObjectMeta{Name: hostName, Labels: map[string]string{"node-role.kubernetes.io/master": ""}}, | ||
Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: ipAddress}}}, | ||
}, | ||
}) | ||
} |
Oops, something went wrong.