Bug 1771986: encryption: backup/restore support + speedup + fixes

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/kubernetes-sigs/kube-storage-version-migrator v0.0.0-00010101000000-000000000000 // indirect
 	github.com/openshift/api v3.9.1-0.20191112184635-86def77f6f90+incompatible
 	github.com/openshift/client-go v0.0.0
-	github.com/openshift/library-go v0.0.0-20191114153449-9563a4e8e5fb
+	github.com/openshift/library-go v0.0.0-20191129090103-d5944b425412
 	github.com/pkg/profile v1.3.0 // indirect
 	github.com/prometheus/client_golang v1.1.0
 	github.com/spf13/cobra v0.0.5
@@ -47,7 +47,7 @@ replace (
 	github.com/kubernetes-sigs/kube-storage-version-migrator => github.com/openshift/kubernetes-kube-storage-version-migrator v0.0.3-0.20191031181212-bdca3bf7d454
 	github.com/openshift/api => github.com/openshift/api v3.9.1-0.20191114191201-4efd1a50a74e+incompatible
 	github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20191022152013-2823239d2298
-	github.com/openshift/library-go => github.com/openshift/library-go v0.0.0-20191121124438-7c776f7cc17a
+	github.com/openshift/library-go => github.com/openshift/library-go v0.0.0-20191129090103-d5944b425412
 	github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.2
 	k8s.io/api => k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783

go.sum

@@ -251,6 +251,16 @@ github.com/openshift/library-go v0.0.0-20191118102510-4e2c7112d252 h1:GY3oBSyQIk
 github.com/openshift/library-go v0.0.0-20191118102510-4e2c7112d252/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo=
 github.com/openshift/library-go v0.0.0-20191121124438-7c776f7cc17a h1:AUH7F2j7/piVDQ6Y7jGmIxTVur0H5pz4oGc50zMHrRk=
 github.com/openshift/library-go v0.0.0-20191121124438-7c776f7cc17a/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo=
+github.com/openshift/library-go v0.0.0-20191126125523-e708126e63db h1:+FyLID2Ha+XcgfCTfs3I7kEHJSgG6W3S30C2f0xcp5c=
+github.com/openshift/library-go v0.0.0-20191126125523-e708126e63db/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo=
+github.com/openshift/library-go v0.0.0-20191127111209-9a4c27565703 h1:P4Xn1AaqXY6yde8scvJsicb5pT8bnuUfFv3acsi2PAU=
+github.com/openshift/library-go v0.0.0-20191127111209-9a4c27565703/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo=
+github.com/openshift/library-go v0.0.0-20191127171859-96ab19e87694 h1:6GsmXEVM/vWDWPZdD3jq6JCeCXo9xpbmcIrbMnkK1eE=
+github.com/openshift/library-go v0.0.0-20191127171859-96ab19e87694/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo=
+github.com/openshift/library-go v0.0.0-20191128143151-1bcd3a3233c1 h1:nkOcd3XSVpQvfGqiaVlUWS6mKAnCpfQt90+jgnOH+Cg=
+github.com/openshift/library-go v0.0.0-20191128143151-1bcd3a3233c1/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo=
+github.com/openshift/library-go v0.0.0-20191129090103-d5944b425412 h1:UwqfISVEVPYoFbgzKrZtOPJWyp3iXgOJew1xvgx9DPE=
+github.com/openshift/library-go v0.0.0-20191129090103-d5944b425412/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

pkg/operator/starter.go

@@ -42,7 +42,10 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 	if err != nil {
 		return err
 	}
-	dynamicClient, err := dynamic.NewForConfig(ctx.KubeConfig)
+	migrationClientConfig := dynamic.ConfigFor(ctx.KubeConfig)
+	migrationClientConfig.Burst = 40
+	migrationClientConfig.QPS = 30
+	dynamicClientForMigration, err := dynamic.NewForConfig(migrationClientConfig)
 	if err != nil {
 		return err
 	}
@@ -154,7 +157,7 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 	if err != nil {
 		return err
 	}
-	migrator := migrators.NewInProcessMigrator(dynamicClient, kubeClient.Discovery())
+	migrator := migrators.NewInProcessMigrator(dynamicClientForMigration, kubeClient.Discovery())
 
 	encryptionControllers, err := encryption.NewControllers(
 		operatorclient.TargetNamespace,
@@ -188,21 +191,21 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 	// register config metrics
 	configmetrics.Register(configInformers)
 
-	kubeInformersForNamespaces.Start(ctx.Done())
-	configInformers.Start(ctx.Done())
-	dynamicInformers.Start(ctx.Done())
+	kubeInformersForNamespaces.Start(ctx.Ctx.Done())
+	configInformers.Start(ctx.Ctx.Done())
+	dynamicInformers.Start(ctx.Ctx.Done())
 
-	go staticPodControllers.Run(ctx.Done())
-	go resourceSyncController.Run(1, ctx.Done())
-	go targetConfigReconciler.Run(1, ctx.Done())
-	go configObserver.Run(1, ctx.Done())
-	go clusterOperatorStatus.Run(1, ctx.Done())
-	go certRotationController.Run(1, ctx.Done())
-	go encryptionControllers.Run(ctx.Done())
-	go featureUpgradeableController.Run(1, ctx.Done())
-	go certRotationTimeUpgradeableController.Run(1, ctx.Done())
+	go staticPodControllers.Run(ctx.Ctx, 1)
+	go resourceSyncController.Run(1, ctx.Ctx.Done())
+	go targetConfigReconciler.Run(1, ctx.Ctx.Done())
+	go configObserver.Run(1, ctx.Ctx.Done())
+	go clusterOperatorStatus.Run(1, ctx.Ctx.Done())
+	go certRotationController.Run(1, ctx.Ctx.Done())
+	go encryptionControllers.Run(ctx.Ctx.Done())
+	go featureUpgradeableController.Run(1, ctx.Ctx.Done())
+	go certRotationTimeUpgradeableController.Run(1, ctx.Ctx.Done())
 
-	<-ctx.Done()
+	<-ctx.Ctx.Done()
 	return fmt.Errorf("stopped")
 }
 

test/e2e-encryption-perf/encryption_perf_test.go

@@ -1,7 +1,153 @@
 package e2e_encryption_perf
 
-import "testing"
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
 
-func TestImplementMe(t *testing.T) {
-	t.Log("implement me")
+	"github.com/stretchr/testify/require"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+
+	operatorv1 "github.com/openshift/api/operator/v1"
+	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient"
+	operatorencryption "github.com/openshift/cluster-kube-apiserver-operator/test/library/encryption"
+	library "github.com/openshift/library-go/test/library/encryption"
+)
+
+const (
+	cmStatsKey      = "created configmaps"
+	secretsStatsKey = "created secrets"
+)
+
+func TestPerfEncryptionTypeAESCBC(tt *testing.T) {
+	operatorClient := operatorencryption.GetOperator(tt)
+	library.TestPerfEncryptionTypeAESCBC(tt, library.PerfScenario{
+		BasicScenario: library.BasicScenario{
+			Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
+			LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,
+			EncryptionConfigSecretName:      fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace),
+			EncryptionConfigSecretNamespace: operatorclient.GlobalMachineSpecifiedConfigNamespace,
+			OperatorNamespace:               operatorclient.OperatorNamespace,
+			TargetGRs:                       operatorencryption.DefaultTargetGRs,
+			AssertFunc:                      operatorencryption.AssertSecretsAndConfigMaps,
+		},
+		GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) {
+			apiServerOperator, err := operatorClient.Get("cluster", metav1.GetOptions{})
+			if err != nil {
+				return nil, err
+			}
+			return apiServerOperator.Status.Conditions, nil
+		},
+		AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) {
+			secretsCount, ok := statStore[secretsStatsKey]
+			if !ok {
+				err := errors.New("missing secrets count stats, can't continue the test")
+				require.NoError(t, err)
+			}
+			if secretsCount < 25000 {
+				err := fmt.Errorf("expected to create at least 25000 secrets but %d were created", secretsCount)
+				require.NoError(t, err)
+			}
+			t.Logf("Created %d secrets", secretsCount)
+
+			configMpasCount, ok := statStore[cmStatsKey]
+			if !ok {
+				err := errors.New("missing configmaps count stats, can't continue the test")
+				require.NoError(t, err)
+			}
+			if configMpasCount < 14000 {
+				err := fmt.Errorf("expected to create at least 14000 configmaps but %d were created", configMpasCount)
+				require.NoError(t, err)
+			}
+			t.Logf("Created %d configmaps", configMpasCount)
+
+		},
+		AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) {
+			t.Logf("migration took %v", migrationTime)
+			expectedMigrationTime := 28 * time.Minute
+			if migrationTime > expectedMigrationTime {
+				t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime)
+			}
+		},
+		DBLoaderWorkers: 3,
+		DBLoaderFunc: library.DBLoaderRepeat(1, true,
+			createNamespace,
+			waitUntilNamespaceActive,
+			library.DBLoaderRepeatParallel(5010, 50, false, createConfigMap, reportConfigMap),
+			library.DBLoaderRepeatParallel(9010, 50, false, createSecret, reportSecret)),
+	})
+}
+
+func createSecret(kubeClient kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:    namespace,
+			GenerateName: "encryption-",
+		},
+		Data: map[string][]byte{
+			"quote": []byte("I have no special talents. I am only passionately curious"),
+		},
+	}
+	_, err := kubeClient.CoreV1().Secrets(namespace).Create(secret)
+	return err
+}
+
+func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error {
+	statsCollector(secretsStatsKey)
+	return nil
+}
+
+func createConfigMap(kubeClient kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error {
+	cm := &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:    namespace,
+			GenerateName: "encryption-",
+		},
+		Data:       nil,
+		BinaryData: nil,
+	}
+
+	_, err := kubeClient.CoreV1().ConfigMaps(namespace).Create(cm)
+	return err
+}
+
+func reportConfigMap(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error {
+	statsCollector(cmStatsKey)
+	return nil
+}
+
+func createNamespace(kubeClient kubernetes.Interface, name string, errorCollector func(error), statsCollector func(string)) error {
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: "",
+		},
+		Status: corev1.NamespaceStatus{},
+	}
+	_, err := kubeClient.CoreV1().Namespaces().Create(ns)
+	return err
+}
+
+func waitUntilNamespaceActive(kubeClient kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error {
+	err := wait.Poll(10*time.Millisecond, 30*time.Second, func() (bool, error) {
+		ns, err := kubeClient.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if ns.Status.Phase == corev1.NamespaceActive {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		err = fmt.Errorf("failed waiting for ns to become ready, err %v", err)
+		errorCollector(err)
+	}
+	return err
 }