Implement KMS v2alpha1

- add feature gate - add encrypted object and run generated_files - generate protobuf for encrypted object and add unit tests - move parse endpoint to util and refactor - refactor interface and remove unused interceptor - add protobuf generate to update-generated-kms.sh - add integration tests - add defaulting for apiVersion in kmsConfiguration - handle v1/v2 and default in encryption config parsing - move metrics to own pkg and reuse for v2 - use Marshal and Unmarshal instead of serializer - add context for all service methods - check version and keyid for healthz Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>
openshift · Aug 3, 2022 · f19f3f4 · f19f3f4
1 parent 448e48b
commit f19f3f4
Show file tree

Hide file tree

Showing 40 changed files with 2,446 additions and 103 deletions.
diff --git a/hack/update-generated-kms-dockerized.sh b/hack/update-generated-kms-dockerized.sh
@@ -21,7 +21,9 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 KUBE_KMS_V1BETA1="${KUBE_ROOT}/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/"
 KUBE_KMS_V2ALPHA1="${KUBE_ROOT}/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v2alpha1/"
+KUBE_KMS_V2="${KUBE_ROOT}/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2alpha1/"
 
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
 kube::protoc::generate_proto "${KUBE_KMS_V1BETA1}"
 kube::protoc::generate_proto "${KUBE_KMS_V2ALPHA1}"
+kube::protoc::generate_proto "${KUBE_KMS_V2}"
diff --git a/hack/verify-generated-kms.sh b/hack/verify-generated-kms.sh
@@ -26,13 +26,15 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 ERROR="KMS gRPC is out of date. Please run hack/update-generated-kms.sh"
 KUBE_KMS_V1BETA1="${KUBE_ROOT}/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/"
 KUBE_KMS_V2ALPHA1="${KUBE_ROOT}/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v2alpha1/"
+KUBE_KMS_V2="${KUBE_ROOT}/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2alpha1/"
 
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
 kube::golang::setup_env
 
 function cleanup {
 	rm -rf "${KUBE_KMS_V1BETA1}/_tmp/"
 	rm -rf "${KUBE_KMS_V2ALPHA1}/_tmp/"
+	rm -rf "${KUBE_KMS_V2}/_tmp/"
 }
 
 trap cleanup EXIT
@@ -41,9 +43,13 @@ mkdir -p "${KUBE_KMS_V1BETA1}/_tmp"
 cp "${KUBE_KMS_V1BETA1}/api.pb.go" "${KUBE_KMS_V1BETA1}/_tmp/"
 mkdir -p "${KUBE_KMS_V2ALPHA1}/_tmp"
 cp "${KUBE_KMS_V2ALPHA1}/api.pb.go" "${KUBE_KMS_V2ALPHA1}/_tmp/"
+mkdir -p "${KUBE_KMS_V2}/_tmp"
+cp "${KUBE_KMS_V2}/api.pb.go" "${KUBE_KMS_V2}/_tmp/"
 
 KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-kms.sh"
 kube::protoc::diff "${KUBE_KMS_V1BETA1}/api.pb.go" "${KUBE_KMS_V1BETA1}/_tmp/api.pb.go" "${ERROR}"
 echo "Generated kms v1beta1 api is up to date."
 kube::protoc::diff "${KUBE_KMS_V2ALPHA1}/api.pb.go" "${KUBE_KMS_V2ALPHA1}/_tmp/api.pb.go" "${ERROR}"
 echo "Generated kms v2alpha1 api is up to date."
+kube::protoc::diff "${KUBE_KMS_V2}/api.pb.go" "${KUBE_KMS_V2}/_tmp/api.pb.go" "${ERROR}"
+echo "Generated kms v2 api is up to date."
diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/types.go b/staging/src/k8s.io/apiserver/pkg/apis/config/types.go
@@ -86,6 +86,9 @@ type IdentityConfiguration struct{}
 
 // KMSConfiguration contains the name, cache size and path to configuration file for a KMS based envelope transformer.
 type KMSConfiguration struct {
+	// apiVersion of KeyManagementService
+	// +optional
+	APIVersion string
 	// name is the name of the KMS plugin to be used.
 	Name string
 	// cachesize is the maximum number of secrets which are cached in memory. The default value is 1000.

diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/defaults.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/defaults.go
@@ -24,8 +24,9 @@ import (
 )
 
 var (
-	defaultTimeout         = &metav1.Duration{Duration: 3 * time.Second}
-	defaultCacheSize int32 = 1000
+	defaultTimeout          = &metav1.Duration{Duration: 3 * time.Second}
+	defaultCacheSize  int32 = 1000
+	defaultAPIVersion       = "v1"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -41,4 +42,8 @@ func SetDefaults_KMSConfiguration(obj *KMSConfiguration) {
 	if obj.CacheSize == nil {
 		obj.CacheSize = &defaultCacheSize
 	}
+
+	if obj.APIVersion == "" {
+		obj.APIVersion = defaultAPIVersion
+	}
 }
diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/defaults_test.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/defaults_test.go
@@ -34,12 +34,12 @@ func TestKMSProviderTimeoutDefaults(t *testing.T) {
 		{
 			desc: "timeout not supplied",
 			in:   &KMSConfiguration{},
-			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &defaultCacheSize},
+			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &defaultCacheSize, APIVersion: defaultAPIVersion},
 		},
 		{
 			desc: "timeout supplied",
 			in:   &KMSConfiguration{Timeout: &v1.Duration{Duration: 1 * time.Minute}},
-			want: &KMSConfiguration{Timeout: &v1.Duration{Duration: 1 * time.Minute}, CacheSize: &defaultCacheSize},
+			want: &KMSConfiguration{Timeout: &v1.Duration{Duration: 1 * time.Minute}, CacheSize: &defaultCacheSize, APIVersion: defaultAPIVersion},
 		},
 	}
 
@@ -67,17 +67,45 @@ func TestKMSProviderCacheDefaults(t *testing.T) {
 		{
 			desc: "cache size not supplied",
 			in:   &KMSConfiguration{},
-			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &defaultCacheSize},
+			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &defaultCacheSize, APIVersion: defaultAPIVersion},
 		},
 		{
 			desc: "cache of zero size supplied",
 			in:   &KMSConfiguration{CacheSize: &zero},
-			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &zero},
+			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &zero, APIVersion: defaultAPIVersion},
 		},
 		{
 			desc: "positive cache size supplied",
 			in:   &KMSConfiguration{CacheSize: &ten},
-			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &ten},
+			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &ten, APIVersion: defaultAPIVersion},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			SetDefaults_KMSConfiguration(tt.in)
+			if d := cmp.Diff(tt.want, tt.in); d != "" {
+				t.Fatalf("KMS Provider mismatch (-want +got):\n%s", d)
+			}
+		})
+	}
+}
+
+func TestKMSProviderAPIVersionDefaults(t *testing.T) {
+	testCases := []struct {
+		desc string
+		in   *KMSConfiguration
+		want *KMSConfiguration
+	}{
+		{
+			desc: "apiVersion not supplied",
+			in:   &KMSConfiguration{},
+			want: &KMSConfiguration{Timeout: defaultTimeout, CacheSize: &defaultCacheSize, APIVersion: defaultAPIVersion},
+		},
+		{
+			desc: "apiVersion supplied",
+			in:   &KMSConfiguration{Timeout: &v1.Duration{Duration: 1 * time.Minute}, APIVersion: "v2"},
+			want: &KMSConfiguration{Timeout: &v1.Duration{Duration: 1 * time.Minute}, CacheSize: &defaultCacheSize, APIVersion: "v2"},
 		},
 	}
 

diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go
@@ -86,6 +86,9 @@ type IdentityConfiguration struct{}
 
 // KMSConfiguration contains the name, cache size and path to configuration file for a KMS based envelope transformer.
 type KMSConfiguration struct {
+	// apiVersion of KeyManagementService
+	// +optional
+	APIVersion string `json:"apiVersion"`
 	// name is the name of the KMS plugin to be used.
 	Name string `json:"name"`
 	// cachesize is the maximum number of secrets which are cached in memory. The default value is 1000.

diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/zz_generated.conversion.go
diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation.go b/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation.go
@@ -21,22 +21,25 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net/url"
+	"strings"
 
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apiserver/pkg/apis/config"
 )
 
 const (
-	moreThanOneElementErr    = "more than one provider specified in a single element, should split into different list elements"
-	keyLenErrFmt             = "secret is not of the expected length, got %d, expected one of %v"
-	unsupportedSchemeErrFmt  = "unsupported scheme %q for KMS provider, only unix is supported"
-	atLeastOneRequiredErrFmt = "at least one %s is required"
-	invalidURLErrFmt         = "invalid endpoint for kms provider, error: parse %s: net/url: invalid control character in URL"
-	mandatoryFieldErrFmt     = "%s is a mandatory field for a %s"
-	base64EncodingErr        = "secrets must be base64 encoded"
-	zeroOrNegativeErrFmt     = "%s should be a positive value"
-	nonZeroErrFmt            = "%s should be a positive value, or negative to disable"
-	encryptionConfigNilErr   = "EncryptionConfiguration can't be nil"
+	moreThanOneElementErr          = "more than one provider specified in a single element, should split into different list elements"
+	keyLenErrFmt                   = "secret is not of the expected length, got %d, expected one of %v"
+	unsupportedSchemeErrFmt        = "unsupported scheme %q for KMS provider, only unix is supported"
+	unsupportedKMSAPIVersionErrFmt = "unsupported apiVersion %s for KMS provider, only v1 and v2 are supported"
+	atLeastOneRequiredErrFmt       = "at least one %s is required"
+	invalidURLErrFmt               = "invalid endpoint for kms provider, error: parse %s: net/url: invalid control character in URL"
+	mandatoryFieldErrFmt           = "%s is a mandatory field for a %s"
+	base64EncodingErr              = "secrets must be base64 encoded"
+	zeroOrNegativeErrFmt           = "%s should be a positive value"
+	nonZeroErrFmt                  = "%s should be a positive value, or negative to disable"
+	encryptionConfigNilErr         = "EncryptionConfiguration can't be nil"
+	invalidKMSConfigNameErrFmt     = "invalid KMS provider name %s, must not contain ':'"
 )
 
 var (
@@ -174,12 +177,12 @@ func validateKey(key config.Key, fieldPath *field.Path, expectedLen []int) field
 
 func validateKMSConfiguration(c *config.KMSConfiguration, fieldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
-	if c.Name == "" {
-		allErrs = append(allErrs, field.Required(fieldPath.Child("name"), fmt.Sprintf(mandatoryFieldErrFmt, "name", "provider")))
-	}
+
+	allErrs = append(allErrs, validateKMSConfigName(c, fieldPath.Child("name"))...)
 	allErrs = append(allErrs, validateKMSTimeout(c, fieldPath.Child("timeout"))...)
 	allErrs = append(allErrs, validateKMSEndpoint(c, fieldPath.Child("endpoint"))...)
 	allErrs = append(allErrs, validateKMSCacheSize(c, fieldPath.Child("cachesize"))...)
+	allErrs = append(allErrs, validateKMSAPIVersion(c, fieldPath.Child("apiVersion"))...)
 	return allErrs
 }
 
@@ -218,3 +221,25 @@ func validateKMSEndpoint(c *config.KMSConfiguration, fieldPath *field.Path) fiel
 
 	return allErrs
 }
+
+func validateKMSAPIVersion(c *config.KMSConfiguration, fieldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if c.APIVersion != "v1" && c.APIVersion != "v2" {
+		allErrs = append(allErrs, field.Invalid(fieldPath, c.APIVersion, fmt.Sprintf(unsupportedKMSAPIVersionErrFmt, "apiVersion")))
+	}
+
+	return allErrs
+}
+
+func validateKMSConfigName(c *config.KMSConfiguration, fieldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if c.Name == "" {
+		allErrs = append(allErrs, field.Required(fieldPath, fmt.Sprintf(mandatoryFieldErrFmt, "name", "provider")))
+	}
+
+	if c.APIVersion != "v1" && strings.Contains(c.Name, ":") {
+		allErrs = append(allErrs, field.Invalid(fieldPath, c.Name, fmt.Sprintf(invalidKMSConfigNameErrFmt, c.Name)))
+	}
+
+	return allErrs
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation_test.go b/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation_test.go
@@ -350,3 +350,79 @@ func TestKMSProviderCacheSize(t *testing.T) {
 		})
 	}
 }
+
+func TestKMSProviderAPIVersion(t *testing.T) {
+	apiVersionField := field.NewPath("Resource").Index(0).Child("Provider").Index(0).Child("KMS").Child("APIVersion")
+
+	testCases := []struct {
+		desc string
+		in   *config.KMSConfiguration
+		want field.ErrorList
+	}{
+		{
+			desc: "valid v1 api version",
+			in:   &config.KMSConfiguration{APIVersion: "v1"},
+			want: field.ErrorList{},
+		},
+		{
+			desc: "valid v2 api version",
+			in:   &config.KMSConfiguration{APIVersion: "v2"},
+			want: field.ErrorList{},
+		},
+		{
+			desc: "invalid api version",
+			in:   &config.KMSConfiguration{APIVersion: "v3"},
+			want: field.ErrorList{
+				field.Invalid(apiVersionField, "v3", fmt.Sprintf(unsupportedKMSAPIVersionErrFmt, "apiVersion")),
+			},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			got := validateKMSAPIVersion(tt.in, apiVersionField)
+			if d := cmp.Diff(tt.want, got); d != "" {
+				t.Fatalf("KMS Provider validation mismatch (-want +got):\n%s", d)
+			}
+		})
+	}
+}
+
+func TestKMSProviderName(t *testing.T) {
+	nameField := field.NewPath("Resource").Index(0).Child("Provider").Index(0).Child("KMS").Child("name")
+
+	testCases := []struct {
+		desc string
+		in   *config.KMSConfiguration
+		want field.ErrorList
+	}{
+		{
+			desc: "valid name",
+			in:   &config.KMSConfiguration{Name: "foo"},
+			want: field.ErrorList{},
+		},
+		{
+			desc: "empty name",
+			in:   &config.KMSConfiguration{},
+			want: field.ErrorList{
+				field.Required(nameField, fmt.Sprintf(mandatoryFieldErrFmt, "name", "provider")),
+			},
+		},
+		{
+			desc: "invalid name with :",
+			in:   &config.KMSConfiguration{Name: "foo:bar"},
+			want: field.ErrorList{
+				field.Invalid(nameField, "foo:bar", fmt.Sprintf(invalidKMSConfigNameErrFmt, "foo:bar")),
+			},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			got := validateKMSConfigName(tt.in, nameField)
+			if d := cmp.Diff(tt.want, got); d != "" {
+				t.Fatalf("KMS Provider validation mismatch (-want +got):\n%s", d)
+			}
+		})
+	}
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go
@@ -107,6 +107,13 @@ const (
 	// Allows for updating watchcache resource version with progress notify events.
 	EfficientWatchResumption featuregate.Feature = "EfficientWatchResumption"
 
+	// owner: @aramase
+	// kep: http://kep.k8s.io/3299
+	// alpha: v1.25
+	//
+	// Enables KMS v2 API for encryption at rest.
+	KMSv2 featuregate.Feature = "KMSv2"
+
 	// owner: @jiahuif
 	// kep: http://kep.k8s.io/2887
 	// alpha: v1.23
@@ -205,6 +212,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	EfficientWatchResumption: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 
+	KMSv2: {Default: false, PreRelease: featuregate.Alpha},
+
 	OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta},
 
 	OpenAPIV3: {Default: true, PreRelease: featuregate.Beta},